diff --git a/Makefile b/Makefile index 74ca63fc803..1c6a9956120 100644 --- a/Makefile +++ b/Makefile @@ -326,3 +326,13 @@ smoketest/all/cleanup: echo "-> Cleanup $${test_dir} smoke tests..."; \ $(MAKE) smoketest/cleanup TEST_DIR=$${test_dir}; \ done + +.PHONY: api-docs +api-docs: ## Generate bundled OpenAPI documents + @npx @redocly/cli bundle "docs/spec/openapi/apm-openapi.yaml" --ext yaml --output "docs/spec/openapi/bundled.yaml" + @npx @redocly/cli bundle "docs/spec/openapi/apm-openapi.yaml" --ext json --output "docs/spec/openapi/bundled.json" + +.PHONY: api-docs-lint +api-docs-lint: ## Run spectral API docs linter + @npx @stoplight/spectral-cli lint "docs/spec/openapi/bundled.yaml" --ruleset "docs/spec/openapi/.spectral.yaml" + diff --git a/NOTICE.txt b/NOTICE.txt index 075e1dc2d58..4bf55700323 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -362,11 +362,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/apm-data -Version: v1.8.1 +Version: v1.8.4 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.8.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.8.4/LICENSE: Apache License Version 2.0, January 2004 @@ -573,11 +573,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/apm-data@v1.8.1 -------------------------------------------------------------------------------- Dependency : github.com/elastic/beats/v7 -Version: v7.0.0-alpha2.0.20240705174358-6e97683404dd +Version: v7.0.0-alpha2.0.20240712104211-aa5ec880ee71 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20240705174358-6e97683404dd/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/beats/v7@v7.0.0-alpha2.0.20240712104211-aa5ec880ee71/LICENSE.txt: Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Elastic License. Outside of @@ -6249,11 +6249,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.26.0 +Version: v0.27.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.26.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.27.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -6323,11 +6323,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.21.0 +Version: v0.22.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.21.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.22.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -9834,11 +9834,11 @@ limitations under the License. -------------------------------------------------------------------------------- Dependency : github.com/elastic/opentelemetry-lib -Version: v0.7.0 +Version: v0.7.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-lib@v0.7.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/opentelemetry-lib@v0.7.2/LICENSE: Apache License Version 2.0, January 2004 @@ -16934,11 +16934,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.24.0 +Version: v0.25.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.24.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.25.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -17008,11 +17008,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.21.0 +Version: v0.22.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.21.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.22.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/changelogs/8.14.asciidoc b/changelogs/8.14.asciidoc index fe7e019649d..cdb469aaa11 100644 --- a/changelogs/8.14.asciidoc +++ b/changelogs/8.14.asciidoc @@ -26,7 +26,8 @@ https://github.com/elastic/apm-server/compare/v8.14.1\...v8.14.2[View commits] No significant changes. -=== Added +[float] +==== Added - Optimize memory usage and disk read IO for Tail Based Sampling {pull}13464[13464] [float] diff --git a/changelogs/8.15.asciidoc b/changelogs/8.15.asciidoc index f3f4e495947..1b205224465 100644 --- a/changelogs/8.15.asciidoc +++ b/changelogs/8.15.asciidoc @@ -6,7 +6,7 @@ [[apm-release-notes-8.15.0]] === APM version 8.15.0 -https://github.com/elastic/apm-server/compare/v8.14.2\...v8.15.0[View commits] +https://github.com/elastic/apm-server/compare/v8.14.3\...v8.15.0[View commits] [float] ==== Bug fixes @@ -33,6 +33,6 @@ https://github.com/elastic/apm-server/compare/v8.14.2\...v8.15.0[View commits] - Upgraded bundled APM Java agent attacher CLI to version 1.50.0 {pull}13326[13326] - Enable Kibana curated UIs to work with hostmetrics from OpenTelemetry's https://pkg.go.dev/go.opentelemetry.io/collector/receiver/hostmetricsreceiver[hostmetricsreceiver] {pull}13196[13196] - Add require data stream to bulk index requests {pull}13398[13398] -- Support self-instrumentation when in managed mode by getting tracing configs via reloader {pull}13514[13514] +- Support self-instrumentation when in managed mode by getting tracing configs via reloader {pull}13514[13514] {pull}13653[13653] - Add mapping for OpenTelemetry attribute `messaging.destination.name` to derive `service.target` correctly {pull}13472[13472] - APM Server now automatically retries document-level 429s from Elasticsearch to avoid dropping data. `output.elasticsearch.max_retries` now controls both request-level and document-level retries, and defaults to `3`. {pull}13620[13620] diff --git a/docker-compose.yml b/docker-compose.yml index 17cb5c07474..4d05496a341 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ x-logging: &default-logging max-size: "1g" services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-474afc1d-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-9cf398b8-SNAPSHOT ports: - 9200:9200 healthcheck: @@ -41,7 +41,7 @@ services: logging: *default-logging kibana: - image: docker.elastic.co/kibana/kibana:8.15.0-474afc1d-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.15.0-9cf398b8-SNAPSHOT ports: - 5601:5601 healthcheck: @@ -60,7 +60,7 @@ services: logging: *default-logging metricbeat: - image: docker.elastic.co/beats/metricbeat:8.15.0-474afc1d-SNAPSHOT + image: docker.elastic.co/beats/metricbeat:8.15.0-9cf398b8-SNAPSHOT environment: ELASTICSEARCH_HOSTS: '["http://elasticsearch:9200"]' ELASTICSEARCH_USERNAME: "${KIBANA_ES_USER:-admin}" diff --git a/docs/README.md b/docs/README.md index 67ed86a4049..b810f90f253 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,10 +9,10 @@ > * **Sample data sets** that are injected into the docs are in the [`docs/data/`](/docs/data/) directory. > * **Specifications** that are injected into the docs are in the [`docs/spec/`](/docs/spec/) directory. -To generate the bundled files in the [`docs/spec/openapi`](/docs/spec/openapi) directory, use [redocly bundle](https://redocly.com/docs/cli/commands/bundle/). For example: +To generate the bundled files in the [`docs/spec/openapi`](/docs/spec/openapi) directory, use the following make commands: ``` -npx @redocly/cli bundle apm-openapi.yaml --output bundled.yaml --ext yaml -npx @redocly/cli bundle apm-openapi.yaml --output bundled.json --ext json +make api-docs +make api-docs-lint ``` diff --git a/docs/spec/openapi/.spectral.yaml b/docs/spec/openapi/.spectral.yaml new file mode 100644 index 00000000000..57eb29c3e1d --- /dev/null +++ b/docs/spec/openapi/.spectral.yaml @@ -0,0 +1,104 @@ +extends: ["spectral:oas"] +rules: +# Built-in rules + # Descriptions + oas3-parameter-description: warn + oas2-parameter-description: warn + tag-description: info + # Document info + info-contact: info + info-description: warn + info-license: warn + # Examples + oas3-valid-media-example: false + oas3-valid-schema-example: false + oas2-valid-media-example: false + # Operations + operation-operationId: false + operation-operationId-unique: false + operation-operationId-valid-in-url: false + operation-tag-defined: warn + operation-tags: warn + # Responses + operation-success-response: warn + # Schema + oas3-schema: error + oas2-schema: error + array-items: false + # Tags + openapi-tags: warn + openapi-tags-alphabetical: info + # Turn off some built-in rules + operation-description: false + operation-singular-tag: false +# Custom rules + # Descriptions + avoid-problematic-words: + description: Ban certain words from descriptions + message: "Use appropriate replacements for problematic terms" + severity: warn + given: "$..*.description" + then: + function: pattern + functionOptions: + notMatch: /(blacklist|whitelist|execute|kill)/i + # Examples + operation-success-examples: + formats: ["oas3_1"] + description: Response code 200 should have at least one example. + message: "Each response body should have a realistic example. It must not contain any sensitive or confidential data." + severity: info + given: $.paths[*].[*].responses.[200].content.[application/json] + then: + field: examples + function: defined + # Extensions + internal-extension: + description: Operations should not have x-internal extension. + message: "Do not publish x-internal operations" + severity: error + given: $.paths[*].[get,put,post,delete,options,head,patch,trace] + then: + field: x-internal + function: undefined + # Operations + operation-summary: + description: Operations should have summaries. + message: "Each operation should have a summary" + severity: error + recommended: true + given: $.paths[*].[get,put,post,delete,options,head,patch,trace] + then: + field: summary + function: defined + operation-summary-length: + description: Operation summary should be between 5 and 45 characters + given: "$.paths[*].[get,put,post,delete,options,head,patch,trace]" + then: + field: summary + function: length + functionOptions: + max: 45 + min: 5 + severity: warn + simple-verbs-in-summary: + given: + - "$.paths[*][*].summary" + then: + function: pattern + functionOptions: + notMatch: "Retrieve|Return|List *" + severity: warn + description: Summaries should use common verbs. + message: "Summaries should use common verbs like Get, Update, Delete whenever possible" + # NOTE: This one hiccups on acronyms so perhaps too noisy + # docs-operation-summary-sentence-case: + # description: Operation summary should be sentence cased + # given: "$.paths[*].[get,put,post,delete,options,head,patch,trace]" + # then: + # field: summary + # function: pattern + # functionOptions: + # match: /^[A-Z]+[^A-Z]+$/ + # severity: warn + diff --git a/docs/spec/openapi/apm-openapi.yaml b/docs/spec/openapi/apm-openapi.yaml index d4a9a84db3f..34a86f7876a 100644 --- a/docs/spec/openapi/apm-openapi.yaml +++ b/docs/spec/openapi/apm-openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: - title: APM Server - description: OpenAPI schema for APM Server APIs + title: Observability Intake APIs + description: OpenAPI schema for Observability Intake APIs version: '0.1' license: name: Elastic License 2.0 diff --git a/docs/spec/openapi/bundled.json b/docs/spec/openapi/bundled.json index db24a1f41ab..ed6b85683fb 100644 --- a/docs/spec/openapi/bundled.json +++ b/docs/spec/openapi/bundled.json @@ -1,8 +1,8 @@ { "openapi": "3.1.0", "info": { - "title": "APM Server", - "description": "OpenAPI schema for APM Server APIs", + "title": "Observability Intake APIs", + "description": "OpenAPI schema for Observability Intake APIs", "version": "0.1", "license": { "name": "Elastic License 2.0", diff --git a/go.mod b/go.mod index 4c28b5fdca4..e60e828cc75 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dustin/go-humanize v1.0.1 github.com/elastic/apm-aggregation v1.0.1 - github.com/elastic/apm-data v1.8.1 - github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240705174358-6e97683404dd + github.com/elastic/apm-data v1.8.4 + github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240712104211-aa5ec880ee71 github.com/elastic/elastic-agent-client/v7 v7.13.0 github.com/elastic/elastic-agent-libs v0.9.13 github.com/elastic/elastic-agent-system-metrics v0.10.3 @@ -50,9 +50,9 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.28.0 go.uber.org/automaxprocs v1.5.3 go.uber.org/zap v1.27.0 - golang.org/x/net v0.26.0 + golang.org/x/net v0.27.0 golang.org/x/sync v0.7.0 - golang.org/x/term v0.21.0 + golang.org/x/term v0.22.0 golang.org/x/time v0.5.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 @@ -90,7 +90,7 @@ require ( github.com/elastic/go-structform v0.0.10 // indirect github.com/elastic/go-windows v1.0.1 // indirect github.com/elastic/gosigar v0.14.3 // indirect - github.com/elastic/opentelemetry-lib v0.7.0 // indirect + github.com/elastic/opentelemetry-lib v0.7.2 // indirect github.com/elastic/pkcs8 v1.0.0 // indirect github.com/fatih/color v1.16.0 // indirect github.com/frankban/quicktest v1.14.0 // indirect @@ -150,12 +150,12 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.25.0 // indirect golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect - golang.org/x/mod v0.18.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.22.0 // indirect + golang.org/x/tools v0.23.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect diff --git a/go.sum b/go.sum index 251644c7d38..9f5c02436f8 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,10 @@ github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2 h1:FDif4R1+UUR+00q6wquyX90K7A8dN+R5E8GEadoP7sU= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2/go.mod h1:aiYBYui4BJ/BJCAIKs92XiPyQfTaBWqvHujDwKb6CBU= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.7.0 h1:rTfKOCZGy5ViVrlA74ZPE99a+SgoEE2K/yg3RyW9dFA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.7.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 h1:1nGuui+4POelzDwI7RG56yfQJHCnKvwfMoU7VsEp+Zg= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0/go.mod h1:99EvauvlcJ1U06amZiksfYz/3aFGyIhWGHVyiZXtBAI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1 h1:Xy/qV1DyOhhqsU/z0PyFMJfYCxnzna+vBEUtFW0ksQo= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.1/go.mod h1:oib6iWdC+sILvNUoJbbBn3xv7TXow7mEp/WRcsYvmow= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0 h1:AAIdAyPkFff6XTct2lQCxOWN/+LnA41S7kIkzKaMbyE= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.6.0/go.mod h1:noQIdW75SiQFB3mSFJBr4iRRH83S9skaFiBv4C0uEs0= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= @@ -125,10 +125,10 @@ github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/elastic/apm-aggregation v1.0.1 h1:t4VKR6axu620Y9CtQE9kmZZcx+3BlfnqGywnvgWFcmA= github.com/elastic/apm-aggregation v1.0.1/go.mod h1:hmRuhkV01qMHk/Eu3xe9W/cNtBcRVf0DpFf9USVAE2Y= -github.com/elastic/apm-data v1.8.1 h1:q21kFswSUJsQ6SQBjpnj4yv1wi2SEFozajSOkH0rfB0= -github.com/elastic/apm-data v1.8.1/go.mod h1:NE5PyUTnIADrV8yIRa5hRwOxlRI1hbnEVGb470veU8s= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240705174358-6e97683404dd h1:Yb4ki43RUdw0hdkkexx/kRRUCFjCvAUHw9ky1wW+ZZw= -github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240705174358-6e97683404dd/go.mod h1:2I8Zlyzdl1oYau3ojZP2VmzLOQkLVfpKMXzwLUbwd/k= +github.com/elastic/apm-data v1.8.4 h1:6IVC1zKejhhRt9+akgBR4jI17SCt/FRQIx5r0U1EMtc= +github.com/elastic/apm-data v1.8.4/go.mod h1:TOlqsDVhESSpg4VxGZBlld7C1JeaX9H29opgzrK5UM0= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240712104211-aa5ec880ee71 h1:eKYSv3CuPcXuIR2RnrlhlbPmdBbyGNd7mVIt4e4vtcU= +github.com/elastic/beats/v7 v7.0.0-alpha2.0.20240712104211-aa5ec880ee71/go.mod h1:cN5agJpe7JuQ1OF6uET1IT5bL2U1OHfcnhIuapoRSpo= github.com/elastic/elastic-agent-autodiscover v0.7.0 h1:FCrHXh5AZGrPlpAx8kBu/s/guw9d/EXt+GKlFCnrgsc= github.com/elastic/elastic-agent-autodiscover v0.7.0/go.mod h1:zLf0SDdQXisVZxzXPxKXdj3Fa+H4bsu4HHbTEQImDz8= github.com/elastic/elastic-agent-client/v7 v7.13.0 h1:ENCfV5XIMmjWo9/0J7t//5N7xgm43Ktg0SyIomupRcA= @@ -157,8 +157,8 @@ github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUt github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss= github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= -github.com/elastic/opentelemetry-lib v0.7.0 h1:sarbwqZJ9sx+B1zUl6qmf93/j7ggDAidc5CYKsIsQQk= -github.com/elastic/opentelemetry-lib v0.7.0/go.mod h1:/kKvHbJLVo/NcKMPHI8/RZKL64fushmnRUzn+arQpjg= +github.com/elastic/opentelemetry-lib v0.7.2 h1:3M+2Ks62qzc/XUxw32+lYniXSyhg6A58X0RD+H8xBQc= +github.com/elastic/opentelemetry-lib v0.7.2/go.mod h1:/kKvHbJLVo/NcKMPHI8/RZKL64fushmnRUzn+arQpjg= github.com/elastic/pkcs8 v1.0.0 h1:HhitlUKxhN288kcNcYkjW6/ouvuwJWd9ioxpjnD9jVA= github.com/elastic/pkcs8 v1.0.0/go.mod h1:ipsZToJfq1MxclVTwpG7U/bgeDtf+0HkUiOxebk95+0= github.com/elastic/sarama v1.19.1-0.20210823122811-11c3ef800752 h1:5/RUNg7rkIvayjPhAIoI3v8p45NfWcfWs5DZSElycis= @@ -478,8 +478,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -488,8 +488,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -504,8 +504,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -540,16 +540,16 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -570,8 +570,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/beater/beater.go b/internal/beater/beater.go index 0c715af5223..3c94c336086 100644 --- a/internal/beater/beater.go +++ b/internal/beater/beater.go @@ -292,7 +292,7 @@ func (s *Runner) Run(ctx context.Context) error { } } - instrumentation, err := instrumentation.New(s.rawConfig, "apm-server", version.Version) + instrumentation, err := newInstrumentation(s.rawConfig) if err != nil { return err } @@ -537,6 +537,57 @@ func (s *Runner) Run(ctx context.Context) error { return errors.Join(result, closeErr) } +// newInstrumentation is a thin wrapper around libbeat instrumentation that +// sets missing tracer configuration from elastic agent. +func newInstrumentation(rawConfig *agentconfig.C) (instrumentation.Instrumentation, error) { + // This config struct contains missing fields from elastic agent APMConfig + // https://github.com/elastic/elastic-agent/blob/main/internal/pkg/core/monitoring/config/config.go#L127 + // that are not directly handled by libbeat instrumentation below. + // + // Note that original config keys were additionally marshalled by + // https://github.com/elastic/elastic-agent/blob/main/pkg/component/runtime/apm_config_mapper.go#L18 + // that's why the keys are different from the original APMConfig struct. + var apmCfg struct { + GlobalLabels string `config:"globallabels"` + TLS struct { + SkipVerify bool `config:"skipverify"` + ServerCertificate string `config:"servercert"` + ServerCA string `config:"serverca"` + } `config:"tls"` + } + cfg, err := rawConfig.Child("instrumentation", -1) + if err != nil { + // Fallback to instrumentation.New if the configs are not present. + return instrumentation.New(rawConfig, "apm-server", version.Version) + } + if err := cfg.Unpack(&apmCfg); err != nil { + return nil, err + } + const ( + envVerifyServerCert = "ELASTIC_APM_VERIFY_SERVER_CERT" + envServerCert = "ELASTIC_APM_SERVER_CERT" + envCACert = "ELASTIC_APM_SERVER_CA_CERT_FILE" + envGlobalLabels = "ELASTIC_APM_GLOBAL_LABELS" + ) + if apmCfg.TLS.SkipVerify { + os.Setenv(envVerifyServerCert, "false") + defer os.Unsetenv(envVerifyServerCert) + } + if apmCfg.TLS.ServerCertificate != "" { + os.Setenv(envServerCert, apmCfg.TLS.ServerCertificate) + defer os.Unsetenv(envServerCert) + } + if apmCfg.TLS.ServerCA != "" { + os.Setenv(envCACert, apmCfg.TLS.ServerCA) + defer os.Unsetenv(envCACert) + } + if len(apmCfg.GlobalLabels) > 0 { + os.Setenv(envGlobalLabels, apmCfg.GlobalLabels) + defer os.Unsetenv(envGlobalLabels) + } + return instrumentation.New(rawConfig, "apm-server", version.Version) +} + func maxConcurrentDecoders(memLimitGB float64) uint { // Allow 128 concurrent decoders for each 1GB memory, limited to at most 2048. const max = 2048 diff --git a/internal/beater/beater_test.go b/internal/beater/beater_test.go index b035fe9c8cf..c7aa453c163 100644 --- a/internal/beater/beater_test.go +++ b/internal/beater/beater_test.go @@ -18,11 +18,15 @@ package beater import ( + "compress/zlib" "context" "encoding/json" + "encoding/pem" "fmt" "net/http" "net/http/httptest" + "os" + "path/filepath" "testing" "time" @@ -231,3 +235,43 @@ func TestRunnerNewDocappenderConfig(t *testing.T) { }) } } + +func TestNewInstrumentation(t *testing.T) { + labels := make(chan map[string]string, 1) + defer close(labels) + s := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/intake/v2/events" { + var b struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` + } + zr, _ := zlib.NewReader(r.Body) + _ = json.NewDecoder(zr).Decode(&b) + labels <- b.Metadata.Labels + } + w.WriteHeader(http.StatusOK) + })) + defer s.Close() + certPath := filepath.Join(t.TempDir(), "cert.pem") + f, err := os.Create(certPath) + assert.NoError(t, err) + err = pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: s.Certificate().Raw}) + assert.NoError(t, err) + cfg := agentconfig.MustNewConfigFrom(map[string]interface{}{ + "instrumentation": map[string]interface{}{ + "enabled": true, + "hosts": []string{s.URL}, + "tls": map[string]interface{}{ + "servercert": certPath, + }, + "globallabels": "k1=val,k2=new val", + }, + }) + i, err := newInstrumentation(cfg) + require.NoError(t, err) + tracer := i.Tracer() + tracer.StartTransaction("name", "type").End() + tracer.Flush(nil) + assert.Equal(t, map[string]string{"k1": "val", "k2": "new val"}, <-labels) +} diff --git a/testing/infra/k8s/base/stack/apm-server.yaml b/testing/infra/k8s/base/stack/apm-server.yaml index 2b01b26bd08..e58e16ce92e 100644 --- a/testing/infra/k8s/base/stack/apm-server.yaml +++ b/testing/infra/k8s/base/stack/apm-server.yaml @@ -3,7 +3,7 @@ kind: ApmServer metadata: name: apm-server spec: - version: 8.15.0-474afc1d-SNAPSHOT + version: 8.15.0-9cf398b8-SNAPSHOT count: 1 http: tls: diff --git a/testing/infra/k8s/base/stack/elasticsearch.yaml b/testing/infra/k8s/base/stack/elasticsearch.yaml index f58e63c2f54..478e8af49ce 100644 --- a/testing/infra/k8s/base/stack/elasticsearch.yaml +++ b/testing/infra/k8s/base/stack/elasticsearch.yaml @@ -3,7 +3,7 @@ kind: Elasticsearch metadata: name: elasticsearch spec: - version: 8.15.0-474afc1d-SNAPSHOT + version: 8.15.0-9cf398b8-SNAPSHOT auth: fileRealm: - secretName: elasticsearch-admin diff --git a/testing/infra/k8s/base/stack/kibana.yaml b/testing/infra/k8s/base/stack/kibana.yaml index 4f6f18f23aa..1b57f53b5a6 100644 --- a/testing/infra/k8s/base/stack/kibana.yaml +++ b/testing/infra/k8s/base/stack/kibana.yaml @@ -3,7 +3,7 @@ kind: Kibana metadata: name: kibana spec: - version: 8.15.0-474afc1d-SNAPSHOT + version: 8.15.0-9cf398b8-SNAPSHOT count: 1 elasticsearchRef: name: elasticsearch