Remove apmpackage (#12411)

Remove apmpackage from the apm-server repo, as we are moving the
integration package to elastic/integrations.

A couple of things have led to this:
 - We are currently making minimal changes to the integration
   package.
 - We are going to switch to an Elasticsearch plugin (apm-data)
   for installing index templates and ingest pipelines, and use
   dynamic mapping more heavily to make the apm-server code and
   templates more loosely coupled.

The integration package will still exist, but will be "input-only",
meaning that it will not define data streams; it will only be
used to define the apm-server configuration/policy vars. To align
with other packages, we will move the integration package to the
elastic/integrations repo. We're doing it now, rather than waiting
for the switchover to the ES plugin, to make progress on phasing
out some legacy CI infrastructure.

What does this mean for development?

 - We won't be able to atomically make changes to the data stream
   templates and pipelines along with code. This will be the case
   when we are switched over to the plugin, we're just bringing
   that forward.
 - System tests, and automated benchmarks and Elastic Cloud
   deployments will use the integration package bundled with
   Kibana, rather than building and installing the package by
   upload.

.ci/packaging.groovy

@@ -77,9 +77,10 @@ pipeline {
                 // Build a preview package which includes the Git commit timestamp, and upload it to package storage.
                 // Note, we intentionally do not sign or upload the "release" package, as it does not include a timestamp,
                 // and will break package storage's immutability requirement.
-                sh(label: 'make build-package-snapshot', script: 'make build-package-snapshot')
-                packageStoragePublish('build/packages', 'apm-*-preview-*.zip')
-                archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/packages/*.zip')
+                // TODO remove the pipeline
+                //sh(label: 'make build-package-snapshot', script: 'make build-package-snapshot')
+                //packageStoragePublish('build/packages', 'apm-*-preview-*.zip')
+                //archiveArtifacts(allowEmptyArchive: false, artifacts: 'build/packages/*.zip')
               }
             }
           }

.github/pull_request_template.md

@@ -28,7 +28,6 @@ Once the PR is ready for review there should be no unticked boxes.
 -->
 
 - [ ] Update [CHANGELOG.asciidoc](https://github.com/elastic/apm-server/blob/main/CHANGELOG.asciidoc)
-- [ ] Update [package changelog.yml](https://github.com/elastic/apm-server/blob/main/apmpackage/apm/changelog.yml) (only if changes to `apmpackage` have been made)
 - [ ] Documentation has been updated
 
 For functional changes, consider:

CONTRIBUTING.md

@@ -32,7 +32,6 @@ Development instructions are available in the project [readme](README.md#apm-ser
 
 When submitting changes, add an entry to the [changelog](changelogs/head.asciidoc),
 describing the enhancement or fix that your PR is adding.
-If you also make code changes to the [apmpackage](apmpackage), you also need to update the apmpackage specific changelog.
 
 Please read our [pull request template](.github/pull_request_template.md), which includes the information we care about the most when submitting new changes.
 

Makefile

@@ -13,14 +13,6 @@ GOTESTFLAGS?=-v
 # Prevent unintended modifications of go.[mod|sum]
 GOMODFLAG?=-mod=readonly
 
-# Define the github.com/elastic/ecs ref used for the integration package for
-# resolving ECS fields. The top-level "value" file in the repo will be used
-# for populating the `ecs.version` field added to documents.
-#
-# TODO(axw) when the device.* fields we're using have been added to a release,
-# we should pin to a release tag here.
-ECS_REF?=266cf6aa62e46bff1965342a61191ce5ffe1b0d7
-
 PYTHON_ENV?=.
 PYTHON_VENV_DIR:=$(PYTHON_ENV)/build/ve/$(shell $(GO) env GOOS)
 PYTHON_BIN:=$(PYTHON_VENV_DIR)/bin
@@ -109,7 +101,7 @@ check-full: update check staticcheck check-docker-compose
 check-approvals:
 	@$(GO) run -modfile=tools/go.mod github.com/elastic/apm-tools/cmd/check-approvals
 
-check: check-fmt check-headers check-git-diff check-package
+check: check-fmt check-headers check-git-diff
 
 .PHONY: check-git-diff
 check-git-diff:
@@ -128,7 +120,7 @@ bench:
 tidy:
 	@go mod tidy # make sure go.sum is complete
 
-update: tidy go-generate add-headers build-package notice apm-server.docker.yml docs/spec
+update: tidy go-generate add-headers notice apm-server.docker.yml docs/spec
 
 apm-server.docker.yml: apm-server.yml
 	sed -e 's/127.0.0.1:8200/0.0.0.0:8200/' -e 's/localhost:9200/elasticsearch:9200/' $< > $@
@@ -156,29 +148,6 @@ get-version:
 update-go-version:
 	$(GITROOT)/script/update_go_version.sh
 
-##############################################################################
-# Integration package generation.
-##############################################################################
-
-ECS_REF_FILE:=build/ecs/$(ECS_REF).txt
-$(ECS_REF_FILE):
-	@mkdir -p $(@D)
-	@curl --fail --silent -o $@ https://raw.githubusercontent.com/elastic/ecs/$(ECS_REF)/version
-
-build-package: build/packages/apm-$(APM_SERVER_VERSION).zip
-build-package-snapshot: build/packages/apm-$(APM_SERVER_VERSION)-preview-$(GITCOMMITTIMESTAMPUNIX).zip
-build/packages/apm-$(APM_SERVER_VERSION).zip: build/apmpackage
-build/packages/apm-$(APM_SERVER_VERSION)-preview-$(GITCOMMITTIMESTAMPUNIX).zip: build/apmpackage-snapshot
-build/packages/apm-%.zip: $(ELASTICPACKAGE)
-	cd $(filter build/apmpackage%, $^) && $(ELASTICPACKAGE) build
-
-.PHONY: build/apmpackage build/apmpackage-snapshot
-build/apmpackage: PACKAGE_VERSION=$(APM_SERVER_VERSION)
-build/apmpackage-snapshot: PACKAGE_VERSION=$(APM_SERVER_VERSION)-preview-$(GITCOMMITTIMESTAMPUNIX)
-build/apmpackage build/apmpackage-snapshot: $(ECS_REF_FILE)
-	@mkdir -p $(@D) && rm -fr $@
-	@$(GO) run ./apmpackage/cmd/genpackage -o $@ -version=$(PACKAGE_VERSION) -ecs=$$(cat $(ECS_REF_FILE)) -ecsref=git@$(ECS_REF)
-
 ##############################################################################
 # Documentation.
 ##############################################################################
@@ -250,10 +219,6 @@ endif
 check-docker-compose:
 	./script/check_docker_compose.sh $(BEATS_VERSION)
 
-check-package: build-package $(ELASTICPACKAGE)
-	@(cd build/apmpackage && $(ELASTICPACKAGE) format --fail-fast && $(ELASTICPACKAGE) lint)
-	@go run cmd/check-internal-metrics/main.go
-
 .PHONY: check-gofmt gofmt
 check-fmt: check-gofmt
 fmt: gofmt

Tiltfile

@@ -84,20 +84,6 @@ else:
   k8s_kind('Kibana')
   k8s_resource('kibana', port_forwards=default_kibana_port, resource_deps=['elasticsearch'])
 
-# Build and install the APM integration package whenever source under
-# "apmpackage" changes.
-run_with_go_ver = os.path.join(script_dir, 'run_with_go_ver')
-
-
-local_resource(
-  'apmpackage',
-  cmd = [os.path.join(script_dir, 'run_with_go_ver'), 'go', 'run', './cmd/runapm -init'],
-  dir = 'systemtest',
-  deps = ['apmpackage'],
-  resource_deps=['kibana'],
-  env={ "KIBANA_HOST": kibana_host, "KIBANA_BASE_PATH": kibana_base_path, "KIBANA_PORT": str(default_kibana_port) }
-)
-
 k8s_resource('elastic-operator', objects=['eck-trial-license:Secret:elastic-system'])
 k8s_resource('apm-server', port_forwards=8200)
 k8s_resource('elasticsearch', port_forwards=9200, objects=['elasticsearch-admin:Secret:default'])
@@ -111,8 +97,8 @@ if config.tilt_subcommand == "down":
   print(local("kubectl delete --ignore-not-found namespace/elastic-system"))
 
 # Add a button for sending trace events and metrics to APM Server.
+run_with_go_ver = os.path.join(script_dir, 'run_with_go_ver')
 load('ext://uibutton', 'cmd_button')
-
 cmd_button(
   'apm-server:sendotlp',
   argv=['sh', '-c', 'cd systemtest && %s go run ./cmd/sendotlp' % run_with_go_ver],