Add elastic-agent-service container to packaging (#5349)

Adds the elastic-agent-service container with the py-connectors component enabled.

.buildkite/integration.pipeline.yml

@@ -116,7 +116,9 @@ steps:
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
+      machineType: "c2-standard-16"
       image: "family/core-ubuntu-2204"
+      diskSizeGb: 400
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Kubernetes Integration tests"

dev-tools/mage/checksums.go

@@ -108,9 +108,9 @@ func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, ver
 			// Only care about packages that match the required package constraint (os/arch)
 			if strings.Contains(pkgName, requiredPackage) {
 				// Iterate over the external binaries that we care about for packaging agent
-				for binary := range manifest.ExpectedBinaries {
+				for _, spec := range manifest.ExpectedBinaries {
 					// If the individual package doesn't match the expected prefix, then continue
-					if !strings.HasPrefix(pkgName, binary) {
+					if !strings.HasPrefix(pkgName, spec.BinaryName) {
 						continue
 					}
 
@@ -215,14 +215,14 @@ func getComponentVersion(componentName string, requiredPackage string, component
 	// Iterate over all the packages in the component project
 	for pkgName := range componentProject.Packages {
 		// Only care about the external binaries that we want to package
-		for binary, project := range manifest.ExpectedBinaries {
+		for _, spec := range manifest.ExpectedBinaries {
 			// If the given component name doesn't match the external binary component, skip
-			if componentName != project.Name {
+			if componentName != spec.ProjectName {
 				continue
 			}
 
 			// Split the package name on the binary name prefix plus a dash
-			firstSplit := strings.Split(pkgName, binary+"-")
+			firstSplit := strings.Split(pkgName, spec.BinaryName+"-")
 			if len(firstSplit) < 2 {
 				continue
 			}

dev-tools/mage/dockervariants.go

@@ -17,6 +17,7 @@ const (
 	complete      = "complete"
 	completeWolfi = "complete-wolfi"
 	cloud         = "cloud"
+	service       = "service"
 )
 
 // DockerVariant defines the docker variant to build.
@@ -31,6 +32,7 @@ const (
 	WolfiComplete
 	Complete
 	Cloud
+	Service
 )
 
 // String returns the name of the docker variant type.
@@ -50,6 +52,8 @@ func (typ DockerVariant) String() string {
 		return complete
 	case Cloud:
 		return cloud
+	case Service:
+		return service
 	default:
 		return invalid
 	}
@@ -77,6 +81,8 @@ func (typ *DockerVariant) UnmarshalText(text []byte) error {
 		*typ = Complete
 	case cloud:
 		*typ = Cloud
+	case service:
+		*typ = Service
 	default:
 		return fmt.Errorf("unknown docker variant: %v", string(text))
 	}

dev-tools/mage/manifest/manifest.go

@@ -94,20 +94,23 @@ var PlatformPackages = map[string]string{
 // ExpectedBinaries  is a map of binaries agent needs to their project in the unified-release manager.
 // The project names are those used in the "projects" list in the unified release manifest.
 // See the sample manifests in the testdata directory.
-var ExpectedBinaries = map[string]BinarySpec{
-	"agentbeat":             {Name: "beats", Platforms: AllPlatforms},
-	"apm-server":            {Name: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}},
-	"cloudbeat":             {Name: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
-	"endpoint-security":     {Name: "endpoint-dev", Platforms: AllPlatforms},
-	"fleet-server":          {Name: "fleet-server", Platforms: AllPlatforms},
-	"pf-elastic-collector":  {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
-	"pf-elastic-symbolizer": {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
-	"pf-host-agent":         {Name: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
+var ExpectedBinaries = []BinarySpec{
+	{BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms},
+	{BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}},
+	{BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
+	{BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true},
+	{BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms},
+	{BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms},
+	{BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
+	{BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
+	{BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
 }
 
 type BinarySpec struct {
-	Name      string
-	Platforms []Platform
+	BinaryName  string
+	ProjectName string
+	Platforms   []Platform
+	PythonWheel bool
 }
 
 func (proj BinarySpec) SupportsPlatform(platform string) bool {
@@ -119,6 +122,13 @@ func (proj BinarySpec) SupportsPlatform(platform string) bool {
 	return false
 }
 
+func (proj BinarySpec) GetPackageName(version string, platform string) string {
+	if proj.PythonWheel {
+		return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version)
+	}
+	return fmt.Sprintf("%s-%s-%s", proj.BinaryName, version, PlatformPackages[platform])
+}
+
 type Platform struct {
 	OS   string
 	Arch string
@@ -187,27 +197,27 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string
 
 	errGrp, downloadsCtx := errgroup.WithContext(ctx)
 	// for project, pkgs := range expectedProjectPkgs() {
-	for binary, project := range ExpectedBinaries {
+	for _, spec := range ExpectedBinaries {
 		for _, platform := range platforms {
 			targetPath := filepath.Join(dropPath)
 			err := os.MkdirAll(targetPath, 0755)
 			if err != nil {
 				return fmt.Errorf("failed to create directory %s", targetPath)
 			}
-			log.Printf("+++ Prepare to download project [%s] for [%s]", project.Name, platform)
+			log.Printf("+++ Prepare to download [%s] project [%s] for [%s]", spec.BinaryName, spec.ProjectName, platform)
 
-			if !project.SupportsPlatform(platform) {
-				log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", binary, platform)
+			if !spec.SupportsPlatform(platform) {
+				log.Printf(">>>>>>>>> Binary [%s] does not support platform [%s] ", spec.BinaryName, platform)
 				continue
 			}
 
-			pkgURL, err := resolveManifestPackage(projects[project.Name], binary, PlatformPackages[platform], majorMinorPatchVersion)
+			pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform)
 			if err != nil {
 				return err
 			}
 
 			for _, p := range pkgURL {
-				log.Printf(">>>>>>>>> Downloading [%s] [%s] ", binary, p)
+				log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p)
 				pkgFilename := path.Base(p)
 				downloadTarget := filepath.Join(targetPath, pkgFilename)
 				if _, err := os.Stat(downloadTarget); err != nil {
@@ -228,35 +238,35 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string
 	return nil
 }
 
-func resolveManifestPackage(project Project, binary string, platformPkg string, version string) ([]string, error) {
+func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) {
 	var val Package
 	var ok bool
 
 	// Try the normal/easy case first
-	packageName := fmt.Sprintf("%s-%s-%s", binary, version, platformPkg)
+	packageName := spec.GetPackageName(version, platform)
 	val, ok = project.Packages[packageName]
 	if !ok {
 		// If we didn't find it, it may be an Independent Agent Release, where
 		// the opted-in projects will have a patch version one higher than
 		// the rest of the projects, so we need to seek that out
 		if mg.Verbose() {
-			log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", binary, platformPkg)
+			log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform])
 		}
 
 		var foundIt bool
 		for pkgName := range project.Packages {
-			if strings.HasPrefix(pkgName, binary) {
-				firstSplit := strings.Split(pkgName, binary+"-")
+			if strings.HasPrefix(pkgName, spec.BinaryName) {
+				firstSplit := strings.Split(pkgName, spec.BinaryName+"-")
 				if len(firstSplit) < 2 {
 					continue
 				}
 
 				secondHalf := firstSplit[1]
 				// Make sure we're finding one w/ the same required package type
-				if strings.Contains(secondHalf, platformPkg) {
+				if strings.Contains(secondHalf, PlatformPackages[platform]) {
 
 					// Split again after the version with the required package string
-					secondSplit := strings.Split(secondHalf, "-"+platformPkg)
+					secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform])
 					if len(secondSplit) < 2 {
 						continue
 					}
@@ -268,7 +278,7 @@ func resolveManifestPackage(project Project, binary string, platformPkg string,
 					}
 
 					// Create a project/package key with the package, derived version, and required package
-					foundPkgKey := fmt.Sprintf("%s-%s-%s", binary, pkgVersion, platformPkg)
+					foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform])
 					if mg.Verbose() {
 						log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey)
 					}

dev-tools/mage/manifest/manifest_test.go

@@ -135,15 +135,15 @@ func TestResolveManifestPackage(t *testing.T) {
 			projects := manifestJson.Projects
 
 			// Verify the component name is in the list of expected packages.
-			project, ok := ExpectedBinaries[tc.binary]
+			spec, ok := findBinarySpec(tc.binary)
 			assert.True(t, ok)
 
-			if !project.SupportsPlatform(tc.platform) {
-				t.Logf("Project %s does not support platform %s", project.Name, tc.platform)
+			if !spec.SupportsPlatform(tc.platform) {
+				t.Logf("Project %s does not support platform %s", spec.ProjectName, tc.platform)
 				return
 			}
 
-			urlList, err := resolveManifestPackage(projects[tc.projectName], tc.binary, PlatformPackages[tc.platform], manifestJson.Version)
+			urlList, err := resolveManifestPackage(projects[tc.projectName], spec, manifestJson.Version, tc.platform)
 			require.NoError(t, err)
 
 			assert.Len(t, urlList, 3)
@@ -153,3 +153,12 @@ func TestResolveManifestPackage(t *testing.T) {
 		})
 	}
 }
+
+func findBinarySpec(name string) (BinarySpec, bool) {
+	for _, spec := range ExpectedBinaries {
+		if spec.BinaryName == name {
+			return spec, true
+		}
+	}
+	return BinarySpec{}, false
+}

dev-tools/packaging/files/linux/connectors.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+PY_AGENT_CLIENT_PATH=/usr/share/connectors
+PYTHON_PATH=$PY_AGENT_CLIENT_PATH/.venv/bin/python
+COMPONENT_PATH=$PY_AGENT_CLIENT_PATH/connectors/agent/cli.py
+$PYTHON_PATH $COMPONENT_PATH

dev-tools/packaging/packages.yml

@@ -291,6 +291,20 @@ shared:
         source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/agentbeat-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
         mode: 0755
 
+  # service build is based on previous cloud variant
+  - &agent_docker_service_spec
+    docker_variant: 'service'
+    files:
+      'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip':
+        source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip'
+        mode: 0755
+      'data/{{.BeatName}}-{{ commit_short }}/components/connectors':
+        source: '{{ elastic_beats_dir }}/dev-tools/packaging/files/linux/connectors.sh'
+        mode: 0755
+      'data/{{.BeatName}}-{{ commit_short }}/components/connectors.spec.yml':
+        source: '{{ elastic_beats_dir }}/specs/connectors.spec.yml'
+        mode: 0644
+
   # includes nodejs with @elastic/synthetics
   - &agent_docker_complete_spec
     <<: *agent_docker_spec
@@ -1025,6 +1039,35 @@ specs:
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+    #### Service specific docker images ####
+    - os: linux
+      arch: amd64
+      types: [ docker ]
+      spec:
+        <<: *agent_docker_spec
+        # The service image is always based on Wolfi
+        <<: *docker_wolfi_spec
+        <<: *docker_builder_spec
+        <<: *agent_docker_cloud_spec
+        <<: *agent_docker_service_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+    - os: linux
+      arch: arm64
+      types: [ docker ]
+      spec:
+        <<: *agent_docker_spec
+        # The service image is always based on Wolfi
+        <<: *docker_wolfi_arm_spec
+        <<: *docker_builder_arm_spec
+        <<: *agent_docker_cloud_spec
+        <<: *agent_docker_service_spec
+        <<: *elastic_license_for_binaries
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
     - os: linux
       arch: amd64
       types: [docker]

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -43,7 +43,7 @@ RUN true && \
     chmod 0775 {{ $beatHome}}/{{ $modulesd }} && \
 {{- end }}
 
-{{- if eq .Variant "cloud" }}
+{{- if or (eq .Variant "cloud") (eq .Variant "service") }}
     mkdir -p /opt/agentbeat /opt/filebeat /opt/metricbeat && \
     cp -f {{ $beatHome }}/data/cloud_downloads/filebeat.sh /opt/filebeat/filebeat && \
     chmod +x /opt/filebeat/filebeat && \
@@ -170,13 +170,21 @@ RUN mkdir /licenses
 COPY --from=home {{ $beatHome }}/LICENSE.txt /licenses
 COPY --from=home {{ $beatHome }}/NOTICE.txt /licenses
 
-{{- if eq .Variant "cloud" }}
+{{- if or (eq .Variant "cloud") (eq .Variant "service") }}
 COPY --from=home /opt /opt
 # Generate folder for a stub command that will be overwritten at runtime
 RUN mkdir /app && \
     chown {{ .user }}:{{ .user }} /app
 {{- end }}
 
+{{- if eq .Variant "service" }}
+RUN apk add --no-cache git make python-3.11 py3.11-pip && \
+    unzip {{ $beatHome }}/data/service/connectors-*.zip -d {{ $beatHome }}/data/service && \
+    mv {{ $beatHome }}/data/service/elasticsearch_connectors-* /usr/share/connectors && \
+    PYTHON=python3.11 make -C /usr/share/connectors clean install install-agent && \
+    chmod 0755 {{ $beatHome }}/data/elastic-agent-*/components/connectors
+{{- end }}
+
 {{- if (and (eq .Variant "complete") (contains .from "ubuntu"))  }}
 USER root
 ENV NODE_PATH={{ $beatHome }}/.node
@@ -284,7 +292,7 @@ ENV LIBBEAT_MONITORING_CGROUPS_HIERARCHY_OVERRIDE=/
 
 WORKDIR {{ $beatHome }}
 
-{{- if eq .Variant "cloud" }}
+{{- if or (eq .Variant "cloud") (eq .Variant "service") }}
 ENTRYPOINT ["/usr/bin/tini", "--"]
 CMD ["/app/apm.sh"]
 # Generate a stub command that will be overwritten at runtime