Add manifest to package (#3910)

* Add manifest to package
* Add tests for manifest in zip packages
* Add manifest to rpm and deb packages

Author: pchila

dev-tools/mage/common.go

@@ -480,6 +480,14 @@ func untar(sourceFile, destinationDir string) error {
 			if err = writer.Close(); err != nil {
 				return err
 			}
+		case tar.TypeSymlink:
+			if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+				return err
+			}
+			if err := os.Symlink(header.Linkname, path); err != nil {
+				return fmt.Errorf("error creating symlink %s pointing to %s: %w", path, header.Linkname, err)
+			}
+
 		default:
 			return fmt.Errorf("unable to untar type=%c in file=%s", header.Typeflag, path)
 		}
@@ -861,21 +869,21 @@ var parseVersionRegex = regexp.MustCompile(`(?m)^[^\d]*(?P<major>\d+)\.(?P<minor
 
 // ParseVersion extracts the major, minor, and optional patch number from a
 // version string.
-func ParseVersion(version string) (major, minor, patch int, err error) {
+func ParseVersion(version string) (int, int, int, error) {
 	names := parseVersionRegex.SubexpNames()
 	matches := parseVersionRegex.FindStringSubmatch(version)
 	if len(matches) == 0 {
-		err = fmt.Errorf("failed to parse version '%v'", version)
-		return
+		err := fmt.Errorf("failed to parse version '%v'", version)
+		return 0, 0, 0, err
 	}
 
 	data := map[string]string{}
 	for i, match := range matches {
 		data[names[i]] = match
 	}
-	major, _ = strconv.Atoi(data["major"])
-	minor, _ = strconv.Atoi(data["minor"])
-	patch, _ = strconv.Atoi(data["patch"])
+	major, _ := strconv.Atoi(data["major"])
+	minor, _ := strconv.Atoi(data["minor"])
+	patch, _ := strconv.Atoi(data["patch"])
 	return major, minor, patch, nil
 }
 

dev-tools/mage/settings.go

@@ -10,6 +10,7 @@ import (
 	"go/build"
 	"log"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -19,11 +20,13 @@ import (
 
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
+	"gopkg.in/yaml.v3"
 
 	"github.com/magefile/mage/sh"
 	"golang.org/x/tools/go/vcs"
 
 	"github.com/elastic/elastic-agent/dev-tools/mage/gotool"
+	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
 )
 
 const (
@@ -43,7 +46,8 @@ const (
 	agentPackageVersionEnvVar = "AGENT_PACKAGE_VERSION"
 
 	// Mapped functions
-	agentPackageVersionMappedFunc = "agent_package_version"
+	agentPackageVersionMappedFunc    = "agent_package_version"
+	agentManifestGeneratorMappedFunc = "manifest"
 )
 
 // Common settings with defaults derived from files, CWD, and environment.
@@ -91,18 +95,19 @@ var (
 	ManifestURL string
 
 	FuncMap = map[string]interface{}{
-		"beat_doc_branch":             BeatDocBranch,
-		"beat_version":                BeatQualifiedVersion,
-		"commit":                      CommitHash,
-		"commit_short":                CommitHashShort,
-		"date":                        BuildDate,
-		"elastic_beats_dir":           ElasticBeatsDir,
-		"go_version":                  GoVersion,
-		"repo":                        GetProjectRepoInfo,
-		"title":                       func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
-		"tolower":                     strings.ToLower,
-		"contains":                    strings.Contains,
-		agentPackageVersionMappedFunc: AgentPackageVersion,
+		"beat_doc_branch":                BeatDocBranch,
+		"beat_version":                   BeatQualifiedVersion,
+		"commit":                         CommitHash,
+		"commit_short":                   CommitHashShort,
+		"date":                           BuildDate,
+		"elastic_beats_dir":              ElasticBeatsDir,
+		"go_version":                     GoVersion,
+		"repo":                           GetProjectRepoInfo,
+		"title":                          func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) },
+		"tolower":                        strings.ToLower,
+		"contains":                       strings.Contains,
+		agentPackageVersionMappedFunc:    AgentPackageVersion,
+		agentManifestGeneratorMappedFunc: PackageManifest,
 	}
 )
 
@@ -296,6 +301,40 @@ func AgentPackageVersion() (string, error) {
 	return BeatQualifiedVersion()
 }
 
+func PackageManifest() (string, error) {
+	m := v1.NewManifest()
+	m.Package.Snapshot = Snapshot
+	packageVersion, err := AgentPackageVersion()
+	if err != nil {
+		return "", fmt.Errorf("retrieving agent package version: %w", err)
+	}
+	m.Package.Version = packageVersion
+	commitHashShort, err := CommitHashShort()
+	if err != nil {
+		return "", fmt.Errorf("retrieving agent commit hash: %w", err)
+	}
+
+	versionedHomePath := path.Join("data", fmt.Sprintf("%s-%s", BeatName, commitHashShort))
+	m.Package.VersionedHome = versionedHomePath
+	m.Package.PathMappings = []map[string]string{{}}
+	m.Package.PathMappings[0][versionedHomePath] = fmt.Sprintf("data/elastic-agent-%s%s-%s", m.Package.Version, SnapshotSuffix(), commitHashShort)
+	m.Package.PathMappings[0]["manifest.yaml"] = fmt.Sprintf("data/elastic-agent-%s%s-%s/manifest.yaml", m.Package.Version, SnapshotSuffix(), commitHashShort)
+	yamlBytes, err := yaml.Marshal(m)
+	if err != nil {
+		return "", fmt.Errorf("marshaling manifest: %w", err)
+
+	}
+	return string(yamlBytes), nil
+}
+
+func SnapshotSuffix() string {
+	if !Snapshot {
+		return ""
+	}
+
+	return "-SNAPSHOT"
+}
+
 var (
 	elasticBeatsDirValue string
 	elasticBeatsDirErr   error

dev-tools/packaging/package_test.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"slices"
@@ -27,6 +28,12 @@ import (
 
 	"github.com/blakesmith/ar"
 	"github.com/cavaliercoder/go-rpm"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+
+	"github.com/elastic/elastic-agent/dev-tools/mage"
+	v1 "github.com/elastic/elastic-agent/pkg/api/v1"
 )
 
 const (
@@ -159,6 +166,14 @@ func checkTar(t *testing.T, file string) {
 	checkModulesPermissions(t, p)
 	checkModulesOwner(t, p, true)
 	checkLicensesPresent(t, "", p)
+
+	t.Run(p.Name+"_check_manifest_file", func(t *testing.T) {
+		tempExtractionPath := t.TempDir()
+		err = mage.Extract(file, tempExtractionPath)
+		require.NoError(t, err, "error extracting tar archive")
+		containingDir := strings.TrimSuffix(path.Base(file), ".tar.gz")
+		checkManifestFileContents(t, filepath.Join(tempExtractionPath, containingDir))
+	})
 }
 
 func checkZip(t *testing.T, file string) {
@@ -174,6 +189,48 @@ func checkZip(t *testing.T, file string) {
 	checkModulesDPresent(t, "", p)
 	checkModulesPermissions(t, p)
 	checkLicensesPresent(t, "", p)
+
+	t.Run(p.Name+"_check_manifest_file", func(t *testing.T) {
+		tempExtractionPath := t.TempDir()
+		err = mage.Extract(file, tempExtractionPath)
+		require.NoError(t, err, "error extracting zip archive")
+		containingDir := strings.TrimSuffix(path.Base(file), ".zip")
+		checkManifestFileContents(t, filepath.Join(tempExtractionPath, containingDir))
+	})
+}
+
+func checkManifestFileContents(t *testing.T, extractedPackageDir string) {
+	t.Log("Checking file manifest.yaml")
+	manifestReadCloser, err := os.Open(filepath.Join(extractedPackageDir, "manifest.yaml"))
+	if err != nil {
+		t.Errorf("opening manifest %s : %v", "manifest.yaml", err)
+	}
+	defer func(closer io.ReadCloser) {
+		err := closer.Close()
+		assert.NoError(t, err, "error closing manifest file")
+	}(manifestReadCloser)
+
+	var m v1.PackageManifest
+	err = yaml.NewDecoder(manifestReadCloser).Decode(&m)
+	if err != nil {
+		t.Errorf("unmarshaling package manifest: %v", err)
+	}
+
+	assert.Equal(t, v1.ManifestKind, m.Kind, "manifest specifies wrong kind")
+	assert.Equal(t, v1.VERSION, m.Version, "manifest specifies wrong api version")
+
+	if assert.NotEmpty(t, m.Package.PathMappings, "path mappings in manifest are empty") {
+		versionedHome := m.Package.VersionedHome
+		assert.DirExistsf(t, filepath.Join(extractedPackageDir, versionedHome), "versionedHome directory %q not found in %q", versionedHome, extractedPackageDir)
+		if assert.Contains(t, m.Package.PathMappings[0], versionedHome, "path mappings in manifest do not contain the extraction path for versionedHome") {
+			// the first map should have the mapping for the data/elastic-agent-****** path)
+			mappedPath := m.Package.PathMappings[0][versionedHome]
+			assert.Contains(t, mappedPath, m.Package.Version, "mapped path for versionedHome does not contain the package version")
+			if m.Package.Snapshot {
+				assert.Contains(t, mappedPath, "SNAPSHOT", "mapped path for versionedHome does not contain the snapshot qualifier")
+			}
+		}
+	}
 }
 
 const (
@@ -667,9 +724,9 @@ func readTarContents(tarName string, data io.Reader) (*packageFile, error) {
 type inspector func(pkg, file string, contents io.Reader) error
 
 func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFile, error) {
-	r, err := zip.OpenReader(zipFile)
+	r, err := openZip(zipFile)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("opening zip: %w", err)
 	}
 	defer r.Close()
 
@@ -699,6 +756,14 @@ func readZip(t *testing.T, zipFile string, inspectors ...inspector) (*packageFil
 	return p, nil
 }
 
+func openZip(zipFile string) (*zip.ReadCloser, error) {
+	r, err := zip.OpenReader(zipFile)
+	if err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
 func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) {
 	// Read the manifest file first so that the config file and layer
 	// names are known in advance.

dev-tools/packaging/packages.yml

@@ -74,6 +74,10 @@ shared:
         mode: 0755
         config_mode: 0644
         skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/manifest.yaml:
+        mode: 0644
+        content: >
+          {{ manifest }}
 
   # MacOS pkg spec for community beats.
   - &macos_agent_pkg_spec
@@ -159,6 +163,10 @@ shared:
       content: >
         {{ commit }}
       mode: 0644
+    'manifest.yaml':
+      mode: 0644
+      content: >
+        {{ manifest }}
 
   - &agent_binary_files
     '{{.BeatName}}{{.BinaryExt}}':

pkg/api/v1/api.go

@@ -0,0 +1,13 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// Package v1 contains definitions for elastic-agent/v1 objects
+package v1
+
+const VERSION = "co.elastic.agent/v1"
+
+type apiObject struct {
+	Version string `yaml:"version" json:"version"`
+	Kind    string `yaml:"kind" json:"kind"`
+}

pkg/api/v1/manifest.go

@@ -0,0 +1,45 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package v1
+
+import (
+	"fmt"
+	"io"
+
+	"gopkg.in/yaml.v2"
+)
+
+const ManifestKind = "PackageManifest"
+
+type PackageDesc struct {
+	Version       string              `yaml:"version,omitempty" json:"version,omitempty"`
+	Snapshot      bool                `yaml:"snapshot,omitempty" json:"snapshot,omitempty"`
+	VersionedHome string              `yaml:"versioned-home,omitempty" json:"versionedHome,omitempty"`
+	PathMappings  []map[string]string `yaml:"path-mappings,omitempty" json:"pathMappings,omitempty"`
+}
+
+type PackageManifest struct {
+	apiObject `yaml:",inline"`
+	Package   PackageDesc `yaml:"package" json:"package"`
+}
+
+func NewManifest() *PackageManifest {
+	return &PackageManifest{
+		apiObject: apiObject{
+			Version: VERSION,
+			Kind:    ManifestKind,
+		},
+	}
+}
+
+func ParseManifest(r io.Reader) (*PackageManifest, error) {
+	m := new(PackageManifest)
+	err := yaml.NewDecoder(r).Decode(m)
+	if err != nil {
+		return nil, fmt.Errorf("decoding package manifest: %w", err)
+	}
+
+	return m, nil
+}

pkg/api/v1/manifest_test.go

@@ -0,0 +1,49 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package v1
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEmptyManifest(t *testing.T) {
+	m := NewManifest()
+	assert.Equal(t, VERSION, m.Version)
+	assert.Equal(t, ManifestKind, m.Kind)
+}
+
+func TestParseManifest(t *testing.T) {
+
+	manifest := `
+# version and kind to uniquely identify the schema
+version: co.elastic.agent/v1
+kind: PackageManifest
+
+# description of the package itself
+package:
+  version: 8.12.0
+  snapshot: false
+  versioned-home: data/elastic-agent-4f2d39/
+  # generic path mapping:
+  # - key is a prefix representing a path relative to the top of the archive 
+  # - value is the substitution to be applied when extracting the files
+  path-mappings:
+    - data/elastic-agent-4f2d39/ : data/elastic-agent-8.12.0/
+      foo: bar
+    - manifest.yaml : data/elastic-agent-8.12.0/manifest.yaml 
+`
+	m, err := ParseManifest(strings.NewReader(manifest))
+	assert.NoError(t, err)
+	assert.Equal(t, VERSION, m.Version)
+	assert.Equal(t, ManifestKind, m.Kind)
+
+	assert.Equal(t, m.Package.Version, "8.12.0")
+	assert.Equal(t, m.Package.Snapshot, false)
+	assert.Equal(t, m.Package.VersionedHome, "data/elastic-agent-4f2d39/")
+	assert.Equal(t, m.Package.PathMappings, []map[string]string{{"data/elastic-agent-4f2d39/": "data/elastic-agent-8.12.0/", "foo": "bar"}, {"manifest.yaml": "data/elastic-agent-8.12.0/manifest.yaml"}})
+}