DEB packages build (opensearch-project#2526)

* Introduce deb package build

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* fix: debian control file dependencies

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Add customized rules file for building the deb package and fix the control.tar.zst issue when building on Ubuntu and trying installing the deb package on Debian

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Extract same code from bundle deb & rpm into new bundle_linux_distro file

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Remove unnecessary import from bundle_deb

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Simplify install.sh script

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix opensearch-dashboards package build

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Improve deb package building and fixing all tests

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix missing copyright header for new files

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix dependency import order

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix test bundle linux distro for mypy

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix test bundle linux deb on windows

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Replace /tmp string in Makefile

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Improve bundle linux deb changelog generation

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Remove BundleLinuxDistro

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Adopt changes for Makefile and build.sh

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Remove unused imports

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix linux-deb-arm64 build.sh

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Fix homepage in deb package control

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

* Remove —output argument

The argument `—output` can’t be used in the binutils which are delivered for the Amazon Linux distribution.

The `cwd=dest` are enough to extract the data.tar.gz file into the right directory.

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

Signed-off-by: Martin Wilhelmi <mnin@mnin.de>

.gitignore

@@ -21,8 +21,10 @@ out.txt
 /dist/
 /test-results/
 
+/deb/
 /rpm/
 /tar/
+/tar.gz/
 /zip/
 
 /.vscode/

README.md

@@ -231,7 +231,7 @@ At this moment there's no official MacOS distribution. However, this project doe
 The [checkout workflow](src/checkout_workflow) checks out source code for a given manifest for further examination.
 
 ```bash
-./checkout.sh manfiests/1.3.0/opensearch-1.3.0.yml
+./checkout.sh manifests/1.3.0/opensearch-1.3.0.yml
 ```
 
 See [src/checkout_workflow](./src/checkout_workflow) for more information.

scripts/components/OpenSearch-Dashboards/build.sh

@@ -95,6 +95,18 @@ case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in
         BUILD_PARAMS="build-platform"
         EXTRA_PARAMS="--skip-os-packages"
         SUFFIX="$PLATFORM-x64"
+    linux-deb-arm64)
+        TARGET="--$DISTRIBUTION-arm"
+        EXT="$DISTRIBUTION"
+        BUILD_PARAMS="build"
+        EXTRA_PARAMS="--skip-archives"
+        SUFFIX="arm64"
+    linux-deb-x64)
+        TARGET="--$DISTRIBUTION"
+        EXT="$DISTRIBUTION"
+        BUILD_PARAMS="build"
+        EXTRA_PARAMS="--skip-archives"
+        SUFFIX="amd64"
         ;;
     linux-rpm-x64)
         TARGET="--$DISTRIBUTION"

scripts/components/OpenSearch-Dashboards/install.sh

@@ -79,16 +79,15 @@ echo $DIR
 cd $DIR
 
 ## Setup default config
-
 MAJOR_VERSION=`echo $VERSION | cut -d. -f1`
 if ! (ls ../../../config/ | grep -E "opensearch_dashboards-.*.x.yml" | grep $MAJOR_VERSION); then
     MAJOR_VERSION="default"
 fi
 
-if [ "$DISTRIBUTION" = "rpm" ]; then
+if [ "$DISTRIBUTION" = "rpm" -o "$DISTRIBUTION" = "deb" ]; then
     cp -v ../../../config/opensearch_dashboards-$MAJOR_VERSION.x.yml "$OUTPUT/../etc/opensearch-dashboards/opensearch_dashboards.yml"
     cp -a ../../../scripts/pkg/service_templates/opensearch-dashboards/* "$OUTPUT/../"
-    cp -a ../../../scripts/pkg/build_templates/opensearch-dashboards/* "$OUTPUT/../"
+    cp -a ../../../scripts/pkg/build_templates/opensearch-dashboards/$DISTRIBUTION/* "$OUTPUT/../"
 else
     cp -v ../../../config/opensearch_dashboards-$MAJOR_VERSION.x.yml "$OUTPUT/config/opensearch_dashboards.yml"
 fi

scripts/components/OpenSearch/build.sh

@@ -103,6 +103,20 @@ case $PLATFORM-$DISTRIBUTION-$ARCHITECTURE in
         TARGET="$PLATFORM-arm64-$PACKAGE"
         SUFFIX="$PLATFORM-arm64"
         ;;
+    linux-deb-x64)
+        PACKAGE="deb"
+        EXT="deb"
+        TYPE="packages"
+        TARGET="deb"
+        SUFFIX="amd64"
+        ;;
+    linux-deb-arm64)
+        PACKAGE="deb"
+        EXT="deb"
+        TYPE="packages"
+        TARGET="arm64-deb"
+        SUFFIX="arm64"
+        ;;
     linux-rpm-x64)
         PACKAGE="rpm"
         EXT="rpm"

scripts/components/OpenSearch/install.sh

@@ -81,10 +81,9 @@ cd $DIR
 ## Copy the tar installation script into the bundle
 if [ "$DISTRIBUTION" = "tar" ]; then
     cp -v ../../../scripts/startup/tar/linux/opensearch-tar-install.sh "$OUTPUT/"
-
-elif [ "$DISTRIBUTION" = "rpm" ]; then
+elif [ "$DISTRIBUTION" = "deb" -o "$DISTRIBUTION" = "rpm" ]; then
     cp -va ../../../scripts/pkg/service_templates/opensearch/* "$OUTPUT/../"
-    cp -va ../../../scripts/pkg/build_templates/opensearch/* "$OUTPUT/../"
+    cp -va ../../../scripts/pkg/build_templates/opensearch/$DISTRIBUTION/* "$OUTPUT/../"
 elif [ "$DISTRIBUTION" = "zip" ] && [ "$PLATFORM" = "windows" ]; then
     cp -v ../../../scripts/startup/zip/windows/opensearch-windows-install.bat "$OUTPUT/"
 fi

scripts/pkg/build_templates/opensearch-dashboards/deb/Makefile

@@ -0,0 +1,13 @@
+all: install
+
+install:
+	mkdir -p $(CURDIR)/debian/opensearch
+	cp -a $(CURDIR)/etc $(CURDIR)/debian/opensearch/etc
+	cp -a $(CURDIR)/usr $(CURDIR)/debian/opensearch/usr
+	cp -a $(CURDIR)/var $(CURDIR)/debian/opensearch/var
+
+clean: ;
+
+distclean: clean
+
+.PHONY: all clean distclean install

scripts/pkg/build_templates/opensearch-dashboards/deb/debian/control

@@ -0,0 +1,12 @@
+Source: opensearch-dashboards
+Section: database
+Priority: optional
+Maintainer: OpenSearch Team <opensearch@amazon.com>
+Build-Depends: debhelper-compat (= 12)
+Standards-Version: 4.5.0
+Homepage: https://opensearch.org/
+
+Package: opensearch-dashboards
+Architecture: any
+Multi-Arch: foreign
+Description: OpenSearch

scripts/pkg/build_templates/opensearch-dashboards/deb/debian/copyright

@@ -0,0 +1,38 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: opensearch-dashboards
+Upstream-Contact: opensearch@amazon.com
+Source: https://opensearch.org
+Files: *
+Copyright: 2020-2022 OpenSearch (https://opensearch.org/)
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+      http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache License, Version 2
+ can be found in "/usr/share/common-licenses/Apache-2.0".
+
+Files: debian/*
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+      http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache License, Version 2
+ can be found in "/usr/share/common-licenses/Apache-2.0".

scripts/pkg/build_templates/opensearch-dashboards/deb/debian/rules

@@ -0,0 +1,18 @@
+#!/usr/bin/make -f
+# You must remove unused comment lines for the released package.
+#export DH_VERBOSE = 1
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+%:
+	dh $@
+
+override_dh_builddeb:
+	dh_builddeb -- -Zgzip
+
+#override_dh_auto_install:
+#	dh_auto_install -- prefix=/usr
+
+#override_dh_install:
+#	dh_install --list-missing -X.pyc -X.pyo

scripts/pkg/build_templates/opensearch/deb/Makefile

@@ -0,0 +1,13 @@
+all: install
+
+install:
+	mkdir -p $(CURDIR)/debian/opensearch
+	cp -a $(CURDIR)/etc $(CURDIR)/debian/opensearch/etc
+	cp -a $(CURDIR)/usr $(CURDIR)/debian/opensearch/usr
+	cp -a $(CURDIR)/var $(CURDIR)/debian/opensearch/var
+
+clean: ;
+
+distclean: clean
+
+.PHONY: all clean distclean install

scripts/pkg/build_templates/opensearch/deb/debian/control

@@ -0,0 +1,12 @@
+Source: opensearch
+Section: database
+Priority: optional
+Maintainer: OpenSearch Team <opensearch@amazon.com>
+Build-Depends: debhelper-compat (= 12)
+Standards-Version: 4.5.0
+Homepage: https://opensearch.org/
+
+Package: opensearch
+Architecture: any
+Multi-Arch: foreign
+Description: OpenSearch

scripts/pkg/build_templates/opensearch/deb/debian/copyright

@@ -0,0 +1,38 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: opensearch
+Upstream-Contact: opensearch@amazon.com
+Source: https://opensearch.org
+Files: *
+Copyright: 2020-2022 OpenSearch (https://opensearch.org/)
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+      http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache License, Version 2
+ can be found in "/usr/share/common-licenses/Apache-2.0".
+
+Files: debian/*
+License: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+      http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the complete text of the Apache License, Version 2
+ can be found in "/usr/share/common-licenses/Apache-2.0".

scripts/pkg/build_templates/opensearch/deb/debian/rules

@@ -0,0 +1,18 @@
+#!/usr/bin/make -f
+# You must remove unused comment lines for the released package.
+#export DH_VERBOSE = 1
+#export DEB_BUILD_MAINT_OPTIONS = hardening=+all
+#export DEB_CFLAGS_MAINT_APPEND  = -Wall -pedantic
+#export DEB_LDFLAGS_MAINT_APPEND = -Wl,--as-needed
+
+%:
+	dh $@  
+
+override_dh_builddeb:
+	dh_builddeb -- -Zgzip
+
+#override_dh_auto_install:
+#	dh_auto_install -- prefix=/usr
+
+#override_dh_install:
+#	dh_install --list-missing -X.pyc -X.pyo

src/assemble_workflow/bundle_linux_deb.py

@@ -0,0 +1,126 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import logging
+import os
+import shutil
+import subprocess
+from typing import List
+
+from manifests.build_manifest import BuildManifest
+from system.os import deb_architecture
+
+
+class BundleLinuxDeb:
+
+    def __init__(self, filename: str, package_path: str, min_path: str) -> None:
+        self.filename = filename
+        self.package_path = package_path
+        self.min_path = min_path
+
+    def changelog_content(self, version: str) -> List[str]:
+        # instead of 'UNRELEASED' its possible to use 'stable'
+        # which will use gpg to sign with the given from 'OpenSearch Team <opensearch@amazon.com>'
+
+        return [
+            f"{self.filename} ({version}) UNRELEASED; urgency=low",
+            "",
+            "  * Initial release.",
+            "",
+            " -- OpenSearch Team <opensearch@amazon.com>  Fri, 14 Oct 2022 10:06:23 +0000"
+        ]
+
+    def generate_changelog_file(self, dest: str, version: str) -> None:
+        changelog_path = os.path.join(dest, 'debian', 'changelog')
+        logging.info(f"Write debian changelog to: {changelog_path}")
+
+        with open(changelog_path, 'w') as file:
+            for line in self.changelog_content(version):
+                file.write(f"{line}\n")
+
+    def extract(self, dest: str) -> None:
+        data_path = os.path.join(dest, "data.tar.gz")
+        min_source_path = os.path.join(dest, 'usr', 'share', self.filename)
+        min_dest_path = os.path.join(dest, self.min_path)
+        min_config_path = os.path.join(dest, 'etc', self.filename)
+
+        # Extract data.tar.gz from deb so we can extract the content
+        logging.info(f"Extract data.tar.gz from deb for extraction: {self.package_path} to {data_path}")
+        with open(data_path, 'wb') as fp:
+            subprocess.check_call(
+                [
+                    'ar',
+                    '-xf',
+                    self.package_path,
+                    'data.tar.gz'
+                ],
+                stdout=fp,
+                cwd=dest,
+            )
+
+        # Extract data.tar.gz archive based on the deb package
+        logging.info(f"Extract data.tar.gz content to {dest}")
+        with open(data_path, 'rb') as fp:
+            subprocess.check_call(
+                [
+                    'tar',
+                    '-zvxf',
+                    '-'
+                ],
+                stdin=fp,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.STDOUT,
+                cwd=dest,
+            )
+
+        # Move core folder destination so plugin install can proceed
+        logging.info(f"Move {min_source_path} to {min_dest_path} for plugin installation")
+        shutil.move(min_source_path, min_dest_path)
+
+        # Multiple modifications and env vars setups before install plugins
+        # As bin/opensearch-env is different between archive and package
+        # https://github.com/opensearch-project/OpenSearch/issues/2092
+        os.environ[f"{self.filename.upper()}_PATH_CONF"] = min_config_path
+
+    def build(self, name: str, dest: str, archive_path: str, build_cls: BuildManifest.Build) -> None:
+        # extract dest and build dest are not the same, this is restoring the extract dest
+        # mainly due to deb requires several different setups compares to tarball and zip
+        ext_dest = os.path.dirname(archive_path)
+        min_source_path = os.path.join(ext_dest, 'usr', 'share', self.filename)
+        min_dest_path = os.path.join(ext_dest, self.min_path)
+        bundle_artifact_path: str = None
+
+        # Remove env var
+        logging.info('Organize folder structure before generating deb')
+        os.environ.pop('OPENSEARCH_PATH_CONF', None)
+
+        logging.info(f"Move {min_dest_path} to {min_source_path} for building the debian package")
+        shutil.move(min_dest_path, min_source_path)
+
+        deb_version = build_cls.version.replace('-', '.')
+        self.generate_changelog_file(ext_dest, deb_version)
+
+        bundle_cmd = " ".join(
+            [
+                'debmake',
+                '-f "OpenSearch Team"',
+                '-e "opensearch@amazon.com"',
+                '-i debuild',
+                f'-p {self.filename}',
+                '-n',
+                '-r 1',
+                f"-u {deb_version}"
+            ]
+        )
+
+        logging.info(f"Execute {bundle_cmd} in {ext_dest}")
+        subprocess.check_call(bundle_cmd, cwd=ext_dest, shell=True)
+
+        # Move artifact to repo root before being published to {dest}
+        # for dirpath, dirnames, filenames in os.walk(os.path.join('/tmp/opensearch*')):
+        logging.info(f"Found deb file: {bundle_artifact_path}")
+        shutil.move(f"/tmp/{self.filename}_{deb_version}_{deb_architecture(build_cls.architecture)}.deb", name)

src/assemble_workflow/bundle_rpm.py → src/assemble_workflow/bundle_linux_rpm.py

@@ -14,7 +14,7 @@
 from system.os import rpm_architecture
 
 
-class BundleRpm:
+class BundleLinuxRpm:
 
     def __init__(self, filename: str, package_path: str, min_path: str) -> None:
         self.filename = filename