fix aarch64 build,verify and strip binaries (#15)

* fix aarch64 build,verify and strip binaries

* fix strip cmd

* fix verification

* build zstd aarch64 with zlib support

* support liblzma

* fix path

---------

Co-authored-by: ppaul <pritam.paul@yahooinc.com>

Author: pritamstyz4ever

build_skopeo.sh

@@ -23,6 +23,7 @@ cd skopeo-${SKOPEO_VERSION}
 build_skopeo() {
     local arch=$1
     local output_name=$2
+    local verify_arch=$3
     echo "Building skopeo for architecture: ${arch}"
 
     # Build with the specified architecture
@@ -45,19 +46,19 @@ build_skopeo() {
     fi
 
     # Check architecture
-    if file ../${output_name} | grep -q "${arch}"; then
-        echo "The binary ${output_name} is for the correct architecture (${arch})."
+    if file ${CURR_DIR}/${output_name} | grep -q "${verify_arch}, version 1 (SYSV), statically linked"; then
+        echo "The binary ${output_name} is for the correct architecture (${verify_arch})."
     else
-        echo "Error: The binary ${output_name} is not for the correct architecture (expected ${arch})."
+        echo "Error: The binary ${output_name} is not for the correct architecture (expected ${verify_arch})."
         exit 1
     fi
 }
 
 # Build for amd64
-build_skopeo "amd64" "${SKOPEO_PACKAGE_AMD64}"
+build_skopeo "amd64" "${SKOPEO_PACKAGE_AMD64}" "x86-64"
 
 # Build for arm64
-build_skopeo "arm64" "${SKOPEO_PACKAGE_ARM64}"
+build_skopeo "arm64" "${SKOPEO_PACKAGE_ARM64}" "ARM aarch64"
 
 cd ..
 echo "Builds completed. Binaries located in ${CURR_DIR}."

build_zstd.sh

@@ -5,13 +5,21 @@ apt-get update && apt-get install -y build-essential wget cmake gcc-aarch64-linu
 
 ARCHITECTURES=("x86_64" "aarch64")
 COMPILERS=("gcc" "aarch64-linux-gnu-gcc")
-OUTPUT_FILES=("${BUILD_DIR}/zstd-linux-x86_64" "${BUILD_DIR}/zstd-linux-aarch64")
+OUTPUT_FILES=("zstd-linux-x86_64" "zstd-linux-aarch64")
+VERIFY=("x86-64" "ARM aarch64")
+STRIP=("strip" "aarch64-linux-gnu-strip")
+CURR_DIR=$(pwd)
 
 # Function to check if binary is static and has correct architecture
 check_binary() {
     local binary="$1"
     local arch="$2"
 
+    # Verify the binary
+    echo "Verifying $binary..."
+    ldd $binary
+    file $binary
+    
     # Check if binary is statically linked
     if ldd "$binary" 2>&1 | grep -q "not a dynamic executable"; then
         echo "The binary $binary is statically linked."
@@ -21,42 +29,87 @@ check_binary() {
     fi
 
     # Check architecture
-    if file "$binary" | grep -q "$arch"; then
+    if file "$binary" | grep -q "$arch, version 1 (GNU/Linux), statically linked"; then
         echo "The binary $binary is for the correct architecture $arch."
     else
         echo "Error: The binary $binary is not for the correct architecture (expected $arch)."
         exit 1
     fi
 }
 
+# build zlib for aarch64
+build_zlib() {
+    wget https://zlib.net/zlib-1.3.1.tar.gz
+    tar -xzf zlib-1.3.1.tar.gz
+    cd zlib-1.3.1
+    CC=aarch64-linux-gnu-gcc ./configure --prefix=/tmp/zlib-aarch64 --static
+    make
+    make install
+    file /tmp/zlib-aarch64/lib/libz.a
+    aarch64-linux-gnu-objdump -f /tmp/zlib-aarch64/lib/libz.a | grep architecture
+    cd ..
+}
+
+build_liblzma() {
+    wget https://tukaani.org/xz/xz-5.4.4.tar.gz
+    tar -xzf xz-5.4.4.tar.gz
+    cd xz-5.4.4
+    CC=aarch64-linux-gnu-gcc ./configure --prefix=/tmp/liblzma-aarch64 --host=aarch64-linux-gnu --enable-static --disable-shared
+    make
+    make install
+    file /tmp/liblzma-aarch64/lib/liblzma.a
+    aarch64-linux-gnu-objdump -f /tmp/liblzma-aarch64/lib/liblzma.a | grep architecture
+    cd ..
+}
+
+
 # Ensure BUILD_DIR and ZSTD_VERSION are set
 if [ -z "$BUILD_DIR" ] || [ -z "$ZSTD_VERSION" ]; then
     echo "BUILD_DIR and ZSTD_VERSION must be set."
     exit 1
 fi
 
-mkdir -p "$BUILD_DIR"
-cd "$BUILD_DIR"
-
 if [ ! -d "zstd-${ZSTD_VERSION}" ]; then
     echo "Downloading zstd version ${ZSTD_VERSION}..."
     wget https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz
     tar -xzf zstd-${ZSTD_VERSION}.tar.gz
 fi
-cd "zstd-${ZSTD_VERSION}"
+
+mkdir -p "$BUILD_DIR"
 
 # Build and verify binaries for each architecture
 for i in "${!ARCHITECTURES[@]}"; do
     arch="${ARCHITECTURES[$i]}"
+    verify="${VERIFY[$i]}"
     compiler="${COMPILERS[$i]}"
     output_file="${OUTPUT_FILES[$i]}"
+    strip_cmd="${STRIP[$i]}"
+
+    cf_flags="-static -O2 -pthread"
+    ld_flags="-static"
+
+    if [ "$arch" == "aarch64" ]; then
+        build_zlib
+        build_liblzma
+        cf_flags="-static -O2 -pthread -I/tmp/zlib-aarch64/include -I/tmp/liblzma-aarch64/include"
+        ld_flags="-static -L/tmp/zlib-aarch64/lib -lz -L/tmp/liblzma-aarch64/lib -llzma"
+        cd $CURR_DIR
+    fi
+
+    cp -r "zstd-${ZSTD_VERSION}" "$BUILD_DIR"
+    cd "$BUILD_DIR"
+    ls -lrt
 
     echo "Building zstd statically for $arch..."
-    make clean
-    CC="$compiler" CFLAGS="-static -O2 -pthread" LDFLAGS="-static" make -j4 zstd
-    cp zstd "$output_file"
+    make clean -C "zstd-${ZSTD_VERSION}"
+    CC="$compiler" CFLAGS="${cf_flags}" LDFLAGS="${ld_flags}" make -j4 -C "zstd-${ZSTD_VERSION}" zstd
+    "$strip_cmd" -s "zstd-${ZSTD_VERSION}/programs/zstd"
+    mv "zstd-${ZSTD_VERSION}/programs/zstd" "$CURR_DIR/$output_file"
 
+    chmod +x $CURR_DIR/$output_file
+    rm -rf *
+    cd "$CURR_DIR"
     echo "Verifying $output_file..."
-    check_binary "$output_file" "$arch"
+    check_binary "$output_file" "$verify"
 done
-echo "Binaries are located at ${OUTPUT_FILES[*]}."
+echo "Binaries are located at ${OUTPUT_FILES[*]}."

screwdriver.yaml

@@ -46,14 +46,14 @@ jobs:
     image: buildpack-deps:oracular
     environment:
       ZSTD_VERSION: 1.5.6
-      BUILD_DIR: "/tmp/ztsd_static_build"
+      BUILD_DIR: "/tmp/zstd_static_build"
       USER_SHELL_BIN: bash
     steps:
       - make: ./build_zstd.sh
       - package: |
           a=($RELEASE_FILES)
-          tar -czf ${a[1]} ${BUILD_DIR}/zstd-linux-x86_64 && store-cli set ${a[1]} --type=cache --scope=event
-          tar -czf ${a[2]} ${BUILD_DIR}/zstd-linux-aarch64 && store-cli set ${a[2]} --type=cache --scope=event
+          tar -czf ${a[1]} ./zstd-linux-x86_64 && store-cli set ${a[1]} --type=cache --scope=event
+          tar -czf ${a[2]} ./zstd-linux-aarch64 && store-cli set ${a[2]} --type=cache --scope=event
           
   zstd-mac:
     requires: [~pr, ~commit]