V29 cleanup dependencies (#33)

* fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * finalize v29
nebulastream · Jan 30, 2023 · be22af1 · be22af1
1 parent 111dcfc
commit be22af1
Show file tree

Hide file tree

Showing 23 changed files with 98 additions and 485 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -6,23 +6,41 @@ on:
       - 'v*'
 
 jobs:
-  build-x64-linux:
+  build-linux:
     timeout-minutes: 360
-    env:
-      VCPKG_DEP_LIST: x64-linux
-      VCPKG_FEATURE_FLAGS: -binarycaching
-    runs-on: [self-hosted, linux, X64, dep-builder]
+    name: ${{ matrix.osversion }} ${{ matrix.arch }}
+    runs-on: [self-hosted, linux, '${{ matrix.arch }}', dep-builder]
     strategy:
+      fail-fast: false
       matrix:
-        osversion: [ubuntu-18.04, ubuntu-20.04, ubuntu-22.04]
+        include:
+          - osversion: ubuntu-18.04
+            arch: "x64"
+            force_system_binary: 0
+          - osversion: ubuntu-20.04
+            arch: "x64"
+            force_system_binary: 0
+          - osversion: ubuntu-22.04
+            arch: "x64"
+            force_system_binary: 0
+          - osversion: ubuntu-20.04
+            arch: "arm64"
+            force_system_binary: 1
+          - osversion: ubuntu-22.04
+            arch: "arm64"
+            force_system_binary: 1
+    env:
+      VCPKG_DEP_LIST: ${{ matrix.arch }}-linux
+      VCPKG_FORCE_SYSTEM_BINARIES: ${{ matrix.force_system_binary }}
+      VCPKG_FEATURE_FLAGS: -binarycaching
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: true
           fetch-depth: 0
       - name: Checkout vcokg submodules
         working-directory: ${{ github.workspace }}/vcpkg/
-        run: git fetch origin daa7215f3d68c2a2f2e18c398cb73dab63cda829
+        run: git fetch --unshallow || true
       - name: Set output vars to the actual tag name
         id: vars
         run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
@@ -33,12 +51,12 @@ jobs:
       - name: Install vcpkg
         id: bootstrapvcpkg
         run: |
-          docker run -v ${{ github.workspace }}:/build_dir --rm nes_dep_build_${{ matrix.osversion }} ./build_dir/vcpkg/bootstrap-vcpkg.sh -disableMetrics
+          docker run -e ARCH=${{ matrix.arch }} -v ${{ github.workspace }}:/build_dir --rm nes_dep_build_${{ matrix.osversion }} ./build_dir/vcpkg/bootstrap-vcpkg.sh -disableMetrics
         shell: bash
       - name: Install dependencies in manifest mode
         id: installdeps
         run: |
-          docker run -v ${{ github.workspace }}:/build_dir --rm nes_dep_build_${{ matrix.osversion }} \
+          docker run -e ARCH=${{ matrix.arch }} -v ${{ github.workspace }}:/build_dir --rm nes_dep_build_${{ matrix.osversion }}  \
           ./build_dir/vcpkg/vcpkg install --triplet=${{ env.VCPKG_DEP_LIST }}-nes \
           --host-triplet=${{ env.VCPKG_DEP_LIST }}-nes --overlay-triplets=/build_dir/custom-triplets/ --x-manifest-root=/build_dir/ --overlay-ports=/build_dir/vcpkg-registry/ports
       - name: Compress artifacts
@@ -77,70 +95,19 @@ jobs:
           test -n "$(docker image ls -q nes_dep_build_${{ matrix.osversion }})" && \
           docker image rm $(docker image ls -q nes_dep_build_${{ matrix.osversion }}) || \
           true
-
-  build-arm64-linux:
-    timeout-minutes: 360
-    env:
-      VCPKG_DEP_LIST: arm64-linux
-      VCPKG_FORCE_SYSTEM_BINARIES: 1
-      VCPKG_FEATURE_FLAGS: -binarycaching
-    runs-on: [self-hosted, linux, ARM64, dep-builder]
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-      - name: Checkout vcokg submodules
-        working-directory: ${{ github.workspace }}/vcpkg/
-        run: git fetch origin daa7215f3d68c2a2f2e18c398cb73dab63cda829
-      - name: Set output vars to the actual tag name
-        id: vars
-        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
-      - name: Install vcpkg
-        id: bootstrapvcpkg
-        run: |
-          ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh -disableMetrics
-        shell: bash
-      - name: Install dependencies in manifest mode
-        id: installdeps
-        run: |
-          ${{ github.workspace }}/vcpkg/vcpkg install --triplet=${{ env.VCPKG_DEP_LIST }}-nes --host-triplet=${{ env.VCPKG_DEP_LIST }}-nes --overlay-triplets=custom-triplets/ --overlay-ports=vcpkg-registry/ports
-      - name: Compress artifacts
-        id: compressdeps
-        run: |
-          mkdir nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes && \
-          mv ${{ github.workspace }}/vcpkg_installed nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/installed && \
-          mkdir -p nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/scripts/buildsystems && \
-          cp ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake ${{ github.workspace }}/nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/scripts/buildsystems/ && \
-          touch ${{ github.workspace }}/nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/.vcpkg-root && \
-          7z a nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes.7z nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes -mx9 -aoa
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        id: createrelease
-        with:
-          files: |
-            nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes.7z
-      - name: Upload GitHub Actions artifact of vcpkg build logs
-        uses: actions/upload-artifact@v2
-        if: failure()
-        id: uploadlogs
-        with:
-          name: vcpkg-logs-${{ env.VCPKG_DEP_LIST }}
-          path: ${{ github.workspace }}/vcpkg/buildtrees/**/*.log
-      - name: Clean build artifacts
-        id: cleanbuildartifacts
-        if: always()
-        run: |
-          rm -rf nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes \
-          nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-${{ matrix.osversion }}-nes.7z \
-          ${{ github.workspace }}/vcpkg/{buildtrees,downloads,packages}
-
-  build-arm-osx:
+  build-osx:
     timeout-minutes: 360
+    name: osx ${{ matrix.arch }}
+    runs-on: [self-hosted, macOS, '${{ matrix.arch }}']
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: "x64"
+          - arch: "arm64"
     env:
-      VCPKG_DEP_LIST: arm64-osx
+      VCPKG_DEP_LIST: ${{ matrix.arch }}-osx
       VCPKG_FEATURE_FLAGS: -binarycaching
-    runs-on: [ self-hosted, macos, ARM64]
     steps:
       - uses: lukka/get-cmake@latest
       - uses: actions/checkout@v3
@@ -149,7 +116,7 @@ jobs:
           fetch-depth: 0
       - name: Checkout vcokg submodules
         working-directory: ${{ github.workspace }}/vcpkg/
-        run: git fetch origin daa7215f3d68c2a2f2e18c398cb73dab63cda829
+        run: git fetch --unshallow || true
       - uses: actions/setup-python@v4
         with:
           python-version: "3.11"
@@ -188,66 +155,6 @@ jobs:
         with:
           name: vcpkg-logs-${{ env.VCPKG_DEP_LIST }}
           path: ${{ github.workspace }}/vcpkg/buildtrees/**/*.log
-      - name: Clean build artifacts
-        id: cleanbuildartifacts
-        if: always()
-        run: |
-          rm -rf nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes \
-          nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-${{ matrix.osversion }}-nes.7z \
-          ${{ github.workspace }}/vcpkg/{buildtrees,downloads,packages}
-
-  build-x64-osx:
-    timeout-minutes: 360
-    env:
-      VCPKG_DEP_LIST: x64-osx
-      VCPKG_FEATURE_FLAGS: -binarycaching
-    runs-on: [ self-hosted, macos, X64, dep-builder ]
-    steps:
-      - uses: lukka/get-cmake@latest
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.11"
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-          fetch-depth: 0
-      - name: Checkout vcokg submodules
-        working-directory: ${{ github.workspace }}/vcpkg/
-        run: git fetch origin daa7215f3d68c2a2f2e18c398cb73dab63cda829
-      - name: Set output vars to the actual tag name
-        id: vars
-        run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
-      - name: Install vcpkg
-        id: bootstrapvcpkg
-        run: |
-          ${{ github.workspace }}/vcpkg/bootstrap-vcpkg.sh -disableMetrics
-        shell: bash
-      - name: Install dependencies in manifest mode
-        id: installdeps
-        run: |
-          ${{ github.workspace }}/vcpkg/vcpkg install --triplet=${{ env.VCPKG_DEP_LIST }}-nes --host-triplet=${{ env.VCPKG_DEP_LIST }}-nes --overlay-triplets=custom-triplets/ --overlay-ports=vcpkg-registry/ports
-      - name: Compress artifacts
-        id: compressdeps
-        run: |
-          mkdir nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes && \
-          mv ${{ github.workspace }}/vcpkg_installed nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/installed && \
-          mkdir -p nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/scripts/buildsystems && \
-          cp ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake ${{ github.workspace }}/nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/scripts/buildsystems/ && \
-          touch ${{ github.workspace }}/nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes/.vcpkg-root && \
-          7z a nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes.7z nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes -mx9 -aoa
-      - name: Release
-        uses: softprops/action-gh-release@v1
-        id: createrelease
-        with:
-          files: |
-            nes-dependencies-${{ steps.vars.outputs.tag }}-${{ env.VCPKG_DEP_LIST }}-nes.7z
-      - name: Upload GitHub Actions artifact of vcpkg build logs
-        uses: actions/upload-artifact@v2
-        if: failure()
-        id: uploadlogs
-        with:
-          name: vcpkg-logs-${{ env.VCPKG_DEP_LIST }}
-          path: ${{ github.workspace }}/vcpkg/buildtrees/**/*.log
       - name: Clean build artifacts
         id: cleanbuildartifacts
         if: always()

diff --git a/custom-triplets/arm64-linux-nes.cmake b/custom-triplets/arm64-linux-nes.cmake
@@ -35,8 +35,4 @@ endif()
 
 if (PORT STREQUAL oatpp)
 	set(VCPKG_LIBRARY_LINKAGE static)
-endif()
-
-if (PORT STREQUAL cppkafka)
-	set(VCPKG_LIBRARY_LINKAGE dynamic)
 endif()
diff --git a/custom-triplets/arm64-osx-nes.cmake b/custom-triplets/arm64-osx-nes.cmake
@@ -28,8 +28,4 @@ endif()
 
 if (PORT STREQUAL oatpp)
 	set(VCPKG_LIBRARY_LINKAGE static)
-endif()
-
-if (PORT STREQUAL cppkafka)
-	set(VCPKG_LIBRARY_LINKAGE dynamic)
 endif()
diff --git a/custom-triplets/x64-linux-musl-nes.cmake b/custom-triplets/x64-linux-musl-nes.cmake
@@ -30,8 +30,4 @@ endif()
 
 if (PORT STREQUAL oatpp)
 	set(VCPKG_LIBRARY_LINKAGE static)
-endif()
-
-if (PORT STREQUAL cppkafka)
-	set(VCPKG_LIBRARY_LINKAGE dynamic)
 endif()
diff --git a/custom-triplets/x64-linux-nes.cmake b/custom-triplets/x64-linux-nes.cmake
@@ -30,8 +30,4 @@ endif()
 
 if (PORT STREQUAL oatpp)
 	set(VCPKG_LIBRARY_LINKAGE static)
-endif()
-
-if (PORT STREQUAL cppkafka)
-	set(VCPKG_LIBRARY_LINKAGE dynamic)
 endif()
diff --git a/custom-triplets/x64-osx-nes.cmake b/custom-triplets/x64-osx-nes.cmake
@@ -28,8 +28,4 @@ endif()
 
 if (PORT STREQUAL oatpp)
 	set(VCPKG_LIBRARY_LINKAGE static)
-endif()
-
-if (PORT STREQUAL cppkafka)
-	set(VCPKG_LIBRARY_LINKAGE dynamic)
 endif()
diff --git a/docker/Dockerfile-ubuntu-18.04 b/docker/Dockerfile-ubuntu-18.04
@@ -13,8 +13,18 @@ RUN apt-get update -qq && DEBIAN_FRONTEND="noninteractive" apt-get install -qq \
   build-essential \
   curl zip unzip tar \  
   pkg-config \
+  autoconf \
+  cmake \
+  ninja-build \
+  flex \
+  bison \
   && apt-get clean -qq
 
+RUN cd ${HOME} && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \
+    && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' \
+    && apt update && apt install -y kitware-archive-keyring && rm /etc/apt/trusted.gpg.d/kitware.gpg && apt update && apt install -y cmake
+
+
 ADD ./entrypoint-build.sh /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/Dockerfile-ubuntu-20.04 b/docker/Dockerfile-ubuntu-20.04
@@ -13,8 +13,18 @@ RUN apt-get update -qq && DEBIAN_FRONTEND="noninteractive" apt-get install -qq \
   build-essential \
   curl zip unzip tar \  
   pkg-config \
+  autoconf \
+  cmake \
+  ninja-build \
+  flex \
+  bison \
   && apt-get clean -qq
 
+RUN cd ${HOME} && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null \
+    && apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' \
+    && apt update && apt install -y kitware-archive-keyring && rm /etc/apt/trusted.gpg.d/kitware.gpg && apt update && apt install -y cmake
+
+
 ADD ./entrypoint-build.sh /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/Dockerfile-ubuntu-21.10 b/docker/Dockerfile-ubuntu-21.10
@@ -13,6 +13,11 @@ RUN apt-get update -qq && DEBIAN_FRONTEND="noninteractive" apt-get install -qq \
   build-essential \
   curl zip unzip tar \  
   pkg-config \
+  autoconf \
+  cmake \
+  ninja-build \
+  flex \
+  bison \
   && apt-get clean -qq
 
 ADD ./entrypoint-build.sh /entrypoint.sh

diff --git a/docker/Dockerfile-ubuntu-22.04 b/docker/Dockerfile-ubuntu-22.04
@@ -13,8 +13,15 @@ RUN apt-get update -qq && DEBIAN_FRONTEND="noninteractive" apt-get install -qq \
   build-essential \
   curl zip unzip tar \  
   pkg-config \
+  autoconf \
+  cmake \
+  ninja-build \
+  flex \
+  bison \
   && apt-get clean -qq
 
+
+
 ADD ./entrypoint-build.sh /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
diff --git a/docker/entrypoint-build.sh b/docker/entrypoint-build.sh
@@ -16,6 +16,14 @@
 #quit if command returns non-zero code
 set -e
 
+if [ "$ARCH" == "arm64" ]
+then
+ echo "USE ARM"
+ export VCPKG_FORCE_SYSTEM_BINARIES=1
+else
+ echo "USE X64"
+fi
+
 if [ $# -eq 0 ]
 then
     cd /build_dir/  

diff --git a/vcpkg-registry/ports/folly/FindLZ4.cmake b/vcpkg-registry/ports/folly/FindLZ4.cmake
diff --git a/vcpkg-registry/ports/folly/FindSnappy.cmake b/vcpkg-registry/ports/folly/FindSnappy.cmake