diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 0ad270e79..2da400e73 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -23,6 +23,24 @@ - [ ] This change is a new feature (non-breaking change which adds functionality). - [ ] This change is a breaking change (fix or feature that will cause existing behavior to change). +## Custom Platform Testing (Optional) + + +**Custom HPCC-Platform Repository:** +repository: + +**Custom HPCC-Platform Branch:** +branch: + + + ## Checklist: - [ ] I have created a corresponding JIRA ticket for this submission - [ ] My code follows the code style of this project. diff --git a/.github/actions/build-k8s/action.yaml b/.github/actions/build-k8s/action.yaml new file mode 100644 index 000000000..d13533b43 --- /dev/null +++ b/.github/actions/build-k8s/action.yaml @@ -0,0 +1,92 @@ +name: 'Build HPCC K8s Image' +description: 'Builds and registers an HPCC K8s image' +inputs: + os: + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + asset-name: + description: 'Asset Name' + required: false + default: 'docker-ubuntu-22_04-containerized' + platform-folder: + description: 'Platform Folder' + required: false + default: './HPCC-Platform' + +runs: + using: "composite" + steps: + - name: Free additional disk space (remove Android SDK + Tools) + run: | + sudo rm -rf /usr/local/lib/android + shell: bash + + - name: Download Package + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.asset-name }} + path: ${{ inputs.asset-name }} + + - name: Download Support Files + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.asset-name }}-support-files + path: ${{ inputs.asset-name }}-support-files + + - name: Wait for registry startup + run: | + kubectl wait pods --for=jsonpath='{.status.phase}'=Running -l app=registry -n container-registry --timeout=90s + shell: bash + + # Note: We need to move the package file due to the Dockerfile expecting it in the platform folder + - name: Find & Move Package + run: | + pwd + echo "Searching for .deb packages..." + find ./ -name "*.deb" -ls + + # Find the most recent .deb file anywhere in the current directory tree + k8s_pkg_path=$(find ./ -name "*.deb" -type f -print 2>/dev/null | sort -nr | head -1) + + if [ -z "$k8s_pkg_path" ]; then + echo "ERROR: No .deb package found!" + exit 1 + fi + + echo "Found package: $k8s_pkg_path" + k8s_pkg_file=$(basename "$k8s_pkg_path") + + mv ${k8s_pkg_path} ${{ inputs.platform-folder }}/${k8s_pkg_file} + echo "k8s_pkg_file=$k8s_pkg_file" >> $GITHUB_ENV + echo "$k8s_pkg_file" + shell: bash + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + with: + driver-opts: | + network=host + + - name: Calculate vars + id: vars + run: | + community_base_ref=${{ github.event.base_ref || github.ref }} + candidate_branch=$(echo $community_base_ref | cut -d'/' -f3) + echo "candidate_branch=$candidate_branch" >> $GITHUB_OUTPUT + echo "candidate_base_branch=$(echo $candidate_branch | awk -F'.' -v OFS='.' '{ $3="x"; print }')" >> $GITHUB_OUTPUT + shell: bash + + - name: Create Docker Image (community) + uses: docker/build-push-action@v4 + with: + builder: ${{ steps.buildx.outputs.name }} + file: ${{ inputs.platform-folder }}/dockerfiles/platform-core-ubuntu-22.04/Dockerfile + context: ${{ inputs.platform-folder }}/ + push: true + tags: localhost:32000/hpccsystems/platform-core:latest + build-args: | + PKG_FILE=${{ env.k8s_pkg_file }} + cache-from: | + type=registry,ref=hpccsystems/platform-core-${{ inputs.os }}:${{ steps.vars.outputs.candidate_base_branch }} diff --git a/.github/actions/deploy-hpcc-k8s/action.yaml b/.github/actions/deploy-hpcc-k8s/action.yaml new file mode 100644 index 000000000..849559eee --- /dev/null +++ b/.github/actions/deploy-hpcc-k8s/action.yaml @@ -0,0 +1,143 @@ +name: 'Deploy HPCC on K8s' +description: 'Builds a docker image and deploys a HPCC cluster on K8s' +inputs: + use-local-image: + description: 'Use Local Docker Image' + required: false + default: 'false' + platform-folder: + description: 'Platform Folder' + required: false + default: './HPCC-Platform' + +runs: + using: "composite" + steps: + + # Note: IP Address range below is the IP address range that will be made available for load balancers + # on the host machine, they aren't actual load balancers so they will not be accessible externally + - name: Enable LoadBalancers + run: | + sudo microk8s enable metallb:10.64.140.43-10.64.140.69 + shell: bash + + - name: Create Root Certificates + run: | + echo "[req] + default_bits = 2048 + default_keyfile = ca.key + distinguished_name = dn + prompt = no + x509_extensions = x509_ca + + [dn] + C = US + ST = GA + L = Alparetta + O = Lexis Nexis Risk + OU = Platform Development + CN = TestCluster + emailAddress = support@lexisnexisrisk.com + + [x509_ca] + basicConstraints=CA:true,pathlen:1" > ca-req.cfg + openssl req -x509 -newkey rsa:2048 -nodes -keyout ca.key -sha256 -days 1825 -out ca.crt -config ca-req.cfg + kubectl create secret tls hpcc-signing-issuer-key-pair --cert=ca.crt --key=ca.key + kubectl create secret tls hpcc-local-issuer-key-pair --cert=ca.crt --key=ca.key + sudo keytool -import -trustcacerts -cacerts -storepass changeit -noprompt -alias hpcc-local-issuer -file ca.crt + shell: bash + + - name: Install JetStack Cert Manager + run: | + sudo microk8s helm repo add jetstack https://charts.jetstack.io + sudo microk8s helm repo update + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml + sudo microk8s helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.11.0 + shell: bash + + - name: Install HPCC Cluster + run: | + cat < values.yaml + certificates: + enabled: true + dafilesrv: + - name: rowservice + disabled: false + application: stream + service: + servicePort: 7600 + visibility: global + - name: direct-access + disabled: true + application: directio + service: + servicePort: 7200 + visibility: local + - name: spray-service + application: spray + service: + servicePort: 7300 + visibility: cluster + EOF + # if use-local-image is true, install from local + if [ "${{ inputs.use-local-image }}" == "true" ]; then + sudo microk8s helm install myhpcc ${{ inputs.platform-folder }}/helm/hpcc --set global.image.root=localhost:32000/hpccsystems --set global.image.version=latest -f values.yaml + else + sudo microk8s helm repo add hpcc https://hpcc-systems.github.io/helm-chart + sudo microk8s helm repo update + sudo microk8s helm install myhpcc hpcc/hpcc -f values.yaml + fi + shell: bash + + - name: Wait for Deployment Rollout & Grab Service IPs + run: | + sleep 10 + deploy=$(kubectl get deploy -o name) + for i in $deploy; do + if ! kubectl rollout status $i -w --timeout=180s; then + echo "Deployment $i failed to roll out within timeout" + echo "kubectl get $i -o yaml" + kubectl get $i -o yaml + echo "kubectl describe $i" + kubectl describe $i + echo "kubectl get pods" + kubectl get pods + echo "Getting information for all pods" + for pod in $(kubectl get pods --no-headers | awk '{ print $1 }'); do + echo "Pod name: $pod" + echo "$pod: kubectl get pod" + kubectl get pod $pod -o yaml || echo "Failed to get pod for $pod" + echo "$pod: kubectl describe pod" + kubectl describe pod $pod || echo "Failed to describe pod for $pod" + echo "$pod: getting recent logs" + kubectl logs --tail=50 $pod || echo "Failed to get logs for $pod" + done + echo "kubectl get rs" + kubectl get rs + exit 1 + fi + done + echo "ECLWATCH_IP=$(kubectl get svc eclwatch -o jsonpath='{.spec.clusterIP}')" >> $GITHUB_ENV + echo "ROWSERVICE_IP=$(kubectl get svc rowservice -o jsonpath='{.spec.clusterIP}')" >> $GITHUB_ENV + echo "SQL_TO_ECL_IP=$(kubectl get svc sql2ecl -o jsonpath='{.spec.clusterIP}')" >> $GITHUB_ENV + kubectl get pods + kubectl get svc + shell: bash + + - name: Add Host File Entries + run: | + sudo -- sh -c -e "echo '${{ env.ECLWATCH_IP }} eclwatch.default' >> /etc/hosts"; + sudo -- sh -c -e "echo '${{ env.ROWSERVICE_IP }} rowservice.default' >> /etc/hosts"; + sudo -- sh -c -e "echo '${{ env.SQL_TO_ECL_IP }} sql2ecl.default' >> /etc/hosts"; + shell: bash + + - name: Trust Certs + run: | + openssl s_client -showcerts -connect eclwatch.default:8010 < /dev/null | openssl x509 -outform DER > cert.der + sudo keytool -import -keystore $JAVA_HOME/lib/security/cacerts -storepass changeit -noprompt -alias eclwatch-tls -file cert.der + openssl s_client -showcerts -connect rowservice.default:7600 < /dev/null | openssl x509 -outform DER > cert.der + sudo keytool -import -keystore $JAVA_HOME/lib/security/cacerts -storepass changeit -noprompt -alias dafilesrv-tls -file cert.der + openssl s_client -showcerts -connect sql2ecl.default:8510 < /dev/null | openssl x509 -outform DER > cert.der + sudo keytool -import -keystore $JAVA_HOME/lib/security/cacerts -storepass changeit -noprompt -alias sqltoecl-tls -file cert.der + shell: bash + diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 000000000..88afb2a39 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,255 @@ +name: Build Package (Docker) + +on: + workflow_call: + inputs: + should-build: + type: boolean + description: 'Should the build be executed' + required: false + default: true + repository: + type: string + description: 'Repository to checkout (if not HPCC-Platform)' + required: false + default: 'HPCC-Platform' + branch: + type: string + description: 'Branch to checkout (if not base branch)' + required: false + default: '' + os: + type: string + description: 'Operating System' + required: false + default: 'ubuntu-22.04' + ln: + type: boolean + description: 'Internal Build' + required: false + default: false + single-package: + type: boolean + description: 'Single Package' + required: false + default: true + build-type: + type: string + description: 'CMake Build Type' + required: false + default: 'RelWithDebInfo' + containerized: + type: boolean + description: 'Containerized Build' + required: false + default: false + strip-files: + type: boolean + description: 'Strip Debug Symbols' + required: false + default: true + cmake-configuration: + type: string + description: 'CMake Configuration' + required: false + default: '-DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DUSE_CPPUNIT=ON -DSUPPRESS_V8EMBED=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache' + cmake-configuration-ex: + type: string + description: 'CMake Configuration Extra' + required: false + default: '' + update-cache: + type: boolean + description: 'Update Cache' + required: false + default: true + upload-package: + type: boolean + description: 'Upload Package as Asset' + required: false + default: false + asset-name: + type: string + description: 'Asset Name (if upload-package is true)' + required: false + default: 'build-docker-package' + tag_postfix: + type: string + description: 'Either -arm or empty string' + required: false + default: '' + secrets: + LNB_TOKEN: + required: false + +jobs: + + build-docker: + runs-on: ubuntu-22.04${{ inputs.tag_postfix }} + steps: + - name: Free additional disk space (remove Android SDK + Tools) + if: ${{ inputs.should-build }} + continue-on-error: true + run: | + df -h + sudo rm -rf ${ANDROID_HOME} + sudo rm -rf ${JAVA_HOME_8_X64} + sudo rm -rf ${JAVA_HOME_11_X64} + sudo rm -rf ${JAVA_HOME_17_X64} + sudo rm -rf ${JAVA_HOME_21_X64} + sudo rm -rf ${CHROMEWEBDRIVER} + sudo rm -rf ${EDGEWEBDRIVER} + sudo rm -rf ${GECKOWEBDRIVER} + sudo rm -rf ${SELENIUM_JAR_PATH} + + - name: Display disk space (post free) + if: ${{ inputs.should-build }} + run: | + df -h + + - name: Checkout HPCC-Platform + if: ${{ (inputs.repository == '' || inputs.branch == '') && inputs.should-build == true }} + uses: actions/checkout@v4 + with: + submodules: recursive + path: ${{ github.workspace }}/HPCC-Platform + + - name: Checkout HPCC-Platform + if: ${{ inputs.repository != '' || inputs.branch != '' && inputs.should-build == true }} + uses: actions/checkout@v4 + with: + submodules: recursive + path: ${{ github.workspace }}/HPCC-Platform + repository: ${{ inputs.repository }} + ref: ${{ inputs.branch }} + + + - name: Checkout LN + if: ${{ inputs.ln == true && inputs.should-build == true }} + uses: actions/checkout@v4 + with: + repository: ${{ github.repository_owner }}/LN + token: ${{ secrets.LNB_TOKEN }} + ref: ${{ inputs.branch }} + submodules: recursive + path: ${{ github.workspace }}/LN + + - name: Calculate vars + if: ${{ inputs.should-build }} + id: vars + working-directory: ${{ github.workspace }}/HPCC-Platform/vcpkg + shell: "bash" + run: | + branch_label_1=${{ inputs.branch }} + branch_label_2=$(echo ${{ github.ref }} | cut -d'/' -f3) + branch_label=${branch_label_1:-$branch_label_2} + echo "branch_label=$branch_label" >> $GITHUB_OUTPUT + vcpkg_sha_short=$(git rev-parse --short=8 HEAD) + echo "vcpkg_sha_short=$vcpkg_sha_short" >> $GITHUB_OUTPUT + docker_build_label=hpccsystems/platform-build-${{ inputs.os }} + echo "docker_tag=$docker_build_label:$vcpkg_sha_short${{ inputs.tag_postfix }}" >> $GITHUB_OUTPUT + echo "docker_tag_candidate_base=$docker_build_label:$branch_label${{ inputs.tag_postfix }}" >> $GITHUB_OUTPUT + + - name: Print vars + if: ${{ inputs.should-build }} + shell: "bash" + run: | + echo "${{ toJSON(steps.vars.outputs) }}" + + - name: Set up Docker Buildx + if: ${{ inputs.should-build }} + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Pull Build Image + if: ${{ inputs.should-build }} + run: | + docker pull ${{ steps.vars.outputs.docker_tag }} || true + docker pull ${{ steps.vars.outputs.docker_tag_candidate_base }} || true + + - uses: hendrikmuhs/ccache-action@v1.2 + if: ${{ inputs.should-build }} + with: + save: ${{ inputs.update-cache == true }} + key: docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} + restore-keys: | + docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}-${{ inputs.containerized == true && 'k8s' || 'bare-metal' }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}-${{ steps.vars.outputs.branch_label }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}-${{ inputs.build-type }}- + docker${{ inputs.tag_postfix }}-${{ inputs.os }}- + + - name: Docker build image + if: ${{ inputs.should-build }} + uses: docker/build-push-action@v5 + with: + builder: ${{ steps.buildx.outputs.name }} + file: HPCC-Platform/dockerfiles/${{ inputs.os }}.dockerfile + context: HPCC-Platform/dockerfiles/. + push: false + load: true + build-args: | + VCPKG_REF=${{ steps.vars.outputs.vcpkg_sha_short }}${{ inputs.tag_postfix }} + tags: | + ${{ steps.vars.outputs.docker_tag }} + ${{ steps.vars.outputs.docker_tag_candidate_base }} + cache-from: | + type=registry,ref=${{ steps.vars.outputs.docker_tag }} + type=registry,ref=${{ steps.vars.outputs.docker_tag_candidate_base }} + + - name: CMake Configure and Build + if: ${{ inputs.should-build }} + shell: "bash" + run: | + mkdir -p ${{ github.workspace }}/LN + mkdir -p ${{ github.workspace }}/build + mkdir -p ${{ github.workspace }}/.ccache + declare -a plugins + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED"' }}) + for plugin in "${plugins[@]}"; do + sudo rm -f ${{ github.workspace }}/build/CMakeCache.txt + sudo rm -rf ${{ github.workspace }}/build/CMakeFiles + docker run --rm \ + --mount source="${{ github.workspace }}/HPCC-Platform",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=delegated \ + --mount source="${{ github.workspace }}/LN",target=/hpcc-dev/LN,type=bind,consistency=delegated \ + --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=delegated \ + --mount source="${{ github.workspace }}/.ccache",target=/root/.ccache,type=bind,consistency=delegated \ + ${{ steps.vars.outputs.docker_tag }} "\ + cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \ + cmake --build /hpcc-dev/build --parallel ${{ inputs.upload-package == true && '--target package' || ''}}" + done + + - name: Upload Package + if: ${{ inputs.upload-package == true && inputs.should-build == true }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.asset-name }} + path: | + ${{ github.workspace }}/build/*.deb + ${{ github.workspace }}/build/*.rpm + if-no-files-found: error + + - name: Upload Support Files + if: ${{ inputs.upload-package == true && inputs.should-build == true }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.asset-name }}-support-files + path: | + ${{ github.workspace }}/HPCC-Platform/.github/workflows/smoketest-preabort.sh + ${{ github.workspace }}/HPCC-Platform/.github/workflows/timeoutcmd + if-no-files-found: error + + - name: Upload ECL Watch UI Test Files + if: ${{ inputs.upload-package == true && inputs.should-build == true }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.asset-name }}-ecl_watch_ui_tests + path: | + ${{ github.workspace }}/HPCC-Platform/esp/src/test-ui/tests/**/* + if-no-files-found: error + + - name: Upload Error Logs + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.os }}-${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }}-logs + path: ${{ github.workspace }}/build/**/*.log diff --git a/.github/workflows/k8s-regression-suite.yml b/.github/workflows/k8s-regression-suite.yml index 4a5b284ad..ea08379e1 100644 --- a/.github/workflows/k8s-regression-suite.yml +++ b/.github/workflows/k8s-regression-suite.yml @@ -9,9 +9,67 @@ on: workflow_dispatch: jobs: - test-against-platform: + parse-pr-body: runs-on: ubuntu-latest + outputs: + repository: ${{ steps.parse-pr-body.outputs.repository }} + branch: ${{ steps.parse-pr-body.outputs.branch }} + has-custom-platform: ${{ steps.parse-pr-body.outputs.has-custom-platform == 'true' }} + steps: + - name: Parse PR Body + id: parse-pr-body + uses: actions/github-script@v7 + with: + script: | + if (context.eventName === 'workflow_dispatch') { + core.info('Manual trigger - using default platform'); + core.setOutput("repository", ""); + core.setOutput("branch", ""); + core.setOutput("has-custom-platform", false); + return; + } + + const body = context.payload.pull_request?.body || ""; + console.log(`PR body: ${body}`); + + let repository = ""; + let branch = ""; + + const repoMatch = body.match(/repository:\s*([^\n]+)/); + if (repoMatch) repository = repoMatch[1].trim(); + + const branchMatch = body.match(/branch:\s*([^\n]+)/); + if (branchMatch) branch = branchMatch[1].trim(); + + const hasCustomPlatform = repository != "" && branch != ""; + + if (!hasCustomPlatform) { + console.log(`No custom repository/branch specified. Using default platform.`); + } else { + console.log(`Custom platform found: ${repository}@${branch}`); + } + + core.setOutput("repository", repository); + core.setOutput("branch", branch); + core.setOutput("has-custom-platform", hasCustomPlatform); + + build-docker: + name: build-docker-ubuntu-22.04 + needs: parse-pr-body + uses: ./.github/workflows/build-docker.yml + with: + os: ubuntu-22.04 + should-build: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} + repository: ${{ needs.parse-pr-body.outputs.repository }} + branch: ${{ needs.parse-pr-body.outputs.branch }} + upload-package: true + containerized: true + asset-name: docker-ubuntu-22_04-containerized + secrets: inherit + test-against-platform: + runs-on: ubuntu-latest + needs: [parse-pr-body, build-docker] steps: - name: Setup JDK 11 @@ -26,114 +84,37 @@ jobs: devMode: 'true' addons: '["dns", "rbac", "hostpath-storage", "registry"]' - # Note: IP Address range below is the IP address range that will be made available for load balancers - # on the host machine, they aren't actual load balancers so they will not be accessible externally - - name: Enable LoadBalancers - run: | - sudo microk8s enable metallb:10.64.140.43-10.64.140.49 - - - name: Install JetStack Cert Manager - run: | - helm repo add jetstack https://charts.jetstack.io - helm repo update - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.0/cert-manager.crds.yaml - helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.11.0 - - - name: Create Root Certificates - run: | - echo "[req] - default_bits = 2048 - default_keyfile = ca.key - distinguished_name = dn - prompt = no - x509_extensions = x509_ca - - [dn] - C = US - ST = GA - L = Alparetta - O = Lexis Nexis Risk - OU = Platform Development - CN = TestCluster - emailAddress = support@lexisnexisrisk.com - - [x509_ca] - basicConstraints=CA:true,pathlen:1" > ca-req.cfg - openssl req -x509 -newkey rsa:2048 -nodes -keyout ca.key -sha256 -days 1825 -out ca.crt -config ca-req.cfg - kubectl create secret tls hpcc-signing-issuer-key-pair --cert=ca.crt --key=ca.key - kubectl create secret tls hpcc-local-issuer-key-pair --cert=ca.crt --key=ca.key - sudo keytool -import -trustcacerts -cacerts -storepass changeit -noprompt -alias hpcc-local-issuer -file ca.crt - - - name: Install HPCC Cluster - run: | - cat < values.yaml - certificates: - enabled: true - dafilesrv: - - name: rowservice - disabled: false - application: stream - service: - servicePort: 7600 - visibility: global - - name: direct-access - disabled: true - application: directio - service: - servicePort: 7200 - visibility: local - - name: spray-service - application: spray - service: - servicePort: 7300 - visibility: cluster - EOF - helm repo add hpcc https://hpcc-systems.github.io/helm-chart - helm repo update - helm install myhpcc hpcc/hpcc -f values.yaml - - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - - name: Rebase + - name: Checkout HPCC-Platform + if: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} + uses: actions/checkout@v4 + with: + submodules: recursive + path: ./HPCC-Platform + repository: ${{ needs.parse-pr-body.outputs.repository }} + ref: ${{ needs.parse-pr-body.outputs.branch }} + + - name: Set up Git run: | git config user.email 'hpccsystems@lexisnexisrisk.com' git config user.name 'hpccsystems development' - git rebase origin/${{ github.event.pull_request.base.ref }} git log --pretty=one -n 15 - - name: Wait for ECLWatch Startup - run: | - echo "Waiting for ECLWatch startup" && kubectl wait --for=condition=ready pod --timeout=180s -l app=eclwatch - echo "Waiting for Rowservice startup" && kubectl wait --for=condition=ready pod --timeout=180s -l server=rowservice - echo "Waiting for SQL2ECL startup" && kubectl wait --for=condition=ready pod --timeout=180s -l app=sql2ecl - echo "ECLWATCH_IP=$(kubectl get svc eclwatch -o jsonpath='{.spec.clusterIP}')" >> $GITHUB_ENV - echo "ROWSERVICE_IP=$(kubectl get svc rowservice -o jsonpath='{.spec.clusterIP}')" >> $GITHUB_ENV - echo "SQL_TO_ECL_IP=$(kubectl get svc sql2ecl -o jsonpath='{.spec.clusterIP}')" >> $GITHUB_ENV - kubectl get pods - kubectl get svc - - - name: Add Host File Entries - run: | - sudo -- sh -c -e "echo '${{ env.ECLWATCH_IP }} eclwatch.default' >> /etc/hosts"; - sudo -- sh -c -e "echo '${{ env.ROWSERVICE_IP }} rowservice.default' >> /etc/hosts"; - sudo -- sh -c -e "echo '${{ env.SQL_TO_ECL_IP }} sql2ecl.default' >> /etc/hosts"; - - # Notes: - # Using keytool -import -cacerts doesn't work as expected, need to specify the cacerts path explicitly - # Path changed between JDK 8 & 11, Command for JDK 8: - # sudo keytool -import -keystore $JAVA_HOME/jre/lib/security/cacerts -storepass changeit -noprompt -alias eclwatch-tls -file cert.der - - name: Trust Certs - run: | - openssl s_client -showcerts -connect eclwatch.default:8010 < /dev/null | openssl x509 -outform DER > cert.der - sudo keytool -import -keystore $JAVA_HOME/lib/security/cacerts -storepass changeit -noprompt -alias eclwatch-tls -file cert.der - openssl s_client -showcerts -connect rowservice.default:7600 < /dev/null | openssl x509 -outform DER > cert.der - sudo keytool -import -keystore $JAVA_HOME/lib/security/cacerts -storepass changeit -noprompt -alias dafilesrv-tls -file cert.der - openssl s_client -showcerts -connect sql2ecl.default:8510 < /dev/null | openssl x509 -outform DER > cert.der - sudo keytool -import -keystore $JAVA_HOME/lib/security/cacerts -storepass changeit -noprompt -alias sqltoecl-tls -file cert.der + - name: Build HPCC Docker Image + if: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} + uses: ./.github/actions/build-k8s + with: + platform-folder: ./HPCC-Platform + - name: Deploy HPCC on K8s + uses: ./.github/actions/deploy-hpcc-k8s + with: + use-local-image: ${{ needs.parse-pr-body.outputs.has-custom-platform == 'true' }} + platform-folder: ./HPCC-Platform # speed things up with caching from https://docs.github.com/en/actions/guides/building-and-testing-java-with-maven - name: Cache Maven packages @@ -151,19 +132,32 @@ jobs: echo "DFSClient tests failed - collecting rowservice pod logs" ROWSERVICE_POD=$(kubectl get pods -l server=rowservice -o jsonpath='{.items[0].metadata.name}') echo "Rowservice pod: $ROWSERVICE_POD" - kubectl logs $ROWSERVICE_POD | grep -v "meta info did not mark file" + kubectl logs $ROWSERVICE_POD > rowservice.log - name: ECLWatch Logs run: | ESP_POD=$(kubectl get pods -l app=eclwatch -o jsonpath='{.items[0].metadata.name}') echo "ESP pod: $ESP_POD" - kubectl logs $ESP_POD + kubectl logs $ESP_POD > eclwatch.log - name: SQL2ECL Logs run: | SQL_TO_ECL_POD=$(kubectl get pods -l app=sql2ecl -o jsonpath='{.items[0].metadata.name}') echo "SQL to ECL pod: $SQL_TO_ECL_POD" - kubectl logs $SQL_TO_ECL_POD + kubectl logs $SQL_TO_ECL_POD > sql2ecl.log + + - name: Upload Logs and Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: k8s-test-results-${{ github.run_number }} + path: | + rowservice.log + eclwatch.log + sql2ecl.log + **/FailedTests.csv + **/*.log + retention-days: 30 - name: Process Errors shell: python @@ -199,3 +193,32 @@ jobs: if hadErrors: sys.exit(1) + + - name: Extract Certificates and Collect Files + if: always() + run: | + echo "Extracting certificates from Kubernetes secret..." + kubectl get secret hpcc-signing-issuer-key-pair -o jsonpath='{.data.tls\.crt}' | base64 -d > extracted.crt + kubectl get secret hpcc-signing-issuer-key-pair -o jsonpath='{.data.tls\.key}' | base64 -d > extracted.key + + echo "Checking for filePartBlob.txt and other relevant files..." + find . -name "filePartBlob.txt" -type f -exec echo "Found blob file: {}" \; + find . -name "*.csv" -path "*/FailedTests.csv" -exec echo "Found failed test file: {}" \; + + echo "Certificate files created:" + ls -la extracted.crt extracted.key + + echo "Certificate info:" + openssl x509 -in extracted.crt -text -noout | head -20 + + - name: Upload Blob & Key + if: always() + uses: actions/upload-artifact@v4 + with: + name: upload-blob-key-${{ github.run_number }} + path: | + extracted.crt + extracted.key + **/filePartBlob.txt + **/FailedTests.csv + retention-days: 30 \ No newline at end of file diff --git a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java index f0fbe7b0a..389f85e52 100644 --- a/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java +++ b/dfsclient/src/test/java/org/hpccsystems/dfs/client/DFSReadWriteTest.java @@ -347,63 +347,77 @@ public void readResumeTest() throws Exception Assert.fail("Invalid or null record definition"); } - ArrayList resumeInfo = new ArrayList(); - ArrayList resumeFilePart = new ArrayList(); ArrayList records = new ArrayList(); - for (int i = 0; i < fileParts.length; i++) + ArrayList resumedRecords = new ArrayList(); + try { - HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition()); - HpccRemoteFileReader fileReader = new HpccRemoteFileReader(fileParts[i], originalRD, recordBuilder); + ArrayList resumeInfo = new ArrayList(); + ArrayList resumeFilePart = new ArrayList(); - while (fileReader.hasNext()) + for (int i = 0; i < fileParts.length; i++) { - resumeInfo.add(fileReader.getFileReadResumeInfo()); - resumeFilePart.add(i); - HPCCRecord record = fileReader.next(); + HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition()); + HpccRemoteFileReader fileReader = new HpccRemoteFileReader(fileParts[i], originalRD, recordBuilder); - if (record == null) + while (fileReader.hasNext()) { - Assert.fail("Received null record during read"); - } + resumeInfo.add(fileReader.getFileReadResumeInfo()); + resumeFilePart.add(i); + HPCCRecord record = fileReader.next(); - records.add(record); - } - fileReader.close(); + if (record == null) + { + Assert.fail("Received null record during read"); + } - if (fileReader.getRemoteReadMessageCount() > 0) - System.out.println("Messages from file part (" + i + ") read operation:\n" + fileReader.getRemoteReadMessages()); - } + records.add(record); + } + fileReader.close(); - Runtime runtime = Runtime.getRuntime(); - int readSizeKB = 100; - ArrayList resumedRecords = new ArrayList(); - for (int i = 0; i < resumeInfo.size(); i++) - { - HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition()); - HpccRemoteFileReader fileReader = new HpccRemoteFileReader( - fileParts[resumeFilePart.get(i)], originalRD, recordBuilder, 1000000, -1, true, readSizeKB, resumeInfo.get(i)); + if (fileReader.getRemoteReadMessageCount() > 0) + System.out.println("Messages from file part (" + i + ") read operation:\n" + fileReader.getRemoteReadMessages()); + } - if (fileReader.hasNext()) + Runtime runtime = Runtime.getRuntime(); + int readSizeKB = 100; + for (int i = 0; i < resumeInfo.size(); i++) { - HPCCRecord record = fileReader.next(); - if (record == null) + HPCCRecordBuilder recordBuilder = new HPCCRecordBuilder(file.getProjectedRecordDefinition()); + HpccRemoteFileReader fileReader = new HpccRemoteFileReader( + fileParts[resumeFilePart.get(i)], originalRD, recordBuilder, 1000000, -1, true, readSizeKB, resumeInfo.get(i)); + + if (fileReader.hasNext()) { - Assert.fail("Received null record during resumed read"); + HPCCRecord record = fileReader.next(); + if (record == null) + { + Assert.fail("Received null record during resumed read"); + } + + resumedRecords.add(record); } - resumedRecords.add(record); - } + // Periodically run garbage collector to prevent buffers in remote file readers from exhausting free memory + // This is only needed due to rapidly creating / destroying thousands of HpccRemoteFileReaders + if ((i % 10) == 0) + { + runtime.gc(); + } + fileReader.close(); - // Periodically run garbage collector to prevent buffers in remote file readers from exhausting free memory - // This is only needed due to rapidly creating / destroying thousands of HpccRemoteFileReaders - if ((i % 10) == 0) - { - runtime.gc(); + fileReader = null; + recordBuilder = null; } - fileReader.close(); + } + catch (Exception e) + { + // Write fileParts[0].fileAccessBlob to a local file part + Path filePartPath = Paths.get("filePartBlob.txt"); + Files.write(filePartPath, fileParts[0].getFileAccessBlob().getBytes()); + System.out.println("File part access blob written to: " + filePartPath.toAbsolutePath()); - fileReader = null; - recordBuilder = null; + e.printStackTrace(); + Assert.fail("Exception during read resume test: " + e.getMessage()); } assertEquals("Number of records did not match during read resume.", records.size(), resumedRecords.size());