diff --git a/.github/workflows/build-yocto.yml b/.github/workflows/build-yocto.yml new file mode 100644 index 00000000..c4acbdae --- /dev/null +++ b/.github/workflows/build-yocto.yml @@ -0,0 +1,216 @@ +name: Build Yocto + +on: + workflow_call: + outputs: + artifacts_url: + description: "URL to retrieve build artifacts" + value: ${{ jobs.create-output.outputs.url }} + +env: + CACHE_DIR: /efs/qli/meta-qcom + KAS_REPO_REF_DIR: /efs/qli/meta-qcom/kas-mirrors + KAS_CONTAINER: /efs/qli/meta-qcom/kas-mirrors/kas-container + +jobs: + kas-setup: + if: github.repository_owner == 'qualcomm-linux' + runs-on: [self-hosted, qcom-u2404, amd64-ssd] + steps: + - name: Update kas-container + run: | + LATEST=$(git ls-remote --tags --refs --sort="v:refname" https://github.com/siemens/kas | tail -n1 | sed 's/.*\///') + wget -qO ${KAS_CONTAINER} https://raw.githubusercontent.com/siemens/kas/refs/tags/$LATEST/kas-container + chmod +x ${KAS_CONTAINER} + + - name: Update kas mirrors + run: | + for r in $(find ${KAS_REPO_REF_DIR}/* -maxdepth 0 -type d); do + echo "pre-fetch: $r" + git -C $r fetch --prune origin '+refs/*:refs/*' + done + + - uses: actions/checkout@v4 + with: + repository: qualcomm-linux/meta-qcom + ref: master + + - name: Run kas lock + run: | + ${KAS_CONTAINER} lock --update ci/base.yml:ci/qcom-distro.yml + + - uses: actions/upload-artifact@v4 + with: + name: kas-lock + path: ci/*.lock.yml + + yocto-run-checks: + needs: kas-setup + if: github.repository_owner == 'qualcomm-linux' + runs-on: [self-hosted, qcom-u2404, amd64-ssd] + steps: + - uses: actions/checkout@v4 + with: + repository: qualcomm-linux/meta-qcom + path: meta-qcom + ref: master + + - uses: actions/checkout@v4 + with: + path: meta-qcom-distro + + - uses: actions/download-artifact@v6 + with: + name: kas-lock + path: meta-qcom/ci/ + + - name: Run yocto-check-layer + run: | + cd meta-qcom-distro + ../meta-qcom/ci/kas-container-shell-helper.sh ci/yocto-check-layer.sh ../meta-qcom-distro + + - name: Run Yocto patchreview + run: | + cd meta-qcom + ci/kas-container-shell-helper.sh ci/yocto-patchreview.sh ../meta-qcom-distro + + compile: + needs: [kas-setup, yocto-run-checks] + if: github.repository_owner == 'qualcomm-linux' + runs-on: [self-hosted, qcom-u2404, amd64-ssd] + outputs: + url: ${{ steps.compile_kas.outputs.url }} + strategy: + fail-fast: true + matrix: + machine: + - iq-8275-evk + - iq-9075-evk + - iq-x7181-evk + - kaanapali-mtp + - qcm6490-idp + - qcs615-ride + - qcs8300-ride-sx + - qcs9100-ride-sx + - qcom-armv8a + - rb1-core-kit + - rb3gen2-core-kit + - sm8750-mtp + distro: + - name: qcom-distro + yamlfile: ':ci/qcom-distro-prop-image.yml' + - name: qcom-distro-sota + yamlfile: ':ci/qcom-distro-sota.yml' + kernel: + - type: default + dirname: "" + yamlfile: "" + - type: 6.18 + dirname: "+linux-qcom-6.18" + yamlfile: ":ci/linux-qcom-6.18.yml" + include: + # Additional builds for specific machines + - machine: qcom-armv8a + distro: + name: qcom-distro + yamlfile: ':ci/qcom-distro-prop-image.yml' + kernel: + type: additional + dirname: "+linux-yocto-dev" + yamlfile: ":ci/linux-yocto-dev.yml" + # include kvm compatible machines for the builds + - machine: qcom-armv8a + distro: + name: qcom-distro-kvm + yamlfile: ':ci/qcom-distro-kvm.yml' + kernel: + type: default + dirname: "" + yamlfile: "" + - machine: qcom-armv7a + distro: + name: qcom-distro + yamlfile: ':ci/qcom-distro.yml' + kernel: + type: default + dirname: "" + yamlfile: "" + name: ${{ matrix.machine }}/${{ matrix.distro.name }}${{ matrix.kernel.dirname }} + steps: + - uses: actions/checkout@v4 + with: + repository: qualcomm-linux/meta-qcom + ref: master + + - name: Run kas build + uses: ./.github/actions/compile + id: compile_kas + with: + machine: ${{matrix.machine}} + distro_yaml: ${{matrix.distro.yamlfile}} + distro_name: ${{matrix.distro.name}} + kernel_yaml: ${{matrix.kernel.yamlfile}} + kernel_dirname: ${{matrix.kernel.dirname}} + cache_dir: ${CACHE_DIR} + kas: ${KAS_CONTAINER} + + publish_summary: + needs: compile + runs-on: [self-hosted, qcom-u2404, amd64-ssd] + steps: + - name: 'Download build URLs' + uses: actions/download-artifact@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + pattern: build-url* + path: urlfiles + merge-multiple: true + + - name: "Print output" + shell: python + id: print-output + run: | + import os + ftable = {} + oslist = set() + machinelist = set() + for fname in os.listdir("./urlfiles"): + if fname.startswith("build-url"): + b, m, o = fname.split("_", 2) + oslist.add(o) + machinelist.add(m) + url = "" + with open(f"./urlfiles/{fname}", "r") as urlfile: + url = urlfile.read() + if not o in ftable: + ftable.update({o:{m: url}}) + else: + ftable[o].update({m: url}) + + table_str = "| |" + + for m in sorted(machinelist): + table_str += f" {m} |" + + table_str += "\n|" + for i in range(len(machinelist) + 1): + table_str += " ---- |" + + table_str += "\n" + + for o in sorted(ftable.keys()): + table_str += f"| {o} |" + for m in sorted(machinelist): + url = ftable[o].get(m) + if url: + url = url.strip() + table_str += f" [Files]({url}/{o}/{m}/) |" + else: + table_str += " |" + table_str += "\n" + summary_file_name = os.environ.get("GITHUB_STEP_SUMMARY") + if summary_file_name: + with open(summary_file_name, "a") as summaryfile: + summaryfile.write("## Download URLs\n") + summaryfile.write(table_str) + print(table_str) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 00000000..64843511 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,28 @@ +name: Build on PR + +on: + pull_request: + paths-ignore: + - 'README.md' + - 'README' + - 'SECURITY.md' + +permissions: + checks: write + pull-requests: write + contents: read + packages: read + +jobs: + event-file: + name: "Upload event file" + runs-on: ubuntu-latest + steps: + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: Event File + path: ${{ github.event_path }} + build-pr: + uses: ./.github/workflows/build-yocto.yml + diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml new file mode 100644 index 00000000..fe90f58b --- /dev/null +++ b/.github/workflows/publish-results.yml @@ -0,0 +1,66 @@ +name: Publish test results + +on: + workflow_call: + inputs: + workflow_id: + required: true + type: string + event_name: + required: true + type: string + event_file: + required: true + type: string + commit: + required: true + type: string + +permissions: + checks: write + pull-requests: write + contents: read + packages: read + +jobs: + publish-test-results: + name: "Publish Tests Results" + runs-on: ubuntu-latest + steps: + - name: Download result files + uses: actions/download-artifact@v6 + with: + run-id: ${{ inputs.workflow_id }} + path: artifacts + github-token: ${{ github.token }} + + - name: Download result files PR + if: ${{ github.run_id != inputs.workflow_id }} + uses: actions/download-artifact@v6 + with: + path: artifacts + github-token: ${{ github.token }} + + - name: "List files" + run: | + echo $GITHUB_WORKSPACE + ls -R $GITHUB_WORKSPACE + + - id: app_token + uses: actions/create-github-app-token@v2 + if: always() + with: + app-id: 2291458 + private-key: ${{ secrets.TEST_REPORTING_APP_TOKEN }} + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + if: always() + with: + commit: ${{ inputs.commit }} + event_file: ${{ inputs.event_file}} + event_name: ${{ inputs.event_name }} + files: "${{ github.workspace }}/artifacts/**/*.xml" + action_fail: true + action_fail_on_inconclusive: true + github_token: ${{ steps.app_token.outputs.token }} diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 00000000..61ad7719 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,31 @@ +name: Build on push + +on: + push: + branches: + - main + +permissions: + checks: write + pull-requests: write + contents: read + packages: read + +jobs: + build: + uses: ./.github/workflows/build-yocto.yml + test: + uses: ./.github/workflows/test.yml + needs: [build] + secrets: inherit + with: + build_id: ${{ github.run_id }} + publish-test-results: + uses: ./.github/workflows/publish-results.yml + needs: test + secrets: inherit + with: + workflow_id: ${{ github.run_id }} + event_name: ${{ github.event_name }} + event_file: ${{ github.event_path }} + commit: ${{ github.sha }} diff --git a/.github/workflows/repolinter.yml b/.github/workflows/repolinter.yml new file mode 100644 index 00000000..aff3ca2e --- /dev/null +++ b/.github/workflows/repolinter.yml @@ -0,0 +1,31 @@ +name: QuIC Organization Repolinter + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + repolinter: + if: github.repository_owner == 'qualcomm-linux' + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Verify repolinter config file is present + id: check_files + uses: andstor/file-existence-action@v3 + with: + files: "repolint.json" + - name: Run Repolinter with local repolint.json + if: steps.check_files.outputs.files_exists == 'true' + uses: todogroup/repolinter-action@v1 + with: + config_file: "repolint.json" + - name: Run Repolinter with default ruleset + if: steps.check_files.outputs.files_exists == 'false' + uses: todogroup/repolinter-action@v1 + with: + config_url: "https://raw.githubusercontent.com/quic/.github/main/repolint.json" diff --git a/.github/workflows/stales.yml b/.github/workflows/stales.yml new file mode 100644 index 00000000..e9fed721 --- /dev/null +++ b/.github/workflows/stales.yml @@ -0,0 +1,25 @@ +name: 'Close stale issues and pull requests with no recent activity' +on: + schedule: + - cron: "30 1 * * *" + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue has been marked as stale due to 30 days of inactivity. To prevent automatic closure in 5 days, remove the stale label or add a comment. You can reopen a closed issue at any time.' + stale-pr-message: 'This pull request has been marked as stale due to 30 days of inactivity. To prevent automatic closure in 5 days, remove the stale label or add a comment. You can reopen a closed pull request at any time.' + exempt-issue-labels: bug,enhancement + exempt-pr-labels: bug,enhancement + days-before-stale: 30 + days-before-close: 5 + days-before-issue-close: -1 + remove-stale-when-updated: true + remove-issue-stale-when-updated: true + remove-pr-stale-when-updated: true diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml new file mode 100644 index 00000000..d68291fe --- /dev/null +++ b/.github/workflows/test-pr.yml @@ -0,0 +1,83 @@ +name: Test PR build + +run-name: "Tests triggered by PR: ${{ github.event.workflow_run.display_title }}" + +on: + workflow_run: + workflows: ["Build on PR"] + types: + - completed + +permissions: + checks: write + pull-requests: write + contents: read + packages: read + +jobs: + test: + if: ${{ github.event.workflow_run.conclusion == 'success' }} + uses: ./.github/workflows/test.yml + secrets: inherit + with: + build_id: ${{ github.event.workflow_run.id }} + + comment-on-pr: + name: "Comment on PR" + needs: test + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Download Artifacts + uses: actions/download-artifact@v6 + with: + path: artifacts + + - name: Download event file + uses: actions/download-artifact@v6 + with: + run-id: ${{ github.event.workflow_run.id }} + path: artifacts + github-token: ${{ github.token }} + + - name: "List files" + run: | + echo $GITHUB_WORKSPACE + ls -R $GITHUB_WORKSPACE + + - name: Prepare PR comment + id: pr_comment_prep + run: | + echo "## Test run [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" > pr-comment.txt + echo "## Test jobs for commit ${{ github.event.workflow_run.head_sha }}" >> pr-comment.txt + for json_file in $(find ${{ github.workspace }} -name "test-job-*.json") + do + DEVICE_TYPE=$(cat "$json_file" | jq -r ".requested_device_type") + URL=$(cat "$json_file" | jq -r ".url") + JOB_ID=$(cat "$json_file" | jq -r ".id") + JOB_TITLE=$(cat "$json_file"| jq -r ".description") + echo " * [Job $JOB_ID on $DEVICE_TYPE]($URL) - ${JOB_TITLE}" + echo " * [Job $JOB_ID on $DEVICE_TYPE]($URL) - ${JOB_TITLE}" >> pr-comment.txt + done + PR_NUMBER=$(cat "artifacts/Event File/event.json" | jq -r ".number") + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + + - name: Comment on PR + uses: thollander/actions-comment-pull-request@v3 + with: + file-path: pr-comment.txt + pr-number: ${{ steps.pr_comment_prep.outputs.pr_number }} + + publish-test-results: + uses: ./.github/workflows/publish-results.yml + secrets: inherit + needs: [test] + with: + commit: ${{ github.event.workflow_run.sha }} + event_file: artifacts/Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + workflow_id: ${{ github.event.workflow_run.id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..cda91619 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,146 @@ +name: Tests + +on: + workflow_call: + inputs: + build_id: + required: true + type: string + +jobs: + prepare-jobs: + runs-on: ubuntu-latest + strategy: + matrix: + machine: + - iq-8275-evk + - iq-9075-evk + - qcm6490-idp + - qcs615-ride + - rb3gen2-core-kit + - qcs8300-ride-sx + - qcs9100-ride-sx + - qcom-armv8a + - qcom-armv7a + - rb1-core-kit + distro: + - name: qcom-distro + steps: + - uses: actions/checkout@v4 + with: + repository: qualcomm-linux/meta-qcom + ref: master + fetch-depth: 1 + + - name: 'Download build URLs' + uses: actions/download-artifact@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ inputs.build_id }} + pattern: build-url* + + - name: "Generate testjobs" + run: | + export BUILD_URL_FILE="$GITHUB_WORKSPACE/build-url_${{ matrix.machine }}_${{ matrix.distro.name }}/build-url_${{ matrix.machine }}_${{ matrix.distro.name }}" + echo "${BUILD_URL_FILE}" + if [ -f "$BUILD_URL_FILE" ]; then + # install dependencies + export BUILD_URL=$(cat "${BUILD_URL_FILE}") + pip install jinja2 + if [ "${{ matrix.machine }}" = "qcom-armv8a" ]; then + python3 ci/generatetests.py --templates ci/lava --os ${{ matrix.distro.name }} --device dragonboard-410c --build-url "${BUILD_URL}" + python3 ci/generatetests.py --templates ci/lava --os ${{ matrix.distro.name }} --device dragonboard-820c --build-url "${BUILD_URL}" + echo "MACHINE=dragonboard" >> $GITHUB_ENV + else + python3 ci/generatetests.py --templates ci/lava --os ${{ matrix.distro.name }} --device ${{ matrix.machine }} --build-url "${BUILD_URL}" + echo "MACHINE=${{ matrix.machine }}" >> $GITHUB_ENV + fi + fi + - name: 'Upload test jobs' + uses: actions/upload-artifact@v4 + with: + name: testjobs-${{ matrix.machine }}-${{ matrix.distro.name }} + path: ${{ env.MACHINE}}*-${{ matrix.distro.name }}-*.yaml + + prepare-job-list: + needs: prepare-jobs + runs-on: ubuntu-latest + outputs: + jobmatrix: ${{ steps.listjobs.outputs.jobmatrix }} + steps: + - name: 'Download job templates' + uses: actions/download-artifact@v6 + with: + pattern: testjobs-* + + - name: "List jobs" + id: listjobs + run: | + JOBFILES=$(find . -name *.yaml) + JOBFILES=$(echo "$JOBFILES" | sed -e "s/^/\"/" | sed -e "s/$/\",/" | tr -d "\n" | sed -e "s/.$//") + JOBFILES="[${JOBFILES}]" + J=$(jq -cn --argjson jobfiles "$JOBFILES" '{target: $jobfiles}') + echo "jobmatrix=$J" >> $GITHUB_OUTPUT + echo "Preparing testjob files" + + submit-job: + needs: prepare-job-list + runs-on: ubuntu-latest + continue-on-error: true + strategy: + matrix: ${{ fromJson(needs.prepare-job-list.outputs.jobmatrix) }} + steps: + - name: 'Download job templates' + uses: actions/download-artifact@v6 + with: + pattern: testjobs-* + + - name: 'Prepare output file name' + run: | + OUTPUT_NAME=$(echo "${{ matrix.target }}" | sed "s|\/|-|g") + echo "RESULT_NAME=${OUTPUT_NAME#??}" >> $GITHUB_ENV + + - name: Submit ${{ matrix.target }} + timeout-minutes: 20 + uses: foundriesio/lava-action@v9 + with: + lava_token: ${{ secrets.LAVATOKEN }} + lava_url: 'lava.infra.foundries.io' + job_definition: ${{ matrix.target }} + wait_for_job: true + fail_action_on_failure: true + fail_action_on_incomplete: true + save_result_as_artifact: true + save_job_details: true + result_file_name: "${{ env.RESULT_NAME }}" + + publish-test-summary: + name: "Publish Tests Summary" + needs: submit-job + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + + steps: + - name: Download Artifacts + uses: actions/download-artifact@v6 + with: + path: artifacts + + - name: "List files" + run: | + echo $GITHUB_WORKSPACE + ls -R $GITHUB_WORKSPACE + + - name: Publish Test Job Details + run: | + for json_file in $(find ${{ github.workspace }} -name "test-job-*.json") + do + DEVICE_TYPE=$(cat "$json_file" | jq -r ".requested_device_type") + URL=$(cat "$json_file" | jq -r ".url") + JOB_ID=$(cat "$json_file" | jq -r ".id") + JOB_TITLE=$(cat "$json_file"| jq -r ".description") + echo " * [Job $JOB_ID on $DEVICE_TYPE]($URL) - ${JOB_TITLE}" + echo " * [Job $JOB_ID on $DEVICE_TYPE]($URL) - ${JOB_TITLE}" >> $GITHUB_STEP_SUMMARY + done diff --git a/README.md b/README.md index fe4bf398..5f172d78 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # meta-qcom-distro +[![Build on push](https://img.shields.io/github/actions/workflow/status/qualcomm-linux/meta-qcom-distro/push.yml?label=Build%20on%20push)](https://github.com/qualcomm-linux/meta-qcom-distro/actions/workflows/push.yml) + ## Introduction OpenEmbedded/Yocto Project Reference Distro layer for Qualcomm based platforms. diff --git a/ci/yocto-check-layer.sh b/ci/yocto-check-layer.sh new file mode 100755 index 00000000..e027fec2 --- /dev/null +++ b/ci/yocto-check-layer.sh @@ -0,0 +1,41 @@ +#!/bin/bash -e +# Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved. +# SPDX-License-Identifier: MIT + +if [ -z $1 ] || [ -z $2 ] ; then + echo "The REPO_DIR or WORK_DIR is empty and it needs to point to the corresponding directories." + echo "Please run it with:" + echo " $0 REPO_DIR WORK_DIR" + exit 1 +fi + +REPO_DIR="$1" +WORK_DIR="$2" + +_is_dir(){ + test -d "$1" && return + echo "The '$1' is not a directory." + exit 1 +} + +_is_dir "$REPO_DIR" +_is_dir "$WORK_DIR" + +# Creates a temporary build directory to run the yocto-check-layer +# script to avoid a contaminated environment. +BUILDDIR="$(mktemp -p $WORK_DIR -d -t build-yocto-check-layer-XXXX)" +source $WORK_DIR/oe-core/oe-init-build-env $BUILDDIR +git -c advice.detachedHead=false -c init.defaultBranch=master clone --quiet --shared $REPO_DIR meta-qcom-distro + +# Yocto Project layer checking tool +CMD="yocto-check-layer" +# Layer to check +CMD="$CMD meta-qcom-distro" +# Disable auto layer discovery +CMD="$CMD --no-auto" +# Layers to process for dependencies +CMD="$CMD --dependency $WORK_DIR/oe-core/meta" +# Disable automatic testing of dependencies +CMD="$CMD --no-auto-dependency" + +exec $CMD