diff --git a/.github/actions/on_device_tests/action.yaml b/.github/actions/on_device_tests/action.yaml index c981288dafe7f..f9aac1aa568f0 100644 --- a/.github/actions/on_device_tests/action.yaml +++ b/.github/actions/on_device_tests/action.yaml @@ -4,7 +4,7 @@ description: Runs on-device tests. runs: using: "composite" steps: - - name: Install requirements + - name: Install Requirements run: | pip3 install grpcio==1.38.0 grpcio-tools==1.38.0 shell: bash @@ -12,15 +12,14 @@ runs: run: | python -m grpc_tools.protoc -Itools/ --python_out=tools/ --grpc_python_out=tools/ tools/on_device_tests_gateway.proto shell: bash - - name: Set up Cloud SDK + - name: Set Up Cloud SDK uses: isarkis/setup-gcloud@40dce7857b354839efac498d3632050f568090b6 # v1.1.1 - name: Set env vars - env: - WORKFLOW: ${{ github.workflow }} run: | echo "PROJECT_NAME=$(gcloud config get-value project)" >> $GITHUB_ENV - echo "GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER}" >> $GITHUB_ENV - echo "WORKFLOW=${WORKFLOW}" >> $GITHUB_ENV + + # Test results and logs + echo "GCS_RESULTS_PATH=gs://cobalt-unittest-storage/results/${{ matrix.name }}/${{ github.run_id }}_${{ matrix.shard }}" >> $GITHUB_ENV # Boot loader env if [ "${COBALT_EVERGREEN_LOADER}" != "null" ]; then @@ -43,8 +42,9 @@ runs: echo "USE_SHARDING=1" >> $GITHUB_ENV fi shell: bash - - name: run ${{ env.SHARD_NAME }} tests on ${{ matrix.platform }} platform + - name: Run ${{ env.SHARD_NAME }} Tests on ${{ matrix.platform }} Platform env: + GCS_PATH: gs://${{ env.PROJECT_NAME }}-test-artifacts/${{ github.workflow }}/${{ github.run_number }}/${{ matrix.platform }}_${{ matrix.config }} GITHUB_SHA: ${{ github.sha }} GITHUB_TOKEN: ${{ github.token }} GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -57,13 +57,17 @@ runs: GITHUB_PR_HEAD_USER_ID: ${{ github.event.pull_request.head.user.id }} GITHUB_COMMIT_AUTHOR_USERNAME: ${{ github.event.commits[0].author.username }} GITHUB_COMMIT_AUTHOR_EMAIL: ${{ github.event.commits[0].author.email }} + GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + GITHUB_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GITHUB_WORKFLOW: ${{ github.workflow }} run: | set -uxe python3 tools/on_device_tests_gateway_client.py \ --token ${GITHUB_TOKEN} \ --change_id "${GITHUB_PR_HEAD_SHA:-$GITHUB_SHA}" \ trigger \ - --test_type ${{ env.TEST_TYPE }} \ + --test_type ${TEST_TYPE} \ --platform ${{ matrix.target_platform }} \ --config ${{ matrix.config }} \ --tag cobalt_github_${GITHUB_EVENT_NAME} \ @@ -74,10 +78,11 @@ runs: ${DIMENSION:+"--dimension" "$DIMENSION"} \ ${USE_SHARDING:+"--unittest_shard_index" "${{ matrix.shard }}"} \ ${ON_DEVICE_TEST_ATTEMPTS:+"--test_attempts" "$ON_DEVICE_TEST_ATTEMPTS"} \ - --archive_path gs://${PROJECT_NAME}-test-artifacts/${WORKFLOW}/${GITHUB_RUN_NUMBER}/${{ matrix.platform }}_${{ matrix.config }}/artifacts.tar \ + --archive_path "${GCS_PATH}/artifacts.tar" \ + --gcs_result_path "${GCS_RESULTS_PATH}" \ --label github \ --label ${GITHUB_EVENT_NAME} \ - --label ${WORKFLOW} \ + --label ${GITHUB_WORKFLOW} \ --label actor-${GITHUB_ACTOR} \ --label actor_id-${GITHUB_ACTOR_ID} \ --label triggering_actor-${GITHUB_TRIGGERING_ACTOR} \ @@ -86,3 +91,115 @@ runs: --label author-${GITHUB_PR_HEAD_USER_LOGIN:-$GITHUB_COMMIT_AUTHOR_USERNAME} \ --label author_id-${GITHUB_PR_HEAD_USER_ID:-$GITHUB_COMMIT_AUTHOR_EMAIL} shell: bash + - name: Download ${{ matrix.platform }} Test Results + if: always() && env.TEST_TYPE == 'unit_test' + run: | + # Don't break on error (-e), some commands are expected to fail. + set -ux + + COBALT_LOGS_DIR="${GITHUB_WORKSPACE}/cobalt_logs" + UNIT_TEST_RESULT_PATH="${GITHUB_WORKSPACE}/unit-test-results" + COBALT_XMLS_FILENAME="cobalt_xmls.zip" + + # Forward environment variables for uploading artifacts in later steps. + echo "UNIT_TEST_RESULT_PATH=${UNIT_TEST_RESULT_PATH}" >> $GITHUB_ENV + echo "COBALT_LOGS_DIR=${COBALT_LOGS_DIR}" >> $GITHUB_ENV + + mkdir -p "${GITHUB_WORKSPACE}/test_results" + cd "${GITHUB_WORKSPACE}/test_results" + + i=0 + # Try downloading the results for 6x 10 seconds before giving up. + while [ $i -lt 6 ]; do + # The results are uploaded after the test has completed. + sleep 10 + + # The log files are named by the device lab test driver. + COBALT_ERROR_LOG_FILENAME="webDriverTestLog.ERROR" + COBALT_INFO_LOG_FILENAME="webDriverTestLog.INFO" + + # This command will fail until the results have been uploaded. + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_ERROR_LOG_FILENAME}" . + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_INFO_LOG_FILENAME}" . + gsutil cp "${GCS_RESULTS_PATH}/${COBALT_XMLS_FILENAME}" . + + # Break if all files were downloaded. + if [[ -f "${COBALT_XMLS_FILENAME}" && -f "${COBALT_ERROR_LOG_FILENAME}" && -f "${COBALT_INFO_LOG_FILENAME}" ]]; then + break + fi + + i=$(( ${i} + 1 )) + done + + # Rename log files for archiving to not expose legacy weirdness. + mkdir -p "${COBALT_LOGS_DIR}/${{ matrix.platform }}/" + cp "${COBALT_ERROR_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stderr_${{ matrix.shard }}.log" + cp "${COBALT_INFO_LOG_FILENAME}" "${COBALT_LOGS_DIR}/${{ matrix.platform }}/stdout_${{ matrix.shard }}.log" + + # Prepare unit test results for DataDog upload. + RESULT_PATH=${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/${{ matrix.shard }}/ + mkdir -p ${RESULT_PATH} + + # Set tags for test differentiation. + tags="os.platform:${{ matrix.platform }}" + echo $tags > ${UNIT_TEST_RESULT_PATH}/${{ matrix.platform }}/TAGS + + unzip ${COBALT_XMLS_FILENAME} -d ${RESULT_PATH} + shell: bash + - name: Archive Unit Test Logs + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: Device logs + path: ${{ env.COBALT_LOGS_DIR }}/ + - name: Archive Unit Test Results + uses: actions/upload-artifact@v3 + if: always() && env.TEST_TYPE == 'unit_test' + with: + name: unit-test-results + path: ${{ env.UNIT_TEST_RESULT_PATH }}/ + - name: Get Datadog CLI + if: always() && env.TEST_TYPE == 'unit_test' && github.event == 'push' + shell: bash + env: + DD_VERSION: 'v2.18.0' + DD_SHA256SUM: 'adbe9b3a41faaf0b1d9702ba256cf8fa9e474c0cc8216f25e5b489c53d6f0a70 datadog-ci' + run: | + set -e + download_url="https://github.com/DataDog/datadog-ci/releases/download/${DD_VERSION}/datadog-ci_linux-x64" + curl -L --fail $download_url --output datadog-ci + echo ${DD_SHA256SUM} | sha256sum --check + chmod +x datadog-ci + - name: Upload to DataDog + if: always() && env.TEST_TYPE == 'unit_test' && github.event == 'push' + env: + DATADOG_API_KEY: ${{ secrets.DD_API_KEY }} + DATADOG_SITE: us5.datadoghq.com + DD_ENV: ci + DD_SERVICE: ${{ github.event.repository.name }} + # Need to populate git info via env vars as we don't have the repo to look at. + DD_GIT_REPOSITORY_URL: ${{ github.event.repository.git_url }} + DD_GIT_COMMIT_SHA: ${{ github.event.workflow_run.head_sha }} + DD_GIT_BRANCH: ${{ github.event.workflow_run.head_branch }} + DD_GIT_COMMIT_MESSAGE: ${{ github.event.workflow_run.head_commit.message }} + DD_GIT_COMMIT_AUTHOR_NAME: ${{ github.event.workflow_run.head_commit.author.name }} + DD_GIT_COMMIT_AUTHOR_EMAIL: ${{ github.event.workflow_run.head_commit.author.email }} + DD_GIT_COMMIT_AUTHOR_DATE: ${{ github.event.workflow_run.head_commit.timestamp }} + DD_GIT_COMMIT_COMMITTER_NAME: ${{ github.event.workflow_run.head_commit.committer.name }} + DD_GIT_COMMIT_COMMITTER_EMAIL: ${{ github.event.workflow_run.head_commit.committer.email }} + DD_GIT_COMMIT_COMMITTER_DATE: ${{ github.event.workflow_run.head_commit.timestamp }} + run: | + # Loop over each platform, extract the tags and upload xml results. + echo "Uploading $dir test report" + export DD_TAGS=`cat ${dir}TAGS` + ./datadog-ci junit upload unit-test-results/*.xml + shell: bash + - name: Print device logs + if: always() + run: | + if ls ${COBALT_LOGS_DIR}/**/*.log 1> /dev/null 2>&1; then + cat ${COBALT_LOGS_DIR}/**/*.log + else + echo "No device logs found" + fi + shell: bash diff --git a/.github/actions/on_host_test/action.yaml b/.github/actions/on_host_test/action.yaml index c751de0de0f97..745dc512a684e 100644 --- a/.github/actions/on_host_test/action.yaml +++ b/.github/actions/on_host_test/action.yaml @@ -84,16 +84,17 @@ runs: xvfb-run -a --server-args="-screen 0 1920x1080x24i +render +extension GLX -noreset" python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_runner.py --platform ${{matrix.target_platform}} --config ${{matrix.config}} -s ${{matrix.shard}} -r ${loader_args} --xml_output_dir=${TEST_RESULTS_DIR} fi fi - - name: Process unit test results - if: failure() + - name: Populate TAGS for unit test report + if: always() && steps.run-tests.outputs.test_type == 'unit_tests' shell: bash run: | - set -x - echo "Saving unit test report to ${TEST_REPORT_FILE}" - python3 ${GITHUB_WORKSPACE}/starboard/tools/testing/test_report_parser.py ${TEST_RESULTS_DIR} > ${TEST_REPORT_FILE} - - name: Upload unit test report + # Set tags for test differentiation. + tags="os.platform:${{ matrix.platform }}" + tags="${tags},os.family:${{ inputs.os }}" + echo $tags > ${TEST_RESULTS_DIR}/${{ matrix.platform }}/TAGS + - name: Archive unit test results uses: actions/upload-artifact@v3 - if: failure() + if: always() && steps.run-tests.outputs.test_type == 'unit_tests' with: name: unit-test-reports path: ${{env.TEST_REPORT_FILE}} diff --git a/tools/on_device_tests_gateway.proto b/tools/on_device_tests_gateway.proto index 456f1a9384660..e4907f53a2936 100644 --- a/tools/on_device_tests_gateway.proto +++ b/tools/on_device_tests_gateway.proto @@ -52,6 +52,10 @@ message OnDeviceTestsCommand { string unittest_shard_index = 17; string test_attempts = 18; string retry_level = 19; + string start_timeout = 20; + string test_timeout = 21; + string builder_url = 22; + string gcs_result_path = 23; } // Working directory and command line arguments to be passed to the gateway. diff --git a/tools/on_device_tests_gateway_client.py b/tools/on_device_tests_gateway_client.py index 6be7de8785c6b..bf2c05f74e407 100644 --- a/tools/on_device_tests_gateway_client.py +++ b/tools/on_device_tests_gateway_client.py @@ -77,6 +77,7 @@ def run_trigger_command(self, workdir: str, args: argparse.Namespace): config=args.config, tag=args.tag, labels=args.label, + gcs_result_path=args.gcs_result_path, builder_name=args.builder_name, change_id=args.change_id, build_number=args.build_number, @@ -189,6 +190,12 @@ def main(): type=str, help='Name of the builder that built the artifacts, ' 'if any. Saved with performance test results') + trigger_parser.add_argument( + '--builder_url', type=str, help='Url to the run, if any.') + trigger_parser.add_argument( + '--gcs_result_path', + type=str, + help='GCS url where test result files should be uploaded.') trigger_parser.add_argument( '-n', '--build_number',