From d0f41202e4369e7b90a21f4dc3f81c33b165c82e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 5 Dec 2024 11:37:20 -0800 Subject: [PATCH 1/7] ci: provide a gcs report bucket name --- .github/workflows/connector-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 0f7033f1f..8fed7d764 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -119,7 +119,7 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' with: repository: airbytehq/airbyte - ref: master + ref: aj/airbyte-ci/allow-report-uploads-with-fewer-permissions path: airbyte - name: Test Connector if: steps.no_changes.outputs.status != 'cancelled' @@ -131,6 +131,7 @@ jobs: run: | cd airbyte make tools.airbyte-ci-dev.install + --ci-report-bucket-name=airbyte-ci-reports-multi airbyte-ci-dev connectors \ --name ${{matrix.connector}} \ --use-local-cdk \ From 57d4683e9be24975593120fe2a5ca4d0057395a6 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 5 Dec 2024 11:44:19 -0800 Subject: [PATCH 2/7] dummy change --- airbyte_cdk/logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte_cdk/logger.py b/airbyte_cdk/logger.py index 78061b605..5c8bf5306 100644 --- a/airbyte_cdk/logger.py +++ b/airbyte_cdk/logger.py @@ -80,6 +80,7 @@ def format(self, record: logging.LogRecord) -> str: ) return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode() + @staticmethod def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]: """ From 3899e683a601a9a2eaad2203cf6ffd1930940c89 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 5 Dec 2024 11:46:23 -0800 Subject: [PATCH 3/7] fix airbyte-ci syntax --- .github/workflows/connector-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 8fed7d764..7ac027395 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -131,8 +131,9 @@ jobs: run: | cd airbyte make tools.airbyte-ci-dev.install - --ci-report-bucket-name=airbyte-ci-reports-multi - airbyte-ci-dev connectors \ + airbyte-ci-dev \ + --ci-report-bucket-name=airbyte-ci-reports-multi \ + connectors \ --name ${{matrix.connector}} \ --use-local-cdk \ test \ From 44a5a56e0af6e64469887bedb92db57d03795b74 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 5 Dec 2024 12:06:20 -0800 Subject: [PATCH 4/7] evaluate results and post the url for easy viewing --- .github/workflows/connector-tests.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 7ac027395..ff29b7435 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -94,6 +94,8 @@ jobs: # cdk_extra: n/a name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs['src'] == 'false' || needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" + permissions: + checks: write steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) id: no_changes @@ -141,7 +143,8 @@ jobs: --skip-step qa_checks \ --skip-step connector_live_tests - - name: Evaluate Test Output + - name: Evaluate Test + id: evaluate_output if: always() && steps.no_changes.outputs.status != 'cancelled' run: | # save job output json file as ci step output @@ -150,7 +153,9 @@ jobs: success=$(echo ${job_output} | jq -r '.success') failed_step=$(echo ${job_output} | jq -r '.failed_steps | select(length > 0) | .[0] // "None"') run_duration=$(echo ${job_output} | jq -r '.run_duration') + html_report_url=$(echo ${job_output} | jq -r '.html_report_url') echo "## Job Output for ${{matrix.connector}}" >> $GITHUB_STEP_SUMMARY + echo "- [HTML Report](${html_report_url})" >> $GITHUB_STEP_SUMMARY echo "- Success: ${success}" >> $GITHUB_STEP_SUMMARY echo "- Test Duration: $(printf "%.0f" ${run_duration})s" >> $GITHUB_STEP_SUMMARY if [ "${success}" != "true" ]; then @@ -161,6 +166,8 @@ jobs: echo "::error::Test failed for connector '${{ matrix.connector }}' on step '${failed_step}'. Check the logs for more details." exit 1 fi + echo "success=${success}" >> $GITHUB_OUTPUT + echo "html_report_url=${html_report_url}" >> $GITHUB_OUTPUT # Upload the job output to the artifacts - name: Upload Job Output @@ -170,3 +177,14 @@ jobs: with: name: ${{matrix.connector}}-job-output path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports + + - name: Override Details URL for GitHub Check Run + if: steps.evaluate_output.outputs.html_report_url != '' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh api \ + -X PATCH \ + -H "Accept: application/vnd.github.v3+json" \ + "/repos/${{ github.repository }}/check-runs/${{ github.run_id }}" \ + -f details_url="${{ steps.evaluate_output.outputs.html_report_url }}" From a50c1b74f6f4bd9863c8432fa5cfe27210eacbc8 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 5 Dec 2024 12:13:44 -0800 Subject: [PATCH 5/7] remove details url override attempt (not working) --- .github/workflows/connector-tests.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index ff29b7435..d30fec53d 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -177,14 +177,3 @@ jobs: with: name: ${{matrix.connector}}-job-output path: airbyte/airbyte-ci/connectors/pipelines/pipeline_reports - - - name: Override Details URL for GitHub Check Run - if: steps.evaluate_output.outputs.html_report_url != '' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - gh api \ - -X PATCH \ - -H "Accept: application/vnd.github.v3+json" \ - "/repos/${{ github.repository }}/check-runs/${{ github.run_id }}" \ - -f details_url="${{ steps.evaluate_output.outputs.html_report_url }}" From 491896df211f406d476aa8397258b1574757c604 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Thu, 5 Dec 2024 20:27:27 +0000 Subject: [PATCH 6/7] Auto-fix lint and format issues --- airbyte_cdk/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte_cdk/logger.py b/airbyte_cdk/logger.py index 5c8bf5306..78061b605 100644 --- a/airbyte_cdk/logger.py +++ b/airbyte_cdk/logger.py @@ -80,7 +80,6 @@ def format(self, record: logging.LogRecord) -> str: ) return orjson.dumps(AirbyteMessageSerializer.dump(log_message)).decode() - @staticmethod def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]: """ From 9748eae3afff11f0f02a1ab171bce382363d0386 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Thu, 5 Dec 2024 13:11:22 -0800 Subject: [PATCH 7/7] Apply suggestions from code review --- .github/workflows/connector-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index d30fec53d..666b722e5 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -121,7 +121,7 @@ jobs: if: steps.no_changes.outputs.status != 'cancelled' with: repository: airbytehq/airbyte - ref: aj/airbyte-ci/allow-report-uploads-with-fewer-permissions + ref: master path: airbyte - name: Test Connector if: steps.no_changes.outputs.status != 'cancelled'