Merge remote-tracking branch 'remotes/airbyte/main' into 50395-2-2

* remotes/airbyte/main:
  fix(airbyte-cdk): Fix RequestOptionsProvider for PerPartitionWithGlobalCursor (airbytehq#254)
  feat(low-code): add profile assertion flow to oauth authenticator component (airbytehq#236)
  feat(Low-Code Concurrent CDK): Add ConcurrentPerPartitionCursor (airbytehq#111)
  fix: don't mypy unit_tests (airbytehq#241)
  fix: handle backoff_strategies in CompositeErrorHandler (airbytehq#225)
  feat(concurrent cursor): attempt at clamping datetime (airbytehq#234)
  ci: use `ubuntu-24.04` explicitly (resolves CI warnings) (airbytehq#244)
  Fix(sdm): module ref issue in python components import (airbytehq#243)
  feat(source-declarative-manifest): add support for custom Python components from dynamic text input (airbytehq#174)
  chore(deps): bump avro from 1.11.3 to 1.12.0 (airbytehq#133)
  docs: comments on what the `Dockerfile` is for (airbytehq#240)
  chore: move ruff configuration to dedicated ruff.toml file (airbytehq#237)

Author: Rusi Popov

.github/workflows/connector-tests.yml

@@ -25,7 +25,7 @@ concurrency:
 jobs:
   cdk_changes:
     name: Get Changes
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     permissions:
       statuses: write
       pull-requests: read
@@ -62,7 +62,7 @@ jobs:
     # Forked PRs are handled by the community_ci.yml workflow
     # If the condition is not met the job will be skipped (it will not fail)
     # runs-on: connector-test-large
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 360 # 6 hours
     strategy:
       fail-fast: false
@@ -123,6 +123,10 @@ jobs:
           repository: airbytehq/airbyte
           ref: master
           path: airbyte
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
       - name: Test Connector
         if: steps.no_changes.outputs.status != 'cancelled'
         timeout-minutes: 90
@@ -131,7 +135,7 @@ jobs:
           POETRY_DYNAMIC_VERSIONING_BYPASS: "0.0.0"
         run: |
           cd airbyte
-          make tools.airbyte-ci-binary.install
+          make tools.airbyte-ci-dev.install
           airbyte-ci \
           --ci-report-bucket-name=airbyte-ci-reports-multi \
           connectors \

.github/workflows/pdoc_preview.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   preview_docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout code

.github/workflows/pdoc_publish.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   publish_docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     environment:
       name: "github-pages"
       url: ${{ steps.deployment.outputs.page_url }}

.github/workflows/pypi_publish.yml

@@ -35,7 +35,7 @@ on:
 jobs:
   build:
     name: Build Python Package
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Detect Release Tag Version
         if: startsWith(github.ref, 'refs/tags/v')
@@ -107,7 +107,7 @@ jobs:
 
   publish_cdk:
     name: Publish CDK version to PyPI
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: [build]
     permissions:
       id-token: write
@@ -156,7 +156,7 @@ jobs:
       (github.event_name == 'workflow_dispatch' &&
        github.event.inputs.publish_to_dockerhub == 'true'
       )
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: [build]
     environment:
       name: DockerHub
@@ -257,7 +257,7 @@ jobs:
     env:
       VERSION: ${{ needs.build.outputs.VERSION }}
       IS_PRERELEASE: ${{ needs.build.outputs.IS_PRERELEASE }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/setup-python@v5
         with:

.github/workflows/pytest_fast.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   test-build:
     name: Build and Inspect Python Package
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -36,7 +36,7 @@ jobs:
 
   pytest-fast:
     name: Pytest (Fast)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       # Common steps:
       - name: Checkout code

.github/workflows/python_lint.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   ruff-lint-check:
     name: Ruff Lint Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       # Common steps:
       - name: Checkout code
@@ -32,7 +32,7 @@ jobs:
 
   ruff-format-check:
     name: Ruff Format Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       # Common steps:
       - name: Checkout code
@@ -55,7 +55,7 @@ jobs:
 
   mypy-check:
     name: MyPy Check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       # Common steps:
       - name: Checkout code

.github/workflows/release_drafter.yml

@@ -16,7 +16,7 @@ jobs:
     permissions:
       contents: write
       pull-requests: write
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       # Drafts the next Release notes as Pull Requests are merged into "main"
       - uses: release-drafter/release-drafter@v6

.github/workflows/semantic_pr_check.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
   validate_pr_title:
     name: Validate PR title
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - uses: amannn/action-semantic-pull-request@v5
         if: ${{ github.event.pull_request.draft == false }}

.github/workflows/slash_command_dispatch.yml

@@ -8,7 +8,7 @@ jobs:
   slashCommandDispatch:
     # Only allow slash commands on pull request (not on issues)
     if: ${{ github.event.issue.pull_request }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Slash Command Dispatch
         id: dispatch

.github/workflows/test-command.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   start-workflow:
     name: Append 'Starting' Comment
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Get PR JSON
         id: pr-info
@@ -127,7 +127,7 @@ jobs:
   log-success-comment:
     name: Append 'Success' Comment
     needs: [pytest-on-demand]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Append success comment
         uses: peter-evans/create-or-update-comment@v4
@@ -143,7 +143,7 @@ jobs:
     # This job will only run if the workflow fails
     needs: [pytest-on-demand, start-workflow]
     if: always() && needs.pytest-on-demand.result == 'failure'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Append failure comment
         uses: peter-evans/create-or-update-comment@v4

Dockerfile

@@ -1,3 +1,10 @@
+# This Dockerfile is used to build `airbyte/source-declarative-manifest` image that in turn is used
+# 1. to build Manifest-only connectors themselves
+# 2. to run manifest (Builder) connectors published into a particular user's workspace in Airbyte
+#
+# A new version of source-declarative-manifest is built for every new Airbyte CDK release, and their versions are kept in sync.
+#
+
 FROM docker.io/airbyte/python-connector-base:3.0.0@sha256:1a0845ff2b30eafa793c6eee4e8f4283c2e52e1bbd44eed6cb9e9abd5d34d844
 
 WORKDIR /airbyte/integration_code

airbyte_cdk/cli/source_declarative_manifest/_run.py

@@ -171,6 +171,12 @@ def create_declarative_source(
                 "Invalid config: `__injected_declarative_manifest` should be provided at the root "
                 f"of the config but config only has keys: {list(config.keys() if config else [])}"
             )
+        if not isinstance(config["__injected_declarative_manifest"], dict):
+            raise ValueError(
+                "Invalid config: `__injected_declarative_manifest` should be a dictionary, "
+                f"but got type: {type(config['__injected_declarative_manifest'])}"
+            )
+
         return ConcurrentDeclarativeSource(
             config=config,
             catalog=catalog,

airbyte_cdk/connector_builder/connector_builder_handler.py

@@ -52,6 +52,7 @@ def get_limits(config: Mapping[str, Any]) -> TestReadLimits:
 def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> ManifestDeclarativeSource:
     manifest = config["__injected_declarative_manifest"]
     return ManifestDeclarativeSource(
+        config=config,
         emit_connector_builder_messages=True,
         source_config=manifest,
         component_factory=ModelToComponentFactory(

airbyte_cdk/sources/declarative/auth/oauth.py

@@ -3,11 +3,12 @@
 #
 
 from dataclasses import InitVar, dataclass, field
-from typing import Any, List, Mapping, Optional, Union
+from typing import Any, List, Mapping, MutableMapping, Optional, Union
 
 import pendulum
 
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
+from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository
@@ -44,10 +45,10 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
         message_repository (MessageRepository): the message repository used to emit logs on HTTP requests
     """
 
-    client_id: Union[InterpolatedString, str]
-    client_secret: Union[InterpolatedString, str]
     config: Mapping[str, Any]
     parameters: InitVar[Mapping[str, Any]]
+    client_id: Optional[Union[InterpolatedString, str]] = None
+    client_secret: Optional[Union[InterpolatedString, str]] = None
     token_refresh_endpoint: Optional[Union[InterpolatedString, str]] = None
     refresh_token: Optional[Union[InterpolatedString, str]] = None
     scopes: Optional[List[str]] = None
@@ -66,6 +67,8 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut
     grant_type_name: Union[InterpolatedString, str] = "grant_type"
     grant_type: Union[InterpolatedString, str] = "refresh_token"
     message_repository: MessageRepository = NoopMessageRepository()
+    profile_assertion: Optional[DeclarativeAuthenticator] = None
+    use_profile_assertion: Optional[Union[InterpolatedBoolean, str, bool]] = False
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         super().__init__()
@@ -76,11 +79,19 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         else:
             self._token_refresh_endpoint = None
         self._client_id_name = InterpolatedString.create(self.client_id_name, parameters=parameters)
-        self._client_id = InterpolatedString.create(self.client_id, parameters=parameters)
+        self._client_id = (
+            InterpolatedString.create(self.client_id, parameters=parameters)
+            if self.client_id
+            else self.client_id
+        )
         self._client_secret_name = InterpolatedString.create(
             self.client_secret_name, parameters=parameters
         )
-        self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters)
+        self._client_secret = (
+            InterpolatedString.create(self.client_secret, parameters=parameters)
+            if self.client_secret
+            else self.client_secret
+        )
         self._refresh_token_name = InterpolatedString.create(
             self.refresh_token_name, parameters=parameters
         )
@@ -99,7 +110,12 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self.grant_type_name = InterpolatedString.create(
             self.grant_type_name, parameters=parameters
         )
-        self.grant_type = InterpolatedString.create(self.grant_type, parameters=parameters)
+        self.grant_type = InterpolatedString.create(
+            "urn:ietf:params:oauth:grant-type:jwt-bearer"
+            if self.use_profile_assertion
+            else self.grant_type,
+            parameters=parameters,
+        )
         self._refresh_request_body = InterpolatedMapping(
             self.refresh_request_body or {}, parameters=parameters
         )
@@ -115,6 +131,13 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
             if self.token_expiry_date
             else pendulum.now().subtract(days=1)  # type: ignore # substract does not have type hints
         )
+        self.use_profile_assertion = (
+            InterpolatedBoolean(self.use_profile_assertion, parameters=parameters)
+            if isinstance(self.use_profile_assertion, str)
+            else self.use_profile_assertion
+        )
+        self.assertion_name = "assertion"
+
         if self.access_token_value is not None:
             self._access_token_value = InterpolatedString.create(
                 self.access_token_value, parameters=parameters
@@ -126,9 +149,20 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
             self._access_token_value if self.access_token_value else None
         )
 
+        if not self.use_profile_assertion and any(
+            client_creds is None for client_creds in [self.client_id, self.client_secret]
+        ):
+            raise ValueError(
+                "OAuthAuthenticator configuration error: Both 'client_id' and 'client_secret' are required for the "
+                "basic OAuth flow."
+            )
+        if self.profile_assertion is None and self.use_profile_assertion:
+            raise ValueError(
+                "OAuthAuthenticator configuration error: 'profile_assertion' is required when using the profile assertion flow."
+            )
         if self.get_grant_type() == "refresh_token" and self._refresh_token is None:
             raise ValueError(
-                "OAuthAuthenticator needs a refresh_token parameter if grant_type is set to `refresh_token`"
+                "OAuthAuthenticator configuration error: A 'refresh_token' is required when the 'grant_type' is set to 'refresh_token'."
             )
 
     def get_token_refresh_endpoint(self) -> Optional[str]:
@@ -145,19 +179,21 @@ def get_client_id_name(self) -> str:
         return self._client_id_name.eval(self.config)  # type: ignore # eval returns a string in this context
 
     def get_client_id(self) -> str:
-        client_id: str = self._client_id.eval(self.config)
+        client_id = self._client_id.eval(self.config) if self._client_id else self._client_id
         if not client_id:
             raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter")
-        return client_id
+        return client_id  # type: ignore # value will be returned as a string, or an error will be raised
 
     def get_client_secret_name(self) -> str:
         return self._client_secret_name.eval(self.config)  # type: ignore # eval returns a string in this context
 
     def get_client_secret(self) -> str:
-        client_secret: str = self._client_secret.eval(self.config)
+        client_secret = (
+            self._client_secret.eval(self.config) if self._client_secret else self._client_secret
+        )
         if not client_secret:
             raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter")
-        return client_secret
+        return client_secret  # type: ignore # value will be returned as a string, or an error will be raised
 
     def get_refresh_token_name(self) -> str:
         return self._refresh_token_name.eval(self.config)  # type: ignore # eval returns a string in this context
@@ -192,6 +228,27 @@ def get_token_expiry_date(self) -> pendulum.DateTime:
     def set_token_expiry_date(self, value: Union[str, int]) -> None:
         self._token_expiry_date = self._parse_token_expiration_date(value)
 
+    def get_assertion_name(self) -> str:
+        return self.assertion_name
+
+    def get_assertion(self) -> str:
+        if self.profile_assertion is None:
+            raise ValueError("profile_assertion is not set")
+        return self.profile_assertion.token
+
+    def build_refresh_request_body(self) -> Mapping[str, Any]:
+        """
+        Returns the request body to set on the refresh request
+
+        Override to define additional parameters
+        """
+        if self.use_profile_assertion:
+            return {
+                self.get_grant_type_name(): self.get_grant_type(),
+                self.get_assertion_name(): self.get_assertion(),
+            }
+        return super().build_refresh_request_body()
+
     @property
     def access_token(self) -> str:
         if self._access_token is None: