Skip to content

Commit 6cce87f

Browse files
authoredJul 18, 2023
Merge pull request #25 from lsst-sqre/tickets/DM-39646
DM-39646: Add FastAPI app integration through dependencies
2 parents fbe6900 + 8e3705c commit 6cce87f

14 files changed

+271
-21
lines changed
 

‎.github/workflows/ci.yaml

+35-10
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: Python CI
22

33
"on":
4+
merge_group: {}
45
push:
56
branches-ignore:
67
# These should always correspond to pull requests, so ignore them for
@@ -11,9 +12,9 @@ name: Python CI
1112
- "renovate/**"
1213
- "tickets/**"
1314
- "u/**"
14-
tags:
15-
- "*"
1615
pull_request: {}
16+
release:
17+
types: [published]
1718

1819
jobs:
1920
lint:
@@ -37,8 +38,6 @@ jobs:
3738
strategy:
3839
matrix:
3940
python:
40-
- "3.8"
41-
- "3.9"
4241
- "3.10"
4342
- "3.11"
4443

@@ -81,22 +80,48 @@ jobs:
8180
username: ${{ secrets.LTD_USERNAME }}
8281
password: ${{ secrets.LTD_PASSWORD }}
8382
if: >
84-
github.event_name != 'pull_request'
85-
|| startsWith(github.head_ref, 'tickets/')
83+
github.event_name != 'merge_group'
84+
&& (github.event_name != 'pull_request'
85+
|| startsWith(github.head_ref, 'tickets/'))
86+
87+
test-packaging:
88+
89+
name: Test packaging
90+
runs-on: ubuntu-latest
91+
92+
steps:
93+
- uses: actions/checkout@v3
94+
with:
95+
fetch-depth: 0 # full history for setuptools_scm
96+
97+
- name: Build and publish
98+
uses: lsst-sqre/build-and-publish-to-pypi@v2
99+
with:
100+
python-version: "3.11"
101+
upload: false
86102

87103
pypi:
88104

105+
# This job requires set up:
106+
# 1. Set up a trusted publisher for PyPI
107+
# 2. Set up a "pypi" environment in the repository
108+
# See https://github.com/lsst-sqre/build-and-publish-to-pypi
109+
name: Upload release to PyPI
89110
runs-on: ubuntu-latest
90-
needs: [lint, test, docs]
111+
needs: [lint, test, docs, test-packaging]
112+
environment:
113+
name: pypi
114+
url: https://pypi.org/p/kafkit
115+
permissions:
116+
id-token: write
117+
if: github.event_name == 'release' && github.event.action == 'published'
91118

92119
steps:
93120
- uses: actions/checkout@v3
94121
with:
95122
fetch-depth: 0 # full history for setuptools_scm
96123

97124
- name: Build and publish
98-
uses: lsst-sqre/build-and-publish-to-pypi@v1
125+
uses: lsst-sqre/build-and-publish-to-pypi@v2
99126
with:
100-
pypi-token: ${{ secrets.PYPI_SQRE_ADMIN }}
101127
python-version: "3.11"
102-
upload: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}

‎.github/workflows/dependencies.yaml

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Dependency Update
2+
3+
"on":
4+
schedule:
5+
- cron: "0 12 * * 1"
6+
workflow_dispatch: {}
7+
8+
jobs:
9+
update:
10+
runs-on: ubuntu-latest
11+
timeout-minutes: 10
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
16+
- name: Run neophile
17+
uses: lsst-sqre/run-neophile@v1
18+
with:
19+
python-version: "3.11"
20+
mode: pr
21+
types: pre-commit
22+
app-id: ${{ secrets.NEOPHILE_APP_ID }}
23+
app-secret: ${{ secrets.NEOPHILE_PRIVATE_KEY }}
24+
25+
- name: Report status
26+
if: always()
27+
uses: ravsamhq/notify-slack-action@v2
28+
with:
29+
status: ${{ job.status }}
30+
notify_when: "failure"
31+
notification_title: "Periodic dependency update for {repo} failed"
32+
env:
33+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_ALERT_WEBHOOK }}

‎.github/workflows/periodic-ci.yaml

+3-5
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,6 @@ jobs:
1515
strategy:
1616
matrix:
1717
python:
18-
- "3.8"
19-
- "3.9"
2018
- "3.10"
2119
- "3.11"
2220

@@ -43,17 +41,17 @@ jobs:
4341
tox-envs: "docs,docs-linkcheck"
4442
use-cache: false
4543

46-
pypi:
44+
test-packaging:
4745
runs-on: ubuntu-latest
46+
timeout-minutes: 10
4847

4948
steps:
5049
- uses: actions/checkout@v3
5150
with:
5251
fetch-depth: 0 # full history for setuptools_scm
5352

5453
- name: Build and publish
55-
uses: lsst-sqre/build-and-publish-to-pypi@v1
54+
uses: lsst-sqre/build-and-publish-to-pypi@v2
5655
with:
57-
pypi-token: ""
5856
python-version: "3.11"
5957
upload: false

‎.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
.vscode
2+
13
# Byte-compiled / optimized / DLL files
24
__pycache__/
35
*.py[cod]

‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ help:
55

66
.PHONY: init
77
init:
8-
pip install -e ".[aiohttp,httpx,pydantic,dev]"
9-
pip install -U tox pre-commit
8+
pip install -e ".[aiohttp,httpx,pydantic,aiokafka,dev]"
9+
pip install -U tox pre-commit scriv
1010
pre-commit install
1111
rm -rf .tox
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
### Backwards-incompatible changes
2+
3+
- Only Python 3.10 or later is supported.
4+
5+
### New features
6+
7+
- Integration into FastAPI apps through dependencies in `kafkit.fastapi.dependencies`:
8+
9+
- `AioKafkaProducerDependency` provides a Kafka producer based on aiokafka's `AIOKafkaProducer` (requires the `aiokafka` extra).
10+
- `PydanticSchemaManager` provides a Pydantic-based schema manager for Avro schemas, `kafkit.schema.manager.PydanticSchemaManager`.
11+
- `RegistryApiDependency` provides an HTTPX-based Schema Registry client, `kafkit.registry.httpx.RegistryApi`.
12+
13+
### Other changes
14+
15+
- Adopt PyPI's trusted publishers mechanism for releases.
16+
- Adopt the new [Neophile](https://github.com/lsst-sqre/neophile) workflow for keeping pre-commit hooks up-to-date.

‎docs/api.rst

+12
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,18 @@
22
Kafkit API reference
33
####################
44

5+
.. automodapi:: kafkit.fastapi.dependencies.aiokafkaproducer
6+
:no-inheritance-diagram:
7+
:include-all-objects:
8+
9+
.. automodapi:: kafkit.fastapi.dependencies.pydanticschemamanager
10+
:no-inheritance-diagram:
11+
:include-all-objects:
12+
13+
.. automodapi:: kafkit.fastapi.dependencies.registryapi
14+
:no-inheritance-diagram:
15+
:include-all-objects:
16+
517
.. automodapi:: kafkit.registry
618
:no-inheritance-diagram:
719

‎pyproject.toml

+3-4
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,22 @@ classifiers = [
1414
"License :: OSI Approved :: MIT License",
1515
"Programming Language :: Python",
1616
"Programming Language :: Python :: 3",
17-
"Programming Language :: Python :: 3.8",
18-
"Programming Language :: Python :: 3.9",
1917
"Programming Language :: Python :: 3.10",
2018
"Programming Language :: Python :: 3.11",
2119
"Intended Audience :: Developers",
2220
"Natural Language :: English",
2321
"Operating System :: POSIX",
2422
"Typing :: Typed",
2523
]
26-
requires-python = ">=3.8"
24+
requires-python = ">=3.10"
2725
dependencies = ["fastavro", "uritemplate"]
2826
dynamic = ["version"]
2927

3028
[project.optional-dependencies]
3129
aiohttp = ["aiohttp"]
3230
httpx = ["httpx"]
3331
pydantic = ["pydantic", "dataclasses-avroschema[pydantic]"]
32+
aiokafka = ["aiokafka"]
3433
dev = [
3534
# Testing
3635
"coverage[toml]",
@@ -82,7 +81,7 @@ exclude_lines = [
8281

8382
[tool.black]
8483
line-length = 79
85-
target-version = ['py38']
84+
target-version = ['py310']
8685
exclude = '''
8786
/(
8887
\.eggs

‎src/kafkit/fastapi/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Kafkit integration with FastApi applications."""
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""FastAPI dependencies for Kafkit applications."""
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
"""A FastAPI dependency that provides an aiokafka Producer."""
2+
3+
import aiokafka # patched for testing
4+
5+
from kafkit.settings import KafkaConnectionSettings
6+
7+
__all__ = ["kafka_producer_dependency", "AioKafkaProducerDependency"]
8+
9+
10+
class AioKafkaProducerDependency:
11+
"""A FastAPI dependency that provides an aiokafka Producer."""
12+
13+
def __init__(self) -> None:
14+
self._producer: aiokafka.AIOKafkaProducer | None = None
15+
16+
async def initialize(self, settings: KafkaConnectionSettings) -> None:
17+
"""Initialize the dependency (call during FastAPI startup).
18+
19+
Parameters
20+
----------
21+
settings
22+
The Kafka connection settings.
23+
"""
24+
security_protocol = settings.security_protocol.value
25+
sasl_mechanism = (
26+
settings.sasl_mechanism.value if settings.sasl_mechanism else None
27+
)
28+
self._producer = aiokafka.AIOKafkaProducer(
29+
bootstrap_servers=settings.bootstrap_servers,
30+
security_protocol=security_protocol,
31+
ssl_context=settings.ssl_context,
32+
sasl_mechanism=sasl_mechanism,
33+
sasl_plain_password=(
34+
settings.sasl_password.get_secret_value()
35+
if settings.sasl_password
36+
else None
37+
),
38+
sasl_plain_username=settings.sasl_username,
39+
)
40+
await self._producer.start()
41+
42+
async def __call__(self) -> aiokafka.AIOKafkaProducer:
43+
"""Get the dependency (call during FastAPI request handling)."""
44+
if self._producer is None:
45+
raise RuntimeError("Dependency not initialized")
46+
return self._producer
47+
48+
async def stop(self) -> None:
49+
"""Stop the dependency (call during FastAPI shutdown)."""
50+
if self._producer is None:
51+
raise RuntimeError("Dependency not initialized")
52+
await self._producer.stop()
53+
54+
55+
kafka_producer_dependency = AioKafkaProducerDependency()
56+
"""The FastAPI dependency callable that provides an AIOKafkaProducer."""
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
"""A FastAPI dependency that provides a Kafkit PydanticSchemaManager
2+
for serializing Pydantic models into Avro.
3+
"""
4+
5+
from collections.abc import Iterable
6+
from typing import Type
7+
8+
from dataclasses_avroschema.avrodantic import AvroBaseModel
9+
from httpx import AsyncClient
10+
11+
from kafkit.registry import manager # this is patched in tests
12+
from kafkit.registry.httpx import RegistryApi
13+
14+
__all__ = [
15+
"pydantic_schema_manager_dependency",
16+
"PydanticSchemaManagerDependency",
17+
]
18+
19+
20+
class PydanticSchemaManagerDependency:
21+
"""A FastAPI dependency that provides a Kafkit PydanticSchemaManager
22+
for serializing Pydantic models into Avro.
23+
"""
24+
25+
def __init__(self) -> None:
26+
self._schema_manager: manager.PydanticSchemaManager | None = None
27+
28+
async def initialize(
29+
self,
30+
*,
31+
http_client: AsyncClient,
32+
registry_url: str,
33+
models: Iterable[Type[AvroBaseModel]],
34+
suffix: str = "",
35+
compatibility: str = "FORWARD",
36+
) -> None:
37+
"""Initialize the dependency (call during FastAPI startup).
38+
39+
Parameters
40+
----------
41+
http_client
42+
The httpx AsyncClient instance to use for HTTP requests.
43+
registry_url
44+
The URL of the Schema Registry.
45+
models
46+
The Pydantic models to register.
47+
suffix
48+
A suffix that is added to the schema name (and thus subject name),
49+
for example ``_dev1``.
50+
compatibility
51+
The compatibility level to use when registering the schemas.
52+
"""
53+
registry_api = RegistryApi(http_client=http_client, url=registry_url)
54+
self._schema_manager = manager.PydanticSchemaManager(
55+
registry=registry_api, suffix=suffix
56+
)
57+
58+
await self._schema_manager.register_models(
59+
models, compatibility=compatibility
60+
)
61+
62+
async def __call__(self) -> manager.PydanticSchemaManager:
63+
"""Get the dependency (call during FastAPI request handling)."""
64+
if self._schema_manager is None:
65+
raise RuntimeError("Dependency not initialized")
66+
return self._schema_manager
67+
68+
69+
pydantic_schema_manager_dependency = PydanticSchemaManagerDependency()
70+
"""The FastAPI dependency callable that provides a Kafkit PydanticSchemaManager
71+
instance for serializing Pydantic models into Avro.
72+
"""

0 commit comments

Comments
 (0)