diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 84bf96f62..14bbd78da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -40,7 +40,7 @@ jobs: run: | python -m pip install --upgrade pip pip install poetry==2.1.3 - poetry install --with test -E pg && poetry run pytest papermerge/ + poetry install --with test -E cloud -E pg && poetry run pytest papermerge/ env: PAPERMERGE__DATABASE__URL: 'postgresql://pmguser:pmgpass@localhost:5432/test_pmgdb' PAPERMERGE__MAIN__API_PREFIX: '' diff --git a/papermerge/core/config.py b/papermerge/core/config.py index 3f727832c..2852b7305 100644 --- a/papermerge/core/config.py +++ b/papermerge/core/config.py @@ -33,6 +33,12 @@ class Settings(BaseSettings): papermerge__ocr__automatic: bool = False papermerge__search__url: str | None = None + aws_access_key_id: str | None = None + aws_secret_access_key: str | None = None + aws_region_name: str | None = None + aws_endpoint_url: str | None = None + papermerge__s3__bucket_name: str | None = None + settings = Settings() def get_settings(): diff --git a/papermerge/core/features/document/s3.py b/papermerge/core/features/document/s3.py index 5383ab628..6fa8a93b8 100644 --- a/papermerge/core/features/document/s3.py +++ b/papermerge/core/features/document/s3.py @@ -1,30 +1,49 @@ -from uuid import UUID -from urllib.parse import quote from pathlib import Path +from urllib.parse import quote +from uuid import UUID + +import boto3 +from botocore.config import Config -from papermerge.core.types import ImagePreviewSize -from papermerge.core import pathlib as plib from papermerge.core import config +from papermerge.core import pathlib as plib +from papermerge.core.types import ImagePreviewSize settings = config.get_settings() VALID_FOR_SECONDS = 600 -def resource_sign_url(prefix, resource_path: Path): - from papermerge.core.cloudfront import sign_url +def generate_s3_signed_url(path: str): + client = boto3.client( + "s3", + aws_access_key_id=settings.aws_access_key_id, + aws_secret_access_key=settings.aws_secret_access_key, + region_name=settings.aws_region_name, + endpoint_url=settings.aws_endpoint_url, + config=Config(signature_version="s3v4"), + ) + return client.generate_presigned_url( + ClientMethod="get_object", + Params={"Bucket": settings.papermerge__s3__bucket_name, "Key": path}, + ExpiresIn=VALID_FOR_SECONDS, + ) - encoded_path = quote(str(resource_path)) - if prefix: - url = f"https://{settings.papermerge__main__cf_domain}/{prefix}/{encoded_path}" +def resource_sign_url(prefix, resource_path: Path): + encoded_path = quote(str(resource_path)) + path = encoded_path if not prefix else f"{prefix}/{encoded_path}" + # if a cloudFront domain is configured -> generate a signed url via cloudFront + # else -> generate a direct signed url to object storage + if settings.papermerge__main__cf_domain is not None: + from papermerge.core.cloudfront import sign_url + + return sign_url( + f"https://{settings.papermerge__main__cf_domain}/{path}", + valid_for=VALID_FOR_SECONDS, + ) else: - url = f"https://{settings.papermerge__main__cf_domain}/{encoded_path}" - - return sign_url( - url, - valid_for=VALID_FOR_SECONDS, - ) + return generate_s3_signed_url(path=path) def doc_thumbnail_signed_url(uid: UUID) -> str: diff --git a/papermerge/core/features/document/schema.py b/papermerge/core/features/document/schema.py index a7ec9e72b..040b8f4a4 100644 --- a/papermerge/core/features/document/schema.py +++ b/papermerge/core/features/document/schema.py @@ -214,6 +214,7 @@ def thumbnail_url_validator(cls, value, info): return f"/api/thumbnails/{info.data['id']}" # if it is not local, then it is s3 + CDN/cloudfront + # or just plain s3 without CDN if ( "preview_status" in info.data and info.data["preview_status"] == ImagePreviewStatus.ready