From 3a6017c04a63848e32a5944dd1691b45b5f5d1fc Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Thu, 30 May 2024 12:01:40 +1200 Subject: [PATCH 1/3] test: Clarify subtest name --- scripts/stac/imagery/tests/collection_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/stac/imagery/tests/collection_test.py b/scripts/stac/imagery/tests/collection_test.py index 9ad0d8125..358587342 100644 --- a/scripts/stac/imagery/tests/collection_test.py +++ b/scripts/stac/imagery/tests/collection_test.py @@ -146,7 +146,7 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] for property_name in ["created", "updated"]: - with subtests.test(msg=f"{property_name} property"): + with subtests.test(msg=f"item assets.visual.{property_name}"): assert item.stac["assets"]["visual"][property_name] == "2001-02-03T04:05:06Z" From ba70ab9d0de63e424aa1ddc5c9672fdbaee238e5 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Thu, 30 May 2024 14:18:05 +1200 Subject: [PATCH 2/3] feat: Set created/updated item properties TDE-1147 When creating datasets, we want to set these to the current datetime. Injects the function to get the current datetime, to allow testing the exact values. --- scripts/datetimes.py | 4 +++ scripts/stac/imagery/create_stac.py | 3 +- scripts/stac/imagery/item.py | 7 ++-- scripts/stac/imagery/tests/collection_test.py | 34 ++++++++++++++----- scripts/stac/imagery/tests/item_test.py | 5 +-- scripts/tests/datetimes_test.py | 12 ++++++- 6 files changed, 50 insertions(+), 15 deletions(-) diff --git a/scripts/datetimes.py b/scripts/datetimes.py index f3f8a30b4..d84de6e6e 100644 --- a/scripts/datetimes.py +++ b/scripts/datetimes.py @@ -39,6 +39,10 @@ def format_rfc_3339_nz_midnight_datetime_string(datetime_object: datetime) -> st return format_rfc_3339_datetime_string(datetime_utc) +def utc_now() -> datetime: + return datetime.now(tz=timezone.utc) + + class NaiveDatetimeError(Exception): def __init__(self) -> None: super().__init__("Can't convert naive datetime to UTC") diff --git a/scripts/stac/imagery/create_stac.py b/scripts/stac/imagery/create_stac.py index 1acb232a2..fa71148cf 100644 --- a/scripts/stac/imagery/create_stac.py +++ b/scripts/stac/imagery/create_stac.py @@ -2,6 +2,7 @@ from linz_logger import get_log +from scripts.datetimes import utc_now from scripts.files.files_helper import get_file_name_from_path from scripts.files.geotiff import get_extents from scripts.gdal.gdal_helper import gdal_info @@ -35,7 +36,7 @@ def create_item( geometry, bbox = get_extents(gdalinfo_result) - item = ImageryItem(id_, file) + item = ImageryItem(id_, file, utc_now) item.update_datetime(start_datetime, end_datetime) item.update_spatial(geometry, bbox) item.add_collection(collection_id) diff --git a/scripts/stac/imagery/item.py b/scripts/stac/imagery/item.py index 651f6b553..1efae2716 100644 --- a/scripts/stac/imagery/item.py +++ b/scripts/stac/imagery/item.py @@ -1,5 +1,6 @@ import os -from typing import Any, Dict, Tuple +from datetime import datetime +from typing import Any, Callable, Dict, Tuple from scripts.datetimes import format_rfc_3339_datetime_string from scripts.files import fs @@ -12,9 +13,10 @@ class ImageryItem: stac: Dict[str, Any] - def __init__(self, id_: str, file: str) -> None: + def __init__(self, id_: str, file: str, now: Callable[[], datetime]) -> None: file_content = fs.read(file) file_modified_datetime = format_rfc_3339_datetime_string(modified(file)) + now_string = format_rfc_3339_datetime_string(now()) self.stac = { "type": "Feature", "stac_version": STAC_VERSION, @@ -32,6 +34,7 @@ def __init__(self, id_: str, file: str) -> None: } }, "stac_extensions": [StacExtensions.file.value], + "properties": {"created": now_string, "updated": now_string}, } def update_datetime(self, start_datetime: str, end_datetime: str) -> None: diff --git a/scripts/stac/imagery/tests/collection_test.py b/scripts/stac/imagery/tests/collection_test.py index 358587342..ef12b5720 100644 --- a/scripts/stac/imagery/tests/collection_test.py +++ b/scripts/stac/imagery/tests/collection_test.py @@ -4,12 +4,13 @@ from datetime import datetime, timezone from shutil import rmtree from tempfile import mkdtemp -from typing import Generator +from typing import Callable, Generator import pytest import shapely.geometry from pytest_subtests import SubTests +from scripts.datetimes import format_rfc_3339_datetime_string from scripts.files.fs import read from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.item import ImageryItem @@ -113,12 +114,20 @@ def test_interval_updated_from_existing(metadata: CollectionMetadata) -> None: assert collection.stac["extent"]["temporal"]["interval"] == [["2021-01-27T00:00:00Z", "2021-02-20T00:00:00Z"]] +def fixed_now_function(now: datetime) -> Callable[[], datetime]: + def func() -> datetime: + return now + + return func + + def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: collection = ImageryCollection(metadata) item_file_path = "./scripts/tests/data/empty.tiff" modified_datetime = datetime(2001, 2, 3, hour=4, minute=5, second=6, tzinfo=timezone.utc) os.utime(item_file_path, times=(any_epoch_datetime().timestamp(), modified_datetime.timestamp())) - item = ImageryItem("BR34_5000_0304", item_file_path) + now = any_epoch_datetime() + item = ImageryItem("BR34_5000_0304", item_file_path, fixed_now_function(now)) geometry = { "type": "Polygon", "coordinates": [[1799667.5, 5815977.0], [1800422.5, 5815977.0], [1800422.5, 5814986.0], [1799667.5, 5814986.0]], @@ -131,13 +140,17 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: collection.add_item(item.stac) - with subtests.test(): - assert { - "file:checksum": "122097b5d2b049c6ffdf608af28c4ba2744fad7f03046d1f58b2523402f30577f618", - "rel": "item", - "href": "./BR34_5000_0304.json", - "type": "application/json", - } in collection.stac["links"] + links = collection.stac["links"].copy() + + with subtests.test(msg="File checksum heuristic"): + # The checksum changes based on the contents + assert links[1].pop("file:checksum").startswith("1220") + + with subtests.test(msg="Main links content"): + assert [ + {"href": "./collection.json", "rel": "self", "type": "application/json"}, + {"rel": "item", "href": "./BR34_5000_0304.json", "type": "application/json"}, + ] == links with subtests.test(): assert collection.stac["extent"]["temporal"]["interval"] == [[start_datetime, end_datetime]] @@ -146,6 +159,9 @@ def test_add_item(metadata: CollectionMetadata, subtests: SubTests) -> None: assert collection.stac["extent"]["spatial"]["bbox"] == [bbox] for property_name in ["created", "updated"]: + with subtests.test(msg=f"item properties.{property_name}"): + assert item.stac["properties"][property_name] == format_rfc_3339_datetime_string(now) + with subtests.test(msg=f"item assets.visual.{property_name}"): assert item.stac["assets"]["visual"][property_name] == "2001-02-03T04:05:06Z" diff --git a/scripts/stac/imagery/tests/item_test.py b/scripts/stac/imagery/tests/item_test.py index 4b3be4858..e180ccd4b 100644 --- a/scripts/stac/imagery/tests/item_test.py +++ b/scripts/stac/imagery/tests/item_test.py @@ -7,6 +7,7 @@ from scripts.stac.imagery.collection import ImageryCollection from scripts.stac.imagery.item import ImageryItem from scripts.stac.imagery.metadata_constants import CollectionMetadata +from scripts.tests.datetimes_test import any_epoch_datetime def test_imagery_stac_item(mocker: MockerFixture, subtests: SubTests) -> None: @@ -23,7 +24,7 @@ def test_imagery_stac_item(mocker: MockerFixture, subtests: SubTests) -> None: start_datetime = "2021-01-27T00:00:00Z" end_datetime = "2021-01-27T00:00:00Z" - item = ImageryItem(id_, path) + item = ImageryItem(id_, path, any_epoch_datetime) item.update_spatial(geometry, bbox) item.update_datetime(start_datetime, end_datetime) # checks @@ -77,7 +78,7 @@ def test_imagery_add_collection(mocker: MockerFixture, subtests: SubTests) -> No path = "./scripts/tests/data/empty.tiff" id_ = get_file_name_from_path(path) mocker.patch("scripts.files.fs.read", return_value=b"") - item = ImageryItem(id_, path) + item = ImageryItem(id_, path, any_epoch_datetime) item.add_collection(collection.stac["id"]) diff --git a/scripts/tests/datetimes_test.py b/scripts/tests/datetimes_test.py index ffb8d3b95..0ad27a834 100644 --- a/scripts/tests/datetimes_test.py +++ b/scripts/tests/datetimes_test.py @@ -2,7 +2,7 @@ from random import randint from dateutil.tz import tz -from pytest import raises +from pytest import approx, raises from pytest_subtests import SubTests from scripts.datetimes import ( @@ -11,6 +11,7 @@ format_rfc_3339_nz_midnight_datetime_string, parse_rfc_3339_date, parse_rfc_3339_datetime, + utc_now, ) @@ -49,6 +50,15 @@ def test_should_format_rfc_3339_nz_midnight_datetime_string() -> None: assert format_rfc_3339_nz_midnight_datetime_string(datetime_object) == "2001-02-02T11:00:00Z" +def test_system_clock_should_return_current_time() -> None: + # At most five second difference + assert utc_now().timestamp() == approx(datetime.now().timestamp(), abs=5) + + +def test_system_clock_should_return_utc_datetime() -> None: + assert utc_now().tzinfo == timezone.utc + + def any_epoch_datetime() -> datetime: """ Get arbitrary datetime From fe8372cfe7cc5c63c75880f33e5bef2f000a5e96 Mon Sep 17 00:00:00 2001 From: Victor Engmark Date: Tue, 4 Jun 2024 14:52:18 +1200 Subject: [PATCH 3/3] refactor: Use doctests --- scripts/datetimes.py | 12 ++++++++++++ scripts/tests/datetimes_test.py | 12 +----------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/scripts/datetimes.py b/scripts/datetimes.py index d84de6e6e..df80709c2 100644 --- a/scripts/datetimes.py +++ b/scripts/datetimes.py @@ -40,6 +40,18 @@ def format_rfc_3339_nz_midnight_datetime_string(datetime_object: datetime) -> st def utc_now() -> datetime: + """ + Get the current datetime with UTC time zone + + Should return something close to the current time: + >>> current_timestamp = datetime.now().timestamp() + >>> current_timestamp - 5 < utc_now().timestamp() < current_timestamp + 5 + True + + Should have UTC time zone: + >>> utc_now().tzname() + 'UTC' + """ return datetime.now(tz=timezone.utc) diff --git a/scripts/tests/datetimes_test.py b/scripts/tests/datetimes_test.py index 0ad27a834..ffb8d3b95 100644 --- a/scripts/tests/datetimes_test.py +++ b/scripts/tests/datetimes_test.py @@ -2,7 +2,7 @@ from random import randint from dateutil.tz import tz -from pytest import approx, raises +from pytest import raises from pytest_subtests import SubTests from scripts.datetimes import ( @@ -11,7 +11,6 @@ format_rfc_3339_nz_midnight_datetime_string, parse_rfc_3339_date, parse_rfc_3339_datetime, - utc_now, ) @@ -50,15 +49,6 @@ def test_should_format_rfc_3339_nz_midnight_datetime_string() -> None: assert format_rfc_3339_nz_midnight_datetime_string(datetime_object) == "2001-02-02T11:00:00Z" -def test_system_clock_should_return_current_time() -> None: - # At most five second difference - assert utc_now().timestamp() == approx(datetime.now().timestamp(), abs=5) - - -def test_system_clock_should_return_utc_datetime() -> None: - assert utc_now().tzinfo == timezone.utc - - def any_epoch_datetime() -> datetime: """ Get arbitrary datetime