Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add /timeseries endpoints #33

Merged
merged 7 commits into from
Nov 7, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
simplify datetime parameters
  • Loading branch information
hrodmn committed Nov 4, 2024
commit d2d17944134ff7640a3a73ce7607f2f1b52ae3fe
5 changes: 2 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -40,6 +40,7 @@ dependencies = [
"python-dateutil>=2.9.0.post0",
"httpx>=0.27.2",
"pillow>=11.0.0",
hrodmn marked this conversation as resolved.
Show resolved Hide resolved
"isodate>=0.7.2",
]
dynamic = ["version"]

@@ -49,7 +50,6 @@ uvicorn = [
]
dev = [
"folium",
"httpx",
"jupyterlab>=4.2.5",
"matplotlib",
"pre-commit",
@@ -58,10 +58,9 @@ test = [
"pytest>=8.3.3",
"pytest-cov>=5.0.0",
"pytest-asyncio>=0.24.0",
"httpx>=0.27.2",
"pytest-mock>=3.14.0",
"pytest-recording>=0.13.2",
"asgi-lifespan>=2.1.0",
"freezegun>=1.5.1",
]

[project.urls]
23 changes: 17 additions & 6 deletions tests/test_app.py
Original file line number Diff line number Diff line change
@@ -306,8 +306,7 @@ async def mock_timestep_request(url: str, **kwargs) -> Response:
"/timeseries/statistics",
params={
**xarray_query_params,
"start_datetime": "2024-10-11T00:00:00Z",
"end_datetime": "2024-10-12T23:59:59Z",
"datetime": "2024-10-11T00:00:00Z/2024-10-12T23:59:59Z",
"step": "P1D",
},
json=arctic_geojson,
@@ -353,8 +352,7 @@ async def mock_timestep_request(url: str, **kwargs) -> Response:
"/timeseries/WebMercatorQuad/tilejson.json",
params={
**xarray_query_params,
"start_datetime": "2024-10-11T00:00:00Z",
"end_datetime": "2024-10-12T23:59:59Z",
"datetime": "2024-10-11T00:00:00Z/2024-10-12T23:59:59Z",
"step": "P1D",
},
)
@@ -396,11 +394,24 @@ async def mock_timestep_request(url: str, **kwargs) -> Response:
f"/timeseries/bbox/{','.join(str(coord) for coord in arctic_bounds)}.gif",
params={
**xarray_query_params,
"start_datetime": "2024-10-11T00:00:00Z",
"end_datetime": "2024-10-12T23:59:59Z",
"datetime": "2024-10-11T00:00:00Z/2024-10-12T23:59:59Z",
"step": "P1D",
},
)

assert response.status_code == 200
assert response.headers["content-type"] == TimeseriesMediaType.gif


def test_unbounded_start(app, xarray_query_params) -> None:
"""Make sure a datetime interval with an unbounded start returns a 400"""
response = app.get(
"/timeseries",
params={
**xarray_query_params,
"datetime": "../2024-10-12T23:59:59Z",
"step": "P1D",
},
)

assert response.status_code == 400
26 changes: 20 additions & 6 deletions tests/test_dependencies.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
"""test titiler-pgstac dependencies."""

from datetime import datetime, timezone

import pytest
from starlette.requests import Request

@@ -105,20 +107,32 @@ def test_output_type():
assert dependencies.OutputType(req, f="json") == MediaType.json


test_datetime = datetime(year=2018, month=2, day=12, hour=9, tzinfo=timezone.utc)


@pytest.mark.parametrize(
"temporal,res",
[
(
"2018-02-12T09:00:00Z",
("2018-02-12T09:00:00+00:00", "2018-02-13T09:00:00+00:00"),
test_datetime,
),
(
"2018-02-12T09:00:00Z/",
(test_datetime, None),
),
(
"2018-02-12T09:00:00Z/..",
(test_datetime, None),
),
("2018-02-12T09:00:00Z/", ("2018-02-12T09:00:00+00:00", None)),
("2018-02-12T09:00:00Z/..", ("2018-02-12T09:00:00+00:00", None)),
("/2018-02-12T09:00:00Z", (None, "2018-02-12T09:00:00+00:00")),
("../2018-02-12T09:00:00Z", (None, "2018-02-12T09:00:00+00:00")),
("/2018-02-12T09:00:00Z", (None, test_datetime)),
("../2018-02-12T09:00:00Z", (None, test_datetime)),
(
"2018-02-12T09:00:00Z/2019-02-12T09:00:00Z",
("2018-02-12T09:00:00+00:00", "2019-02-12T09:00:00+00:00"),
(
test_datetime,
datetime(year=2019, month=2, day=12, hour=9, tzinfo=timezone.utc),
),
),
],
)
95 changes: 39 additions & 56 deletions tests/test_timeseries.py
Original file line number Diff line number Diff line change
@@ -4,37 +4,17 @@
from typing import Dict, Tuple

import pytest
from dateutil.relativedelta import relativedelta
from fastapi import HTTPException
from freezegun import freeze_time

# Import your functions here
from titiler.cmr.timeseries import (
TemporalMode,
TimeseriesParams,
generate_datetime_ranges,
parse_duration,
timeseries_query,
timeseries_cmr_query,
)


def test_parse_duration():
"""Test durations"""
assert parse_duration("P1Y") == relativedelta(years=1)
assert parse_duration("P2M") == relativedelta(months=2)
assert parse_duration("P3D") == relativedelta(days=3)
assert parse_duration("PT4H") == relativedelta(hours=4)
assert parse_duration("PT5M") == relativedelta(minutes=5)
assert parse_duration("PT6S") == relativedelta(seconds=6)
assert parse_duration("PT1S") == relativedelta(seconds=1)
assert parse_duration("P1Y2M3DT4H5M6S") == relativedelta(
years=1, months=2, days=3, hours=4, minutes=5, seconds=6
)

with pytest.raises(ValueError):
parse_duration("P1G")
with pytest.raises(ValueError):
parse_duration("invalid")


def test_generate_datetime_ranges():
"""Test datetime ranges"""
start = datetime(2023, 1, 1)
@@ -87,17 +67,17 @@ def test_generate_datetime_ranges():
assert len(large_step_ranges) == 1
assert large_step_ranges[0] == (start, datetime(2023, 1, 2))

# Test exact=True
# Test point-in-time mode
exact_end_datetime = datetime(2023, 5, 1)
exact_ranges = generate_datetime_ranges(
start, exact_end_datetime, "P1M", exact=True
start, exact_end_datetime, "P1M", temporal_mode=TemporalMode.point
)
assert len(exact_ranges) == 5
assert exact_ranges[-1] == (exact_end_datetime,)

exact_end_datetime = datetime(2023, 10, 25)
exact_ranges = generate_datetime_ranges(
start, exact_end_datetime, "P1W", exact=True
start, exact_end_datetime, "P1W", temporal_mode=TemporalMode.point
)
assert len(exact_ranges) == 43

@@ -185,54 +165,49 @@ def test_timeseries_query(
) -> None:
"""Test timeseries_query"""
start_datetime, end_datetime = xarray_query_params["datetime"].split("/")
query = timeseries_query(
query = timeseries_cmr_query(
concept_id=xarray_query_params["concept_id"],
timeseries_params=TimeseriesParams(
start_datetime=start_datetime,
end_datetime=end_datetime,
datetime=xarray_query_params["datetime"],
step="P1D",
),
)
assert len(query) == 1

query = timeseries_query(
query = timeseries_cmr_query(
concept_id=xarray_query_params["concept_id"],
timeseries_params=TimeseriesParams(
start_datetime=start_datetime,
end_datetime=end_datetime,
datetime=xarray_query_params["datetime"],
step="PT1H",
),
)
assert len(query) == 24

query = timeseries_query(
query = timeseries_cmr_query(
concept_id=xarray_query_params["concept_id"],
timeseries_params=TimeseriesParams(
start_datetime=start_datetime,
end_datetime="2024-10-31T23:59:59Z",
datetime=f"{start_datetime}/2024-10-31T23:59:59Z",
step="P1W",
),
)
assert len(query) == 3

# no step parameter will force a CMR query to get unique
# datetimes from available granules
query = timeseries_query(
query = timeseries_cmr_query(
concept_id=xarray_query_params["concept_id"],
timeseries_params=TimeseriesParams(
start_datetime=start_datetime,
end_datetime=end_datetime,
datetime=xarray_query_params["datetime"],
),
)
assert len(query) == 1

# query CMR to get the actual timesteps from a collection
geographically_limited_concept_id = "C2623694361-GES_DISC"
query = timeseries_query(
query = timeseries_cmr_query(
concept_id=geographically_limited_concept_id,
timeseries_params=TimeseriesParams(
start_datetime=start_datetime,
end_datetime=end_datetime,
datetime=xarray_query_params["datetime"],
),
minx=-100,
miny=30,
@@ -242,11 +217,10 @@ def test_timeseries_query(
assert len(query) == 8

# run a bbox query that returns no granules
query = timeseries_query(
query = timeseries_cmr_query(
concept_id=geographically_limited_concept_id,
timeseries_params=TimeseriesParams(
start_datetime=start_datetime,
end_datetime=end_datetime,
datetime=xarray_query_params["datetime"],
),
minx=1,
miny=1,
@@ -255,20 +229,29 @@ def test_timeseries_query(
)
assert len(query) == 0

# expect an error if only start_datetime or end_datetime provided
with pytest.raises(HTTPException):
timeseries_query(
concept_id=geographically_limited_concept_id,
timeseries_params=TimeseriesParams(
start_datetime="2024-01-01T00:00:00Z",
step="P1W",
),
)

@freeze_time("2024-10-01T00:00:00Z")
def test_timeseries_query_unbounded_intervals(
xarray_query_params: Dict[str, str],
arctic_bounds: Tuple[float, float, float, float],
) -> None:
"""Test unbounded intervals"""
# expect an error if an interval is provided with an unbounded start datetime
with pytest.raises(HTTPException):
timeseries_query(
concept_id=geographically_limited_concept_id,
timeseries_cmr_query(
concept_id=xarray_query_params["concept_id"],
timeseries_params=TimeseriesParams(
end_datetime="2024-01-01T00:00:00Z",
datetime="../2024-01-01T00:00:00Z",
step="P1W",
),
)

unbounded_query = timeseries_cmr_query(
concept_id=xarray_query_params["concept_id"],
timeseries_params=TimeseriesParams(
datetime="2024-01-01T00:00:00Z/..",
step="P1W",
),
)

assert len(unbounded_query) == 40
4 changes: 1 addition & 3 deletions tests/test_timeseries_extension.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,12 @@
"""Tests for the timeseries extension"""

import pytest
from fastapi import FastAPI

from titiler.cmr.factory import Endpoints
from titiler.cmr.timeseries import TimeseriesExtension


@pytest.mark.asyncio
async def test_timeseries_extension() -> None:
def test_timeseries_extension() -> None:
"""Test timeseries extension endpoints"""
tiler = Endpoints()
tiler_plus_timeseries = Endpoints(extensions=[TimeseriesExtension()])
41 changes: 3 additions & 38 deletions titiler/cmr/dependencies.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
"""titiler-cmr dependencies."""

import datetime as python_datetime
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Union, get_args

from ciso8601 import parse_rfc3339
from fastapi import Query
from rio_tiler.types import RIOResampling, WarpResampling
from starlette.requests import Request
from typing_extensions import Annotated

from titiler.cmr.enums import MediaType
from titiler.cmr.errors import InvalidDatetime
from titiler.cmr.utils import parse_datetime
from titiler.core.dependencies import DefaultDependency

ResponseType = Literal["json", "html"]
@@ -81,13 +79,6 @@ def OutputType(
return accept_media_type(request.headers.get("accept", ""), accepted_media)


def _parse_date(date: str) -> python_datetime.datetime:
try:
return parse_rfc3339(date)
except Exception as e:
raise InvalidDatetime(f"Invalid datetime {date}") from e


ConceptID = Annotated[
str,
Query(
@@ -117,34 +108,8 @@ def cmr_query(
query: Dict[str, Any] = {"concept_id": concept_id}

if datetime:
dt = datetime.split("/")
if len(dt) == 1:
start_datetime = _parse_date(dt[0])
end_datetime = start_datetime + python_datetime.timedelta(days=1)
query["temporal"] = (
start_datetime.isoformat(),
end_datetime.isoformat(),
)

elif len(dt) == 2:
dates: List[Optional[str]] = [None, None]
dates[0] = dt[0] if dt[0] not in ["..", ""] else None
dates[1] = dt[1] if dt[1] not in ["..", ""] else None

# TODO: once https://github.com/nsidc/earthaccess/pull/451 is publish
# we can move to Datetime object instead of String
start: Optional[str] = None
end: Optional[str] = None

if dates[0]:
start = _parse_date(dates[0]).isoformat()

if dates[1]:
end = _parse_date(dates[1]).isoformat()

query["temporal"] = (start, end)
else:
raise InvalidDatetime("Invalid datetime: {datetime}")
datetime_, start, end = parse_datetime(datetime)
query["temporal"] = datetime_ if datetime_ else (start, end)

return query

2 changes: 1 addition & 1 deletion titiler/cmr/errors.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""tipg.errors: Error classes."""
"""titiler.cmr.errors: Error classes."""

import logging

Loading
Loading