From f94624733feb608a449f762083e999a045b526ce Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 21 Nov 2023 12:48:00 +0800 Subject: [PATCH 01/11] update es drivers to 8.11.0 --- stac_fastapi/elasticsearch/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 6ff8cd86..c0f941f0 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -13,8 +13,8 @@ "stac-fastapi.types==2.4.8", "stac-fastapi.api==2.4.8", "stac-fastapi.extensions==2.4.8", - "elasticsearch[async]==7.17.9", - "elasticsearch-dsl==7.4.1", + "elasticsearch[async]==8.11.0", + "elasticsearch-dsl==8.11.0", "pystac[validation]", "uvicorn", "orjson", From 9e4fa7a83432b9416b44bd524f13a05cca1a90e6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 21 Nov 2023 12:48:09 +0800 Subject: [PATCH 02/11] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d50d6ff9..66dd8d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +- Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169) + ### Fixed - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166) From 05423b55179d93d6ae8b5697a4a39c91833ad2e1 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 12:48:30 +0800 Subject: [PATCH 03/11] update config --- .../stac_fastapi/elasticsearch/config.py | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index e1630d67..f6b04551 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -1,5 +1,6 @@ """API configuration.""" import os +import ssl from typing import Any, Dict, Set from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore @@ -7,28 +8,37 @@ def _es_config() -> Dict[str, Any]: + # Determine the scheme (http or https) + use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + scheme = "https" if use_ssl else "http" + + # Configure the hosts parameter with the correct scheme + hosts = [f"{scheme}://{os.getenv('ES_HOST')}:{os.getenv('ES_PORT')}"] + + # Initialize the configuration dictionary config = { - "hosts": [{"host": os.getenv("ES_HOST"), "port": os.getenv("ES_PORT")}], - "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"}, - "use_ssl": True, - "verify_certs": True, + "hosts": hosts, + "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"} } - if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): - config["http_auth"] = (u, p) + # Explicitly exclude SSL settings when not using SSL + if not use_ssl: + return config - if (v := os.getenv("ES_USE_SSL")) and v == "false": - config["use_ssl"] = False + # Include SSL settings if using https + config["ssl_version"] = ssl.TLSVersion.TLSv1_2 + config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" - if (v := os.getenv("ES_VERIFY_CERTS")) and v == "false": - config["verify_certs"] = False + # Include CA Certificates if verifying certs + if config["verify_certs"]: + config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") - if v := os.getenv("CURL_CA_BUNDLE"): - config["ca_certs"] = v + # Handle authentication + if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): + config["http_auth"] = (u, p) return config - _forbidden_fields: Set[str] = {"type"} From eb586c6b37ecb54fa9be2e65d9daaf503a59e307 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 12:48:41 +0800 Subject: [PATCH 04/11] ues options --- .../stac_fastapi/elasticsearch/database_logic.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6b2cd433..da16b7bf 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -170,18 +170,12 @@ def indices(collection_ids: Optional[List[str]]) -> str: async def create_collection_index() -> None: - """Create the index for Collections in Elasticsearch. - - This function creates the Elasticsearch index for the `Collections` with the predefined mapping. - If the index already exists, the function ignores the error and continues execution. - """ client = AsyncElasticsearchSettings().create_client - await client.indices.create( + await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", aliases={COLLECTIONS_INDEX: {}}, - mappings=ES_COLLECTIONS_MAPPINGS, - ignore=400, # ignore 400 already exists code + mappings=ES_COLLECTIONS_MAPPINGS ) await client.close() @@ -200,12 +194,11 @@ async def create_item_index(collection_id: str): client = AsyncElasticsearchSettings().create_client index_name = index_by_collection_id(collection_id) - await client.indices.create( + await client.options(ignore_status=400).indices.create( index=f"{index_by_collection_id(collection_id)}-000001", aliases={index_name: {}}, mappings=ES_ITEMS_MAPPINGS, settings=ES_ITEMS_SETTINGS, - ignore=400, # ignore 400 already exists code ) await client.close() From 7a23ed7a4e8308b5d113cf0372eaefacdc916b90 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 12:48:48 +0800 Subject: [PATCH 05/11] update test --- stac_fastapi/elasticsearch/tests/api/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index c50256da..74f0bb55 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -363,6 +363,7 @@ async def test_search_polygon_intersects_get(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_point_intersects_post(app_client, ctx): point = [150.04, -33.14] intersects = {"type": "Point", "coordinates": point} From 96904fa48ace47cad2dab50ad48f526547c5c630 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 13:01:28 +0800 Subject: [PATCH 06/11] pre-commit --- .../stac_fastapi/elasticsearch/config.py | 11 +++++++---- .../stac_fastapi/elasticsearch/database_logic.py | 9 ++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index f6b04551..ecfa8e4b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -18,7 +18,7 @@ def _es_config() -> Dict[str, Any]: # Initialize the configuration dictionary config = { "hosts": hosts, - "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"} + "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"}, } # Explicitly exclude SSL settings when not using SSL @@ -26,12 +26,14 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_2 - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" + config["ssl_version"] = ssl.TLSVersion.TLSv1_2 # type: ignore + config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore # Include CA Certificates if verifying certs if config["verify_certs"]: - config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") + config["ca_certs"] = os.getenv( + "CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt" + ) # Handle authentication if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): @@ -39,6 +41,7 @@ def _es_config() -> Dict[str, Any]: return config + _forbidden_fields: Set[str] = {"type"} diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index da16b7bf..4554e74c 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -170,12 +170,19 @@ def indices(collection_ids: Optional[List[str]]) -> str: async def create_collection_index() -> None: + """ + Create the index for a Collection. + + Returns: + None + + """ client = AsyncElasticsearchSettings().create_client await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", aliases={COLLECTIONS_INDEX: {}}, - mappings=ES_COLLECTIONS_MAPPINGS + mappings=ES_COLLECTIONS_MAPPINGS, ) await client.close() From e7fd47de46f61941c097e8a55c366d23412565c7 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 13:02:02 +0800 Subject: [PATCH 07/11] es containers to 8.11.0 --- .github/workflows/cicd.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index caeae818..e63b5097 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -16,7 +16,7 @@ jobs: services: elasticsearch_8_svc: - image: docker.elastic.co/elasticsearch/elasticsearch:8.10.4 + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 env: cluster.name: stac-cluster node.name: es01 diff --git a/docker-compose.yml b/docker-compose.yml index 1cad4dee..db3352fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: elasticsearch: container_name: es-container - image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.10.4} + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0} environment: ES_JAVA_OPTS: -Xms512m -Xmx1g volumes: From 2efdd41470214e116c3c46512eb563c4b890ef85 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 13:07:34 +0800 Subject: [PATCH 08/11] use TLSv1_3 --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index ecfa8e4b..8634d3b9 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -26,7 +26,7 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_2 # type: ignore + config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore # Include CA Certificates if verifying certs From 0e027411171bc47e21f3d0c7572d48a05fa9ff34 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 9 Dec 2023 11:37:40 +0800 Subject: [PATCH 09/11] remove test --- .../elasticsearch/tests/resources/test_mgmt.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 stac_fastapi/elasticsearch/tests/resources/test_mgmt.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py deleted file mode 100644 index 2b7d9728..00000000 --- a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest - - -@pytest.mark.asyncio -async def test_ping_no_param(app_client): - """ - Test ping endpoint with a mocked client. - Args: - app_client (TestClient): mocked client fixture - """ - res = await app_client.get("/_mgmt/ping") - assert res.status_code == 200 - assert res.json() == {"message": "PONG"} From eef2b3b7d3e7a6671d571dbad4daec95cdf4bc65 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 9 Dec 2023 11:55:15 +0800 Subject: [PATCH 10/11] event loop --- stac_fastapi/elasticsearch/tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index f4b49928..9ddbf691 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -64,7 +64,14 @@ class Config: @pytest.fixture(scope="session") def event_loop(): + # asyncio.new_event_loop().run_until_complete() + + # loop = asyncio.new_event_loop() + # yield loop + # loop.close() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) yield loop loop.close() From 4e3835c3457239497e0cd1a6f8cf6c14749507c6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 9 Dec 2023 12:01:46 +0800 Subject: [PATCH 11/11] add test back --- stac_fastapi/elasticsearch/tests/conftest.py | 6 ------ .../elasticsearch/tests/resources/test_mgmt.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 stac_fastapi/elasticsearch/tests/resources/test_mgmt.py diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index 9ddbf691..fa093af2 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -64,12 +64,6 @@ class Config: @pytest.fixture(scope="session") def event_loop(): - # asyncio.new_event_loop().run_until_complete() - - # loop = asyncio.new_event_loop() - # yield loop - # loop.close() - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) yield loop diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py new file mode 100644 index 00000000..2b7d9728 --- /dev/null +++ b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py @@ -0,0 +1,13 @@ +import pytest + + +@pytest.mark.asyncio +async def test_ping_no_param(app_client): + """ + Test ping endpoint with a mocked client. + Args: + app_client (TestClient): mocked client fixture + """ + res = await app_client.get("/_mgmt/ping") + assert res.status_code == 200 + assert res.json() == {"message": "PONG"}