From dd4edfad0216f3c3b069e860de70d29c35a351e1 Mon Sep 17 00:00:00 2001 From: aschroed Date: Thu, 25 Sep 2025 14:41:51 -0400 Subject: [PATCH 1/3] added checks for local es testing to es functions --- dcicutils/ff_utils.py | 21 +++++++++++++-------- pyproject.toml | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index d2456fc1f..6198fdcde 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -896,12 +896,9 @@ def _get_es_metadata(uuids, es_client, filters, sources, chunk_size, auth): used to create the generator. Should NOT be used directly """ - def get_es_host_local() -> Optional[str]: - return os.environ.get("ES_HOST_LOCAL", None) health = get_health_page(key=auth) if es_client is None: - if not (es_url := get_es_host_local()): - es_url = health['elasticsearch'] + es_url = _get_es_url(healthpage=health) es_client = es_utils.create_es_client(es_url, use_aws_auth=True) namespace_star = health.get('namespace', '') + '*' # match all given uuids to _id fields @@ -1079,6 +1076,14 @@ def get_schema_names(key=None, ff_env=None, allow_abstract=False) -> Dict[str, s } +def _get_es_url(auth=None, healthpage=None) -> str: + if not (es_url := os.environ.get("ES_HOST_LOCAL", None)): + if not (healthpage or auth): + raise ValueError("If ES_HOST_LOCAL is not set, healthpage or auth must be provided to get ES URL from health page.") + es_url = healthpage.get('elasticsearch') or get_health_page(key=auth)['elasticsearch'] + return es_url + + def expand_es_metadata(uuid_list, key=None, ff_env=None, store_frame='raw', add_pc_wfr=False, ignore_field=None, use_generator=False, es_client=None): """ @@ -1126,9 +1131,9 @@ def remove_keys(my_dict, remove_list): return my_dict auth = get_authentication_with_server(key, ff_env) - es_url = None - if es_client is None: # set up an es client if none is provided - es_url = get_health_page(key=auth)['elasticsearch'] + es_url = _get_es_url(auth=auth) + + if es_client is None: es_client = es_utils.create_es_client(es_url, use_aws_auth=True) # creates a dictionary of schema names to collection names @@ -1156,7 +1161,7 @@ def remove_keys(my_dict, remove_list): if es_url: es_client = es_utils.create_es_client(es_url, use_aws_auth=True) else: # recreate client and try again - if we fail here, exception should propagate - es_url = get_health_page(key=auth)['elasticsearch'] + es_url = _get_es_url(auth=auth) es_client = es_utils.create_es_client(es_url, use_aws_auth=True) current_page = get_es_metadata(uuid_list, es_client=es_client, chunk_size=chunk, diff --git a/pyproject.toml b/pyproject.toml index 6e2a22878..7e62db89c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicutils" -version = "8.18.3" +version = "8.18.4" description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources" authors = ["4DN-DCIC Team "] license = "MIT" From 425585b716243414a710428e6ff3a25eca641b84 Mon Sep 17 00:00:00 2001 From: aschroed Date: Thu, 25 Sep 2025 14:55:28 -0400 Subject: [PATCH 2/3] flake fix --- dcicutils/ff_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dcicutils/ff_utils.py b/dcicutils/ff_utils.py index 6198fdcde..5fc077ab5 100644 --- a/dcicutils/ff_utils.py +++ b/dcicutils/ff_utils.py @@ -1079,7 +1079,7 @@ def get_schema_names(key=None, ff_env=None, allow_abstract=False) -> Dict[str, s def _get_es_url(auth=None, healthpage=None) -> str: if not (es_url := os.environ.get("ES_HOST_LOCAL", None)): if not (healthpage or auth): - raise ValueError("If ES_HOST_LOCAL is not set, healthpage or auth must be provided to get ES URL from health page.") + raise ValueError("If ES_HOST_LOCAL is not set, healthpage or auth must be provided to get ES URL.") es_url = healthpage.get('elasticsearch') or get_health_page(key=auth)['elasticsearch'] return es_url From 2b524d1bf69e979c9163cbd1488fe87cbcec7ab8 Mon Sep 17 00:00:00 2001 From: aschroed Date: Tue, 30 Sep 2025 12:28:25 -0400 Subject: [PATCH 3/3] changelog --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f206702b4..4d6f98597 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,12 @@ dcicutils Change Log ---------- +8.18.4 +====== +* ajs / 2025-09-30 / branch: ajs_upd_es_metadata_fxns_250925 / PR-330 + - Updated ff_utils.get_es_metadata and ff_utils.expand_es_metadata to work locally through aws-proxy. + + 8.18.3 ====== * dmichaels / 2025-03-05 / branch: dmichaels-20250305-add-portal-get-schema-super-types / PR-328