From 17f436c50682717a765aa20a7fced8cf179c744a Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe Date: Tue, 23 Apr 2024 16:08:46 -0700 Subject: [PATCH 1/2] Update catalog store public status during validation --- leap_data_management_utils/catalog.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/leap_data_management_utils/catalog.py b/leap_data_management_utils/catalog.py index 504b744..e54b67a 100644 --- a/leap_data_management_utils/catalog.py +++ b/leap_data_management_utils/catalog.py @@ -5,6 +5,7 @@ import pydantic import pydantic_core import upath +import xarray as xr import yaml @@ -13,6 +14,7 @@ class Store(pydantic.BaseModel): name: str = pydantic.Field(None, description='Name of the store') url: str = pydantic.Field(..., description='URL of the store') rechunking: list[dict[str, str]] | None = pydantic.Field(None, alias='ncviewjs:rechunking') + public: bool | None = pydantic.Field(None, description='Whether the store is public') class Link(pydantic.BaseModel): @@ -118,6 +120,10 @@ def validate_feedstocks(*, feedstocks: list[upath.UPath]) -> list[Feedstock]: for feedstock in feedstocks: try: feed = Feedstock.from_yaml(convert_to_raw_github_url(feedstock)) + print('🔄 Checking stores') + for index, store in enumerate(feed.stores): + print(f' 🚦 {store.id} ({index + 1}/{len(feed.stores)})') + feed.stores[index].public = is_store_public(store.url) valid.append({'feedstock': str(feedstock), 'status': 'valid'}) catalog.append(feed) except Exception: @@ -136,6 +142,16 @@ def validate_feedstocks(*, feedstocks: list[upath.UPath]) -> list[Feedstock]: return catalog +def is_store_public(store) -> bool: + try: + xr.open_dataset(store, engine='zarr', chunks={}) + return True + except Exception: + print(f'Store {store} is not public') + # print(traceback.format_exc()) + return False + + def validate(args): if args.single: # If single file path is provided, validate just this one feedstock From dacb74c5d69812e3fb87b64137dca04193b71b68 Mon Sep 17 00:00:00 2001 From: Anderson Banihirwe Date: Tue, 23 Apr 2024 16:13:07 -0700 Subject: [PATCH 2/2] remove print statements --- leap_data_management_utils/catalog.py | 1 - 1 file changed, 1 deletion(-) diff --git a/leap_data_management_utils/catalog.py b/leap_data_management_utils/catalog.py index e54b67a..eb864d1 100644 --- a/leap_data_management_utils/catalog.py +++ b/leap_data_management_utils/catalog.py @@ -148,7 +148,6 @@ def is_store_public(store) -> bool: return True except Exception: print(f'Store {store} is not public') - # print(traceback.format_exc()) return False