From d8161c4acd4e2c7e3d42ed36556f31a7d0284c9e Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:14:54 -0400 Subject: [PATCH 01/27] core to toml --- pctasks/core/pyproject.toml | 67 ++++++++++++ pctasks/core/requirements.txt | 188 ++++++++++++++++++++++++++++++++++ pctasks/core/setup.cfg | 2 - pctasks/core/setup.py | 57 ----------- 4 files changed, 255 insertions(+), 59 deletions(-) create mode 100644 pctasks/core/pyproject.toml create mode 100644 pctasks/core/requirements.txt delete mode 100644 pctasks/core/setup.cfg delete mode 100644 pctasks/core/setup.py diff --git a/pctasks/core/pyproject.toml b/pctasks/core/pyproject.toml new file mode 100644 index 00000000..c7822a71 --- /dev/null +++ b/pctasks/core/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.core" +dynamic = ["version"] +description = "Planetary Computer Tasks framework." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "aiohttp>=3.9", + "azure-cosmos>=4.3.0,<4.4.0", + "azure-data-tables>=12.0.0,<13", + "azure-identity>=1.0.0,<2", + "azure-storage-blob", + "azure-storage-queue>=12.6.0", + "idna>=3.7.0", + "jsonschema>=4.0.1,<4.18", + "opencensus-ext-azure==1.1.0", + "opencensus-ext-logging==0.1.1", + "orjson>=3.0.0,<4", + "planetary-computer>=0.4.0", + "pydantic>=1.9,<2.0.0", + "pyyaml>=5.3", + "stac-validator>=3.1.0", + "strictyaml>=1.6", + "typing-extensions; python_version< '3.10.0'", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.version] +path = "pctasks/core/version.py" + +[tool.hatch.build] +include = [ + "/pctasks/core", +] diff --git a/pctasks/core/requirements.txt b/pctasks/core/requirements.txt new file mode 100644 index 00000000..f904ea9f --- /dev/null +++ b/pctasks/core/requirements.txt @@ -0,0 +1,188 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=pctasks/core/requirements.txt ./pctasks/core/setup.py +# +aiohttp==3.9.5 + # via pctasks.core (pctasks/core/setup.py) +aiosignal==1.3.1 + # via aiohttp +async-timeout==4.0.3 + # via aiohttp +attrs==23.2.0 + # via + # aiohttp + # jsonschema +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure +azure-cosmos==4.3.1 + # via pctasks.core (pctasks/core/setup.py) +azure-data-tables==12.5.0 + # via pctasks.core (pctasks/core/setup.py) +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks.core (pctasks/core/setup.py) +azure-storage-blob==12.19.1 + # via pctasks.core (pctasks/core/setup.py) +azure-storage-queue==12.9.0 + # via pctasks.core (pctasks/core/setup.py) +cachetools==5.3.3 + # via google-auth +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # planetary-computer + # stac-validator +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +google-api-core==2.18.0 + # via opencensus +google-auth==2.29.0 + # via google-api-core +googleapis-common-protos==1.63.0 + # via google-api-core +idna==3.7 + # via + # pctasks.core (pctasks/core/setup.py) + # requests + # yarl +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue +jsonschema==4.17.3 + # via + # pctasks.core (pctasks/core/setup.py) + # pystac + # stac-validator +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +multidict==6.0.5 + # via + # aiohttp + # yarl +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging +opencensus-context==0.1.3 + # via opencensus +opencensus-ext-azure==1.1.0 + # via pctasks.core (pctasks/core/setup.py) +opencensus-ext-logging==0.1.1 + # via pctasks.core (pctasks/core/setup.py) +orjson==3.10.1 + # via pctasks.core (pctasks/core/setup.py) +packaging==24.0 + # via + # msal-extensions + # planetary-computer +planetary-computer==1.0.0 + # via pctasks.core (pctasks/core/setup.py) +portalocker==2.8.2 + # via msal-extensions +proto-plus==1.23.0 + # via google-api-core +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus +psutil==5.9.8 + # via opencensus-ext-azure +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pycparser==2.22 + # via cffi +pydantic==1.10.15 + # via + # pctasks.core (pctasks/core/setup.py) + # planetary-computer +pyjwt[crypto]==2.8.0 + # via + # msal + # pyjwt +pyrsistent==0.20.0 + # via jsonschema +pystac[validation]==1.8.3 + # via + # planetary-computer + # pystac-client +pystac-client==0.7.6 + # via planetary-computer +python-dateutil==2.9.0.post0 + # via + # pystac + # pystac-client + # strictyaml +python-dotenv==1.0.1 + # via planetary-computer +pytz==2024.1 + # via planetary-computer +pyyaml==6.0.1 + # via pctasks.core (pctasks/core/setup.py) +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil +stac-validator==3.3.2 + # via pctasks.core (pctasks/core/setup.py) +strictyaml==1.7.3 + # via pctasks.core (pctasks/core/setup.py) +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic +urllib3==2.2.1 + # via requests +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables diff --git a/pctasks/core/setup.cfg b/pctasks/core/setup.cfg deleted file mode 100644 index 106c50f6..00000000 --- a/pctasks/core/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.core.version.__version__ diff --git a/pctasks/core/setup.py b/pctasks/core/setup.py deleted file mode 100644 index 50683471..00000000 --- a/pctasks/core/setup.py +++ /dev/null @@ -1,57 +0,0 @@ -"""pctasks: types module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "azure-identity>=1.0.0,<2", - "azure-storage-blob", - "azure-storage-queue>=12.6.0", - "azure-data-tables>=12.0.0,<13", - "azure-cosmos>=4.3.0,<4.4.0", - "pydantic>=1.9,<2.0.0", - "orjson>=3.0.0,<4", - "strictyaml>=1.6", - "stac-validator>=3.1.0", - "opencensus-ext-azure==1.1.0", - "opencensus-ext-logging==0.1.1", - "pyyaml>=5.3", - "aiohttp>=3.9", - "planetary-computer>=0.4.0", - "jsonschema>=4.0.1,<4.18", # Matches the pin in https://github.com/stac-utils/pystac/pull/1189/ - "typing-extensions; python_version< '3.10.0'", -] - -extra_reqs = { - "dev": ["pytest", "pytest-cov", "pre-commit"], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.core", - description="Planetary Computer Tasks framework.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author="Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) From 439c8ff83e1936ff386ae84c8ca4920aac4fd03a Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:15:03 -0400 Subject: [PATCH 02/27] cli to TOML --- pctasks/cli/pyproject.toml | 59 ++++++++++++ pctasks/cli/requirements.txt | 182 +++++++++++++++++++++++++++++++++++ 2 files changed, 241 insertions(+) create mode 100644 pctasks/cli/pyproject.toml create mode 100644 pctasks/cli/requirements.txt diff --git a/pctasks/cli/pyproject.toml b/pctasks/cli/pyproject.toml new file mode 100644 index 00000000..98d29d00 --- /dev/null +++ b/pctasks/cli/pyproject.toml @@ -0,0 +1,59 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.cli" +dynamic = ["version"] +description = "Planetary Computer Tasks framework." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "click>=7.0", + # "pctasks.core>=0.1.0", + "pctasks.core @ {root:uri}/../core", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.scripts] +pctasks = "pctasks.cli.cli:cli" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/cli/version.py" + +[tool.hatch.build] +include = [ + "/pctasks/cli", +] diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt new file mode 100644 index 00000000..d49e5e86 --- /dev/null +++ b/pctasks/cli/requirements.txt @@ -0,0 +1,182 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile pyproject.toml -o requirements.txt +aiohttp==3.9.5 + # via pctasks-core +aiosignal==1.3.1 + # via aiohttp +async-timeout==4.0.3 + # via aiohttp +attrs==23.2.0 + # via + # aiohttp + # jsonschema +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure +azure-cosmos==4.3.1 + # via pctasks-core +azure-data-tables==12.5.0 + # via pctasks-core +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core +azure-storage-blob==12.19.1 + # via pctasks-core +azure-storage-queue==12.9.0 + # via pctasks-core +cachetools==5.3.3 + # via google-auth +certifi==2024.2.2 + # via requests +cffi==1.16.0 + # via cryptography +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # planetary-computer + # stac-validator +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal +google-api-core==2.18.0 + # via opencensus +google-auth==2.29.0 + # via google-api-core +googleapis-common-protos==1.63.0 + # via google-api-core +idna==3.7 + # via + # pctasks-core + # requests + # yarl +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator +msal==1.28.0 + # via + # azure-identity + # msal-extensions +msal-extensions==1.1.0 + # via azure-identity +multidict==6.0.5 + # via + # aiohttp + # yarl +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging +opencensus-context==0.1.3 + # via opencensus +opencensus-ext-azure==1.1.0 + # via pctasks-core +opencensus-ext-logging==0.1.1 + # via pctasks-core +orjson==3.10.1 + # via pctasks-core +packaging==24.0 + # via + # msal-extensions + # planetary-computer +pctasks-core @ file:///home/gustavo/code/planetary-computer-tasks/pctasks/core +planetary-computer==1.0.0 + # via pctasks-core +portalocker==2.8.2 + # via msal-extensions +proto-plus==1.23.0 + # via google-api-core +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus +psutil==5.9.8 + # via opencensus-ext-azure +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.0 + # via google-auth +pycparser==2.22 + # via cffi +pydantic==1.10.15 + # via + # pctasks-core + # planetary-computer +pyjwt==2.8.0 + # via msal +pyrsistent==0.20.0 + # via jsonschema +pystac==1.10.0 + # via + # planetary-computer + # pystac-client +pystac-client==0.6.1 + # via planetary-computer +python-dateutil==2.9.0.post0 + # via + # pystac + # pystac-client + # strictyaml +python-dotenv==1.0.1 + # via planetary-computer +pytz==2024.1 + # via planetary-computer +pyyaml==6.0.1 + # via pctasks-core +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator +rsa==4.9 + # via google-auth +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil +stac-validator==3.3.2 + # via pctasks-core +strictyaml==1.7.3 + # via pctasks-core +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic +urllib3==2.2.1 + # via requests +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables From d78d93b91153ca1f4f7a697a0232e9989f4d6ca0 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:21:34 -0400 Subject: [PATCH 03/27] uv pip compile core --- pctasks/core/requirements.txt | 97 ++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/pctasks/core/requirements.txt b/pctasks/core/requirements.txt index f904ea9f..5006471b 100644 --- a/pctasks/core/requirements.txt +++ b/pctasks/core/requirements.txt @@ -1,19 +1,20 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --output-file=pctasks/core/requirements.txt ./pctasks/core/setup.py -# +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/core/pyproject.toml -o pctasks/core/requirements.txt --emit-index-url +--index-url https://pypi.org/simple + aiohttp==3.9.5 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple aiosignal==1.3.1 # via aiohttp + # from https://pypi.org/simple async-timeout==4.0.3 # via aiohttp + # from https://pypi.org/simple attrs==23.2.0 # via # aiohttp # jsonschema + # from https://pypi.org/simple azure-core==1.30.1 # via # azure-cosmos @@ -22,30 +23,35 @@ azure-core==1.30.1 # azure-storage-blob # azure-storage-queue # opencensus-ext-azure + # from https://pypi.org/simple azure-cosmos==4.3.1 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple azure-data-tables==12.5.0 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple azure-identity==1.16.0 - # via - # opencensus-ext-azure - # pctasks.core (pctasks/core/setup.py) + # via opencensus-ext-azure + # from https://pypi.org/simple azure-storage-blob==12.19.1 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple azure-storage-queue==12.9.0 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple cachetools==5.3.3 # via google-auth + # from https://pypi.org/simple certifi==2024.2.2 # via requests + # from https://pypi.org/simple cffi==1.16.0 # via cryptography + # from https://pypi.org/simple charset-normalizer==3.3.2 # via requests + # from https://pypi.org/simple click==8.1.7 # via # planetary-computer # stac-validator + # from https://pypi.org/simple cryptography==42.0.5 # via # azure-identity @@ -53,105 +59,128 @@ cryptography==42.0.5 # azure-storage-queue # msal # pyjwt + # from https://pypi.org/simple frozenlist==1.4.1 # via # aiohttp # aiosignal + # from https://pypi.org/simple google-api-core==2.18.0 # via opencensus + # from https://pypi.org/simple google-auth==2.29.0 # via google-api-core + # from https://pypi.org/simple googleapis-common-protos==1.63.0 # via google-api-core + # from https://pypi.org/simple idna==3.7 # via - # pctasks.core (pctasks/core/setup.py) # requests # yarl + # from https://pypi.org/simple isodate==0.6.1 # via # azure-data-tables # azure-storage-blob # azure-storage-queue + # from https://pypi.org/simple jsonschema==4.17.3 # via - # pctasks.core (pctasks/core/setup.py) # pystac # stac-validator + # from https://pypi.org/simple msal==1.28.0 # via # azure-identity # msal-extensions + # from https://pypi.org/simple msal-extensions==1.1.0 # via azure-identity + # from https://pypi.org/simple multidict==6.0.5 # via # aiohttp # yarl + # from https://pypi.org/simple opencensus==0.11.4 # via # opencensus-ext-azure # opencensus-ext-logging + # from https://pypi.org/simple opencensus-context==0.1.3 # via opencensus + # from https://pypi.org/simple opencensus-ext-azure==1.1.0 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple opencensus-ext-logging==0.1.1 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple orjson==3.10.1 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple packaging==24.0 # via # msal-extensions # planetary-computer + # from https://pypi.org/simple planetary-computer==1.0.0 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple portalocker==2.8.2 # via msal-extensions + # from https://pypi.org/simple proto-plus==1.23.0 # via google-api-core + # from https://pypi.org/simple protobuf==4.25.3 # via # google-api-core # googleapis-common-protos # proto-plus + # from https://pypi.org/simple psutil==5.9.8 # via opencensus-ext-azure + # from https://pypi.org/simple pyasn1==0.6.0 # via # pyasn1-modules # rsa + # from https://pypi.org/simple pyasn1-modules==0.4.0 # via google-auth + # from https://pypi.org/simple pycparser==2.22 # via cffi + # from https://pypi.org/simple pydantic==1.10.15 - # via - # pctasks.core (pctasks/core/setup.py) - # planetary-computer -pyjwt[crypto]==2.8.0 - # via - # msal - # pyjwt + # via planetary-computer + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple pyrsistent==0.20.0 # via jsonschema -pystac[validation]==1.8.3 + # from https://pypi.org/simple +pystac==1.8.3 # via # planetary-computer # pystac-client + # from https://pypi.org/simple pystac-client==0.7.6 # via planetary-computer + # from https://pypi.org/simple python-dateutil==2.9.0.post0 # via # pystac # pystac-client # strictyaml + # from https://pypi.org/simple python-dotenv==1.0.1 # via planetary-computer + # from https://pypi.org/simple pytz==2024.1 # via planetary-computer + # from https://pypi.org/simple pyyaml==6.0.1 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple requests==2.31.0 # via # azure-core @@ -161,18 +190,21 @@ requests==2.31.0 # planetary-computer # pystac-client # stac-validator + # from https://pypi.org/simple rsa==4.9 # via google-auth + # from https://pypi.org/simple six==1.16.0 # via # azure-core # isodate # opencensus # python-dateutil + # from https://pypi.org/simple stac-validator==3.3.2 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple strictyaml==1.7.3 - # via pctasks.core (pctasks/core/setup.py) + # from https://pypi.org/simple typing-extensions==4.11.0 # via # azure-core @@ -180,9 +212,12 @@ typing-extensions==4.11.0 # azure-storage-blob # azure-storage-queue # pydantic + # from https://pypi.org/simple urllib3==2.2.1 # via requests + # from https://pypi.org/simple yarl==1.9.4 # via # aiohttp # azure-data-tables + # from https://pypi.org/simple From 646484978b62da96810432b182d90b067a80d9da Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:28:56 -0400 Subject: [PATCH 04/27] save --- pctasks/cli/requirements.txt | 63 +++++++++++++++++++++++++++++++-- scripts/bin/pctasks-pip-compile | 19 ++++++++++ scripts/generate-requirements | 54 ++++++++++++++++++++++++++++ scripts/install | 5 +-- 4 files changed, 137 insertions(+), 4 deletions(-) create mode 100755 scripts/bin/pctasks-pip-compile create mode 100755 scripts/generate-requirements diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt index d49e5e86..6be6b6f7 100644 --- a/pctasks/cli/requirements.txt +++ b/pctasks/cli/requirements.txt @@ -1,15 +1,21 @@ # This file was autogenerated by uv via the following command: -# uv pip compile pyproject.toml -o requirements.txt +# uv pip compile --emit-index-annotation ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt --emit-index-url +--index-url https://pypi.org/simple + aiohttp==3.9.5 # via pctasks-core + # from https://pypi.org/simple aiosignal==1.3.1 # via aiohttp + # from https://pypi.org/simple async-timeout==4.0.3 # via aiohttp + # from https://pypi.org/simple attrs==23.2.0 # via # aiohttp # jsonschema + # from https://pypi.org/simple azure-core==1.30.1 # via # azure-cosmos @@ -18,30 +24,41 @@ azure-core==1.30.1 # azure-storage-blob # azure-storage-queue # opencensus-ext-azure + # from https://pypi.org/simple azure-cosmos==4.3.1 # via pctasks-core + # from https://pypi.org/simple azure-data-tables==12.5.0 # via pctasks-core + # from https://pypi.org/simple azure-identity==1.16.0 # via # opencensus-ext-azure # pctasks-core + # from https://pypi.org/simple azure-storage-blob==12.19.1 # via pctasks-core + # from https://pypi.org/simple azure-storage-queue==12.9.0 # via pctasks-core + # from https://pypi.org/simple cachetools==5.3.3 # via google-auth + # from https://pypi.org/simple certifi==2024.2.2 # via requests + # from https://pypi.org/simple cffi==1.16.0 # via cryptography + # from https://pypi.org/simple charset-normalizer==3.3.2 # via requests + # from https://pypi.org/simple click==8.1.7 # via # planetary-computer # stac-validator + # from https://pypi.org/simple cryptography==42.0.5 # via # azure-identity @@ -49,103 +66,137 @@ cryptography==42.0.5 # azure-storage-queue # msal # pyjwt + # from https://pypi.org/simple frozenlist==1.4.1 # via # aiohttp # aiosignal + # from https://pypi.org/simple google-api-core==2.18.0 # via opencensus + # from https://pypi.org/simple google-auth==2.29.0 # via google-api-core + # from https://pypi.org/simple googleapis-common-protos==1.63.0 # via google-api-core + # from https://pypi.org/simple idna==3.7 # via # pctasks-core # requests # yarl + # from https://pypi.org/simple isodate==0.6.1 # via # azure-data-tables # azure-storage-blob # azure-storage-queue + # from https://pypi.org/simple jsonschema==4.17.3 # via # pctasks-core # stac-validator + # from https://pypi.org/simple msal==1.28.0 # via # azure-identity # msal-extensions + # from https://pypi.org/simple msal-extensions==1.1.0 # via azure-identity + # from https://pypi.org/simple multidict==6.0.5 # via # aiohttp # yarl + # from https://pypi.org/simple opencensus==0.11.4 # via # opencensus-ext-azure # opencensus-ext-logging + # from https://pypi.org/simple opencensus-context==0.1.3 # via opencensus + # from https://pypi.org/simple opencensus-ext-azure==1.1.0 # via pctasks-core + # from https://pypi.org/simple opencensus-ext-logging==0.1.1 # via pctasks-core + # from https://pypi.org/simple orjson==3.10.1 # via pctasks-core + # from https://pypi.org/simple packaging==24.0 # via # msal-extensions # planetary-computer -pctasks-core @ file:///home/gustavo/code/planetary-computer-tasks/pctasks/core + # from https://pypi.org/simple + planetary-computer==1.0.0 # via pctasks-core + # from https://pypi.org/simple portalocker==2.8.2 # via msal-extensions + # from https://pypi.org/simple proto-plus==1.23.0 # via google-api-core + # from https://pypi.org/simple protobuf==4.25.3 # via # google-api-core # googleapis-common-protos # proto-plus + # from https://pypi.org/simple psutil==5.9.8 # via opencensus-ext-azure + # from https://pypi.org/simple pyasn1==0.6.0 # via # pyasn1-modules # rsa + # from https://pypi.org/simple pyasn1-modules==0.4.0 # via google-auth + # from https://pypi.org/simple pycparser==2.22 # via cffi + # from https://pypi.org/simple pydantic==1.10.15 # via # pctasks-core # planetary-computer + # from https://pypi.org/simple pyjwt==2.8.0 # via msal + # from https://pypi.org/simple pyrsistent==0.20.0 # via jsonschema + # from https://pypi.org/simple pystac==1.10.0 # via # planetary-computer # pystac-client + # from https://pypi.org/simple pystac-client==0.6.1 # via planetary-computer + # from https://pypi.org/simple python-dateutil==2.9.0.post0 # via # pystac # pystac-client # strictyaml + # from https://pypi.org/simple python-dotenv==1.0.1 # via planetary-computer + # from https://pypi.org/simple pytz==2024.1 # via planetary-computer + # from https://pypi.org/simple pyyaml==6.0.1 # via pctasks-core + # from https://pypi.org/simple requests==2.31.0 # via # azure-core @@ -155,18 +206,23 @@ requests==2.31.0 # planetary-computer # pystac-client # stac-validator + # from https://pypi.org/simple rsa==4.9 # via google-auth + # from https://pypi.org/simple six==1.16.0 # via # azure-core # isodate # opencensus # python-dateutil + # from https://pypi.org/simple stac-validator==3.3.2 # via pctasks-core + # from https://pypi.org/simple strictyaml==1.7.3 # via pctasks-core + # from https://pypi.org/simple typing-extensions==4.11.0 # via # azure-core @@ -174,9 +230,12 @@ typing-extensions==4.11.0 # azure-storage-blob # azure-storage-queue # pydantic + # from https://pypi.org/simple urllib3==2.2.1 # via requests + # from https://pypi.org/simple yarl==1.9.4 # via # aiohttp # azure-data-tables + # from https://pypi.org/simple diff --git a/scripts/bin/pctasks-pip-compile b/scripts/bin/pctasks-pip-compile new file mode 100755 index 00000000..211bb463 --- /dev/null +++ b/scripts/bin/pctasks-pip-compile @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +if [[ "${CI}" ]]; then + set -x +fi + +uv pip compile \ + --emit-index-annotation \ + "$@" + +# 3 +# 2 +# 1 +# 0 +# We expect this to be called as pip-compile input -o output +# so $2 should be the output file +sed -i -e 's/.* @ file:\/\/\/.*//' $3 diff --git a/scripts/generate-requirements b/scripts/generate-requirements new file mode 100755 index 00000000..987bb1d9 --- /dev/null +++ b/scripts/generate-requirements @@ -0,0 +1,54 @@ +#!/bin/bash + +set -e + +echo "== Generating requirements.txt ..." +pip_compile_options="--emit-index-url --verbose" + +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/core/pyproject.toml -o pctasks/core/requirements.txt $pip_compile_options + +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt $pip_compile_options + +# docker-compose \ +# -f docker-compose.console.yml \ +# run --rm \ +# dev \ +# scripts/bin/pctasks-pip-compile ./pctasks/task/setup.py -o pctasks/task/requirements.txt $pip_compile_options + +# docker-compose \ +# -f docker-compose.console.yml \ +# run --rm \ +# dev \ +# scripts/bin/pctasks-pip-compile ./pctasks/client/setup.py -o pctasks/client/requirements.txt $pip_compile_options + +# docker-compose \ +# -f docker-compose.console.yml \ +# run --rm \ +# dev \ +# scripts/bin/pctasks-pip-compile ./pctasks/ingest/setup.py -o pctasks/ingest/requirements.txt $pip_compile_options + +# docker-compose \ +# -f docker-compose.console.yml \ +# run --rm \ +# dev \ +# scripts/bin/pctasks-pip-compile ./pctasks/ingest_task/setup.py -o pctasks/ingest_task/requirements.txt $pip_compile_options + +# docker-compose \ +# -f docker-compose.console.yml \ +# run --rm \ +# dev \ +# scripts/bin/pctasks-pip-compile ./pctasks/run/setup.py -o pctasks/run/requirements.txt $pip_compile_options + +# docker-compose \ +# -f docker-compose.console.yml \ +# run --rm \ +# dev \ +# pip-compile ./pctasks/server/setup.py --extra dev -o pctasks/server/requirements.txt $pip_compile_options diff --git a/scripts/install b/scripts/install index b05a398f..2ab731d4 100755 --- a/scripts/install +++ b/scripts/install @@ -29,12 +29,13 @@ while [[ $# -gt 0 ]]; do case $1 in source scripts/env if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + pip install uv for DIR in "${PACKAGE_DIRS[@]}"; do echo "Installing ${DIR}" - pip install -e ${DIR} + uv pip install --python=$(which python3) -e ${DIR} done echo "Installing dev dependencies" - pip install -r requirements-dev.txt + uv pip install --python=$(which python3) -r requirements-dev.txt fi From 757a9876fa93175ec64ba65103cbb0967276c3fa Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:33:16 -0400 Subject: [PATCH 05/27] tctasks --- Dockerfile.dev | 100 ++++++++++++++++++++-------------- pctasks/cli/requirements.txt | 4 +- pctasks/core/requirements.txt | 4 +- pctasks/task/pyproject.toml | 58 ++++++++++++++++++++ pctasks/task/setup.cfg | 6 -- pctasks/task/setup.py | 47 ---------------- scripts/generate-requirements | 12 ++-- 7 files changed, 126 insertions(+), 105 deletions(-) create mode 100644 pctasks/task/pyproject.toml delete mode 100644 pctasks/task/setup.cfg delete mode 100644 pctasks/task/setup.py diff --git a/Dockerfile.dev b/Dockerfile.dev index ea5b8392..4abb79a1 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -23,7 +23,7 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 # See https://github.com/mapbox/rasterio/issues/1289 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt -RUN python -m pip install --upgrade pip +RUN python -m pip install --upgrade pip uv # Install azure-cli RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash @@ -31,55 +31,75 @@ RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash # Install dev dependencies COPY requirements-dev.txt /opt/src/requirements-dev.txt -RUN cd /opt/src && pip install -r requirements-dev.txt +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src && uv pip install --system -r requirements-dev.txt # # Copy and install packages # COPY pctasks/core /opt/src/pctasks/core -RUN cd /opt/src/pctasks/core && \ - pip install -e . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/core && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . COPY pctasks/cli /opt/src/pctasks/cli -RUN cd /opt/src/pctasks/cli && \ - pip install -e . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/cli && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . COPY pctasks/task /opt/src/pctasks/task -RUN cd /opt/src/pctasks/task && \ - pip install -e . - -COPY pctasks/client /opt/src/pctasks/client -RUN cd /opt/src/pctasks/client && \ - pip install -e . - -COPY pctasks/ingest /opt/src/pctasks/ingest -RUN cd /opt/src/pctasks/ingest && \ - pip install -e . - -COPY pctasks/ingest_task /opt/src/pctasks/ingest_task -RUN cd /opt/src/pctasks/ingest_task && \ - pip install -e . - -COPY pctasks/run /opt/src/pctasks/run -RUN cd /opt/src/pctasks/run && \ - pip install -e . - -COPY pctasks/server /opt/src/pctasks/server -RUN cd /opt/src/pctasks/server && \ - pip install -e .[dev] - -COPY pctasks/dataset /opt/src/pctasks/dataset -RUN cd /opt/src/pctasks/dataset && \ - pip install -e . - -COPY pctasks/notify /opt/src/pctasks/notify -RUN cd /opt/src/pctasks/notify && \ - pip install -e . - -COPY pctasks/dev /opt/src/pctasks/dev -RUN cd /opt/src/pctasks/dev && \ - pip install -e .[server] +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/task && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . + +# COPY pctasks/client /opt/src/pctasks/client +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/client && \ +# uv pip install --system -r requirements.txt && \ +# uv pip install --system --no-deps -e . + +# COPY pctasks/ingest /opt/src/pctasks/ingest +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/ingest && \ +# uv pip install --system -r requirements.txt && \ +# uv pip install --system --no-deps -e . + +# COPY pctasks/ingest_task /opt/src/pctasks/ingest_task +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/ingest_task && \ +# uv pip install --system -r requirements.txt && \ +# uv pip install --system --no-deps -e . + +# COPY pctasks/run /opt/src/pctasks/run +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/run && \ +# uv pip install --system -r requirements.txt && \ +# uv pip install --system --no-deps -e . + +# We are here +# COPY pctasks/server /opt/src/pctasks/server +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/server && \ +# pip install -e .[dev] + +# COPY pctasks/dataset /opt/src/pctasks/dataset +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/dataset && \ +# pip install -e . + +# COPY pctasks/notify /opt/src/pctasks/notify +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/notify && \ +# pip install -e . + +# COPY pctasks/dev /opt/src/pctasks/dev +# RUN --mount=type=cache,target=/root/.cache \ +# cd /opt/src/pctasks/dev && \ +# pip install -e .[server] # Setup Python Path to allow import of test modules ENV PYTHONPATH=/opt/src:$PYTHONPATH diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt index 6be6b6f7..667ec2db 100644 --- a/pctasks/cli/requirements.txt +++ b/pctasks/cli/requirements.txt @@ -1,7 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt --emit-index-url ---index-url https://pypi.org/simple - +# uv pip compile --emit-index-annotation ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/core/requirements.txt b/pctasks/core/requirements.txt index 5006471b..4337043e 100644 --- a/pctasks/core/requirements.txt +++ b/pctasks/core/requirements.txt @@ -1,7 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/core/pyproject.toml -o pctasks/core/requirements.txt --emit-index-url ---index-url https://pypi.org/simple - +# uv pip compile --emit-index-annotation ./pctasks/core/pyproject.toml -o pctasks/core/requirements.txt aiohttp==3.9.5 # from https://pypi.org/simple aiosignal==1.3.1 diff --git a/pctasks/task/pyproject.toml b/pctasks/task/pyproject.toml new file mode 100644 index 00000000..67885a31 --- /dev/null +++ b/pctasks/task/pyproject.toml @@ -0,0 +1,58 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.task" +dynamic = ["version"] +description = "Planetary Computer Tasks framework." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "pctasks.core @ {root:uri}/../core", + "pctasks.cli @ {root:uri}/../core", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.entry-points."pctasks.commands"] +task = "pctasks.task.cli:task_cmd" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/task/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/pctasks", +] diff --git a/pctasks/task/setup.cfg b/pctasks/task/setup.cfg deleted file mode 100644 index 04c0f402..00000000 --- a/pctasks/task/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -version = attr: pctasks.task.version.__version__ - -[options.entry_points] -pctasks.commands = - task = pctasks.task.cli:task_cmd \ No newline at end of file diff --git a/pctasks/task/setup.py b/pctasks/task/setup.py deleted file mode 100644 index a0780452..00000000 --- a/pctasks/task/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -"""pctasks: types module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", - "pctasks.cli>=0.1.0", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit" - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.task", - description="Planetary Computer Tasks framework.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/scripts/generate-requirements b/scripts/generate-requirements index 987bb1d9..2d15bb2b 100755 --- a/scripts/generate-requirements +++ b/scripts/generate-requirements @@ -3,7 +3,7 @@ set -e echo "== Generating requirements.txt ..." -pip_compile_options="--emit-index-url --verbose" +pip_compile_options="--verbose" docker-compose \ -f docker-compose.console.yml \ @@ -17,11 +17,11 @@ docker-compose \ dev \ scripts/bin/pctasks-pip-compile ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt $pip_compile_options -# docker-compose \ -# -f docker-compose.console.yml \ -# run --rm \ -# dev \ -# scripts/bin/pctasks-pip-compile ./pctasks/task/setup.py -o pctasks/task/requirements.txt $pip_compile_options +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/task/setup.py -o pctasks/task/requirements.txt $pip_compile_options # docker-compose \ # -f docker-compose.console.yml \ From b15fbcc8b9e5d2c56c3d51b350de98a481330239 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:42:07 -0400 Subject: [PATCH 06/27] client --- pctasks/cli/requirements.txt | 2 +- pctasks/client/pyproject.toml | 63 ++++++++ pctasks/client/requirements.txt | 255 ++++++++++++++++++++++++++++++++ pctasks/core/pyproject.toml | 2 +- pctasks/task/pyproject.toml | 6 +- pctasks/task/requirements.txt | 242 ++++++++++++++++++++++++++++++ 6 files changed, 565 insertions(+), 5 deletions(-) create mode 100644 pctasks/client/pyproject.toml create mode 100644 pctasks/client/requirements.txt create mode 100644 pctasks/task/requirements.txt diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt index 667ec2db..124b9fd9 100644 --- a/pctasks/cli/requirements.txt +++ b/pctasks/cli/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt +# uv pip compile --emit-index-annotation ./pctasks/cli/pyproject.toml -o pctasks/cli/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/client/pyproject.toml b/pctasks/client/pyproject.toml new file mode 100644 index 00000000..27085b07 --- /dev/null +++ b/pctasks/client/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.client" +dynamic = ["version"] +description = "Planetary Computer Tasks framework - Client." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "pctasks.core @ {root:uri}/../core", + "pctasks.cli @ {root:uri}/../cli", + "pydantic[dotenv]>=1.8,<2.0.0", + "rich>=11.2.0", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.entry-points."pctasks.commands"] +profile = "pctasks.client.profile.cli:profile_cmd" +runs = "pctasks.client.runs.cli:runs_cmd" +storage = "pctasks.client.storage.cli:storage_cmd" +workflow = "pctasks.client.workflow.cli:workflow_cmd" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/client/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/client", +] diff --git a/pctasks/client/requirements.txt b/pctasks/client/requirements.txt new file mode 100644 index 00000000..6d360e46 --- /dev/null +++ b/pctasks/client/requirements.txt @@ -0,0 +1,255 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/client/pyproject.toml -o pctasks/client/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + + + # via pctasks-cli +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.9.0.post0 + # via + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rich==13.7.1 + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/pctasks/core/pyproject.toml b/pctasks/core/pyproject.toml index c7822a71..c355335e 100644 --- a/pctasks/core/pyproject.toml +++ b/pctasks/core/pyproject.toml @@ -61,7 +61,7 @@ Homepage = "https://github.com/Microsoft/planetary-computer-tasks" [tool.hatch.version] path = "pctasks/core/version.py" -[tool.hatch.build] +[tool.hatch.build.targets.wheel] include = [ "/pctasks/core", ] diff --git a/pctasks/task/pyproject.toml b/pctasks/task/pyproject.toml index 67885a31..e83290e3 100644 --- a/pctasks/task/pyproject.toml +++ b/pctasks/task/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ "pctasks.core @ {root:uri}/../core", - "pctasks.cli @ {root:uri}/../core", + "pctasks.cli @ {root:uri}/../cli", ] [project.optional-dependencies] @@ -52,7 +52,7 @@ allow-direct-references = true [tool.hatch.version] path = "pctasks/task/version.py" -[tool.hatch.build.targets.sdist] +[tool.hatch.build.targets.wheel] include = [ - "/pctasks", + "/pctasks/task", ] diff --git a/pctasks/task/requirements.txt b/pctasks/task/requirements.txt new file mode 100644 index 00000000..2da8ddd8 --- /dev/null +++ b/pctasks/task/requirements.txt @@ -0,0 +1,242 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/task/pyproject.toml -o pctasks/task/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple +pctasks-cli @ file:///opt/src/pctasks/task/../cli +pctasks-core @ file:///opt/src/pctasks/task/../core + # via pctasks-cli +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.9.0.post0 + # via + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via planetary-computer + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple From 3313d5c41cf1ee2e189f6112d90ad512c54e4fcb Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:47:28 -0400 Subject: [PATCH 07/27] ingest --- Dockerfile.dev | 20 ++++++++++---------- pctasks/task/requirements.txt | 4 ++-- scripts/generate-requirements | 28 ++++++++++++++-------------- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 4abb79a1..ae1b1d23 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -56,17 +56,17 @@ RUN --mount=type=cache,target=/root/.cache \ uv pip install --system -r requirements.txt && \ uv pip install --system --no-deps -e . -# COPY pctasks/client /opt/src/pctasks/client -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/client && \ -# uv pip install --system -r requirements.txt && \ -# uv pip install --system --no-deps -e . +COPY pctasks/client /opt/src/pctasks/client +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/client && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . -# COPY pctasks/ingest /opt/src/pctasks/ingest -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/ingest && \ -# uv pip install --system -r requirements.txt && \ -# uv pip install --system --no-deps -e . +COPY pctasks/ingest /opt/src/pctasks/ingest +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/ingest && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . # COPY pctasks/ingest_task /opt/src/pctasks/ingest_task # RUN --mount=type=cache,target=/root/.cache \ diff --git a/pctasks/task/requirements.txt b/pctasks/task/requirements.txt index 2da8ddd8..56d2abd0 100644 --- a/pctasks/task/requirements.txt +++ b/pctasks/task/requirements.txt @@ -132,8 +132,8 @@ packaging==24.0 # msal-extensions # planetary-computer # from https://pypi.org/simple -pctasks-cli @ file:///opt/src/pctasks/task/../cli -pctasks-core @ file:///opt/src/pctasks/task/../core + + # via pctasks-cli planetary-computer==1.0.0 # via pctasks-core diff --git a/scripts/generate-requirements b/scripts/generate-requirements index 2d15bb2b..51b48b3b 100755 --- a/scripts/generate-requirements +++ b/scripts/generate-requirements @@ -15,25 +15,25 @@ docker-compose \ -f docker-compose.console.yml \ run --rm \ dev \ - scripts/bin/pctasks-pip-compile ./pctasks/cli/setup.py -o pctasks/cli/requirements.txt $pip_compile_options + scripts/bin/pctasks-pip-compile ./pctasks/cli/pyproject.toml -o pctasks/cli/requirements.txt $pip_compile_options docker-compose \ -f docker-compose.console.yml \ run --rm \ dev \ - scripts/bin/pctasks-pip-compile ./pctasks/task/setup.py -o pctasks/task/requirements.txt $pip_compile_options + scripts/bin/pctasks-pip-compile ./pctasks/task/pyproject.toml -o pctasks/task/requirements.txt $pip_compile_options -# docker-compose \ -# -f docker-compose.console.yml \ -# run --rm \ -# dev \ -# scripts/bin/pctasks-pip-compile ./pctasks/client/setup.py -o pctasks/client/requirements.txt $pip_compile_options +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/client/pyproject.toml -o pctasks/client/requirements.txt $pip_compile_options -# docker-compose \ -# -f docker-compose.console.yml \ -# run --rm \ -# dev \ -# scripts/bin/pctasks-pip-compile ./pctasks/ingest/setup.py -o pctasks/ingest/requirements.txt $pip_compile_options +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/ingest/pyproject.toml -o pctasks/ingest/requirements.txt $pip_compile_options # docker-compose \ # -f docker-compose.console.yml \ @@ -45,10 +45,10 @@ docker-compose \ # -f docker-compose.console.yml \ # run --rm \ # dev \ -# scripts/bin/pctasks-pip-compile ./pctasks/run/setup.py -o pctasks/run/requirements.txt $pip_compile_options +# scripts/bin/pctasks-pip-compile ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt $pip_compile_options # docker-compose \ # -f docker-compose.console.yml \ # run --rm \ # dev \ -# pip-compile ./pctasks/server/setup.py --extra dev -o pctasks/server/requirements.txt $pip_compile_options +# pip-compile ./pctasks/server/pyproject.toml --extra dev -o pctasks/server/requirements.txt $pip_compile_options From d25742120cd3bcbdfdd264cff95a416cc38634cb Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:47:36 -0400 Subject: [PATCH 08/27] actually ingest --- pctasks/ingest/pyproject.toml | 59 +++++++ pctasks/ingest/requirements.txt | 268 ++++++++++++++++++++++++++++++++ 2 files changed, 327 insertions(+) create mode 100644 pctasks/ingest/pyproject.toml create mode 100644 pctasks/ingest/requirements.txt diff --git a/pctasks/ingest/pyproject.toml b/pctasks/ingest/pyproject.toml new file mode 100644 index 00000000..d4e74974 --- /dev/null +++ b/pctasks/ingest/pyproject.toml @@ -0,0 +1,59 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.ingest" +dynamic = ["version"] +description = "Planetary Computer Tasks framework - Ingest" +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "jinja2==3.0.3", + "marko==1.0.1", + "pctasks.client @ {root:uri}/../client", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.entry-points."pctasks.commands"] +ingest = "pctasks.ingest.cli:ingest_cmd" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/ingest/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/ingest", +] diff --git a/pctasks/ingest/requirements.txt b/pctasks/ingest/requirements.txt new file mode 100644 index 00000000..d64943ba --- /dev/null +++ b/pctasks/ingest/requirements.txt @@ -0,0 +1,268 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/ingest/pyproject.toml -o pctasks/ingest/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jinja2==3.0.3 + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +marko==1.0.1 + # from https://pypi.org/simple +markupsafe==2.1.5 + # via jinja2 + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + + # via pctasks-client + + + # via + # pctasks-cli + # pctasks-client +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-client + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.9.0.post0 + # via + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rich==13.7.1 + # via pctasks-client + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple From 4d01f1d0c0933edd8f630045dc3d899a02506291 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:51:05 -0400 Subject: [PATCH 09/27] ingest_task --- pctasks/ingest_task/pyproject.toml | 61 ++++++ pctasks/ingest_task/requirements.txt | 314 +++++++++++++++++++++++++++ scripts/generate-requirements | 20 +- 3 files changed, 385 insertions(+), 10 deletions(-) create mode 100644 pctasks/ingest_task/pyproject.toml create mode 100644 pctasks/ingest_task/requirements.txt diff --git a/pctasks/ingest_task/pyproject.toml b/pctasks/ingest_task/pyproject.toml new file mode 100644 index 00000000..a82e51ff --- /dev/null +++ b/pctasks/ingest_task/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.ingest-task" +dynamic = ["version"] +description = "Planetary Computer Tasks framework - Ingest Task." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "fire==0.4.0", + "orjson>=3.5.2", + "pctasks.ingest @ {root:uri}/../ingest", + "pctasks.task @ {root:uri}/../task", + "plpygis==0.2.0", + "pypgstac[psycopg]==0.7.10", + "pystac>=1.0.0,<2", + "python-dateutil==2.8.2", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/ingest_task/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/ingest_task", +] diff --git a/pctasks/ingest_task/requirements.txt b/pctasks/ingest_task/requirements.txt new file mode 100644 index 00000000..3e821973 --- /dev/null +++ b/pctasks/ingest_task/requirements.txt @@ -0,0 +1,314 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/ingest_task/setup.py -o pctasks/ingest_task/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via + # google-auth + # pypgstac + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +fire==0.4.0 + # via pypgstac + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jinja2==3.0.3 + # via pctasks-ingest + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +marko==1.0.1 + # via pctasks-ingest + # from https://pypi.org/simple +markupsafe==2.1.5 + # via jinja2 + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via + # pctasks-core + # pypgstac + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + + # via + # pctasks-client + # pctasks-task + + # via pctasks-ingest + + # via + # pctasks-cli + # pctasks-client + # pctasks-task + + +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +plpygis==0.2.0 + # via pypgstac + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +psycopg==3.1.18 + # via pypgstac + # from https://pypi.org/simple +psycopg-binary==3.1.18 + # via psycopg + # from https://pypi.org/simple +psycopg-pool==3.1.9 + # via pypgstac + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-client + # pctasks-core + # planetary-computer + # pypgstac + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pypgstac==0.7.10 + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.8.2 + # via + # pypgstac + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rich==13.7.1 + # via pctasks-client + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # fire + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +smart-open==6.4.0 + # via pypgstac + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +tenacity==8.1.0 + # via pypgstac + # from https://pypi.org/simple +termcolor==2.4.0 + # via fire + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # psycopg + # psycopg-pool + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +version-parser==1.0.1 + # via pypgstac + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/scripts/generate-requirements b/scripts/generate-requirements index 51b48b3b..09e071fb 100755 --- a/scripts/generate-requirements +++ b/scripts/generate-requirements @@ -35,17 +35,17 @@ docker-compose \ dev \ scripts/bin/pctasks-pip-compile ./pctasks/ingest/pyproject.toml -o pctasks/ingest/requirements.txt $pip_compile_options -# docker-compose \ -# -f docker-compose.console.yml \ -# run --rm \ -# dev \ -# scripts/bin/pctasks-pip-compile ./pctasks/ingest_task/setup.py -o pctasks/ingest_task/requirements.txt $pip_compile_options +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/ingest_task/setup.py -o pctasks/ingest_task/requirements.txt $pip_compile_options -# docker-compose \ -# -f docker-compose.console.yml \ -# run --rm \ -# dev \ -# scripts/bin/pctasks-pip-compile ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt $pip_compile_options +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt $pip_compile_options # docker-compose \ # -f docker-compose.console.yml \ From 3a32ed051438b16a049769464bf2a4a688b78b58 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:53:30 -0400 Subject: [PATCH 10/27] run --- pctasks/run/pyproject.toml | 64 +++++++ pctasks/run/requirements.txt | 329 +++++++++++++++++++++++++++++++++++ 2 files changed, 393 insertions(+) create mode 100644 pctasks/run/pyproject.toml create mode 100644 pctasks/run/requirements.txt diff --git a/pctasks/run/pyproject.toml b/pctasks/run/pyproject.toml new file mode 100644 index 00000000..c7642edb --- /dev/null +++ b/pctasks/run/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.run" +dynamic = ["version"] +description = "Planetary Computer Tasks framework: Runner component" +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "argo-workflows>=6.3.0,<6.4", + "azure-batch>=11.0.0,<12", + "azure-keyvault-secrets>=4.0.0,<5", + "kubernetes", + "networkx>=2.0.0,<3", + "pctasks.core @ {root:uri}/../core", + "pctasks.client @ {root:uri}/../client", + "pctasks.task @ {root:uri}/../task", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.entry-points."pctasks.commands"] +run = "pctasks.run.cli:run_cmd" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/run/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/run", +] diff --git a/pctasks/run/requirements.txt b/pctasks/run/requirements.txt new file mode 100644 index 00000000..d5b73ef1 --- /dev/null +++ b/pctasks/run/requirements.txt @@ -0,0 +1,329 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt +adal==1.2.7 + # via msrestazure + # from https://pypi.org/simple +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +argo-workflows==6.3.10 + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-batch==11.0.0 + # from https://pypi.org/simple +azure-common==1.1.28 + # via azure-batch + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # msrest + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-keyvault-secrets==4.8.0 + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via + # kubernetes + # msrest + # requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # adal + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via + # google-api-core + # kubernetes + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # msrest + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +kubernetes==29.0.0 + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +msrest==0.7.1 + # via + # azure-batch + # msrestazure + # from https://pypi.org/simple +msrestazure==0.6.4.post1 + # via azure-batch + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +networkx==2.8.8 + # from https://pypi.org/simple +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + + # via + # pctasks-client + # pctasks-task + + + # via + # pctasks-cli + # pctasks-client + # pctasks-task + +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-client + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via + # adal + # msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.9.0.post0 + # via + # adal + # argo-workflows + # kubernetes + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via + # kubernetes + # pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # adal + # azure-core + # google-api-core + # kubernetes + # msal + # msrest + # opencensus-ext-azure + # planetary-computer + # pystac-client + # requests-oauthlib + # stac-validator + # from https://pypi.org/simple +requests-oauthlib==2.0.0 + # via + # kubernetes + # msrest + # from https://pypi.org/simple +rich==13.7.1 + # via pctasks-client + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # kubernetes + # msrestazure + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via + # argo-workflows + # kubernetes + # requests + # from https://pypi.org/simple +websocket-client==1.8.0 + # via kubernetes + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple From a93ec106101812e463562463e5d2745d80629aac Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 10:53:46 -0400 Subject: [PATCH 11/27] run dev --- Dockerfile.dev | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index ae1b1d23..2fc2e4f7 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -68,17 +68,17 @@ RUN --mount=type=cache,target=/root/.cache \ uv pip install --system -r requirements.txt && \ uv pip install --system --no-deps -e . -# COPY pctasks/ingest_task /opt/src/pctasks/ingest_task -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/ingest_task && \ -# uv pip install --system -r requirements.txt && \ -# uv pip install --system --no-deps -e . +COPY pctasks/ingest_task /opt/src/pctasks/ingest_task +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/ingest_task && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . -# COPY pctasks/run /opt/src/pctasks/run -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/run && \ -# uv pip install --system -r requirements.txt && \ -# uv pip install --system --no-deps -e . +COPY pctasks/run /opt/src/pctasks/run +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/run && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps -e . # We are here # COPY pctasks/server /opt/src/pctasks/server From bfb535b5d0a91864342738c857beb586fb9ff85e Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 11:15:15 -0400 Subject: [PATCH 12/27] delete all setup.py --- pctasks/cli/setup.cfg | 2 -- pctasks/cli/setup.py | 50 --------------------------------- pctasks/client/setup.cfg | 9 ------ pctasks/client/setup.py | 45 ----------------------------- pctasks/dataset/setup.cfg | 6 ---- pctasks/dataset/setup.py | 49 -------------------------------- pctasks/dev/setup.cfg | 2 -- pctasks/dev/setup.py | 43 ---------------------------- pctasks/ingest/setup.cfg | 6 ---- pctasks/ingest/setup.py | 48 ------------------------------- pctasks/ingest_task/setup.cfg | 2 -- pctasks/ingest_task/setup.py | 49 -------------------------------- pctasks/notify/setup.cfg | 2 -- pctasks/notify/setup.py | 46 ------------------------------ pctasks/router/setup.cfg | 2 -- pctasks/router/setup.py | 46 ------------------------------ pctasks/run/setup.cfg | 6 ---- pctasks/run/setup.py | 53 ----------------------------------- pctasks/server/setup.cfg | 2 -- pctasks/server/setup.py | 47 ------------------------------- 20 files changed, 515 deletions(-) delete mode 100644 pctasks/cli/setup.cfg delete mode 100644 pctasks/cli/setup.py delete mode 100644 pctasks/client/setup.cfg delete mode 100644 pctasks/client/setup.py delete mode 100644 pctasks/dataset/setup.cfg delete mode 100644 pctasks/dataset/setup.py delete mode 100644 pctasks/dev/setup.cfg delete mode 100644 pctasks/dev/setup.py delete mode 100644 pctasks/ingest/setup.cfg delete mode 100644 pctasks/ingest/setup.py delete mode 100644 pctasks/ingest_task/setup.cfg delete mode 100644 pctasks/ingest_task/setup.py delete mode 100644 pctasks/notify/setup.cfg delete mode 100644 pctasks/notify/setup.py delete mode 100644 pctasks/router/setup.cfg delete mode 100644 pctasks/router/setup.py delete mode 100644 pctasks/run/setup.cfg delete mode 100644 pctasks/run/setup.py delete mode 100644 pctasks/server/setup.cfg delete mode 100644 pctasks/server/setup.py diff --git a/pctasks/cli/setup.cfg b/pctasks/cli/setup.cfg deleted file mode 100644 index cdbc1a1f..00000000 --- a/pctasks/cli/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.cli.version.__version__ diff --git a/pctasks/cli/setup.py b/pctasks/cli/setup.py deleted file mode 100644 index e0079f70..00000000 --- a/pctasks/cli/setup.py +++ /dev/null @@ -1,50 +0,0 @@ -"""pctasks: cli module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", - "click>=7.0", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit" - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.cli", - description="Planetary Computer Tasks framework.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, - entry_points={ - 'console_scripts': ['pctasks=pctasks.cli.cli:cli'] - } -) diff --git a/pctasks/client/setup.cfg b/pctasks/client/setup.cfg deleted file mode 100644 index a8996a9e..00000000 --- a/pctasks/client/setup.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[metadata] -version = attr: pctasks.client.version.__version__ - -[options.entry_points] -pctasks.commands = - workflow = pctasks.client.workflow.cli:workflow_cmd - runs = pctasks.client.runs.cli:runs_cmd - profile = pctasks.client.profile.cli:profile_cmd - storage = pctasks.client.storage.cli:storage_cmd diff --git a/pctasks/client/setup.py b/pctasks/client/setup.py deleted file mode 100644 index 7c029454..00000000 --- a/pctasks/client/setup.py +++ /dev/null @@ -1,45 +0,0 @@ -"""pctasks: client module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", - "pctasks.cli>=0.1.0", - "pydantic[dotenv]>=1.8,<2.0.0", - "rich>=11.2.0", -] - -extra_reqs = { - "dev": ["pytest", "pytest-cov", "pre-commit"], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.client", - description="Planetary Computer Tasks framework - Client.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/dataset/setup.cfg b/pctasks/dataset/setup.cfg deleted file mode 100644 index c51c44d1..00000000 --- a/pctasks/dataset/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -version = attr: pctasks.dataset.version.__version__ - -[options.entry_points] -pctasks.commands = - dataset = pctasks.dataset.cli:dataset_cmd \ No newline at end of file diff --git a/pctasks/dataset/setup.py b/pctasks/dataset/setup.py deleted file mode 100644 index 5de24c56..00000000 --- a/pctasks/dataset/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -"""pctasks: datasets module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.task>=0.1.0", - "pctasks.client>=0.1.0", - "pctasks.ingest>=0.1.0", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit", - "responses", - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.dataset", - description="Planetary Computer Tasks framework.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/dev/setup.cfg b/pctasks/dev/setup.cfg deleted file mode 100644 index 70555e56..00000000 --- a/pctasks/dev/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.dev.version.__version__ diff --git a/pctasks/dev/setup.py b/pctasks/dev/setup.py deleted file mode 100644 index 04c880f6..00000000 --- a/pctasks/dev/setup.py +++ /dev/null @@ -1,43 +0,0 @@ -"""pctasks: types module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.task>=0.1.0", - "pctasks.client>=0.1.0", - "pctasks.ingest>=0.1.0", - "pctasks.run>=0.1.0", - "pctasks.cli>=0.1.0", -] - -extra_reqs = {"server": ["fastapi>=0.108.0", "uvicorn[standard]>=0.12.0,<0.16.0"]} - - -setup( - name="pctasks.dev", - description="Planetary Computer Tasks framework.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author="Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - extras_require=extra_reqs, - entry_points={"console_scripts": ["pctasks-dev=pctasks.dev.cli:cli"]}, -) diff --git a/pctasks/ingest/setup.cfg b/pctasks/ingest/setup.cfg deleted file mode 100644 index 051c93a2..00000000 --- a/pctasks/ingest/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -version = attr: pctasks.ingest.version.__version__ - -[options.entry_points] -pctasks.commands = - ingest = pctasks.ingest.cli:ingest_cmd \ No newline at end of file diff --git a/pctasks/ingest/setup.py b/pctasks/ingest/setup.py deleted file mode 100644 index 0f97764b..00000000 --- a/pctasks/ingest/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -"""pctasks.ingest: Ingestion messages and submission.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.client>=0.1.0", - "jinja2==3.0.3", - "marko==1.0.1", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit" - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.ingest", - description="Planetary Computer Tasks framework - Ingest", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/ingest_task/setup.cfg b/pctasks/ingest_task/setup.cfg deleted file mode 100644 index ab4dbee2..00000000 --- a/pctasks/ingest_task/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.ingest_task.version.__version__ diff --git a/pctasks/ingest_task/setup.py b/pctasks/ingest_task/setup.py deleted file mode 100644 index f0bd7153..00000000 --- a/pctasks/ingest_task/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -"""pctasks: types module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.task>=0.1.0", - "pctasks.ingest>=0.1.0", - "pypgstac[psycopg]==0.7.10", - "pystac>=1.0.0,<2", - "orjson>=3.5.2", - "python-dateutil==2.8.2", - "fire==0.4.0", - "plpygis==0.2.0", -] - -extra_reqs = { - "dev": ["pytest", "pytest-cov", "pre-commit"], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.ingest_task", - description="Planetary Computer Tasks framework - Ingest Task.", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author="Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/notify/setup.cfg b/pctasks/notify/setup.cfg deleted file mode 100644 index 9f67123d..00000000 --- a/pctasks/notify/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.notify.version.__version__ diff --git a/pctasks/notify/setup.py b/pctasks/notify/setup.py deleted file mode 100644 index a026f1b1..00000000 --- a/pctasks/notify/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -"""pctasks: notify module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit" - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.notify", - description="Planetary Computer Tasks framework: Notification component", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/router/setup.cfg b/pctasks/router/setup.cfg deleted file mode 100644 index 41a75073..00000000 --- a/pctasks/router/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.router.version.__version__ diff --git a/pctasks/router/setup.py b/pctasks/router/setup.py deleted file mode 100644 index 7cb7f129..00000000 --- a/pctasks/router/setup.py +++ /dev/null @@ -1,46 +0,0 @@ -"""pctasks: router module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit" - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.router", - description="Planetary Computer Tasks framework: Orchestration component", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/run/setup.cfg b/pctasks/run/setup.cfg deleted file mode 100644 index 7d7d1ad7..00000000 --- a/pctasks/run/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[metadata] -version = attr: pctasks.run.version.__version__ - -[options.entry_points] -pctasks.commands = - run = pctasks.run.cli:run_cmd \ No newline at end of file diff --git a/pctasks/run/setup.py b/pctasks/run/setup.py deleted file mode 100644 index 01744865..00000000 --- a/pctasks/run/setup.py +++ /dev/null @@ -1,53 +0,0 @@ -"""pctasks: run module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", - "pctasks.task>=0.1.0", - "pctasks.client>=0.1.0", - "azure-batch>=11.0.0,<12", - "azure-keyvault-secrets>=4.0.0,<5", - "argo-workflows>=6.3.0,<6.4", - "networkx>=2.0.0,<3", - "kubernetes", -] - -extra_reqs = { - "dev": [ - "pytest", - "pytest-cov", - "pre-commit" - ], - "docs": ["mkdocs", "mkdocs-material", "pdocs"], -} - - -setup( - name="pctasks.run", - description="Planetary Computer Tasks framework: Runner component", - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author=u"Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) diff --git a/pctasks/server/setup.cfg b/pctasks/server/setup.cfg deleted file mode 100644 index d26da75a..00000000 --- a/pctasks/server/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: pctasks.server.version.__version__ diff --git a/pctasks/server/setup.py b/pctasks/server/setup.py deleted file mode 100644 index fbbe9277..00000000 --- a/pctasks/server/setup.py +++ /dev/null @@ -1,47 +0,0 @@ -"""pctasks: records module.""" - -from setuptools import find_namespace_packages, setup - -with open("README.md") as f: - desc = f.read() - -install_requires = [ - "pctasks.core>=0.1.0", - "pctasks.run>=0.1.0", - "fastapi>=0.108.0", - "python-multipart==0.0.7", -] - -extra_reqs = { - "dev": ["pytest", "pytest-cov", "pre-commit", "httpx"], - "server": [ - "uvicorn[standard]>=0.12.0,<0.16.0", - ], -} - - -setup( - name="pctasks.server", - description=("Planetary Computer Tasks framework: Server."), - long_description=desc, - long_description_content_type="text/markdown", - python_requires=">=3.8", - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Information Technology", - "Intended Audience :: Science/Research", - "Programming Language :: Python :: 3.8", - "License :: OSI Approved :: MIT License", - ], - keywords="Planetary, STAC", - author="Microsoft", - author_email="planetarycomputer@microsoft.com", - url="https://github.com/Microsoft/planetary-computer-tasks", - license="MIT", - packages=find_namespace_packages(exclude=["tests", "scripts"]), - package_data={"": ["py.typed"]}, - zip_safe=False, - install_requires=install_requires, - tests_require=extra_reqs["dev"], - extras_require=extra_reqs, -) From 7e94a177909071af0aaf4df06c8c02398497cae4 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 11:15:29 -0400 Subject: [PATCH 13/27] more tomls --- Dockerfile.dev | 38 +-- pctasks/dataset/pyproject.toml | 61 ++++ pctasks/dev/pyproject.toml | 55 ++++ pctasks/ingest_task/requirements.txt | 2 +- pctasks/notify/pyproject.toml | 54 ++++ pctasks/router/pyproject.toml | 54 ++++ pctasks/server/pyproject.toml | 56 ++++ pctasks/server/requirements.txt | 422 +++++++++++++++++++++++++++ scripts/generate-requirements | 12 +- 9 files changed, 728 insertions(+), 26 deletions(-) create mode 100644 pctasks/dataset/pyproject.toml create mode 100644 pctasks/dev/pyproject.toml create mode 100644 pctasks/notify/pyproject.toml create mode 100644 pctasks/router/pyproject.toml create mode 100644 pctasks/server/pyproject.toml create mode 100644 pctasks/server/requirements.txt diff --git a/Dockerfile.dev b/Dockerfile.dev index 2fc2e4f7..0e90950a 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -81,25 +81,25 @@ RUN --mount=type=cache,target=/root/.cache \ uv pip install --system --no-deps -e . # We are here -# COPY pctasks/server /opt/src/pctasks/server -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/server && \ -# pip install -e .[dev] - -# COPY pctasks/dataset /opt/src/pctasks/dataset -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/dataset && \ -# pip install -e . - -# COPY pctasks/notify /opt/src/pctasks/notify -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/notify && \ -# pip install -e . - -# COPY pctasks/dev /opt/src/pctasks/dev -# RUN --mount=type=cache,target=/root/.cache \ -# cd /opt/src/pctasks/dev && \ -# pip install -e .[server] +COPY pctasks/server /opt/src/pctasks/server +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/server && \ + uv pip install --system -e .[dev] + +COPY pctasks/dataset /opt/src/pctasks/dataset +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/dataset && \ + uv pip install --system -e . + +COPY pctasks/notify /opt/src/pctasks/notify +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/notify && \ + uv pip install --system -e . + +COPY pctasks/dev /opt/src/pctasks/dev +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/dev && \ + uv pip install --system -e .[server] # Setup Python Path to allow import of test modules ENV PYTHONPATH=/opt/src:$PYTHONPATH diff --git a/pctasks/dataset/pyproject.toml b/pctasks/dataset/pyproject.toml new file mode 100644 index 00000000..abf0979a --- /dev/null +++ b/pctasks/dataset/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.dataset" +dynamic = ["version"] +description = "Planetary Computer Tasks framework." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] + +dependencies = [ + "pctasks.client @ {root:uri}/../client", + "pctasks.ingest @ {root:uri}/../ingest", + "pctasks.task @ {root:uri}/../task", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", + "responses", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.entry-points."pctasks.commands"] +dataset = "pctasks.dataset.cli:dataset_cmd" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/dataset/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/dataset", +] diff --git a/pctasks/dev/pyproject.toml b/pctasks/dev/pyproject.toml new file mode 100644 index 00000000..9c0781ba --- /dev/null +++ b/pctasks/dev/pyproject.toml @@ -0,0 +1,55 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.dev" +dynamic = ["version"] +description = "Planetary Computer Tasks framework." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "pctasks.cli @ {root:uri}/../cli", + "pctasks.client @ {root:uri}/../client", + "pctasks.ingest @ {root:uri}/../ingest", + "pctasks.run @ {root:uri}/../run", + "pctasks.task @ {root:uri}/../task", +] + +[project.optional-dependencies] +server = [ + "fastapi>=0.108.0", + "uvicorn[standard]>=0.12.0,<0.16.0", +] + +[project.scripts] +pctasks-dev = "pctasks.dev.cli:cli" + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/dev/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/dev", +] diff --git a/pctasks/ingest_task/requirements.txt b/pctasks/ingest_task/requirements.txt index 3e821973..c3c0cfd1 100644 --- a/pctasks/ingest_task/requirements.txt +++ b/pctasks/ingest_task/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/ingest_task/setup.py -o pctasks/ingest_task/requirements.txt +# uv pip compile --emit-index-annotation ./pctasks/ingest_task/pyproject.toml -o pctasks/ingest_task/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/notify/pyproject.toml b/pctasks/notify/pyproject.toml new file mode 100644 index 00000000..92038c97 --- /dev/null +++ b/pctasks/notify/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.notify" +dynamic = ["version"] +description = "Planetary Computer Tasks framework: Notification component" +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "pctasks.core @ {root:uri}/../core", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/notify/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/pctasks/notify", +] diff --git a/pctasks/router/pyproject.toml b/pctasks/router/pyproject.toml new file mode 100644 index 00000000..7de0e980 --- /dev/null +++ b/pctasks/router/pyproject.toml @@ -0,0 +1,54 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.router" +dynamic = ["version"] +description = "Planetary Computer Tasks framework: Orchestration component" +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "pctasks.core @ {root:uri}/../core", +] + +[project.optional-dependencies] +dev = [ + "pre-commit", + "pytest", + "pytest-cov", +] +docs = [ + "mkdocs", + "mkdocs-material", + "pdocs", +] + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/router/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/pctasks/router", +] diff --git a/pctasks/server/pyproject.toml b/pctasks/server/pyproject.toml new file mode 100644 index 00000000..a7859ccd --- /dev/null +++ b/pctasks/server/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "pctasks.server" +dynamic = ["version"] +description = "Planetary Computer Tasks framework: Server." +readme = "README.md" +license = "MIT" +requires-python = ">=3.8" +authors = [ + { name = "Microsoft", email = "planetarycomputer@microsoft.com" }, +] +keywords = [ + "Planetary", + "STAC", +] +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.8", +] +dependencies = [ + "fastapi>=0.108.0", + "pctasks.core @ {root:uri}/../core", + "pctasks.run @ {root:uri}/../run", + "python-multipart==0.0.7", +] + +[project.optional-dependencies] +dev = [ + "httpx", + "pre-commit", + "pytest", + "pytest-cov", +] +server = [ + "uvicorn[standard]>=0.12.0,<0.16.0", +] + +[project.urls] +Homepage = "https://github.com/Microsoft/planetary-computer-tasks" + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.version] +path = "pctasks/server/version.py" + +[tool.hatch.build.targets.wheel] +include = [ + "/pctasks/run", +] diff --git a/pctasks/server/requirements.txt b/pctasks/server/requirements.txt new file mode 100644 index 00000000..5a451dab --- /dev/null +++ b/pctasks/server/requirements.txt @@ -0,0 +1,422 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/server/pyproject.toml -o pctasks/server/requirements.txt --extra dev +adal==1.2.7 + # via msrestazure + # from https://pypi.org/simple +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +anyio==4.3.0 + # via + # httpx + # starlette + # from https://pypi.org/simple +argo-workflows==6.3.10 + # via pctasks-run + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-batch==11.0.0 + # via pctasks-run + # from https://pypi.org/simple +azure-common==1.1.28 + # via azure-batch + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # msrest + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-keyvault-secrets==4.8.0 + # via pctasks-run + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via + # httpcore + # httpx + # kubernetes + # msrest + # requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +cfgv==3.4.0 + # via pre-commit + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +coverage==7.5.0 + # via pytest-cov + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # adal + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +distlib==0.3.8 + # via virtualenv + # from https://pypi.org/simple +exceptiongroup==1.2.1 + # via + # anyio + # pytest + # from https://pypi.org/simple +fastapi==0.110.2 + # from https://pypi.org/simple +filelock==3.13.4 + # via virtualenv + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via + # google-api-core + # kubernetes + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +h11==0.14.0 + # via httpcore + # from https://pypi.org/simple +httpcore==1.0.5 + # via httpx + # from https://pypi.org/simple +httpx==0.27.0 + # from https://pypi.org/simple +identify==2.5.36 + # via pre-commit + # from https://pypi.org/simple +idna==3.7 + # via + # anyio + # httpx + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +iniconfig==2.0.0 + # via pytest + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # msrest + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +kubernetes==29.0.0 + # via pctasks-run + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +msrest==0.7.1 + # via + # azure-batch + # msrestazure + # from https://pypi.org/simple +msrestazure==0.6.4.post1 + # via azure-batch + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +networkx==2.8.8 + # via pctasks-run + # from https://pypi.org/simple +nodeenv==1.8.0 + # via pre-commit + # from https://pypi.org/simple +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # pytest + # from https://pypi.org/simple + + # via + # pctasks-client + # pctasks-task + + # via pctasks-run + + # via + # pctasks-cli + # pctasks-client + # pctasks-run + # pctasks-task + + + # via pctasks-run +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +platformdirs==4.2.1 + # via virtualenv + # from https://pypi.org/simple +pluggy==1.5.0 + # via pytest + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +pre-commit==3.7.0 + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # fastapi + # pctasks-client + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via + # adal + # msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +pytest==8.1.1 + # via pytest-cov + # from https://pypi.org/simple +pytest-cov==5.0.0 + # from https://pypi.org/simple +python-dateutil==2.9.0.post0 + # via + # adal + # argo-workflows + # kubernetes + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +python-multipart==0.0.7 + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via + # kubernetes + # pctasks-core + # pre-commit + # from https://pypi.org/simple +requests==2.31.0 + # via + # adal + # azure-core + # google-api-core + # kubernetes + # msal + # msrest + # opencensus-ext-azure + # planetary-computer + # pystac-client + # requests-oauthlib + # stac-validator + # from https://pypi.org/simple +requests-oauthlib==2.0.0 + # via + # kubernetes + # msrest + # from https://pypi.org/simple +rich==13.7.1 + # via pctasks-client + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +setuptools==69.5.1 + # via nodeenv + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # kubernetes + # msrestazure + # opencensus + # python-dateutil + # from https://pypi.org/simple +sniffio==1.3.1 + # via + # anyio + # httpx + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +starlette==0.37.2 + # via fastapi + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +tomli==2.0.1 + # via + # coverage + # pytest + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # anyio + # azure-core + # azure-data-tables + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # fastapi + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via + # argo-workflows + # kubernetes + # requests + # from https://pypi.org/simple +virtualenv==20.26.0 + # via pre-commit + # from https://pypi.org/simple +websocket-client==1.8.0 + # via kubernetes + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/scripts/generate-requirements b/scripts/generate-requirements index 09e071fb..d7ba3e7f 100755 --- a/scripts/generate-requirements +++ b/scripts/generate-requirements @@ -39,7 +39,7 @@ docker-compose \ -f docker-compose.console.yml \ run --rm \ dev \ - scripts/bin/pctasks-pip-compile ./pctasks/ingest_task/setup.py -o pctasks/ingest_task/requirements.txt $pip_compile_options + scripts/bin/pctasks-pip-compile ./pctasks/ingest_task/pyproject.toml -o pctasks/ingest_task/requirements.txt $pip_compile_options docker-compose \ -f docker-compose.console.yml \ @@ -47,8 +47,8 @@ docker-compose \ dev \ scripts/bin/pctasks-pip-compile ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt $pip_compile_options -# docker-compose \ -# -f docker-compose.console.yml \ -# run --rm \ -# dev \ -# pip-compile ./pctasks/server/pyproject.toml --extra dev -o pctasks/server/requirements.txt $pip_compile_options +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/server/pyproject.toml -o pctasks/server/requirements.txt $pip_compile_options --extra dev From f772ce17a5b16c43945d7dc8f44c577532449300 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 11:39:25 -0400 Subject: [PATCH 14/27] Use better context formatting --- pctasks/cli/pyproject.toml | 3 +-- pctasks/client/pyproject.toml | 4 ++-- pctasks/dataset/pyproject.toml | 6 +++--- pctasks/dev/pyproject.toml | 10 +++++----- pctasks/ingest/pyproject.toml | 2 +- pctasks/ingest_task/pyproject.toml | 4 ++-- pctasks/notify/pyproject.toml | 4 ++-- pctasks/router/pyproject.toml | 4 ++-- pctasks/run/pyproject.toml | 6 +++--- pctasks/server/pyproject.toml | 4 ++-- pctasks/task/pyproject.toml | 4 ++-- 11 files changed, 25 insertions(+), 26 deletions(-) diff --git a/pctasks/cli/pyproject.toml b/pctasks/cli/pyproject.toml index 98d29d00..0c357f2b 100644 --- a/pctasks/cli/pyproject.toml +++ b/pctasks/cli/pyproject.toml @@ -25,8 +25,7 @@ classifiers = [ ] dependencies = [ "click>=7.0", - # "pctasks.core>=0.1.0", - "pctasks.core @ {root:uri}/../core", + "pctasks.core @ {root:parent:uri}/core", ] [project.optional-dependencies] diff --git a/pctasks/client/pyproject.toml b/pctasks/client/pyproject.toml index 27085b07..8342792f 100644 --- a/pctasks/client/pyproject.toml +++ b/pctasks/client/pyproject.toml @@ -24,8 +24,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] dependencies = [ - "pctasks.core @ {root:uri}/../core", - "pctasks.cli @ {root:uri}/../cli", + "pctasks.core @ {root:parent:uri}/core", + "pctasks.cli @ {root:parent:uri}/cli", "pydantic[dotenv]>=1.8,<2.0.0", "rich>=11.2.0", ] diff --git a/pctasks/dataset/pyproject.toml b/pctasks/dataset/pyproject.toml index abf0979a..5c770ddb 100644 --- a/pctasks/dataset/pyproject.toml +++ b/pctasks/dataset/pyproject.toml @@ -25,9 +25,9 @@ classifiers = [ ] dependencies = [ - "pctasks.client @ {root:uri}/../client", - "pctasks.ingest @ {root:uri}/../ingest", - "pctasks.task @ {root:uri}/../task", + "pctasks.client @ {root:parent:uri}/client", + "pctasks.ingest @ {root:parent:uri}/ingest", + "pctasks.task @ {root:parent:uri}/task", ] [project.optional-dependencies] diff --git a/pctasks/dev/pyproject.toml b/pctasks/dev/pyproject.toml index 9c0781ba..9efad3ca 100644 --- a/pctasks/dev/pyproject.toml +++ b/pctasks/dev/pyproject.toml @@ -24,11 +24,11 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] dependencies = [ - "pctasks.cli @ {root:uri}/../cli", - "pctasks.client @ {root:uri}/../client", - "pctasks.ingest @ {root:uri}/../ingest", - "pctasks.run @ {root:uri}/../run", - "pctasks.task @ {root:uri}/../task", + "pctasks.cli @ {root:parent:uri}/cli", + "pctasks.client @ {root:parent:uri}/client", + "pctasks.ingest @ {root:parent:uri}/ingest", + "pctasks.run @ {root:parent:uri}/run", + "pctasks.task @ {root:parent:uri}/task", ] [project.optional-dependencies] diff --git a/pctasks/ingest/pyproject.toml b/pctasks/ingest/pyproject.toml index d4e74974..a8f03458 100644 --- a/pctasks/ingest/pyproject.toml +++ b/pctasks/ingest/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ dependencies = [ "jinja2==3.0.3", "marko==1.0.1", - "pctasks.client @ {root:uri}/../client", + "pctasks.client @ {root:parent:uri}/client", ] [project.optional-dependencies] diff --git a/pctasks/ingest_task/pyproject.toml b/pctasks/ingest_task/pyproject.toml index a82e51ff..a70e7868 100644 --- a/pctasks/ingest_task/pyproject.toml +++ b/pctasks/ingest_task/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ dependencies = [ "fire==0.4.0", "orjson>=3.5.2", - "pctasks.ingest @ {root:uri}/../ingest", - "pctasks.task @ {root:uri}/../task", + "pctasks.ingest @ {root:parent:uri}/ingest", + "pctasks.task @ {root:parent:uri}/task", "plpygis==0.2.0", "pypgstac[psycopg]==0.7.10", "pystac>=1.0.0,<2", diff --git a/pctasks/notify/pyproject.toml b/pctasks/notify/pyproject.toml index 92038c97..30b72576 100644 --- a/pctasks/notify/pyproject.toml +++ b/pctasks/notify/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] dependencies = [ - "pctasks.core @ {root:uri}/../core", + "pctasks.core @ {root:parent:uri}/core", ] [project.optional-dependencies] @@ -48,7 +48,7 @@ allow-direct-references = true [tool.hatch.version] path = "pctasks/notify/version.py" -[tool.hatch.build.targets.sdist] +[tool.hatch.build.targets.wheel] include = [ "/pctasks/notify", ] diff --git a/pctasks/router/pyproject.toml b/pctasks/router/pyproject.toml index 7de0e980..3b940a16 100644 --- a/pctasks/router/pyproject.toml +++ b/pctasks/router/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] dependencies = [ - "pctasks.core @ {root:uri}/../core", + "pctasks.core @ {root:parent:uri}/core", ] [project.optional-dependencies] @@ -48,7 +48,7 @@ allow-direct-references = true [tool.hatch.version] path = "pctasks/router/version.py" -[tool.hatch.build.targets.sdist] +[tool.hatch.build.targets.wheel] include = [ "/pctasks/router", ] diff --git a/pctasks/run/pyproject.toml b/pctasks/run/pyproject.toml index c7642edb..504f843c 100644 --- a/pctasks/run/pyproject.toml +++ b/pctasks/run/pyproject.toml @@ -29,9 +29,9 @@ dependencies = [ "azure-keyvault-secrets>=4.0.0,<5", "kubernetes", "networkx>=2.0.0,<3", - "pctasks.core @ {root:uri}/../core", - "pctasks.client @ {root:uri}/../client", - "pctasks.task @ {root:uri}/../task", + "pctasks.core @ {root:parent:uri}/core", + "pctasks.client @ {root:parent:uri}/client", + "pctasks.task @ {root:parent:uri}/task", ] [project.optional-dependencies] diff --git a/pctasks/server/pyproject.toml b/pctasks/server/pyproject.toml index a7859ccd..44b9fe09 100644 --- a/pctasks/server/pyproject.toml +++ b/pctasks/server/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ ] dependencies = [ "fastapi>=0.108.0", - "pctasks.core @ {root:uri}/../core", - "pctasks.run @ {root:uri}/../run", + "pctasks.core @ {root:parent:uri}/core", + "pctasks.run @ {root:parent:uri}/run", "python-multipart==0.0.7", ] diff --git a/pctasks/task/pyproject.toml b/pctasks/task/pyproject.toml index e83290e3..1d372d8d 100644 --- a/pctasks/task/pyproject.toml +++ b/pctasks/task/pyproject.toml @@ -24,8 +24,8 @@ classifiers = [ "Programming Language :: Python :: 3.8", ] dependencies = [ - "pctasks.core @ {root:uri}/../core", - "pctasks.cli @ {root:uri}/../cli", + "pctasks.core @ {root:parent:uri}/core", + "pctasks.cli @ {root:parent:uri}/cli", ] [project.optional-dependencies] From 3ec531ec0a79637de48fdc732f0541f48469f222 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 12:08:56 -0400 Subject: [PATCH 15/27] Uncomment dockerfile --- Dockerfile.task_base | 39 ++++++++++++++++++++++++++------------- deployment/Dockerfile | 7 ++++--- pctasks/server/Dockerfile | 17 +++++++++-------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Dockerfile.task_base b/Dockerfile.task_base index b275baa6..75a4d721 100644 --- a/Dockerfile.task_base +++ b/Dockerfile.task_base @@ -12,35 +12,48 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Install common packages COPY requirements-task-base.txt /tmp/requirements.txt -RUN python -m pip install --no-build-isolation -r /tmp/requirements.txt +RUN python -m pip install --no-build-isolation --upgrade pip uv +RUN uv pip install --system -r /tmp/requirements.txt # # Copy and install packages # COPY pctasks/core /opt/src/pctasks/core -RUN cd /opt/src/pctasks/core && \ - pip install . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/core && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . COPY pctasks/cli /opt/src/pctasks/cli -RUN cd /opt/src/pctasks/cli && \ - pip install . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/cli && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . COPY pctasks/task /opt/src/pctasks/task -RUN cd /opt/src/pctasks/task && \ - pip install . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/task && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . COPY pctasks/client /opt/src/pctasks/client -RUN cd /opt/src/pctasks/client && \ - pip install . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/client && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . COPY pctasks/ingest /opt/src/pctasks/ingest -RUN cd /opt/src/pctasks/ingest && \ - pip install . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/ingest && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . COPY pctasks/dataset /opt/src/pctasks/dataset -RUN cd /opt/src/pctasks/dataset && \ - pip install . +RUN --mount=type=cache,target=/root/.cache \ + cd /opt/src/pctasks/dataset && \ + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . # Setup Python Path to allow import of test modules ENV PYTHONPATH=/opt/src:$PYTHONPATH diff --git a/deployment/Dockerfile b/deployment/Dockerfile index 128b92ba..ac853b75 100644 --- a/deployment/Dockerfile +++ b/deployment/Dockerfile @@ -53,12 +53,13 @@ RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash # Install python dependencies COPY deployment/requirements.txt /opt/src/deployment/requirements.txt -RUN pip3 install -r /opt/src/deployment/requirements.txt +RUN pip install uv +RUN uv pip install --system -r /opt/src/deployment/requirements.txt # Instal pctasks.core COPY pctasks/core /opt/src/pctasks/core RUN cd /opt/src/pctasks/core && \ - pip install . - + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps . WORKDIR /opt/deploy diff --git a/pctasks/server/Dockerfile b/pctasks/server/Dockerfile index fad645f7..38e051c0 100644 --- a/pctasks/server/Dockerfile +++ b/pctasks/server/Dockerfile @@ -8,37 +8,38 @@ RUN apt-get update && \ ENV CURL_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt ENV PIP_NO_CACHE_DIR=1 -RUN pip install "setuptools>=65.5.1" +RUN pip install "setuptools>=65.5.1" uv WORKDIR /opt/src COPY core /opt/src/core RUN cd /opt/src/core && \ - pip install . + uv pip install --system --no-deps . COPY cli /opt/src/cli RUN cd /opt/src/cli && \ - pip install . + uv pip install --system --no-deps . COPY client /opt/src/client RUN cd /opt/src/client && \ - pip install . + uv pip install --system --no-deps . COPY task /opt/src/task RUN cd /opt/src/task && \ - pip install . + uv pip install --system --no-deps . COPY run /opt/src/run RUN cd /opt/src/run && \ - pip install . + uv pip install --system --no-deps . COPY server /opt/src/server RUN cd /opt/src/server && \ - pip install .[server] + uv pip install --system -r requirements.txt && \ + uv pip install --system --no-deps .[server] RUN if [ "${DEVELOPMENT}" = "TRUE" ]; then \ echo "INSTALLING DEVELOPMENT DEPENDENCIES"; \ - pip install \ + uv pip install --system \ -e /opt/src/core \ -e /opt/src/cli \ -e /opt/src/client \ From f90af7852a40dc1b1f0cd7c42d739f94da1f8e28 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 12:37:22 -0400 Subject: [PATCH 16/27] datasets requirements --- pctasks/dataset/requirements.txt | 276 +++++++++++++++++++++++++++++++ scripts/generate-requirements | 6 + 2 files changed, 282 insertions(+) create mode 100644 pctasks/dataset/requirements.txt diff --git a/pctasks/dataset/requirements.txt b/pctasks/dataset/requirements.txt new file mode 100644 index 00000000..5762e08f --- /dev/null +++ b/pctasks/dataset/requirements.txt @@ -0,0 +1,276 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/dataset/pyproject.toml -o pctasks/dataset/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jinja2==3.0.3 + # via pctasks-ingest + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +marko==1.0.1 + # via pctasks-ingest + # from https://pypi.org/simple +markupsafe==2.1.5 + # via jinja2 + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + + # via + # pctasks-client + # pctasks-task + + # via pctasks-ingest + + # via + # pctasks-cli + # pctasks-client + # pctasks-task + + +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-client + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.9.0.post0 + # via + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rich==13.7.1 + # via pctasks-client + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/scripts/generate-requirements b/scripts/generate-requirements index d7ba3e7f..58094768 100755 --- a/scripts/generate-requirements +++ b/scripts/generate-requirements @@ -52,3 +52,9 @@ docker-compose \ run --rm \ dev \ scripts/bin/pctasks-pip-compile ./pctasks/server/pyproject.toml -o pctasks/server/requirements.txt $pip_compile_options --extra dev + +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/dataset/pyproject.toml -o pctasks/dataset/requirements.txt $pip_compile_options From 47b59e47715802e6e698f72839384300a1b91012 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 12:45:51 -0400 Subject: [PATCH 17/27] remove version --- docker-compose.cosmosdb.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.cosmosdb.yml b/docker-compose.cosmosdb.yml index 98b1e396..aafe0fe4 100644 --- a/docker-compose.cosmosdb.yml +++ b/docker-compose.cosmosdb.yml @@ -1,4 +1,3 @@ -version: "2.1" services: cosmosdb: container_name: pctasks-cosmosdb From a44b1516d2de9953218856094802d87fd9b9738a Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 13:19:02 -0400 Subject: [PATCH 18/27] wrong packaging --- pctasks/server/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pctasks/server/pyproject.toml b/pctasks/server/pyproject.toml index 44b9fe09..176b6cca 100644 --- a/pctasks/server/pyproject.toml +++ b/pctasks/server/pyproject.toml @@ -52,5 +52,5 @@ path = "pctasks/server/version.py" [tool.hatch.build.targets.wheel] include = [ - "/pctasks/run", + "/pctasks/server", ] From 4bc18a3c51d989c32ca08ce816aa1f2dbdb5df9c Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 13:19:12 -0400 Subject: [PATCH 19/27] edit docker-compose networks --- docker-compose.aux.yml | 6 ++---- docker-compose.console.yml | 2 ++ docker-compose.cosmosdb.yml | 4 ++-- docker-compose.yml | 5 ++--- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docker-compose.aux.yml b/docker-compose.aux.yml index faab3d6d..6bf7055d 100644 --- a/docker-compose.aux.yml +++ b/docker-compose.aux.yml @@ -1,6 +1,4 @@ # Non-essential development enviroment servers - -version: "2.1" services: stac-api: container_name: pctasks-stac-api @@ -57,5 +55,5 @@ services: networks: default: # Network created during scripts/setup - external: - name: pctasks-network + name: pctasks-network + external: true diff --git a/docker-compose.console.yml b/docker-compose.console.yml index d0c9a344..175ce268 100644 --- a/docker-compose.console.yml +++ b/docker-compose.console.yml @@ -111,3 +111,5 @@ services: networks: default: name: pctasks-network + external: true + diff --git a/docker-compose.cosmosdb.yml b/docker-compose.cosmosdb.yml index aafe0fe4..0511b3b9 100644 --- a/docker-compose.cosmosdb.yml +++ b/docker-compose.cosmosdb.yml @@ -33,8 +33,8 @@ services: networks: default: # Network created during scripts/setup - external: - name: pctasks-network + external: true + name: pctasks-network volumes: pctasks-cosmosdb-data: null diff --git a/docker-compose.yml b/docker-compose.yml index 8ddc3ddb..4dae52d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "2.1" services: azurite: container_name: pctasks-azurite @@ -206,8 +205,8 @@ services: networks: default: # Network created during scripts/setup - external: - name: pctasks-network + external: true + name: pctasks-network volumes: pctasks-azurite-data: null From 1f2dc8cf570f8e773fadaec67e23b840b9e37ca0 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 14:23:23 -0400 Subject: [PATCH 20/27] fix python-dateutil version problem --- pctasks/cli/requirements.txt | 3 +- pctasks/client/requirements.txt | 3 +- pctasks/core/pyproject.toml | 1 + pctasks/core/requirements.txt | 10 +- pctasks/dataset/requirements.txt | 3 +- pctasks/dev/requirements.txt | 351 +++++++++++++++++++++++++++ pctasks/ingest/requirements.txt | 3 +- pctasks/ingest_task/pyproject.toml | 1 - pctasks/ingest_task/requirements.txt | 1 + pctasks/notify/requirements.txt | 240 ++++++++++++++++++ pctasks/router/requirements.txt | 240 ++++++++++++++++++ pctasks/run/requirements.txt | 3 +- pctasks/server/requirements.txt | 3 +- pctasks/task/requirements.txt | 3 +- scripts/generate-requirements | 18 ++ scripts/install | 18 +- 16 files changed, 883 insertions(+), 18 deletions(-) create mode 100644 pctasks/dev/requirements.txt create mode 100644 pctasks/notify/requirements.txt create mode 100644 pctasks/router/requirements.txt diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt index 124b9fd9..b74031d2 100644 --- a/pctasks/cli/requirements.txt +++ b/pctasks/cli/requirements.txt @@ -180,8 +180,9 @@ pystac==1.10.0 pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/pctasks/client/requirements.txt b/pctasks/client/requirements.txt index 6d360e46..3c78c921 100644 --- a/pctasks/client/requirements.txt +++ b/pctasks/client/requirements.txt @@ -192,8 +192,9 @@ pystac==1.10.0 pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/pctasks/core/pyproject.toml b/pctasks/core/pyproject.toml index c355335e..2d7cb223 100644 --- a/pctasks/core/pyproject.toml +++ b/pctasks/core/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "orjson>=3.0.0,<4", "planetary-computer>=0.4.0", "pydantic>=1.9,<2.0.0", + "python-dateutil>=2.8.2,<2.9", "pyyaml>=5.3", "stac-validator>=3.1.0", "strictyaml>=1.6", diff --git a/pctasks/core/requirements.txt b/pctasks/core/requirements.txt index 4337043e..9cbf5357 100644 --- a/pctasks/core/requirements.txt +++ b/pctasks/core/requirements.txt @@ -84,9 +84,7 @@ isodate==0.6.1 # azure-storage-queue # from https://pypi.org/simple jsonschema==4.17.3 - # via - # pystac - # stac-validator + # via stac-validator # from https://pypi.org/simple msal==1.28.0 # via @@ -157,15 +155,15 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.8.3 +pystac==1.10.0 # via # planetary-computer # pystac-client # from https://pypi.org/simple -pystac-client==0.7.6 +pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via # pystac # pystac-client diff --git a/pctasks/dataset/requirements.txt b/pctasks/dataset/requirements.txt index 5762e08f..78268b2a 100644 --- a/pctasks/dataset/requirements.txt +++ b/pctasks/dataset/requirements.txt @@ -212,8 +212,9 @@ pystac==1.10.0 pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/pctasks/dev/requirements.txt b/pctasks/dev/requirements.txt new file mode 100644 index 00000000..2c44de4f --- /dev/null +++ b/pctasks/dev/requirements.txt @@ -0,0 +1,351 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/dev/pyproject.toml -o pctasks/dev/requirements.txt +adal==1.2.7 + # via msrestazure + # from https://pypi.org/simple +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +argo-workflows==6.3.10 + # via pctasks-run + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-batch==11.0.0 + # via pctasks-run + # from https://pypi.org/simple +azure-common==1.1.28 + # via azure-batch + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # msrest + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-keyvault-secrets==4.8.0 + # via pctasks-run + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via + # kubernetes + # msrest + # requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # pctasks-cli + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # adal + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via + # google-api-core + # kubernetes + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # msrest + # from https://pypi.org/simple +jinja2==3.0.3 + # via pctasks-ingest + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +kubernetes==29.0.0 + # via pctasks-run + # from https://pypi.org/simple +markdown-it-py==3.0.0 + # via rich + # from https://pypi.org/simple +marko==1.0.1 + # via pctasks-ingest + # from https://pypi.org/simple +markupsafe==2.1.5 + # via jinja2 + # from https://pypi.org/simple +mdurl==0.1.2 + # via markdown-it-py + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +msrest==0.7.1 + # via + # azure-batch + # msrestazure + # from https://pypi.org/simple +msrestazure==0.6.4.post1 + # via azure-batch + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +networkx==2.8.8 + # via pctasks-run + # from https://pypi.org/simple +oauthlib==3.2.2 + # via + # kubernetes + # requests-oauthlib + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + + # via + # pctasks-client + # pctasks-task + + # via + # pctasks-ingest + # pctasks-run + + # via + # pctasks-cli + # pctasks-client + # pctasks-run + # pctasks-task + + + + # via pctasks-run +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-client + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pygments==2.17.2 + # via rich + # from https://pypi.org/simple +pyjwt==2.8.0 + # via + # adal + # msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.8.2 + # via + # adal + # argo-workflows + # kubernetes + # pctasks-core + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via + # planetary-computer + # pydantic + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via + # kubernetes + # pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # adal + # azure-core + # google-api-core + # kubernetes + # msal + # msrest + # opencensus-ext-azure + # planetary-computer + # pystac-client + # requests-oauthlib + # stac-validator + # from https://pypi.org/simple +requests-oauthlib==2.0.0 + # via + # kubernetes + # msrest + # from https://pypi.org/simple +rich==13.7.1 + # via pctasks-client + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # kubernetes + # msrestazure + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-keyvault-secrets + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via + # argo-workflows + # kubernetes + # requests + # from https://pypi.org/simple +websocket-client==1.8.0 + # via kubernetes + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/pctasks/ingest/requirements.txt b/pctasks/ingest/requirements.txt index d64943ba..df7766b7 100644 --- a/pctasks/ingest/requirements.txt +++ b/pctasks/ingest/requirements.txt @@ -204,8 +204,9 @@ pystac==1.10.0 pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/pctasks/ingest_task/pyproject.toml b/pctasks/ingest_task/pyproject.toml index a70e7868..a99693e3 100644 --- a/pctasks/ingest_task/pyproject.toml +++ b/pctasks/ingest_task/pyproject.toml @@ -31,7 +31,6 @@ dependencies = [ "plpygis==0.2.0", "pypgstac[psycopg]==0.7.10", "pystac>=1.0.0,<2", - "python-dateutil==2.8.2", ] [project.optional-dependencies] diff --git a/pctasks/ingest_task/requirements.txt b/pctasks/ingest_task/requirements.txt index c3c0cfd1..733e393a 100644 --- a/pctasks/ingest_task/requirements.txt +++ b/pctasks/ingest_task/requirements.txt @@ -236,6 +236,7 @@ pystac-client==0.6.1 # from https://pypi.org/simple python-dateutil==2.8.2 # via + # pctasks-core # pypgstac # pystac # pystac-client diff --git a/pctasks/notify/requirements.txt b/pctasks/notify/requirements.txt new file mode 100644 index 00000000..5d164baf --- /dev/null +++ b/pctasks/notify/requirements.txt @@ -0,0 +1,240 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/notify/pyproject.toml -o pctasks/notify/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.8.2 + # via + # pctasks-core + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via planetary-computer + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/pctasks/router/requirements.txt b/pctasks/router/requirements.txt new file mode 100644 index 00000000..13c52334 --- /dev/null +++ b/pctasks/router/requirements.txt @@ -0,0 +1,240 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --emit-index-annotation ./pctasks/router/pyproject.toml -o pctasks/router/requirements.txt +aiohttp==3.9.5 + # via pctasks-core + # from https://pypi.org/simple +aiosignal==1.3.1 + # via aiohttp + # from https://pypi.org/simple +async-timeout==4.0.3 + # via aiohttp + # from https://pypi.org/simple +attrs==23.2.0 + # via + # aiohttp + # jsonschema + # from https://pypi.org/simple +azure-core==1.30.1 + # via + # azure-cosmos + # azure-data-tables + # azure-identity + # azure-storage-blob + # azure-storage-queue + # opencensus-ext-azure + # from https://pypi.org/simple +azure-cosmos==4.3.1 + # via pctasks-core + # from https://pypi.org/simple +azure-data-tables==12.5.0 + # via pctasks-core + # from https://pypi.org/simple +azure-identity==1.16.0 + # via + # opencensus-ext-azure + # pctasks-core + # from https://pypi.org/simple +azure-storage-blob==12.19.1 + # via pctasks-core + # from https://pypi.org/simple +azure-storage-queue==12.9.0 + # via pctasks-core + # from https://pypi.org/simple +cachetools==5.3.3 + # via google-auth + # from https://pypi.org/simple +certifi==2024.2.2 + # via requests + # from https://pypi.org/simple +cffi==1.16.0 + # via cryptography + # from https://pypi.org/simple +charset-normalizer==3.3.2 + # via requests + # from https://pypi.org/simple +click==8.1.7 + # via + # planetary-computer + # stac-validator + # from https://pypi.org/simple +cryptography==42.0.5 + # via + # azure-identity + # azure-storage-blob + # azure-storage-queue + # msal + # pyjwt + # from https://pypi.org/simple +frozenlist==1.4.1 + # via + # aiohttp + # aiosignal + # from https://pypi.org/simple +google-api-core==2.18.0 + # via opencensus + # from https://pypi.org/simple +google-auth==2.29.0 + # via google-api-core + # from https://pypi.org/simple +googleapis-common-protos==1.63.0 + # via google-api-core + # from https://pypi.org/simple +idna==3.7 + # via + # pctasks-core + # requests + # yarl + # from https://pypi.org/simple +isodate==0.6.1 + # via + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # from https://pypi.org/simple +jsonschema==4.17.3 + # via + # pctasks-core + # stac-validator + # from https://pypi.org/simple +msal==1.28.0 + # via + # azure-identity + # msal-extensions + # from https://pypi.org/simple +msal-extensions==1.1.0 + # via azure-identity + # from https://pypi.org/simple +multidict==6.0.5 + # via + # aiohttp + # yarl + # from https://pypi.org/simple +opencensus==0.11.4 + # via + # opencensus-ext-azure + # opencensus-ext-logging + # from https://pypi.org/simple +opencensus-context==0.1.3 + # via opencensus + # from https://pypi.org/simple +opencensus-ext-azure==1.1.0 + # via pctasks-core + # from https://pypi.org/simple +opencensus-ext-logging==0.1.1 + # via pctasks-core + # from https://pypi.org/simple +orjson==3.10.1 + # via pctasks-core + # from https://pypi.org/simple +packaging==24.0 + # via + # msal-extensions + # planetary-computer + # from https://pypi.org/simple + +planetary-computer==1.0.0 + # via pctasks-core + # from https://pypi.org/simple +portalocker==2.8.2 + # via msal-extensions + # from https://pypi.org/simple +proto-plus==1.23.0 + # via google-api-core + # from https://pypi.org/simple +protobuf==4.25.3 + # via + # google-api-core + # googleapis-common-protos + # proto-plus + # from https://pypi.org/simple +psutil==5.9.8 + # via opencensus-ext-azure + # from https://pypi.org/simple +pyasn1==0.6.0 + # via + # pyasn1-modules + # rsa + # from https://pypi.org/simple +pyasn1-modules==0.4.0 + # via google-auth + # from https://pypi.org/simple +pycparser==2.22 + # via cffi + # from https://pypi.org/simple +pydantic==1.10.15 + # via + # pctasks-core + # planetary-computer + # from https://pypi.org/simple +pyjwt==2.8.0 + # via msal + # from https://pypi.org/simple +pyrsistent==0.20.0 + # via jsonschema + # from https://pypi.org/simple +pystac==1.10.0 + # via + # planetary-computer + # pystac-client + # from https://pypi.org/simple +pystac-client==0.6.1 + # via planetary-computer + # from https://pypi.org/simple +python-dateutil==2.8.2 + # via + # pctasks-core + # pystac + # pystac-client + # strictyaml + # from https://pypi.org/simple +python-dotenv==1.0.1 + # via planetary-computer + # from https://pypi.org/simple +pytz==2024.1 + # via planetary-computer + # from https://pypi.org/simple +pyyaml==6.0.1 + # via pctasks-core + # from https://pypi.org/simple +requests==2.31.0 + # via + # azure-core + # google-api-core + # msal + # opencensus-ext-azure + # planetary-computer + # pystac-client + # stac-validator + # from https://pypi.org/simple +rsa==4.9 + # via google-auth + # from https://pypi.org/simple +six==1.16.0 + # via + # azure-core + # isodate + # opencensus + # python-dateutil + # from https://pypi.org/simple +stac-validator==3.3.2 + # via pctasks-core + # from https://pypi.org/simple +strictyaml==1.7.3 + # via pctasks-core + # from https://pypi.org/simple +typing-extensions==4.11.0 + # via + # azure-core + # azure-data-tables + # azure-storage-blob + # azure-storage-queue + # pydantic + # from https://pypi.org/simple +urllib3==2.2.1 + # via requests + # from https://pypi.org/simple +yarl==1.9.4 + # via + # aiohttp + # azure-data-tables + # from https://pypi.org/simple diff --git a/pctasks/run/requirements.txt b/pctasks/run/requirements.txt index d5b73ef1..058c609e 100644 --- a/pctasks/run/requirements.txt +++ b/pctasks/run/requirements.txt @@ -242,11 +242,12 @@ pystac==1.10.0 pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via # adal # argo-workflows # kubernetes + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/pctasks/server/requirements.txt b/pctasks/server/requirements.txt index 5a451dab..14d6b60f 100644 --- a/pctasks/server/requirements.txt +++ b/pctasks/server/requirements.txt @@ -311,11 +311,12 @@ pytest==8.1.1 # from https://pypi.org/simple pytest-cov==5.0.0 # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via # adal # argo-workflows # kubernetes + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/pctasks/task/requirements.txt b/pctasks/task/requirements.txt index 56d2abd0..5f426115 100644 --- a/pctasks/task/requirements.txt +++ b/pctasks/task/requirements.txt @@ -183,8 +183,9 @@ pystac==1.10.0 pystac-client==0.6.1 # via planetary-computer # from https://pypi.org/simple -python-dateutil==2.9.0.post0 +python-dateutil==2.8.2 # via + # pctasks-core # pystac # pystac-client # strictyaml diff --git a/scripts/generate-requirements b/scripts/generate-requirements index 58094768..8c0f8977 100755 --- a/scripts/generate-requirements +++ b/scripts/generate-requirements @@ -58,3 +58,21 @@ docker-compose \ run --rm \ dev \ scripts/bin/pctasks-pip-compile ./pctasks/dataset/pyproject.toml -o pctasks/dataset/requirements.txt $pip_compile_options + +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/notify/pyproject.toml -o pctasks/notify/requirements.txt $pip_compile_options + +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/router/pyproject.toml -o pctasks/router/requirements.txt $pip_compile_options + +docker-compose \ + -f docker-compose.console.yml \ + run --rm \ + dev \ + scripts/bin/pctasks-pip-compile ./pctasks/dev/pyproject.toml -o pctasks/dev/requirements.txt $pip_compile_options \ No newline at end of file diff --git a/scripts/install b/scripts/install index 2ab731d4..18880685 100755 --- a/scripts/install +++ b/scripts/install @@ -30,10 +30,20 @@ source scripts/env if [ "${BASH_SOURCE[0]}" = "${0}" ]; then pip install uv - for DIR in "${PACKAGE_DIRS[@]}"; do - echo "Installing ${DIR}" - uv pip install --python=$(which python3) -e ${DIR} - done + # Dependencies must be mutually compatible across all projects + uv pip install --python=$(which python3) \ + -r "pctasks/core/requirements.txt" \ + -r "pctasks/cli/requirements.txt" \ + -r "pctasks/task/requirements.txt" \ + -r "pctasks/client/requirements.txt" \ + -r "pctasks/ingest/requirements.txt" \ + -r "pctasks/ingest_task/requirements.txt" \ + -r "pctasks/dataset/requirements.txt" \ + -r "pctasks/run/requirements.txt" \ + -r "pctasks/notify/requirements.txt" \ + -r "pctasks/router/requirements.txt" \ + -r "pctasks/server/requirements.txt" \ + -r "pctasks/dev/requirements.txt" echo "Installing dev dependencies" uv pip install --python=$(which python3) -r requirements-dev.txt From ffedeb98953e2c374bf87bd015680c7ccdf968af Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 14:55:52 -0400 Subject: [PATCH 21/27] Pass test? --- Dockerfile.dev | 1 - deployment/requirements.txt | 2 +- pctasks/cli/requirements.txt | 3 ++- pctasks/client/requirements.txt | 3 ++- pctasks/core/pyproject.toml | 1 + pctasks/core/requirements.txt | 2 +- pctasks/dataset/requirements.txt | 3 ++- pctasks/dev/requirements.txt | 3 ++- pctasks/ingest/requirements.txt | 3 ++- pctasks/ingest_task/requirements.txt | 3 ++- pctasks/notify/requirements.txt | 3 ++- pctasks/router/requirements.txt | 3 ++- pctasks/run/requirements.txt | 3 ++- pctasks/server/requirements.txt | 3 ++- pctasks/task/requirements.txt | 3 ++- requirements-dev.txt | 9 ++++----- scripts/install | 8 +++----- 17 files changed, 32 insertions(+), 24 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 0e90950a..38104d56 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -80,7 +80,6 @@ RUN --mount=type=cache,target=/root/.cache \ uv pip install --system -r requirements.txt && \ uv pip install --system --no-deps -e . -# We are here COPY pctasks/server /opt/src/pctasks/server RUN --mount=type=cache,target=/root/.cache \ cd /opt/src/pctasks/server && \ diff --git a/deployment/requirements.txt b/deployment/requirements.txt index eaf89d4e..4dfc7d35 100644 --- a/deployment/requirements.txt +++ b/deployment/requirements.txt @@ -1,2 +1,2 @@ -pypgstac==0.7.3 +pypgstac==0.7.10 Jinja2==3.1.2 diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt index b74031d2..8eab5d2b 100644 --- a/pctasks/cli/requirements.txt +++ b/pctasks/cli/requirements.txt @@ -172,8 +172,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/client/requirements.txt b/pctasks/client/requirements.txt index 3c78c921..83eb43f8 100644 --- a/pctasks/client/requirements.txt +++ b/pctasks/client/requirements.txt @@ -184,8 +184,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/core/pyproject.toml b/pctasks/core/pyproject.toml index 2d7cb223..f1e0d4c0 100644 --- a/pctasks/core/pyproject.toml +++ b/pctasks/core/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "opencensus-ext-logging==0.1.1", "orjson>=3.0.0,<4", "planetary-computer>=0.4.0", + "pystac==1.8.3", "pydantic>=1.9,<2.0.0", "python-dateutil>=2.8.2,<2.9", "pyyaml>=5.3", diff --git a/pctasks/core/requirements.txt b/pctasks/core/requirements.txt index 9cbf5357..8fe9cf06 100644 --- a/pctasks/core/requirements.txt +++ b/pctasks/core/requirements.txt @@ -155,7 +155,7 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via # planetary-computer # pystac-client diff --git a/pctasks/dataset/requirements.txt b/pctasks/dataset/requirements.txt index 78268b2a..abc7a8d5 100644 --- a/pctasks/dataset/requirements.txt +++ b/pctasks/dataset/requirements.txt @@ -204,8 +204,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/dev/requirements.txt b/pctasks/dev/requirements.txt index 2c44de4f..fdcc612a 100644 --- a/pctasks/dev/requirements.txt +++ b/pctasks/dev/requirements.txt @@ -255,8 +255,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/ingest/requirements.txt b/pctasks/ingest/requirements.txt index df7766b7..0c2d2178 100644 --- a/pctasks/ingest/requirements.txt +++ b/pctasks/ingest/requirements.txt @@ -196,8 +196,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/ingest_task/requirements.txt b/pctasks/ingest_task/requirements.txt index 733e393a..0d763259 100644 --- a/pctasks/ingest_task/requirements.txt +++ b/pctasks/ingest_task/requirements.txt @@ -226,8 +226,9 @@ pypgstac==0.7.10 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/notify/requirements.txt b/pctasks/notify/requirements.txt index 5d164baf..fbf841dd 100644 --- a/pctasks/notify/requirements.txt +++ b/pctasks/notify/requirements.txt @@ -172,8 +172,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/router/requirements.txt b/pctasks/router/requirements.txt index 13c52334..c80da126 100644 --- a/pctasks/router/requirements.txt +++ b/pctasks/router/requirements.txt @@ -172,8 +172,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/run/requirements.txt b/pctasks/run/requirements.txt index 058c609e..c8bf7bf2 100644 --- a/pctasks/run/requirements.txt +++ b/pctasks/run/requirements.txt @@ -234,8 +234,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/server/requirements.txt b/pctasks/server/requirements.txt index 14d6b60f..6f74a386 100644 --- a/pctasks/server/requirements.txt +++ b/pctasks/server/requirements.txt @@ -298,8 +298,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/pctasks/task/requirements.txt b/pctasks/task/requirements.txt index 5f426115..7c9e1301 100644 --- a/pctasks/task/requirements.txt +++ b/pctasks/task/requirements.txt @@ -175,8 +175,9 @@ pyjwt==2.8.0 pyrsistent==0.20.0 # via jsonschema # from https://pypi.org/simple -pystac==1.10.0 +pystac==1.8.3 # via + # pctasks-core # planetary-computer # pystac-client # from https://pypi.org/simple diff --git a/requirements-dev.txt b/requirements-dev.txt index 68907a74..236e2742 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,14 +1,13 @@ flake8==4.* black==22.* mypy==1.2.0 -pytest==6.* +pytest==8.1.1 isort==5.* pytest-asyncio==0.18.* kubernetes planetary-computer -pystac[validation]==1.* -jsonschema>=4.0.1,<4.18 # Matches the pin in https://github.com/stac-utils/pystac/pull/1189/ +pystac[validation]==1.8.3 azure-functions azure-functions-durable @@ -28,5 +27,5 @@ types-PyYAML myst-parser pydata-sphinx-theme sphinx -sphinx-autobuild==2021.3.14 -sphinx_design==0.2.0 +sphinx-autobuild +sphinx_design diff --git a/scripts/install b/scripts/install index 18880685..f695e2ba 100755 --- a/scripts/install +++ b/scripts/install @@ -31,7 +31,7 @@ source scripts/env if [ "${BASH_SOURCE[0]}" = "${0}" ]; then pip install uv # Dependencies must be mutually compatible across all projects - uv pip install --python=$(which python3) \ + uv pip install --prerelease allow --python=$(which python3) \ -r "pctasks/core/requirements.txt" \ -r "pctasks/cli/requirements.txt" \ -r "pctasks/task/requirements.txt" \ @@ -43,9 +43,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then -r "pctasks/notify/requirements.txt" \ -r "pctasks/router/requirements.txt" \ -r "pctasks/server/requirements.txt" \ - -r "pctasks/dev/requirements.txt" - - echo "Installing dev dependencies" - uv pip install --python=$(which python3) -r requirements-dev.txt + -r "pctasks/dev/requirements.txt" \ + -r "requirements-dev.txt" fi From 456bf7bc9da29442cc407972af7ab9a318db31bb Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 15:03:46 -0400 Subject: [PATCH 22/27] Require python3.9 --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c7c11590..ad78be05 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" - name: Install local dependencies run: ./scripts/install From 5b682645788921ef198d57daf5c60e362befe24e Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Fri, 26 Apr 2024 15:41:53 -0400 Subject: [PATCH 23/27] install packages into system python as editable installs --- scripts/install | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/install b/scripts/install index f695e2ba..96280436 100755 --- a/scripts/install +++ b/scripts/install @@ -45,5 +45,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then -r "pctasks/server/requirements.txt" \ -r "pctasks/dev/requirements.txt" \ -r "requirements-dev.txt" - + for DIR in "${PACKAGE_DIRS[@]}"; do + echo "Installing ${DIR}" + uv pip install --system --no-deps -e ${DIR} + done fi From 2b82064f9aef838c38518383aa0d4aa5fa660d28 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Mon, 29 Apr 2024 21:46:12 -0400 Subject: [PATCH 24/27] Try setting confirmation required to false --- scripts/citest-integration | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/citest-integration b/scripts/citest-integration index 4381f5dc..a66aec9e 100755 --- a/scripts/citest-integration +++ b/scripts/citest-integration @@ -17,11 +17,12 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then export PCTASKS_CLIENT__ENDPOINT=http://localhost:8500/tasks export PCTASKS_CLIENT__API_KEY=kind-api-key + export PCTASKS_CLIENT__CONFIRMATION_REQUIRED=False export AZURITE_HOST=localhost export AZURITE_PORT=10000 export AZURITE_STORAGE_ACCOUNT=devstoreaccount1 export DEV_DB_CONNECTION_STRING=postgresql://username:password@localhost:5499/postgis export DEV_REMOTE_DB_CONNECTION_STRING=postgresql://username:password@database:5432/postgis - scripts/bin/test-integration + scripts/bin/test-integration --test-only fi \ No newline at end of file From 650ce965b132a7c94569b0ccedac7304e6c17e23 Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Tue, 30 Apr 2024 10:26:14 -0400 Subject: [PATCH 25/27] Add --system and regenerate --- docker-compose.tasks.yml | 1 + pctasks/cli/requirements.txt | 2 +- pctasks/client/requirements.txt | 2 +- pctasks/core/requirements.txt | 2 +- pctasks/dataset/requirements.txt | 2 +- pctasks/dev/requirements.txt | 2 +- pctasks/ingest/requirements.txt | 2 +- pctasks/ingest_task/requirements.txt | 2 +- pctasks/notify/requirements.txt | 2 +- pctasks/router/requirements.txt | 2 +- pctasks/run/requirements.txt | 2 +- pctasks/server/requirements.txt | 2 +- pctasks/task/requirements.txt | 2 +- scripts/bin/pctasks-pip-compile | 1 + 14 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docker-compose.tasks.yml b/docker-compose.tasks.yml index 25b92f47..d3d4773e 100644 --- a/docker-compose.tasks.yml +++ b/docker-compose.tasks.yml @@ -38,5 +38,6 @@ services: networks: default: + external: true name: pctasks-network diff --git a/pctasks/cli/requirements.txt b/pctasks/cli/requirements.txt index 8eab5d2b..8839eb58 100644 --- a/pctasks/cli/requirements.txt +++ b/pctasks/cli/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/cli/pyproject.toml -o pctasks/cli/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/cli/pyproject.toml -o pctasks/cli/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/client/requirements.txt b/pctasks/client/requirements.txt index 83eb43f8..75c0d6ff 100644 --- a/pctasks/client/requirements.txt +++ b/pctasks/client/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/client/pyproject.toml -o pctasks/client/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/client/pyproject.toml -o pctasks/client/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/core/requirements.txt b/pctasks/core/requirements.txt index 8fe9cf06..d1fca775 100644 --- a/pctasks/core/requirements.txt +++ b/pctasks/core/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/core/pyproject.toml -o pctasks/core/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/core/pyproject.toml -o pctasks/core/requirements.txt aiohttp==3.9.5 # from https://pypi.org/simple aiosignal==1.3.1 diff --git a/pctasks/dataset/requirements.txt b/pctasks/dataset/requirements.txt index abc7a8d5..8a32ca60 100644 --- a/pctasks/dataset/requirements.txt +++ b/pctasks/dataset/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/dataset/pyproject.toml -o pctasks/dataset/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/dataset/pyproject.toml -o pctasks/dataset/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/dev/requirements.txt b/pctasks/dev/requirements.txt index fdcc612a..7a17ace7 100644 --- a/pctasks/dev/requirements.txt +++ b/pctasks/dev/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/dev/pyproject.toml -o pctasks/dev/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/dev/pyproject.toml -o pctasks/dev/requirements.txt adal==1.2.7 # via msrestazure # from https://pypi.org/simple diff --git a/pctasks/ingest/requirements.txt b/pctasks/ingest/requirements.txt index 0c2d2178..60065f08 100644 --- a/pctasks/ingest/requirements.txt +++ b/pctasks/ingest/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/ingest/pyproject.toml -o pctasks/ingest/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/ingest/pyproject.toml -o pctasks/ingest/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/ingest_task/requirements.txt b/pctasks/ingest_task/requirements.txt index 0d763259..b13e0d71 100644 --- a/pctasks/ingest_task/requirements.txt +++ b/pctasks/ingest_task/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/ingest_task/pyproject.toml -o pctasks/ingest_task/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/ingest_task/pyproject.toml -o pctasks/ingest_task/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/notify/requirements.txt b/pctasks/notify/requirements.txt index fbf841dd..10d07361 100644 --- a/pctasks/notify/requirements.txt +++ b/pctasks/notify/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/notify/pyproject.toml -o pctasks/notify/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/notify/pyproject.toml -o pctasks/notify/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/router/requirements.txt b/pctasks/router/requirements.txt index c80da126..695b0914 100644 --- a/pctasks/router/requirements.txt +++ b/pctasks/router/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/router/pyproject.toml -o pctasks/router/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/router/pyproject.toml -o pctasks/router/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/pctasks/run/requirements.txt b/pctasks/run/requirements.txt index c8bf7bf2..30118256 100644 --- a/pctasks/run/requirements.txt +++ b/pctasks/run/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/run/pyproject.toml -o pctasks/run/requirements.txt adal==1.2.7 # via msrestazure # from https://pypi.org/simple diff --git a/pctasks/server/requirements.txt b/pctasks/server/requirements.txt index 6f74a386..9a14d7e5 100644 --- a/pctasks/server/requirements.txt +++ b/pctasks/server/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/server/pyproject.toml -o pctasks/server/requirements.txt --extra dev +# uv pip compile --system --emit-index-annotation ./pctasks/server/pyproject.toml -o pctasks/server/requirements.txt --extra dev adal==1.2.7 # via msrestazure # from https://pypi.org/simple diff --git a/pctasks/task/requirements.txt b/pctasks/task/requirements.txt index 7c9e1301..39cb896c 100644 --- a/pctasks/task/requirements.txt +++ b/pctasks/task/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --emit-index-annotation ./pctasks/task/pyproject.toml -o pctasks/task/requirements.txt +# uv pip compile --system --emit-index-annotation ./pctasks/task/pyproject.toml -o pctasks/task/requirements.txt aiohttp==3.9.5 # via pctasks-core # from https://pypi.org/simple diff --git a/scripts/bin/pctasks-pip-compile b/scripts/bin/pctasks-pip-compile index 211bb463..65fa814f 100755 --- a/scripts/bin/pctasks-pip-compile +++ b/scripts/bin/pctasks-pip-compile @@ -7,6 +7,7 @@ if [[ "${CI}" ]]; then fi uv pip compile \ + --system \ --emit-index-annotation \ "$@" From e8170404a21898da76f597dbd8b83554392a2e6b Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Tue, 30 Apr 2024 11:35:59 -0400 Subject: [PATCH 26/27] Add ci auth and oras --- .github/workflows/cicd.yml | 32 ++++++++------ scripts/ciauthenticate | 19 +++++++++ scripts/publish | 86 +++++++++++++++++++++++++++++++++----- 3 files changed, 115 insertions(+), 22 deletions(-) create mode 100644 scripts/ciauthenticate diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index a93aea69..e05e7afe 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -26,7 +26,26 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.8" + python-version: "3.9" + + - name: Get image tag + id: get_image_tag + run: case "${GITHUB_REF}" in + *tags*) + echo "::set-output name=tag::${GITHUB_REF/refs\/tags\//}" ; + echo "acr=pccomponents" >> $GITHUB_OUTPUT + ;; + *) + echo "::set-output name=tag::latest" ; + echo "acr=pccomponentstest" >> $GITHUB_OUTPUT + ;; + esac + + - name: Authenticate + env: + CLIENT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientId }} + CLIENT_SECRET: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientSecret }} + run: ./scripts/ciauthenticate ${{steps.get_image_tag.outputs.acr}} - name: Install local dependencies run: ./scripts/install @@ -61,17 +80,6 @@ jobs: # Publish images - - name: Get image tag - id: get_image_tag - run: case "${GITHUB_REF}" in - *tags*) - echo "::set-output name=tag::${GITHUB_REF/refs\/tags\//}" - ;; - *) - echo "::set-output name=tag::latest" - ;; - esac - - name: Log into the ACR (test) env: CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} diff --git a/scripts/ciauthenticate b/scripts/ciauthenticate new file mode 100644 index 00000000..86736de7 --- /dev/null +++ b/scripts/ciauthenticate @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +if [[ "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +CI authentication for this project. +" +} + +acr=$1 + +az acr login --name $acr +docker login $acr.azurecr.io --username ${CLIENT_ID} --password ${CLIENT_SECRET} \ No newline at end of file diff --git a/scripts/publish b/scripts/publish index 1545e20e..6eff2afc 100755 --- a/scripts/publish +++ b/scripts/publish @@ -58,11 +58,72 @@ while [[ "$#" -gt 0 ]]; do case $1 in ;; esac done -function publish() { - local IMAGE_NAME=$1 - local full_image_name="${ACR_NAME}.azurecr.io/${IMAGE_NAME}:${IMAGE_TAG}" - docker tag ${IMAGE_NAME} ${full_image_name} - docker push ${full_image_name} +function install_oras() { + # https://oras.land/docs/installation/ + VERSION="1.1.0" + curl -LO "https://github.com/oras-project/oras/releases/download/v${VERSION}/oras_${VERSION}_linux_amd64.tar.gz" + mkdir -p oras-install/ + tar -zxf oras_${VERSION}_*.tar.gz -C oras-install/ + sudo mv oras-install/oras /usr/local/bin/ + rm -rf oras_${VERSION}_*.tar.gz oras-install/ +} + +function deprecate_image() { + local full_image_name_with_digest=$1 + deprecated_since=$(date --utc --iso-8601=seconds) + if oras discover $full_image_name_with_digest -o json | jq '.manifests[].annotations' | grep -q "vnd.microsoft.lifecycle.end-of-life.date" ; then + echo "Lifecycle metadata annotation for $full_image_name_with_digest already exists, skip." + else + echo "Deprecating previous image $full_image_name_with_digest" + oras attach \ + --artifact-type "application/vnd.microsoft.artifact.lifecycle" \ + --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$deprecated_since" \ + "$full_image_name_with_digest" + fi +} + +function set_lineage() { + local full_image_name_with_digest=$1 + # if the annotation already exists, do not add it again + if oras discover $full_image_name_with_digest -o json | jq '.manifests[].annotations."vnd.microsoft.artifact.lineage.rolling-tag"' | grep -q $IMAGE_TAG; then + echo "Lineage annotation for $IMAGE_TAG already exists, skip." + else + echo "Adding $full_image_name_with_digest to lineage $IMAGE_TAG" + oras attach \ + --artifact-type "application/vnd.microsoft.artifact.lineage" \ + --annotation "vnd.microsoft.artifact.lineage.rolling-tag=$IMAGE_TAG" \ + "$full_image_name_with_digest" + fi +} + +function publish_image() { + local local_image=$1 + # if $2 is set, use it for published_image otherwise use $1 + if [ -z "$2" ]; then + local published_image=$1 + else + local published_image=$2 + fi + local full_image_name="${ACR_NAME}.azurecr.io/${published_image}:${IMAGE_TAG}" + + local local_image_digest=$(docker inspect --format='{{.RepoDigests}}' "${local_image}" | cut -d'@' -f2 | tr -d \[\]) + local remote_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) + + if [ -z "$remote_image_digest" ]; then + echo "No remote image found, will publish a new image." + elif [ "$local_image_digest" != "$remote_image_digest" ]; then + # Image rolling tag exists in the registry, update the end-of-life + # annotation for the existing image. + deprecate_image "$full_image_name@$remote_image_digest" + else + echo "No changes, licycle metadata annotation will not be attached." + fi + + echo "Publishing ${local_image} to ${full_image_name}" + docker tag "${local_image}" "${full_image_name}" + docker push "${full_image_name}" + local remote_image_digest=$(az acr manifest show-metadata $full_image_name | jq -r .digest) + set_lineage "$full_image_name@$remote_image_digest" } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then @@ -75,13 +136,18 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then az acr login -n ${ACR_NAME} fi + if ! command -v oras &> /dev/null + then + install_oras + fi + if [ "${ONLY_IMAGE}" ]; then - publish ${ONLY_IMAGE} + publish_image ${ONLY_IMAGE} else - publish pctasks-ingest - publish pctasks-server - publish pctasks-run - publish pctasks-task-base + publish_image pctasks-ingest + publish_image pctasks-server + publish_image pctasks-run + publish_image pctasks-task-base fi fi From de5c44d2f4eb01956dbd900c3c921674776cf82b Mon Sep 17 00:00:00 2001 From: Gustavo Hidalgo Date: Wed, 15 May 2024 15:37:42 -0400 Subject: [PATCH 27/27] sync python versions --- .github/workflows/cicd.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index f7c1b38a..6a6356df 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -24,25 +24,6 @@ jobs: with: python-version: "3.9" - - name: Get image tag - id: get_image_tag - run: case "${GITHUB_REF}" in - *tags*) - echo "::set-output name=tag::${GITHUB_REF/refs\/tags\//}" ; - echo "acr=pccomponents" >> $GITHUB_OUTPUT - ;; - *) - echo "::set-output name=tag::latest" ; - echo "acr=pccomponentstest" >> $GITHUB_OUTPUT - ;; - esac - - - name: Authenticate - env: - CLIENT_ID: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientId }} - CLIENT_SECRET: ${{ fromJSON(secrets.AZURE_CREDENTIALS).clientSecret }} - run: ./scripts/ciauthenticate ${{steps.get_image_tag.outputs.acr}} - - name: Install local dependencies run: ./scripts/install @@ -75,6 +56,7 @@ jobs: run: ./scripts/citest-integration # Publish images + - name: Get image tag id: get_image_tag run: case "${GITHUB_REF}" in