Merge branch '517-clean-up-requirements' of github.com:SINTEF/dlite i…

…nto 517-clean-up-requirements

.github/docker/Dockerfile-manylinux.template

@@ -28,6 +28,7 @@ ARG PY_MINORS="7 8 9 10 11"
 
 #COPY requirements.txt /tmp/requirements.txt
 COPY requirements_full.txt /tmp/requirements_full.txt
+COPY requirements_full.txt /tmp/requirements_dev.txt
 
 {{ EXTRA_PRE }}
 
@@ -50,13 +51,17 @@ RUN yum update -y && \
 # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+# Change required version of pydantic to be <2
+  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \
+# Install required Python packages
   mkdir -p /ci/pip_cache && \
   if [ -f "/etc/yum.repos.d/pgdg-91.repo" ]; then export PATH="$PATH:/usr/pgsql-9.1/bin"; fi && \
   for minor in ${PY_MINORS}; do \
     python3.${minor} -m pip install -U pip && \
     python3.${minor} -m pip install -U setuptools wheel && \
     python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
     python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \
+    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_dev.txt; \
   done
 
 {{ EXTRA_POST }}

.github/docker/Dockerfile-musllinux.template

@@ -25,6 +25,7 @@ FROM quay.io/pypa/musllinux{{ TYPE }}_{{ ARCH }}:latest
 
 #COPY requirements.txt /tmp/requirements.txt
 COPY requirements_full.txt /tmp/requirements_full.txt
+COPY requirements_full.txt /tmp/requirements_dev.txt
 
 # Do not use distutils distributed with setuptools
 # This is due to base changes in the distutils API, removing msvccompiler,
@@ -35,16 +36,20 @@ RUN apk add -u \
   redland \
   rasqal \
   hdf5 \
-  swig &&
+  swig && \
 # Unpack static libraries
 # It's necessary to be in /opt/_internal because the internal libraries
 # exist here.
   cd /opt/_internal && \
   tar -Jxvf static-libs-for-embedding-only.tar.xz && \
+# Change required version of pydantic to be <2
+  sed 's/^\(pydantic>.*<\).*$/\12/' -i /tmp/requirements_full.txt && \
+# Install required Python packages
   mkdir -p /ci/pip_cache && \
   for minor in 7 8 9 10; do \
     python3.${minor} -m pip install -U pip && \
     python3.${minor} -m pip install -U setuptools wheel && \
     python3.${minor} -m pip install -U --cache-dir /ci/pip_cache cmake oldest-supported-numpy && \
     python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_full.txt; \
+    python3.${minor} -m pip install --cache-dir /ci/pip_cache --prefer-binary -r /tmp/requirements_dev.txt; \
   done

.github/workflows/cd_docs.yml

@@ -25,7 +25,7 @@ env:
   PUBLISH_DIR: pages
 
 jobs:
-  cd_docs:
+  compile:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -87,7 +87,7 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
 
     runs-on: ubuntu-latest
-    needs: build
+    needs: compile
 
     steps:
       - name: Deploy to GitHub Pages

doc/getting_started/build/runtime_dependencies.md

@@ -8,24 +8,45 @@ On Ubuntu, they can be installed with
     sudo apt install python3 gfortran librdf libhdf5
 
 When DLite is compiled with Python bindings, additional runtime features may be enabled by installing one of more of the following optional Python packages
-- [tripper], optional, (used for property mappings)
-- [PyYAML], optional (used for generic YAML storage plugin)
-- [psycopg2], optional (used for generic PostgreSQL storage plugin)
+- [tripper], used for property mappings
+- [pint], used for units conversion in property mappings
+- [pydantic], used for testing support for pydantic models
+- [typing_extensions], needed by pydantic v2
+- [rdflib], used by `rdf` storage plugin
+- [PyYAML], used by `yaml` storage plugin
+- [psycopg2], used by `postgresql` storage plugin
     Note that in some cases a GSSAPI error is raised when using psycopg2
     by pip installing psycopg2-binary.
     This is solved by installing from source as described in their documentation.
-- [pandas], optional (used for csv storage plugin)
-- [pymongo], optional, (used for mongodb storage plugin)
-- [mongomock], optional, used for testing mongodb storage plugin.
+- [pandas], used by the `csv` storage plugin
+- [requests], used by `http` storage plugin
+- [jinja2], used by `template` storage plugin
+- [pymongo], used by the `mongodb` storage plugin
 
 These optional dependencies can be installed with
 
+    pip install -r requirements_full.txt
+
+Separate requirements can also be installed for development
+
     pip install -r requirements_dev.txt
 
+or for building the documentation
+
+    pip install -r requirements_doc.txt
+
+
 
 [tripper]: https://pypi.org/project/tripper/
+[pint]: https://pint.readthedocs.io/en/stable/
+[pydantic]: https://docs.pydantic.dev/
+[typing_extensions]: https://github.com/python/typing_extensions
+[rdflib]: https://rdflib.readthedocs.io/
 [PyYAML]: https://pypi.org/project/PyYAML/
 [psycopg2]: https://pypi.org/project/psycopg2/
 [pandas]: https://pandas.pydata.org/
 [pymongo]: https://github.com/mongodb/mongo-python-driver
 [mongomock]: https://github.com/mongomock/mongomock
+[openpyxl]: https://openpyxl.readthedocs.io/
+[requests]: https://requests.readthedocs.io/
+[jinja2]: https://jinja.palletsprojects.com/

doc/getting_started/installation.md

@@ -10,21 +10,34 @@ Installing with pip
 If you are using Python, the easiest way to install DLite is with pip:
 
 ```shell
-pip install DLite-Python
+pip install DLite-Python[full]
 ```
 
-This will give you DLite together with all optional Python dependencies (see [runtime dependencies]).
+The bracket `[full]` is optional, but ensures that you install DLite
+together with all optional dependencies needed for additional features
+and storage plugins. (see also [runtime dependencies]).
 
 
 Development installation
 ------------------------
 If you to contribute or develop DLite, you should [build from source].
 
+Python dependencies for development can be installed with
+
+```shell
+pip install DLite-Python[dev]
+```
+
+Install additional Python packages for building documentation with
+
+```shell
+pip install DLite-Python[doc]
+```
 
 
 [Python]: https://www.python.org/
 [Fortran]: https://en.wikipedia.org/wiki/Fortran
 [HDF5]: https://support.hdfgroup.org/HDF5/
 [librdf]: https://librdf.org/
-[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime-dependencies.html
+[runtime dependencies]: https://sintef.github.io/dlite/getting_started/build/runtime_dependencies.html
 [build from source]: https://sintef.github.io/dlite/getting_started/build/build.html

python/setup.py

@@ -158,27 +158,23 @@ def build_extension(self, ext: CMakeExtension) -> None:
 
 requirements = ["numpy"]
 
-# Read extra_requirements from requirements_full.txt
-#with open(SOURCE_DIR / "requirements_full.txt", "r") as f:
-#    extra_requirements = [
-#        line.strip() for line in f.readlines() if not line.startswith("#")
-#    ]
-extra_requirements = [  # TO BE REPLACED WITH THE ABOVE, SEE ISSUE #601
-    "fortran-language-server>=1.12.0,<1.13",
-    "PyYAML>=5.4.1,<7",
-    "psycopg2-binary==2.9.5",
-    "pandas>=1.2,<2.1",
-    "rdflib>=4.2.1,<7",
-    "pint>=0.15,<1",
-    "openpyxl>=3.0.9,<3.2",
-    "pymongo>=4.4.0,<5",
-    "tripper>=0.2.5,<0.3",
-    "requests>=2.10,<3",
-    "pydantic>=1.10.0,<2",
-    #"pydantic>=1.10.0,<3",
-    #"typing_extensions>=4.1,<5",
-    #"jsonschema>=4.0,<4.18",
-]
+# Populate extra_requirements from requirements_*.txt
+extra_requirements = {}
+for name in "full", "dev", "doc":
+    with open(SOURCE_DIR / f"requirements_{name}.txt", "r") as f:
+        extra_requirements[name] = [
+            line.strip() for line in f.readlines() if not line.startswith("#")
+        ]
+
+# Temporary workaround!
+# Require pydantic <2. Needed before we have managed to install rust in
+# the docker image for wheels
+extras = extra_requirements["full"]
+for i, pkg in enumerate(extras):
+    match = re.match("^(pydantic>.*<)", pkg)
+    if match:
+        extras[i] = f"{match.groups()[0]}2"
+
 
 version = re.search(
     r"project\([^)]*VERSION\s+([0-9.]+)",
@@ -215,10 +211,8 @@ def build_extension(self, ext: CMakeExtension) -> None:
         "Programming Language :: Python :: 3.11",
         "Topic :: Software Development :: Libraries :: Python Modules",
     ],
-    install_requires=requirements + extra_requirements,
-    # For now, the extra requirements are hard requirements.
-    # See issue #222: https://github.com/SINTEF/dlite/issues/222
-    # extras_require={"all": extra_requirements},
+    install_requires=requirements,
+    extras_require=extra_requirements,
     packages=["dlite"],
     scripts=[
         str(SOURCE_DIR / "bindings" / "python" / "scripts" / "dlite-validate"),

requirements_dev.txt

@@ -3,4 +3,4 @@ ipython>=7.34.0,<9
 ipykernel>=6.0.1,<7
 ipython_genutils~=0.2.0
 mongomock>=4.1.2,<5
-pydantic>=1.9.0,<2
+openpyxl>=3.0.9,<3.2

requirements_full.txt

@@ -1,14 +1,16 @@
+# Optional requirements - used by various plugins, features or tests
 fortran-language-server>=1.12.0,<1.13
 PyYAML>=5.4.1,<7
 # psycopg2-binary can lead to segfault - so far all seems good with v2.9.5
 psycopg2-binary==2.9.5
 pandas>=1.2,<2.1
 rdflib>=4.2.1,<7
 pint>=0.15,<1
-openpyxl>=3.0.9,<3.2
 pymongo>=4.4.0,<5
 tripper>=0.2.5,<0.3
 requests>=2.10,<3
+jinja2>=3.0,<4
 #pydantic>=1.10.0,<2
 pydantic>=1.10.0,<3
 typing_extensions>=4.1,<5
+#jsonschema>=4.0,<4.18