Merge pull request #838 from MolSSI/rel_55

Release v0.55
MolSSI · May 23, 2024 · eea42a7 · eea42a7
2 parents 3ead276 + 671556d
commit eea42a7
Show file tree

Hide file tree

Showing 19 changed files with 58 additions and 123 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,11 +1,7 @@
-<!-- Thank you for your contribution! -->
-
 ## Description
+<!-- Thank you for your contribution! -->
 <!-- Provide a brief description of the PR's purpose here. -->
 
-## Changelog description
-<!-- Provide a brief single sentence for the changelog. -->
-
 ## Status
 - [ ] Code base linted
 - [ ] Ready to go
diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml
@@ -19,7 +19,7 @@ jobs:
         with:
           fetch-depth: 0 # needed for tag/version
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 

diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml
@@ -28,7 +28,7 @@ jobs:
         run: sudo apt update && sudo apt install postgresql postgresql-contrib
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

diff --git a/.github/workflows/full_tests.yml b/.github/workflows/full_tests.yml
@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -39,11 +39,9 @@ jobs:
       # QCFractal Server
       #################################################
       - name: Configure conda for the QCFractal server
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: qcarchive-server
           environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_server.yaml
           auto-activate-base: false
@@ -72,11 +70,9 @@ jobs:
       # QCFractalCompute worker
       #################################################
       - name: Configure conda for QCFractalCompute worker
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: qcarchive-worker
           environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_worker.yaml
           auto-activate-base: false
@@ -104,11 +100,9 @@ jobs:
       # QCPortal client
       #################################################
       - name: Configure conda for the QCPortal client
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: qcarchive-qcportal
           environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_qcportal.yaml
           auto-activate-base: false

diff --git a/.github/workflows/full_tests_snowflake.yml b/.github/workflows/full_tests_snowflake.yml
@@ -24,7 +24,7 @@ jobs:
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -34,11 +34,9 @@ jobs:
           fetch-depth: 0
 
       - name: Configure conda for the snowflake
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ matrix.python-version }}
-          miniforge-variant: Mambaforge
-          miniforge-version: latest
           activate-environment: qcarchive-snowflake
           environment-file: qcarchivetesting/conda-envs/fulltest_snowflake.yaml
           auto-activate-base: false

diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst
@@ -1,6 +1,35 @@
 Release Notes
 =============
 
+0.55 / 2024-05-23
+-------------------
+
+Some moderate improvements - mainly, the use of ASPW for SQLite, and the removal of the never-used manager log and
+serverinfo log tables. Also adds maintenance jobs for removing old access log and completed internal jobs.
+
+This release also adds the ability to more-strictly handle queue tags (to prevent managers with `*` as a tag from pulling
+everything).
+
+In addition, lots of smaller bug fixes and improvements.
+
+Notable PRs:
+
+- (:pr:`819`) Fix native_files fields return from server & ORM
+- (:pr:`821`) Enable dumping a database from a snowflake 
+- (:pr:`822`) Fix missing client on cached dataset records
+- (:pr:`825`) Fix database constraint violation when program version contains uppercase characters 
+- (:pr:`826`) Fix type of results in ServiceSubtaskRecord
+- (:pr:`829`) Replace sqlite3 (python stdlib module) with APSW
+- (:pr:`830`) Remove server stats & compute manager logs
+- (:pr:`831`) Add internal job to delete old access log entries
+- (:pr:`832`) Add internal job to delete old, finished internal jobs
+- (:pr:`834`) Add property to get errored child records
+- (:pr:`835`) Better handling of queue tags (strict queue tags and case insensitivity)
+- (:pr:`836`) Replace gunicorn with waitress and fix logging issues
+- (:pr:`837`) Commit to database after every returned task is processed (to help prevent deadlocks)
+- (:pr:`838`) Better handling of passwords with `init-db`
+
+
 0.54.1 / 2024-04-12
 -------------------
 

diff --git a/qcarchivetesting/conda-envs/fulltest_qcportal.yaml b/qcarchivetesting/conda-envs/fulltest_qcportal.yaml
@@ -12,7 +12,7 @@ dependencies:
   - pydantic
   - zstandard
   - apsw
-  - qcelemental
+  - qcelemental<0.70a0
   - tabulate
   - tqdm
   - pandas

diff --git a/qcarchivetesting/conda-envs/fulltest_server.yaml b/qcarchivetesting/conda-envs/fulltest_server.yaml
@@ -16,7 +16,7 @@ dependencies:
   - pydantic
   - zstandard
   - apsw
-  - qcelemental
+  - qcelemental<0.70a0
   - tabulate
   - tqdm
   - pandas

diff --git a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml
@@ -1,7 +1,6 @@
 name: qcarchive-snowflake
 channels:
   - conda-forge
-  - conda-forge/label/libint_dev
   - defaults
 
 dependencies:
@@ -17,7 +16,7 @@ dependencies:
   - pydantic
   - zstandard
   - apsw
-  - qcelemental
+  - qcelemental<0.70a0
   - tabulate
   - tqdm
   - pandas
@@ -44,8 +43,8 @@ dependencies:
   - torsiondrive
 
   # Worker codes below
-  - qcengine
-  - psi4>=1.7
+  - qcengine<0.70a0
+  - psi4>=1.9.1
   - rdkit
 
   # Testing packages

diff --git a/qcarchivetesting/conda-envs/fulltest_worker.yaml b/qcarchivetesting/conda-envs/fulltest_worker.yaml
@@ -1,7 +1,6 @@
 name: qcarchive-worker
 channels:
   - conda-forge
-  - conda-forge/label/libint_dev
   - defaults
 
 dependencies:
@@ -16,7 +15,7 @@ dependencies:
   - pydantic
   - zstandard
   - apsw
-  - qcelemental
+  - qcelemental<0.70a0
   - tabulate
   - tqdm
   - pandas
@@ -30,8 +29,8 @@ dependencies:
   - parsl
 
   # Worker codes below
-  - qcengine
-  - psi4>=1.7
+  - qcengine<0.70a0
+  - psi4>=1.9.1
   - rdkit
 
   # Geometric service

diff --git a/qcarchivetesting/pyproject.toml b/qcarchivetesting/pyproject.toml
@@ -22,7 +22,7 @@ classifiers = [
 ]
 dependencies = [
     "qcfractal",
-    "qcengine",
+    "qcengine<0.70a0",
     "pytest",
 ]
 

diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml
diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml
diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml
diff --git a/qcfractal/qcfractal/components/internal_jobs/test_client.py b/qcfractal/qcfractal/components/internal_jobs/test_client.py
@@ -111,7 +111,7 @@ def test_internal_jobs_client_cancel_running(snowflake: QCATestingSnowflake):
 
         job_1 = snowflake_client.get_internal_job(id_1)
         assert job_1.status == InternalJobStatusEnum.cancelled
-        assert job_1.progress < 50
+        assert job_1.progress < 70
         assert job_1.result == "Internal job cancelled"
 
     finally:

diff --git a/qcfractal/qcfractal/components/internal_jobs/test_socket.py b/qcfractal/qcfractal/components/internal_jobs/test_socket.py
@@ -127,7 +127,7 @@ def test_internal_jobs_socket_recover(storage_socket: SQLAlchemySocket, session:
     end_event = threading.Event()
     th = threading.Thread(target=storage_socket.internal_jobs.run_loop, args=(end_event,))
     th.start()
-    time.sleep(15)
+    time.sleep(30)
 
     try:
         session.expire(job_1)

diff --git a/qcfractal/qcfractal/qcfractal_server_cli.py b/qcfractal/qcfractal/qcfractal_server_cli.py
@@ -547,6 +547,12 @@ def server_start_api(config):
 
 
 def server_upgrade_db(config):
+
+    # Always set logging level to INFO, otherwise things are a bit quiet
+    root_logger = logging.getLogger()
+    if root_logger.level > logging.INFO:
+        root_logger.setLevel(logging.INFO)
+
     logger = logging.getLogger(__name__)
 
     # Don't use start_database - we don't want to create the socket (which

diff --git a/qcfractalcompute/qcfractalcompute/test_compute_manager.py b/qcfractalcompute/qcfractalcompute/test_compute_manager.py
@@ -226,13 +226,13 @@ def test_manager_missed_heartbeats_shutdown(snowflake: QCATestingSnowflake):
 
     snowflake.stop_api()
 
-    for i in range(60):
+    for i in range(90):
         time.sleep(1)
 
         if not compute_thread.is_alive():
             break
     else:
-        raise RuntimeError("Compute thread did not stop in 60 seconds")
+        raise RuntimeError("Compute thread did not stop in 90 seconds")
 
     compute_thread._compute_thread.join(5)
     assert compute_thread.is_alive() is False
diff --git a/qcportal/pyproject.toml b/qcportal/pyproject.toml
@@ -28,7 +28,7 @@ dependencies = [
     "pydantic",
     "zstandard",
     "apsw",
-    "qcelemental",
+    "qcelemental<0.70a0",
     "tabulate",
     "tqdm",
     "pandas",