From 1db1a70ba6954eefc5b9a7f2fc21e7a5a84b40c1 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Mon, 20 May 2024 10:54:50 -0400 Subject: [PATCH 01/10] set log level to at least info for upgrade-db --- qcfractal/qcfractal/qcfractal_server_cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/qcfractal/qcfractal/qcfractal_server_cli.py b/qcfractal/qcfractal/qcfractal_server_cli.py index 11bde9442..944a5bb22 100644 --- a/qcfractal/qcfractal/qcfractal_server_cli.py +++ b/qcfractal/qcfractal/qcfractal_server_cli.py @@ -547,6 +547,12 @@ def server_start_api(config): def server_upgrade_db(config): + + # Always set logging level to INFO, otherwise things are a bit quiet + root_logger = logging.getLogger() + if root_logger.level > logging.INFO: + root_logger.setLevel(logging.INFO) + logger = logging.getLogger(__name__) # Don't use start_database - we don't want to create the socket (which From 7b944a1062427cd6db1651cbf2a4ac7eb5017696 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Mon, 20 May 2024 11:33:10 -0400 Subject: [PATCH 02/10] Premptively pin qcengine & qcelemental --- qcarchivetesting/conda-envs/fulltest_qcportal.yaml | 2 +- qcarchivetesting/conda-envs/fulltest_server.yaml | 2 +- qcarchivetesting/conda-envs/fulltest_snowflake.yaml | 4 ++-- qcarchivetesting/conda-envs/fulltest_worker.yaml | 4 ++-- qcarchivetesting/pyproject.toml | 2 +- qcportal/pyproject.toml | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/qcarchivetesting/conda-envs/fulltest_qcportal.yaml b/qcarchivetesting/conda-envs/fulltest_qcportal.yaml index 5f480abd6..5feedf2d6 100644 --- a/qcarchivetesting/conda-envs/fulltest_qcportal.yaml +++ b/qcarchivetesting/conda-envs/fulltest_qcportal.yaml @@ -12,7 +12,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas diff --git a/qcarchivetesting/conda-envs/fulltest_server.yaml b/qcarchivetesting/conda-envs/fulltest_server.yaml index 4d77382bd..dceb74bdf 100644 --- a/qcarchivetesting/conda-envs/fulltest_server.yaml +++ b/qcarchivetesting/conda-envs/fulltest_server.yaml @@ -16,7 +16,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas diff --git a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml index 3dc8ca25d..4baca13bf 100644 --- a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml +++ b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml @@ -17,7 +17,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas @@ -44,7 +44,7 @@ dependencies: - torsiondrive # Worker codes below - - qcengine + - qcengine<0.70a0 - psi4>=1.7 - rdkit diff --git a/qcarchivetesting/conda-envs/fulltest_worker.yaml b/qcarchivetesting/conda-envs/fulltest_worker.yaml index 26afa2c62..18bf0ec9c 100644 --- a/qcarchivetesting/conda-envs/fulltest_worker.yaml +++ b/qcarchivetesting/conda-envs/fulltest_worker.yaml @@ -16,7 +16,7 @@ dependencies: - pydantic - zstandard - apsw - - qcelemental + - qcelemental<0.70a0 - tabulate - tqdm - pandas @@ -30,7 +30,7 @@ dependencies: - parsl # Worker codes below - - qcengine + - qcengine<0.70a0 - psi4>=1.7 - rdkit diff --git a/qcarchivetesting/pyproject.toml b/qcarchivetesting/pyproject.toml index a8cc47de4..eae4055c2 100644 --- a/qcarchivetesting/pyproject.toml +++ b/qcarchivetesting/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ ] dependencies = [ "qcfractal", - "qcengine", + "qcengine<0.70a0", "pytest", ] diff --git a/qcportal/pyproject.toml b/qcportal/pyproject.toml index 3d379e724..5b3db3028 100644 --- a/qcportal/pyproject.toml +++ b/qcportal/pyproject.toml @@ -28,7 +28,7 @@ dependencies = [ "pydantic", "zstandard", "apsw", - "qcelemental", + "qcelemental<0.70a0", "tabulate", "tqdm", "pandas", From e9af2a9ae39f85008f92d33c201470c75e964e46 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Tue, 21 May 2024 12:05:33 -0400 Subject: [PATCH 03/10] Modify PR template (I take the PR one-liner from the title) --- .github/PULL_REQUEST_TEMPLATE.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c01c84bcd..e2bcc4ab2 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,7 @@ - - ## Description + -## Changelog description - - ## Status - [ ] Code base linted - [ ] Ready to go From ea438b026199a79f0f2f73219b510cee13cd0c19 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Tue, 21 May 2024 15:21:08 -0400 Subject: [PATCH 04/10] Remove old compute configs --- .../gha_fractal_oldcompute_dask.yaml | 30 ------------------- .../gha_fractal_oldcompute_parsl.yaml | 30 ------------------- .../gha_fractal_oldcompute_pool.yaml | 26 ---------------- 3 files changed, 86 deletions(-) delete mode 100644 qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml delete mode 100644 qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml delete mode 100644 qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml deleted file mode 100644 index 97b5e57a6..000000000 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_dask.yaml +++ /dev/null @@ -1,30 +0,0 @@ -common: - adapter: dask - cores_per_rank: 1 - cores_per_worker: 2 - max_workers: 2 - memory_per_worker: 2 - nodes_per_job: 1 - nodes_per_task: 1 - retries: 2 - scratch_directory: /tmp - tasks_per_worker: 1 - verbose: true - -manager: - log_file_prefix: qcfractal_compute.log - manager_name: testworker - max_queued_tasks: 3 - update_frequency: 5.0 - queue_tag: - - '*' - -server: - fractal_uri: http://localhost:7900 - password: null - username: null - verify: false - -cluster: - node_exclusivity: True - scheduler: local diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml deleted file mode 100644 index 4e01e649e..000000000 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_parsl.yaml +++ /dev/null @@ -1,30 +0,0 @@ -common: - adapter: parsl - cores_per_rank: 1 - cores_per_worker: 2 - max_workers: 1 - memory_per_worker: 2 - nodes_per_job: 1 - nodes_per_task: 1 - retries: 2 - scratch_directory: /tmp - tasks_per_worker: 1 - verbose: true - -manager: - log_file_prefix: qcfractal_compute.log - manager_name: testworker - max_queued_tasks: 3 - update_frequency: 5.0 - queue_tag: - - '*' - -server: - fractal_uri: http://localhost:7900 - password: null - username: null - verify: false - -cluster: - node_exclusivity: True - scheduler: local diff --git a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml b/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml deleted file mode 100644 index 4bfbcaf89..000000000 --- a/qcarchivetesting/qcarchivetesting/config_files/gha_fractal_oldcompute_pool.yaml +++ /dev/null @@ -1,26 +0,0 @@ -common: - adapter: pool - cores_per_rank: 1 - cores_per_worker: 2 - max_workers: 1 - memory_per_worker: 2 - nodes_per_job: 1 - nodes_per_task: 1 - retries: 2 - scratch_directory: /tmp - tasks_per_worker: 1 - verbose: true - -manager: - log_file_prefix: qcfractal_compute.log - manager_name: testworker - max_queued_tasks: 3 - update_frequency: 5.0 - queue_tag: - - '*' - -server: - fractal_uri: http://localhost:7900 - password: null - username: null - verify: false From 66365c102654f596d08520ea4bfd477b99e88bf3 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Tue, 21 May 2024 16:33:32 -0400 Subject: [PATCH 05/10] Remove libint_dev channel from test envs and update psi4 version --- qcarchivetesting/conda-envs/fulltest_snowflake.yaml | 3 +-- qcarchivetesting/conda-envs/fulltest_worker.yaml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml index 4baca13bf..aa0fe8c48 100644 --- a/qcarchivetesting/conda-envs/fulltest_snowflake.yaml +++ b/qcarchivetesting/conda-envs/fulltest_snowflake.yaml @@ -1,7 +1,6 @@ name: qcarchive-snowflake channels: - conda-forge - - conda-forge/label/libint_dev - defaults dependencies: @@ -45,7 +44,7 @@ dependencies: # Worker codes below - qcengine<0.70a0 - - psi4>=1.7 + - psi4>=1.9.1 - rdkit # Testing packages diff --git a/qcarchivetesting/conda-envs/fulltest_worker.yaml b/qcarchivetesting/conda-envs/fulltest_worker.yaml index 18bf0ec9c..cd0ea2e0b 100644 --- a/qcarchivetesting/conda-envs/fulltest_worker.yaml +++ b/qcarchivetesting/conda-envs/fulltest_worker.yaml @@ -1,7 +1,6 @@ name: qcarchive-worker channels: - conda-forge - - conda-forge/label/libint_dev - defaults dependencies: @@ -31,7 +30,7 @@ dependencies: # Worker codes below - qcengine<0.70a0 - - psi4>=1.7 + - psi4>=1.9.1 - rdkit # Geometric service From a1ee3763cf54b24c93b798890928ae7464fd19dd Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Tue, 21 May 2024 19:22:26 -0400 Subject: [PATCH 06/10] Update GH actions versions --- .github/workflows/build_docs.yml | 2 +- .github/workflows/core_tests.yml | 2 +- .github/workflows/full_tests.yml | 14 ++++---------- .github/workflows/full_tests_snowflake.yml | 6 ++---- 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 3047d3f50..f27950201 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -19,7 +19,7 @@ jobs: with: fetch-depth: 0 # needed for tag/version - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.11' diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml index ceda1fe55..ef2634dc7 100644 --- a/.github/workflows/core_tests.yml +++ b/.github/workflows/core_tests.yml @@ -28,7 +28,7 @@ jobs: run: sudo apt update && sudo apt install postgresql postgresql-contrib - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/full_tests.yml b/.github/workflows/full_tests.yml index 3183474f6..51fe91b5b 100644 --- a/.github/workflows/full_tests.yml +++ b/.github/workflows/full_tests.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -39,11 +39,9 @@ jobs: # QCFractal Server ################################################# - name: Configure conda for the QCFractal server - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-server environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_server.yaml auto-activate-base: false @@ -72,11 +70,9 @@ jobs: # QCFractalCompute worker ################################################# - name: Configure conda for QCFractalCompute worker - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-worker environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_worker.yaml auto-activate-base: false @@ -104,11 +100,9 @@ jobs: # QCPortal client ################################################# - name: Configure conda for the QCPortal client - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-qcportal environment-file: qcarchive/qcarchivetesting/conda-envs/fulltest_qcportal.yaml auto-activate-base: false diff --git a/.github/workflows/full_tests_snowflake.yml b/.github/workflows/full_tests_snowflake.yml index 44e5a47e5..9b6a0138b 100644 --- a/.github/workflows/full_tests_snowflake.yml +++ b/.github/workflows/full_tests_snowflake.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -34,11 +34,9 @@ jobs: fetch-depth: 0 - name: Configure conda for the snowflake - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - miniforge-variant: Mambaforge - miniforge-version: latest activate-environment: qcarchive-snowflake environment-file: qcarchivetesting/conda-envs/fulltest_snowflake.yaml auto-activate-base: false From 304814e720380d4927719a4d6719bda9bf988130 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Wed, 22 May 2024 10:00:24 -0400 Subject: [PATCH 07/10] Fix flaky test --- qcfractal/qcfractal/components/internal_jobs/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcfractal/qcfractal/components/internal_jobs/test_client.py b/qcfractal/qcfractal/components/internal_jobs/test_client.py index ded340ddc..f23aef068 100644 --- a/qcfractal/qcfractal/components/internal_jobs/test_client.py +++ b/qcfractal/qcfractal/components/internal_jobs/test_client.py @@ -111,7 +111,7 @@ def test_internal_jobs_client_cancel_running(snowflake: QCATestingSnowflake): job_1 = snowflake_client.get_internal_job(id_1) assert job_1.status == InternalJobStatusEnum.cancelled - assert job_1.progress < 50 + assert job_1.progress < 70 assert job_1.result == "Internal job cancelled" finally: From 399cc418689b57e15fd7a673de6690a9fec04d5b Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Wed, 22 May 2024 11:27:03 -0400 Subject: [PATCH 08/10] Increase timeout in manager shutdown test Now that there is reconnect options & backoff, this test can sometimes take more than 60 seconds --- qcfractalcompute/qcfractalcompute/test_compute_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qcfractalcompute/qcfractalcompute/test_compute_manager.py b/qcfractalcompute/qcfractalcompute/test_compute_manager.py index 6bad2c813..2b5cdd02e 100644 --- a/qcfractalcompute/qcfractalcompute/test_compute_manager.py +++ b/qcfractalcompute/qcfractalcompute/test_compute_manager.py @@ -226,13 +226,13 @@ def test_manager_missed_heartbeats_shutdown(snowflake: QCATestingSnowflake): snowflake.stop_api() - for i in range(60): + for i in range(90): time.sleep(1) if not compute_thread.is_alive(): break else: - raise RuntimeError("Compute thread did not stop in 60 seconds") + raise RuntimeError("Compute thread did not stop in 90 seconds") compute_thread._compute_thread.join(5) assert compute_thread.is_alive() is False From 5fcd813e92635cb206c303c29cb84e423f410466 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Wed, 22 May 2024 13:38:27 -0400 Subject: [PATCH 09/10] More flaky tests --- qcfractal/qcfractal/components/internal_jobs/test_socket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qcfractal/qcfractal/components/internal_jobs/test_socket.py b/qcfractal/qcfractal/components/internal_jobs/test_socket.py index d1fd26c0e..3b8aa066b 100644 --- a/qcfractal/qcfractal/components/internal_jobs/test_socket.py +++ b/qcfractal/qcfractal/components/internal_jobs/test_socket.py @@ -127,7 +127,7 @@ def test_internal_jobs_socket_recover(storage_socket: SQLAlchemySocket, session: end_event = threading.Event() th = threading.Thread(target=storage_socket.internal_jobs.run_loop, args=(end_event,)) th.start() - time.sleep(15) + time.sleep(30) try: session.expire(job_1) From 671556dada5287206ad80a27b64605b10a8a80a0 Mon Sep 17 00:00:00 2001 From: Benjamin Pritchard Date: Thu, 23 May 2024 10:37:00 -0400 Subject: [PATCH 10/10] Update release notes --- docs/source/release_notes.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index 493710905..26e7ff3f2 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -1,6 +1,35 @@ Release Notes ============= +0.55 / 2024-05-23 +------------------- + +Some moderate improvements - mainly, the use of ASPW for SQLite, and the removal of the never-used manager log and +serverinfo log tables. Also adds maintenance jobs for removing old access log and completed internal jobs. + +This release also adds the ability to more-strictly handle queue tags (to prevent managers with `*` as a tag from pulling +everything). + +In addition, lots of smaller bug fixes and improvements. + +Notable PRs: + +- (:pr:`819`) Fix native_files fields return from server & ORM +- (:pr:`821`) Enable dumping a database from a snowflake +- (:pr:`822`) Fix missing client on cached dataset records +- (:pr:`825`) Fix database constraint violation when program version contains uppercase characters +- (:pr:`826`) Fix type of results in ServiceSubtaskRecord +- (:pr:`829`) Replace sqlite3 (python stdlib module) with APSW +- (:pr:`830`) Remove server stats & compute manager logs +- (:pr:`831`) Add internal job to delete old access log entries +- (:pr:`832`) Add internal job to delete old, finished internal jobs +- (:pr:`834`) Add property to get errored child records +- (:pr:`835`) Better handling of queue tags (strict queue tags and case insensitivity) +- (:pr:`836`) Replace gunicorn with waitress and fix logging issues +- (:pr:`837`) Commit to database after every returned task is processed (to help prevent deadlocks) +- (:pr:`838`) Better handling of passwords with `init-db` + + 0.54.1 / 2024-04-12 -------------------