From 2fb9abc1c4ab6cc320e7a9e2bbab8950ff36d6a2 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 10 Oct 2024 13:23:20 -0600 Subject: [PATCH 1/7] fix #3438 --- CHANGELOG.md | 2 +- qiita_pet/templates/study_ajax/prep_summary.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41c4c821b..438075906 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Deployed on October 14th, 2024 * `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support. * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! -* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425). +* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439). Version 2024.07 diff --git a/qiita_pet/templates/study_ajax/prep_summary.html b/qiita_pet/templates/study_ajax/prep_summary.html index c1b823291..25674e8a1 100644 --- a/qiita_pet/templates/study_ajax/prep_summary.html +++ b/qiita_pet/templates/study_ajax/prep_summary.html @@ -434,7 +434,7 @@

{{name}} - ID {{prep_id}} ({{data_type}}) {% if user_level in ('admin', 'wet-lab admin') and creation_job is not None %} - SampleSheet + SampleSheet {% if creation_job_artifact_summary is not None %} Creation Job Output {% end %} From 54e76c0edf16f1658fcde14ecd65d15e73b52219 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 10 Oct 2024 14:13:52 -0600 Subject: [PATCH 2/7] fix #3436 --- qiita_db/metadata_template/prep_template.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 7ae97858c..bf4992155 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -272,6 +272,11 @@ def delete(cls, id_): "Cannot remove prep template %d because it has an artifact" " associated with it" % id_) + # artifacts that are archived are not returned as part of the code + # above and we need to clean them before moving forward + for aa in self.archived_artifacts(): + qdb.artifact.Artifact.delete(aa.id) + # Delete the prep template filepaths sql = """DELETE FROM qiita.prep_template_filepath WHERE prep_template_id = %s""" From b9ece694174af1414eabf43b33bdb71b5a33f562 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 10 Oct 2024 14:19:38 -0600 Subject: [PATCH 3/7] self->cls --- CHANGELOG.md | 2 +- qiita_db/metadata_template/prep_template.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 438075906..f52e9103f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ Deployed on October 14th, 2024 * `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support. * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! -* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439). +* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). Version 2024.07 diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index bf4992155..d0c3da5d0 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -274,7 +274,7 @@ def delete(cls, id_): # artifacts that are archived are not returned as part of the code # above and we need to clean them before moving forward - for aa in self.archived_artifacts(): + for aa in cls(id_).archived_artifacts(): qdb.artifact.Artifact.delete(aa.id) # Delete the prep template filepaths From 8115b59e24f867acd76f3f63a3e3ec8ccb9d347e Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Thu, 10 Oct 2024 14:56:00 -0600 Subject: [PATCH 4/7] rm () --- qiita_db/metadata_template/prep_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index d0c3da5d0..61dd3e6a6 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -274,7 +274,7 @@ def delete(cls, id_): # artifacts that are archived are not returned as part of the code # above and we need to clean them before moving forward - for aa in cls(id_).archived_artifacts(): + for aa in cls(id_).archived_artifacts: qdb.artifact.Artifact.delete(aa.id) # Delete the prep template filepaths From 959fcbc9d0c561ba6c0ba07f98dda347edc40b68 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 11 Oct 2024 08:36:39 -0600 Subject: [PATCH 5/7] adding tests --- CHANGELOG.md | 2 ++ qiita_db/artifact.py | 3 ++- qiita_db/metadata_template/prep_template.py | 11 +++++++-- qiita_db/test/test_artifact.py | 27 +++++++++++++++++++-- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f52e9103f..d29e6491d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Deployed on October 14th, 2024 * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! * Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). +* General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225). + Version 2024.07 diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index f116236f7..bf81ddf41 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -929,7 +929,8 @@ def can_be_submitted_to_ebi(self): # words has more that one processing step behind it fine_to_send = [] fine_to_send.extend([pt.artifact for pt in self.prep_templates]) - fine_to_send.extend([c for a in fine_to_send for c in a.children]) + fine_to_send.extend([c for a in fine_to_send if a is not None + for c in a.children]) if self not in fine_to_send: return False diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 61dd3e6a6..52f781b9a 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -274,8 +274,15 @@ def delete(cls, id_): # artifacts that are archived are not returned as part of the code # above and we need to clean them before moving forward - for aa in cls(id_).archived_artifacts: - qdb.artifact.Artifact.delete(aa.id) + sql = """SELECT artifact_id + FROM qiita.preparation_artifact + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, args) + archived_artifacts = set( + qdb.sql_connection.TRN.execute_fetchflatten()) + if archived_artifacts: + for aid in archived_artifacts: + qdb.artifact.Artifact.delete(aid) # Delete the prep template filepaths sql = """DELETE FROM qiita.prep_template_filepath diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 76833ed98..6bb18b409 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -23,6 +23,7 @@ from qiita_core.util import qiita_test_checker from qiita_core.testing import wait_for_processing_job import qiita_db as qdb +from qiita_ware.private_plugin import _delete_analysis_artifacts class ArtifactTestsReadOnly(TestCase): @@ -1518,7 +1519,7 @@ def test_archive(self): 'be archived'): A.archive(8) - for aid in range(4, 7): + for aid in range(5, 7): ms = A(aid).merging_scheme A.archive(aid) self.assertEqual(ms, A(aid).merging_scheme) @@ -1526,7 +1527,29 @@ def test_archive(self): self.assertCountEqual(A(1).descendants.nodes(), exp_nodes) obs_artifacts = len(qdb.util.get_artifacts_information([4, 5, 6, 8])) - self.assertEqual(1, obs_artifacts) + self.assertEqual(2, obs_artifacts) + + # now let's try to delete the prep + PT = qdb.metadata_template.prep_template.PrepTemplate + QEE = qdb.exceptions.QiitaDBExecutionError + pt = A(1).prep_templates[0] + # it should fail as this prep is public and have been submitted to ENA + with self.assertRaisesRegex(QEE, 'Cannot remove prep template 1'): + PT.delete(pt.id) + # now, remove those restrictions + analysis + linked artifacts + sql = "DELETE FROM qiita.artifact_processing_job" + qdb.sql_connection.perform_as_transaction(sql) + sql = "DELETE FROM qiita.ebi_run_accession" + qdb.sql_connection.perform_as_transaction(sql) + sql = "UPDATE qiita.artifact SET visibility_id = 1" + qdb.sql_connection.perform_as_transaction(sql) + _delete_analysis_artifacts(qdb.analysis.Analysis(1)) + _delete_analysis_artifacts(qdb.analysis.Analysis(2)) + _delete_analysis_artifacts(qdb.analysis.Analysis(3)) + for aid in [3, 2, 1]: + A.delete(aid) + + PT.delete(1) if __name__ == '__main__': From add088ab6ea9a578621ee5772e382db3dec69acb Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 11 Oct 2024 09:11:15 -0600 Subject: [PATCH 6/7] fix test --- qiita_db/test/test_artifact.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 6bb18b409..1cc2fb632 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -10,7 +10,7 @@ from tempfile import mkstemp, mkdtemp from datetime import datetime from os import close, remove -from os.path import exists, join, basename +from os.path import exists, join, basename, dirname, abspath from shutil import copyfile from functools import partial from json import dumps @@ -1529,7 +1529,22 @@ def test_archive(self): obs_artifacts = len(qdb.util.get_artifacts_information([4, 5, 6, 8])) self.assertEqual(2, obs_artifacts) - # now let's try to delete the prep + # in the tests above we generated and validated archived artifacts + # so this allows us to add tests to delete a prep-info with archived + # artifacts. The first bottleneck to do this is that this tests will + # actually remove files, which we will need for other tests so lets + # make a copy and then restore them + mfolder = dirname(dirname(abspath(__file__))) + mpath = join(mfolder, 'support_files', 'test_data') + mp = partial(join, mpath) + fps = [ + mp('processed_data/1_study_1001_closed_reference_otu_table.biom'), + mp('processed_data/1_study_1001_closed_reference_otu_table_Silva.biom'), + mp('raw_data/1_s_G1_L001_sequences.fastq.gz'), + mp('raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz')] + for fp in fps: + copyfile(fp, f'{fp}.bk') + PT = qdb.metadata_template.prep_template.PrepTemplate QEE = qdb.exceptions.QiitaDBExecutionError pt = A(1).prep_templates[0] @@ -1549,8 +1564,11 @@ def test_archive(self): for aid in [3, 2, 1]: A.delete(aid) - PT.delete(1) + PT.delete(pt.id) + # bringing back the filepaths + for fp in fps: + copyfile(f'{fp}.bk', fp) if __name__ == '__main__': main() From 311d61832e7da7491a8a36119d4b165e2ab09174 Mon Sep 17 00:00:00 2001 From: Antonio Gonzalez Date: Fri, 11 Oct 2024 09:30:02 -0600 Subject: [PATCH 7/7] fix flake8 --- qiita_db/test/test_artifact.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 1cc2fb632..bed71c45a 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -1539,7 +1539,8 @@ def test_archive(self): mp = partial(join, mpath) fps = [ mp('processed_data/1_study_1001_closed_reference_otu_table.biom'), - mp('processed_data/1_study_1001_closed_reference_otu_table_Silva.biom'), + mp('processed_data/' + '1_study_1001_closed_reference_otu_table_Silva.biom'), mp('raw_data/1_s_G1_L001_sequences.fastq.gz'), mp('raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz')] for fp in fps: @@ -1570,5 +1571,6 @@ def test_archive(self): for fp in fps: copyfile(f'{fp}.bk', fp) + if __name__ == '__main__': main()