diff --git a/CHANGELOG.md b/CHANGELOG.md index 438075906..d29e6491d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ Deployed on October 14th, 2024 * `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support. * `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates. * `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks! -* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439). +* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440). +* General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225). + Version 2024.07 diff --git a/qiita_db/artifact.py b/qiita_db/artifact.py index f116236f7..bf81ddf41 100644 --- a/qiita_db/artifact.py +++ b/qiita_db/artifact.py @@ -929,7 +929,8 @@ def can_be_submitted_to_ebi(self): # words has more that one processing step behind it fine_to_send = [] fine_to_send.extend([pt.artifact for pt in self.prep_templates]) - fine_to_send.extend([c for a in fine_to_send for c in a.children]) + fine_to_send.extend([c for a in fine_to_send if a is not None + for c in a.children]) if self not in fine_to_send: return False diff --git a/qiita_db/metadata_template/prep_template.py b/qiita_db/metadata_template/prep_template.py index 7ae97858c..52f781b9a 100644 --- a/qiita_db/metadata_template/prep_template.py +++ b/qiita_db/metadata_template/prep_template.py @@ -272,6 +272,18 @@ def delete(cls, id_): "Cannot remove prep template %d because it has an artifact" " associated with it" % id_) + # artifacts that are archived are not returned as part of the code + # above and we need to clean them before moving forward + sql = """SELECT artifact_id + FROM qiita.preparation_artifact + WHERE prep_template_id = %s""" + qdb.sql_connection.TRN.add(sql, args) + archived_artifacts = set( + qdb.sql_connection.TRN.execute_fetchflatten()) + if archived_artifacts: + for aid in archived_artifacts: + qdb.artifact.Artifact.delete(aid) + # Delete the prep template filepaths sql = """DELETE FROM qiita.prep_template_filepath WHERE prep_template_id = %s""" diff --git a/qiita_db/test/test_artifact.py b/qiita_db/test/test_artifact.py index 76833ed98..bed71c45a 100644 --- a/qiita_db/test/test_artifact.py +++ b/qiita_db/test/test_artifact.py @@ -10,7 +10,7 @@ from tempfile import mkstemp, mkdtemp from datetime import datetime from os import close, remove -from os.path import exists, join, basename +from os.path import exists, join, basename, dirname, abspath from shutil import copyfile from functools import partial from json import dumps @@ -23,6 +23,7 @@ from qiita_core.util import qiita_test_checker from qiita_core.testing import wait_for_processing_job import qiita_db as qdb +from qiita_ware.private_plugin import _delete_analysis_artifacts class ArtifactTestsReadOnly(TestCase): @@ -1518,7 +1519,7 @@ def test_archive(self): 'be archived'): A.archive(8) - for aid in range(4, 7): + for aid in range(5, 7): ms = A(aid).merging_scheme A.archive(aid) self.assertEqual(ms, A(aid).merging_scheme) @@ -1526,7 +1527,49 @@ def test_archive(self): self.assertCountEqual(A(1).descendants.nodes(), exp_nodes) obs_artifacts = len(qdb.util.get_artifacts_information([4, 5, 6, 8])) - self.assertEqual(1, obs_artifacts) + self.assertEqual(2, obs_artifacts) + + # in the tests above we generated and validated archived artifacts + # so this allows us to add tests to delete a prep-info with archived + # artifacts. The first bottleneck to do this is that this tests will + # actually remove files, which we will need for other tests so lets + # make a copy and then restore them + mfolder = dirname(dirname(abspath(__file__))) + mpath = join(mfolder, 'support_files', 'test_data') + mp = partial(join, mpath) + fps = [ + mp('processed_data/1_study_1001_closed_reference_otu_table.biom'), + mp('processed_data/' + '1_study_1001_closed_reference_otu_table_Silva.biom'), + mp('raw_data/1_s_G1_L001_sequences.fastq.gz'), + mp('raw_data/1_s_G1_L001_sequences_barcodes.fastq.gz')] + for fp in fps: + copyfile(fp, f'{fp}.bk') + + PT = qdb.metadata_template.prep_template.PrepTemplate + QEE = qdb.exceptions.QiitaDBExecutionError + pt = A(1).prep_templates[0] + # it should fail as this prep is public and have been submitted to ENA + with self.assertRaisesRegex(QEE, 'Cannot remove prep template 1'): + PT.delete(pt.id) + # now, remove those restrictions + analysis + linked artifacts + sql = "DELETE FROM qiita.artifact_processing_job" + qdb.sql_connection.perform_as_transaction(sql) + sql = "DELETE FROM qiita.ebi_run_accession" + qdb.sql_connection.perform_as_transaction(sql) + sql = "UPDATE qiita.artifact SET visibility_id = 1" + qdb.sql_connection.perform_as_transaction(sql) + _delete_analysis_artifacts(qdb.analysis.Analysis(1)) + _delete_analysis_artifacts(qdb.analysis.Analysis(2)) + _delete_analysis_artifacts(qdb.analysis.Analysis(3)) + for aid in [3, 2, 1]: + A.delete(aid) + + PT.delete(pt.id) + + # bringing back the filepaths + for fp in fps: + copyfile(f'{fp}.bk', fp) if __name__ == '__main__':