diff --git a/reports/anvil-production/README.md b/reports/anvil-production/README.md
index e946bf311..abdbb74cf 100644
--- a/reports/anvil-production/README.md
+++ b/reports/anvil-production/README.md
@@ -13,4 +13,4 @@ After each successful deployment, automated tool tests are also run against the
Latest tool tests for each chunk:
-
+
diff --git a/reports/anvil-production/tool-tests/chunks.json b/reports/anvil-production/tool-tests/chunks.json
index 996d210f4..4c8e54256 100644
--- a/reports/anvil-production/tool-tests/chunks.json
+++ b/reports/anvil-production/tool-tests/chunks.json
@@ -7,11 +7,11 @@
"date2": "Thu Sep 05 00:47:53 2024"
},
"10": {
- "run1": "https://htmlpreview.github.io/?https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-07-00-31-1/results.html",
- "date1": "Sat Sep 07 00:54:23 2024",
- "tools": "https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-07-00-31-1/tools.yml",
- "run2": "https://htmlpreview.github.io/?https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-08-31-00-32-1/results.html",
- "date2": "Sat Aug 31 00:46:38 2024"
+ "run1": "https://htmlpreview.github.io/?https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/results.html",
+ "date1": "Sat Sep 14 00:47:35 2024",
+ "tools": "https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/tools.yml",
+ "run2": "https://htmlpreview.github.io/?https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-07-00-31-1/results.html",
+ "date2": "Sat Sep 07 00:54:23 2024"
},
"11": {
"run1": "https://htmlpreview.github.io/?https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-07-12-30-1/results.html",
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/chunk.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/chunk.json
new file mode 100644
index 000000000..c3da627bd
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/chunk.json
@@ -0,0 +1,7 @@
+{
+ "10": {
+ "run1": "https://htmlpreview.github.io/?https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/results.html",
+ "date1": "Sat Sep 14 00:47:35 2024",
+ "tools": "https://github.com/anvilproject/galaxy-tests/blob/main/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/tools.yml"
+ }
+}
\ No newline at end of file
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/chunk.txt b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/chunk.txt
new file mode 100644
index 000000000..8b52783d8
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/chunk.txt
@@ -0,0 +1,15 @@
+-t annotatemyids --tool-version 44018dd6b447
+-t datamash_reverse --tool-version 0f1724dd59d2
+-t trimmomatic --tool-version d94aff5ee623
+-t samtools_fixmate --tool-version bc0cc7bfbfe9
+-t sam_to_bam --tool-version cf1ffd88f895
+-t bcftools_plugin_setgt --tool-version 8abfcd1a6949
+-t bcftools_call --tool-version 072f606e2e69
+-t vcfbreakcreatemulti --tool-version 9fd5d91eb91b
+-t vcfselectsamples --tool-version 2ba3b87f904f
+-t poretools_occupancy --tool-version 338d15b8d6fb
+-t deeptools_plot_heatmap --tool-version a4dfb122e52b
+-t deeptools_plot_profile --tool-version c361b12d207a
+-t lofreq_filter --tool-version 7dfca164d2e3
+-t salmon --tool-version cc05793bb896
+-t mummer_mummerplot --tool-version ebe3b0f84de9
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/0aec0cc69095fb2b.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/0aec0cc69095fb2b.json
new file mode 100644
index 000000000..47b487adb
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/0aec0cc69095fb2b.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "0aec0cc69095fb2b",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T01:00:42.626264",
+ "create_time": "2024-09-14T01:00:34.879266",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-hp87m",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/53/registry.xml' '/galaxy/server/database/jobs_directory/000/53/upload_params.json' '71:/galaxy/server/database/objects/f/3/7/dataset_f37973fd-efc1-49f4-bca4-2083b7e3bc47_files:/galaxy/server/database/objects/f/3/7/dataset_f37973fd-efc1-49f4-bca4-2083b7e3bc47.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_bx6bgb_t\"",
+ "files_metadata": "{\"__current_case__\": 300, \"file_type\": \"sam\"}",
+ "dbkey": "\"?\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"sam_to_bam_in1.sam\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_fil3ktry\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"sam\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "fa93e67c1e1ebcf1",
+ "src": "hda",
+ "uuid": "f37973fd-efc1-49f4-bca4-2083b7e3bc47"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/219eea5edfdc4ae8.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/219eea5edfdc4ae8.json
new file mode 100644
index 000000000..a44a603af
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/219eea5edfdc4ae8.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "219eea5edfdc4ae8",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T01:00:45.403522",
+ "create_time": "2024-09-14T01:00:35.519784",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-xpxbv",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/54/registry.xml' '/galaxy/server/database/jobs_directory/000/54/upload_params.json' '72:/galaxy/server/database/objects/c/d/3/dataset_cd37bd50-8ef0-44f6-9550-06b20e271e97_files:/galaxy/server/database/objects/c/d/3/dataset_cd37bd50-8ef0-44f6-9550-06b20e271e97.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_f8gboh6h\"",
+ "files_metadata": "{\"__current_case__\": 75, \"file_type\": \"fasta\"}",
+ "dbkey": "\"equCab2\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"chr_m.fasta\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_y8b5xdaw\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"fasta\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "2c20b2126db3a04f",
+ "src": "hda",
+ "uuid": "cd37bd50-8ef0-44f6-9550-06b20e271e97"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/4b3b265db2b6345e.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/4b3b265db2b6345e.json
new file mode 100644
index 000000000..87e83e9b1
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/4b3b265db2b6345e.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "4b3b265db2b6345e",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T01:03:51.267360",
+ "create_time": "2024-09-14T01:03:42.535494",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-pqs99",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/70/registry.xml' '/galaxy/server/database/jobs_directory/000/70/upload_params.json' '88:/galaxy/server/database/objects/e/b/b/dataset_ebbeee57-e587-4c2f-9418-d45c4f97df06_files:/galaxy/server/database/objects/e/b/b/dataset_ebbeee57-e587-4c2f-9418-d45c4f97df06.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_3m1m84ex\"",
+ "files_metadata": "{\"__current_case__\": 328, \"file_type\": \"vcf\"}",
+ "dbkey": "\"?\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"mpileup.vcf\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_tghoa9ch\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"vcf\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "6872c13e0305d080",
+ "src": "hda",
+ "uuid": "ebbeee57-e587-4c2f-9418-d45c4f97df06"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/4f3352db0d5bc390.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/4f3352db0d5bc390.json
new file mode 100644
index 000000000..032de1d05
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/4f3352db0d5bc390.json
@@ -0,0 +1,49 @@
+{
+ "model_class": "Job",
+ "id": "4f3352db0d5bc390",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T01:11:04.701248",
+ "create_time": "2024-09-14T01:10:57.236814",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "toolshed.g2.bx.psu.edu/repos/bgruening/deeptools_plot_profile/deeptools_plot_profile/3.5.4+galaxy0",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-mgxft",
+ "command_line": "plotProfile --matrixFile \"/galaxy/server/database/objects/1/f/2/dataset_1f23dafd-d951-4a12-b9e3-4a38224f484e.dat\" --outFileName \"/galaxy/server/database/objects/4/0/8/dataset_4084c388-b579-4d96-8f18-2068c455f459.dat\" --plotFileFormat 'png'",
+ "traceback": null,
+ "params": {
+ "__input_ext": "\"input\"",
+ "dbkey": "\"?\"",
+ "chromInfo": "\"/cvmfs/data.galaxyproject.org/managed/len/ucsc/?.len\"",
+ "advancedOpt": "{\"__current_case__\": 0, \"showAdvancedOpt\": \"no\"}",
+ "output": "{\"__current_case__\": 0, \"showOutputSettings\": \"no\"}",
+ "scaleRegions": "{\"__current_case__\": 0, \"showScaleRegionsOpt\": \"no\"}"
+ },
+ "inputs": {
+ "matrixFile": {
+ "id": "52d5cbcb026d8518",
+ "src": "hda",
+ "uuid": "1f23dafd-d951-4a12-b9e3-4a38224f484e"
+ }
+ },
+ "outputs": {
+ "outFileName": {
+ "id": "cfa693d8e6b4174a",
+ "src": "hda",
+ "uuid": "4084c388-b579-4d96-8f18-2068c455f459"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/64262c26ef4b4f8c.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/64262c26ef4b4f8c.json
new file mode 100644
index 000000000..60bec279b
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/64262c26ef4b4f8c.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "64262c26ef4b4f8c",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T00:56:01.213807",
+ "create_time": "2024-09-14T00:55:52.824504",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-bh495",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/30/registry.xml' '/galaxy/server/database/jobs_directory/000/30/upload_params.json' '42:/galaxy/server/database/objects/6/c/4/dataset_6c491894-0ad3-4510-a9c4-8a10c6045ac5_files:/galaxy/server/database/objects/6/c/4/dataset_6c491894-0ad3-4510-a9c4-8a10c6045ac5.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_v9lz7191\"",
+ "files_metadata": "{\"__current_case__\": 82, \"file_type\": \"fastqsanger.gz\"}",
+ "dbkey": "\"?\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"Illumina_SG_R2.fastq.gz\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_g7mkis2m\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"fastqsanger.gz\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "3b076214a691ebd7",
+ "src": "hda",
+ "uuid": "6c491894-0ad3-4510-a9c4-8a10c6045ac5"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/877faa65ca7adf3a.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/877faa65ca7adf3a.json
new file mode 100644
index 000000000..c98da0ac4
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/877faa65ca7adf3a.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "877faa65ca7adf3a",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T00:55:45.136973",
+ "create_time": "2024-09-14T00:55:35.975247",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-ww4c7",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/28/registry.xml' '/galaxy/server/database/jobs_directory/000/28/upload_params.json' '40:/galaxy/server/database/objects/d/3/9/dataset_d393355a-1c2b-44f2-8534-fa14c8a5a6c0_files:/galaxy/server/database/objects/d/3/9/dataset_d393355a-1c2b-44f2-8534-fa14c8a5a6c0.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_e2o2iltb\"",
+ "files_metadata": "{\"__current_case__\": 81, \"file_type\": \"fastqsanger\"}",
+ "dbkey": "\"?\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"Illumina_SG_R2.fastq\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_nt66jhoa\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"fastqsanger\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "873462c21322a5f4",
+ "src": "hda",
+ "uuid": "d393355a-1c2b-44f2-8534-fa14c8a5a6c0"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/a6000a46566a855e.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/a6000a46566a855e.json
new file mode 100644
index 000000000..dc9e0da27
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/a6000a46566a855e.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "a6000a46566a855e",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T00:55:09.420570",
+ "create_time": "2024-09-14T00:55:01.221058",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-sf4nz",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/23/registry.xml' '/galaxy/server/database/jobs_directory/000/23/upload_params.json' '35:/galaxy/server/database/objects/b/c/d/dataset_bcda9236-4541-4155-8ce6-4c842913f7b4_files:/galaxy/server/database/objects/b/c/d/dataset_bcda9236-4541-4155-8ce6-4c842913f7b4.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_z3iggls3\"",
+ "files_metadata": "{\"__current_case__\": 84, \"file_type\": \"fastqsolexa\"}",
+ "dbkey": "\"?\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"Illumina_SG_R1.fastqsolexa\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_0z6w7l2r\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"fastqsolexa\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "14174cb4b325ed48",
+ "src": "hda",
+ "uuid": "bcda9236-4541-4155-8ce6-4c842913f7b4"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/da2d40552de3029b.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/da2d40552de3029b.json
new file mode 100644
index 000000000..577fb57e5
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/da2d40552de3029b.json
@@ -0,0 +1,51 @@
+{
+ "model_class": "Job",
+ "id": "da2d40552de3029b",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T00:56:52.038033",
+ "create_time": "2024-09-14T00:56:47.840310",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-q74tk",
+ "command_line": "if [ -z \"$TRIMMOMATIC_JAR_PATH\" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z \"$TRIMMOMATIC_ADAPTERS_PATH\" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/3/7/7/dataset_37778b71-9669-4df5-880b-532f8c4b276b.dat' fastq_in.'fastqsanger' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar SE -threads ${GALAXY_SLOTS:-6} fastq_in.'fastqsanger' fastq_out.'fastqsanger' MAXINFO:75:0.8 2>&1 | tee trimmomatic.log && if [ -z \"$(tail -1 trimmomatic.log | grep \"Completed successfully\")\" ]; then echo \"Trimmomatic did not finish successfully\" >&2 ; exit 1 ; fi && mv fastq_out.'fastqsanger' '/galaxy/server/database/objects/e/6/a/dataset_e6ac6186-9f77-45ca-a14f-43c8894a7811.dat'",
+ "traceback": null,
+ "params": {
+ "__input_ext": "\"fastqsanger\"",
+ "dbkey": "\"?\"",
+ "chromInfo": "\"/cvmfs/data.galaxyproject.org/managed/len/ucsc/?.len\"",
+ "output_err": "false",
+ "output_logs": "false",
+ "operations": "[{\"__index__\": 0, \"operation\": {\"__current_case__\": 7, \"name\": \"MAXINFO\", \"strictness\": \"0.8\", \"target_length\": \"75\"}}]",
+ "illuminaclip": "{\"__current_case__\": 1, \"do_illuminaclip\": false}",
+ "readtype": "{\"__current_case__\": 0, \"fastq_in\": {\"values\": [{\"id\": 45, \"src\": \"hda\"}]}, \"single_or_paired\": \"se\"}"
+ },
+ "inputs": {
+ "readtype|fastq_in": {
+ "id": "346ee1d2fcf7f70c",
+ "src": "hda",
+ "uuid": "37778b71-9669-4df5-880b-532f8c4b276b"
+ }
+ },
+ "outputs": {
+ "fastq_out": {
+ "id": "50872bd7722d341d",
+ "src": "hda",
+ "uuid": "e6ac6186-9f77-45ca-a14f-43c8894a7811"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/f513f03dee6ba111.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/f513f03dee6ba111.json
new file mode 100644
index 000000000..a24ddc5c6
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/f513f03dee6ba111.json
@@ -0,0 +1,56 @@
+{
+ "model_class": "Job",
+ "id": "f513f03dee6ba111",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T01:06:36.828805",
+ "create_time": "2024-09-14T01:06:31.054100",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "toolshed.g2.bx.psu.edu/repos/iuc/bcftools_call/bcftools_call/1.15.1+galaxy5",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-tbxfz",
+ "command_line": "export BCFTOOLS_PLUGINS=`which bcftools | sed 's,bin/bcftools,libexec/bcftools,'`; bgzip -c '/galaxy/server/database/objects/b/8/5/dataset_b851b4ac-86be-4622-bbe1-cc1c4d60f388.dat' > input.vcf.gz && bcftools index input.vcf.gz && bcftools call -m --prior 0.0011 --targets-file \"/galaxy/server/database/objects/f/9/a/dataset_f9ab92a9-992d-4798-8921-d967a0352f8b.dat\" --output-type 'v' --threads ${GALAXY_SLOTS:-4} input.vcf.gz > '/galaxy/server/database/objects/a/1/a/dataset_a1aa4eef-f0a9-4302-bc82-9e4cd2d6448a.dat'",
+ "traceback": null,
+ "params": {
+ "__input_ext": "\"input\"",
+ "dbkey": "\"?\"",
+ "chromInfo": "\"/cvmfs/data.galaxyproject.org/managed/len/ucsc/?.len\"",
+ "output_type": "\"v\"",
+ "sec_input_output": "{\"format_fields\": \"\", \"group_samples\": false, \"keep_alts\": false, \"keep_masked_ref\": false, \"output_tags\": null, \"skip_variants\": null, \"variants_only\": false}",
+ "sec_file_format": "{\"invert_samples\": false, \"invert_samples_file\": false, \"ploidy\": null, \"ploidy_file\": null, \"regions\": {\"__current_case__\": 0, \"regions_src\": \"__none__\"}, \"regions_overlap\": null, \"samples\": \"\", \"samples_file\": null}",
+ "sec_consensus_variant_calling": "{\"variant_calling\": {\"__current_case__\": 0, \"genotypes\": {\"__current_case__\": 0, \"constrain\": \"none\", \"targets\": {\"__current_case__\": 2, \"invert_targets_file\": false, \"targets_file\": {\"values\": [{\"id\": 104, \"src\": \"hda\"}]}, \"targets_src\": \"targets_file\"}, \"targets_overlap\": null}, \"gvcf\": \"\", \"method\": \"multiallelic\", \"prior\": \"0.0011\", \"prior_freqs\": \"\"}}",
+ "sec_restrict": "{\"invert_samples\": false, \"invert_samples_file\": false, \"regions\": {\"__current_case__\": 0, \"regions_src\": \"__none__\"}, \"regions_overlap\": null, \"samples\": \"\", \"samples_file\": null}"
+ },
+ "inputs": {
+ "input_file": {
+ "id": "4f3352db0d5bc390",
+ "src": "hda",
+ "uuid": "b851b4ac-86be-4622-bbe1-cc1c4d60f388"
+ },
+ "sec_consensus_variant_calling|variant_calling|genotypes|targets|targets_file": {
+ "id": "cbd23c4cd10aa88a",
+ "src": "hda",
+ "uuid": "f9ab92a9-992d-4798-8921-d967a0352f8b"
+ }
+ },
+ "outputs": {
+ "output_file": {
+ "id": "fb28fb836c18096d",
+ "src": "hda",
+ "uuid": "a1aa4eef-f0a9-4302-bc82-9e4cd2d6448a"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/fe0a9ea15b54bf6c.json b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/fe0a9ea15b54bf6c.json
new file mode 100644
index 000000000..82d204c85
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/errors/fe0a9ea15b54bf6c.json
@@ -0,0 +1,44 @@
+{
+ "model_class": "Job",
+ "id": "fe0a9ea15b54bf6c",
+ "state": "error",
+ "exit_code": null,
+ "update_time": "2024-09-14T00:53:23.031439",
+ "create_time": "2024-09-14T00:53:14.186609",
+ "galaxy_version": "23.1",
+ "command_version": null,
+ "copied_from_job_id": null,
+ "tool_id": "upload1",
+ "history_id": "ed165837e8ba4cca",
+ "external_id": "gxy-rzcrg",
+ "command_line": "python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/13/registry.xml' '/galaxy/server/database/jobs_directory/000/13/upload_params.json' '16:/galaxy/server/database/objects/d/d/7/dataset_dd7cfbb0-f7a0-489a-868d-01808f00a8fd_files:/galaxy/server/database/objects/d/d/7/dataset_dd7cfbb0-f7a0-489a-868d-01808f00a8fd.dat'",
+ "traceback": null,
+ "params": {
+ "paramfile": "\"/galaxy/server/database/tmp/upload_params_8pius79c\"",
+ "files_metadata": "{\"__current_case__\": 82, \"file_type\": \"fastqsanger.gz\"}",
+ "dbkey": "\"?\"",
+ "force_composite": "\"false\"",
+ "files": "[{\"NAME\": \"Illumina_SG_R1.fastq.gz\", \"__index__\": 0, \"auto_decompress\": \"Yes\", \"dbkey\": \"\", \"file_data\": \"/galaxy/server/database/tmp/upload_file_data_ut5pdeks\", \"file_type\": \"\", \"ftp_files\": null, \"space_to_tab\": null, \"to_posix_lines\": \"Yes\", \"url_paste\": null, \"uuid\": null}]",
+ "file_count": "\"auto\"",
+ "file_type": "\"fastqsanger.gz\""
+ },
+ "inputs": {},
+ "outputs": {
+ "output0": {
+ "id": "7bf7e48be40dc04d",
+ "src": "hda",
+ "uuid": "dd7cfbb0-f7a0-489a-868d-01808f00a8fd"
+ }
+ },
+ "output_collections": {},
+ "tool_stdout": "",
+ "tool_stderr": "",
+ "job_stdout": null,
+ "job_stderr": null,
+ "stderr": "",
+ "stdout": "",
+ "job_messages": null,
+ "dependencies": [],
+ "user_email": "tests@fake.org",
+ "job_metrics": []
+}
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/job-metrics.csv b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/job-metrics.csv
new file mode 100644
index 000000000..5c86eaf6f
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/job-metrics.csv
@@ -0,0 +1,129 @@
+id,history_id,history_name,state,tool_id,invocation_id,workflow_id,cpuacct.usage,galaxy_memory_mb,galaxy_slots,memory.limit_in_bytes,memory.peak,memory.soft_limit_in_bytes,memtotal,processor_count,runtime_seconds
+b7c0a16325039497,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,mummer_mummerplot/4.0.0rc1+galaxy3,,,,4080.0000000,1.0000000,,,,,,0E-7
+2cd1d76bf2ef465f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+8ebbb1e4f11076af,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+234c1bcaae2914ca,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+9a28a94cdc902f04,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,salmon/1.3.0+galaxy1,,,,51539.0000000,6.0000000,,,,,,1.0000000
+54fc498ddf93b220,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+765d4069e7f77bff,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,salmon/1.3.0+galaxy1,,,,51539.0000000,6.0000000,,,,,,1.0000000
+cfa693d8e6b4174a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+52d5cbcb026d8518,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,salmon/1.3.0+galaxy1,,,,51539.0000000,6.0000000,,,,,,2.0000000
+fa3f703e66538a8e,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+066a24ce2001c2a8,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+7f019bb254bd9239,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,salmon/1.3.0+galaxy1,,,,51539.0000000,6.0000000,,,,,,1.0000000
+c51f3c4644daf9a5,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+38d82a18eaba87f9,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,salmon/1.3.0+galaxy1,,,,51539.0000000,6.0000000,,,,,,2.0000000
+03016dc1802ebe2b,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+bd5e06df4cb7f9fc,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,salmon/1.3.0+galaxy1,,,,51539.0000000,6.0000000,,,,,,1.0000000
+a5bab748014374bb,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+22470166765b2f95,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,lofreq_filter/2.1.5+galaxy0,,,,16320.0000000,1.0000000,,,,,,1.0000000
+251b4eb87d395ff4,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+ffe3188988274058,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,lofreq_filter/2.1.5+galaxy0,,,,16320.0000000,1.0000000,,,,,,0E-7
+2ca487e824fa2a6a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+4c4c4e06aa36dd6f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,lofreq_filter/2.1.5+galaxy0,,,,16320.0000000,1.0000000,,,,,,0E-7
+6866e8eee2e30678,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+fb28fb836c18096d,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,deeptools_plot_profile/3.5.4+galaxy0,,,,21474.0000000,1.0000000,,,,,,3.0000000
+cbd23c4cd10aa88a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+4f3352db0d5bc390,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,deeptools_plot_profile/3.5.4+galaxy0,,,,,,,,,,,
+cc384397a197f1fe,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+4807d53d9a2e2e7f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,deeptools_plot_heatmap/3.5.4+galaxy0,,,,26843.0000000,1.0000000,,,,,,3.0000000
+45622d042ca95d2d,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+34c46068446c92bb,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,deeptools_plot_heatmap/3.5.4+galaxy0,,,,26843.0000000,1.0000000,,,,,,3.0000000
+9286c7c1ce47fbe4,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+022a03c7fd9fc863,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,poretools_occupancy/0.6.1a1.1,,,,12240.0000000,1.0000000,,,,,,3.0000000
+b914c5cbdc32f1b4,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+50976ab105970f33,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,poretools_occupancy/0.6.1a1.1,,,,12240.0000000,1.0000000,,,,,,3.0000000
+153dc00d25d9a6d7,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+fb8ab536889f0e94,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,vcfselectsamples/1.0.0_rc3+galaxy0,,,,4080.0000000,1.0000000,,,,,,0E-7
+748366871c7f2141,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+d58abde2fb36f1e8,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,vcfbreakcreatemulti/1.0.0_rc3+galaxy0,,,,4080.0000000,1.0000000,,,,,,0E-7
+c9147cfbb24d024f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+e5e85752534a1f6b,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,vcfbreakcreatemulti/1.0.0_rc3+galaxy0,,,,4080.0000000,1.0000000,,,,,,0E-7
+6872c13e0305d080,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+f513f03dee6ba111,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,bcftools_call/1.15.1+galaxy5,,,,,,,,,,,
+1747158b9458f9f7,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+0ac6b3678845426c,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+16a1aab48891616f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_call/1.15.1+galaxy5,,,,4080.0000000,1.0000000,,,,,,0E-7
+1913c0b28f54e065,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+050fdb9488518af8,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_call/1.15.1+galaxy5,,,,4080.0000000,1.0000000,,,,,,1.0000000
+5c0d3387021faa81,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+0dd086a5fc3869bf,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_call/1.15.1+galaxy5,,,,4080.0000000,1.0000000,,,,,,1.0000000
+6392c2499aa3e408,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+d49c36adeba8557f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+b94e2d0d46c75809,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_call/1.15.1+galaxy5,,,,4080.0000000,1.0000000,,,,,,1.0000000
+b355a78f591d3088,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+dc1abf31b7878b39,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+2611c7230f4ebe6d,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_call/1.15.1+galaxy5,,,,4080.0000000,1.0000000,,,,,,1.0000000
+c12cff306b1f27cd,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+2c20b2126db3a04f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+fa93e67c1e1ebcf1,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+4b3b265db2b6345e,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+87dadb2c71819a0b,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_call/1.15.1+galaxy5,,,,4080.0000000,1.0000000,,,,,,1.0000000
+ab33a4ba552f72dc,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+1a9d1a58818c88c1,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_plugin_setgt/1.15.1+galaxy4,,,,4080.0000000,1.0000000,,,,,,1.0000000
+548270c1cda12d7a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+5138b028115a7f5f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,bcftools_plugin_setgt/1.15.1+galaxy4,,,,4080.0000000,1.0000000,,,,,,1.0000000
+6ec8bf3c5dc3224e,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+24e58a97d82a5b14,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,sam_to_bam/2.1.2,,,,4080.0000000,1.0000000,,,,,,0E-7
+54816788543a0d5a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+1e7e26c0720680d3,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+1762d3613701af86,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,sam_to_bam/2.1.2,,,,4080.0000000,1.0000000,,,,,,0E-7
+9cbc3100f8d963dc,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+58bcb7304f57378f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+95a0a0882149fa40,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,sam_to_bam/2.1.2,,,,4080.0000000,1.0000000,,,,,,0E-7
+5df7adb272208d2d,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+2ea84bc3d4af7b79,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+219eea5edfdc4ae8,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+0aec0cc69095fb2b,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+4fe3ac1409aa61b5,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,samtools_fixmate/1.9,,,,4080.0000000,1.0000000,,,,,,0E-7
+d0d1c4024af98dfb,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+517cff8f48d5d3c0,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,samtools_fixmate/1.9,,,,4080.0000000,1.0000000,,,,,,0E-7
+9184366d0e5e1098,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+169d16a8c5164f68,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,samtools_fixmate/1.9,,,,4080.0000000,1.0000000,,,,,,0E-7
+9907fa81e6c94704,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+50872bd7722d341d,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,samtools_fixmate/1.9,,,,4080.0000000,1.0000000,,,,,,0E-7
+346ee1d2fcf7f70c,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+f77b84ee3fc05e8c,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,samtools_fixmate/1.9,,,,4080.0000000,1.0000000,,,,,,0E-7
+c1f53f965e3f11fc,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+3b076214a691ebd7,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,samtools_fixmate/1.9,,,,4080.0000000,1.0000000,,,,,,0E-7
+9e22d11eadbf9087,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+873462c21322a5f4,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,0E-7
+03ea2d9f5a73c06f,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+4e6201563fcf9362,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+3e0e92ef37a541d1,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,1.0000000
+14174cb4b325ed48,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+4ec971aa3f08aa41,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+da2d40552de3029b,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,trimmomatic/0.38.1,,,,,,,,,,,
+7fd0ebaa51c36236,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+fe9a443ac2bf50b7,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,1.0000000
+f8ff9528d950258c,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+f7e63bf4200f2b12,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+64262c26ef4b4f8c,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+78f113974806f741,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+877faa65ca7adf3a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+9bada2958fd12d58,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,0E-7
+9d1e19ed450e6c86,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+010a92ede45bd68a,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+a6000a46566a855e,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+87943a4e2dd4167e,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,1.0000000
+b4ed4b13837a07b8,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+8b6f5ce3aa2377af,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+584d0aa45b50556b,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,0E-7
+4a12ddb120e74961,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+c0c8a69c8070f677,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+7bf7e48be40dc04d,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,0E-7
+0c8c289d788b4c08,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+defb5211a6a94aa8,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+fe0a9ea15b54bf6c,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,error,upload1,,,,,,,,,,,
+a57289ad4f175228,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,trimmomatic/0.38.1,,,,12884.0000000,6.0000000,,,,,,0E-7
+49702c00cc3a3629,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,3.0000000
+3ec05b8c179927a5,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,datamash_reverse/1.8+galaxy0,,,,4080.0000000,1.0000000,,,,,,0E-7
+24323ccc9ba4a0ff,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+a66ef6e4a4a0e1d4,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,annotatemyids/3.5.0.1,,,,4080.0000000,1.0000000,,,,,,18.0000000
+f560f43330980975,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+cf4669e3d1f60b4e,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,annotatemyids/3.5.0.1,,,,4080.0000000,1.0000000,,,,,,18.0000000
+e39ca091cf5339f4,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+ef71afcbd5eac7e6,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,annotatemyids/3.5.0.1,,,,4080.0000000,1.0000000,,,,,,17.0000000
+198a4099c8492732,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
+a21c742fb7f71c24,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,annotatemyids/3.5.0.1,,,,4080.0000000,1.0000000,,,,,,17.0000000
+ed165837e8ba4cca,ed165837e8ba4cca,anvil-test-2024-09-14-00-47,ok,upload1,,,,4080.0000000,1.0000000,,,,,,4.0000000
diff --git a/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/job.log b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/job.log
new file mode 100644
index 000000000..64881fcd8
--- /dev/null
+++ b/reports/anvil-production/tool-tests/prod-24-09-14-00-32-1/job.log
@@ -0,0 +1,4029 @@
+galaxy.jobs.handler DEBUG 2024-09-14 00:50:31,099 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 5
+tpv.core.entities DEBUG 2024-09-14 00:50:31,124 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:50:31,124 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (5) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:50:31,128 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (5) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:50:31,141 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (5) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:50:31,156 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (5) Working directory for job is: /galaxy/server/database/jobs_directory/000/5
+galaxy.jobs.runners DEBUG 2024-09-14 00:50:31,161 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [5] queued (33.322 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:50:31,164 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (5) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:50:31,168 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 5
+galaxy.jobs DEBUG 2024-09-14 00:50:31,253 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [5] prepared (69.009 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:50:31,272 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/5/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/5/registry.xml' '/galaxy/server/database/jobs_directory/000/5/upload_params.json' '6:/galaxy/server/database/objects/f/e/6/dataset_fe681ec1-6341-4ff3-91be-67bab7337de5_files:/galaxy/server/database/objects/f/e/6/dataset_fe681ec1-6341-4ff3-91be-67bab7337de5.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:50:31,282 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (5) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/5/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/5/galaxy_5.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:50:40,404 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] executing external set_meta script for job 5: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:50:45,071 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] execution of external set_meta for job 5 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:50:45,119 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] loading metadata from file for: HistoryDatasetAssociation 6
+galaxy.jobs INFO 2024-09-14 00:50:45,151 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Collecting metrics for Job 5 in /galaxy/server/database/jobs_directory/000/5
+galaxy.jobs DEBUG 2024-09-14 00:50:45,198 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] job_wrapper.finish for job 5 executed (104.966 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:50:46,510 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 6
+tpv.core.entities DEBUG 2024-09-14 00:50:46,542 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=default, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:50:46,543 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (6) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:50:46,548 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (6) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:50:46,562 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (6) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:50:46,578 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (6) Working directory for job is: /galaxy/server/database/jobs_directory/000/6
+galaxy.jobs.runners DEBUG 2024-09-14 00:50:46,584 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [6] queued (35.817 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:50:46,587 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (6) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:50:46,591 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 6
+galaxy.jobs DEBUG 2024-09-14 00:50:46,664 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [6] prepared (57.870 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:50:46,664 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:50:46,664 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/iuc/annotatemyids/annotatemyids/3.5.0.1: mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:50:46,680 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:50:46,698 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/6/tool_script.sh] for tool command [echo $(R --version | grep version | grep -v GNU)", org.Hs.eg.db version" $(R --vanilla --slave -e "library(org.Hs.eg.db); cat(sessionInfo()\$otherPkgs\$org.Hs.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ")", org.Dr.eg.db version" $(R --vanilla --slave -e "library(org.Dr.eg.db); cat(sessionInfo()\$otherPkgs\$org.Dr.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ")", org.Dm.eg.db version" $(R --vanilla --slave -e "library(org.Dm.eg.db); cat(sessionInfo()\$otherPkgs\$org.Dm.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ")", org.Mm.eg.db version" $(R --vanilla --slave -e "library(org.Mm.eg.db); cat(sessionInfo()\$otherPkgs\$org.Mm.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ") > /galaxy/server/database/jobs_directory/000/6/outputs/COMMAND_VERSION 2>&1;
+Rscript '/galaxy/server/database/jobs_directory/000/6/configs/tmpuk790mlu']
+galaxy.jobs.runners DEBUG 2024-09-14 00:50:46,709 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (6) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/6/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/6/galaxy_6.ec;
+if [ -f "/galaxy/server/database/jobs_directory/000/6/working/"*".tab" ] ; then cp "/galaxy/server/database/jobs_directory/000/6/working/"*".tab" "/galaxy/server/database/objects/2/1/c/dataset_21c89c91-f995-459e-b47e-d2e33ae3547e.dat" ; fi; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:50:46,719 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:50:46,719 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/iuc/annotatemyids/annotatemyids/3.5.0.1: mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:50:46,735 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:08,805 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 6: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:13,419 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 6 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:51:13,470 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 7
+galaxy.jobs INFO 2024-09-14 00:51:13,498 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 6 in /galaxy/server/database/jobs_directory/000/6
+galaxy.objectstore CRITICAL 2024-09-14 00:51:13,537 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] None delete error [Errno 13] Permission denied: 'tmpuk790mlu'
+galaxy.jobs DEBUG 2024-09-14 00:51:13,538 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 6 executed (94.557 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:51:16,149 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 7
+tpv.core.entities DEBUG 2024-09-14 00:51:16,176 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:51:16,177 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (7) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:51:16,180 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (7) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:51:16,193 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (7) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:51:16,208 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (7) Working directory for job is: /galaxy/server/database/jobs_directory/000/7
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:16,214 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [7] queued (33.394 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:51:16,216 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (7) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:51:16,220 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 7
+galaxy.jobs DEBUG 2024-09-14 00:51:16,302 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [7] prepared (67.090 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:51:16,320 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/7/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/7/registry.xml' '/galaxy/server/database/jobs_directory/000/7/upload_params.json' '8:/galaxy/server/database/objects/5/9/4/dataset_594ce8d3-34d3-4e83-ba57-7f6cd4a922d3_files:/galaxy/server/database/objects/5/9/4/dataset_594ce8d3-34d3-4e83-ba57-7f6cd4a922d3.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:16,330 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (7) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/7/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/7/galaxy_7.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:25,017 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 7: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:29,849 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 7 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:51:29,897 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 8
+galaxy.jobs INFO 2024-09-14 00:51:29,931 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 7 in /galaxy/server/database/jobs_directory/000/7
+galaxy.jobs DEBUG 2024-09-14 00:51:29,980 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 7 executed (108.552 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:51:30,476 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 8
+tpv.core.entities DEBUG 2024-09-14 00:51:30,505 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=default, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:51:30,506 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (8) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:51:30,509 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (8) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:51:30,523 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (8) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:51:30,536 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (8) Working directory for job is: /galaxy/server/database/jobs_directory/000/8
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:30,542 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [8] queued (32.672 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:51:30,545 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (8) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:51:30,549 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 8
+galaxy.jobs DEBUG 2024-09-14 00:51:30,615 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [8] prepared (50.597 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:51:30,615 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:51:30,615 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/iuc/annotatemyids/annotatemyids/3.5.0.1: mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:51:30,633 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:51:30,653 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/8/tool_script.sh] for tool command [echo $(R --version | grep version | grep -v GNU)", org.Hs.eg.db version" $(R --vanilla --slave -e "library(org.Hs.eg.db); cat(sessionInfo()\$otherPkgs\$org.Hs.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ")", org.Dr.eg.db version" $(R --vanilla --slave -e "library(org.Dr.eg.db); cat(sessionInfo()\$otherPkgs\$org.Dr.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ")", org.Dm.eg.db version" $(R --vanilla --slave -e "library(org.Dm.eg.db); cat(sessionInfo()\$otherPkgs\$org.Dm.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ")", org.Mm.eg.db version" $(R --vanilla --slave -e "library(org.Mm.eg.db); cat(sessionInfo()\$otherPkgs\$org.Mm.eg.db\$Version)" 2> /dev/null | grep -v -i "WARNING: ") > /galaxy/server/database/jobs_directory/000/8/outputs/COMMAND_VERSION 2>&1;
+Rscript '/galaxy/server/database/jobs_directory/000/8/configs/tmpfbw71315']
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:30,665 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (8) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/8/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/8/galaxy_8.ec;
+if [ -f "/galaxy/server/database/jobs_directory/000/8/working/"*".tab" ] ; then cp "/galaxy/server/database/jobs_directory/000/8/working/"*".tab" "/galaxy/server/database/objects/1/6/f/dataset_16f94c91-8c43-4524-988d-4e562228ca48.dat" ; fi; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:51:30,676 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:51:30,676 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/iuc/annotatemyids/annotatemyids/3.5.0.1: mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:51:30,691 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-fbf0980300c3eec60801bb7996eb16010c38939d:ac85adae9385b3d083688aab93961c503a6f572b-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:53,549 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] executing external set_meta script for job 8: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:51:58,117 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] execution of external set_meta for job 8 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:51:58,162 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 9
+galaxy.jobs INFO 2024-09-14 00:51:58,192 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Collecting metrics for Job 8 in /galaxy/server/database/jobs_directory/000/8
+galaxy.objectstore CRITICAL 2024-09-14 00:51:58,229 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] None delete error [Errno 13] Permission denied: 'tmpfbw71315'
+galaxy.jobs DEBUG 2024-09-14 00:51:58,229 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] job_wrapper.finish for job 8 executed (89.850 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:02,139 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 9
+tpv.core.entities DEBUG 2024-09-14 00:52:02,171 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:52:02,171 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (9) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:02,176 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (9) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:52:02,188 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (9) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:52:02,204 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (9) Working directory for job is: /galaxy/server/database/jobs_directory/000/9
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:02,211 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [9] queued (35.468 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:52:02,215 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (9) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:52:02,218 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Starting queue_job for job 9
+galaxy.jobs DEBUG 2024-09-14 00:52:02,304 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Job wrapper for Job [9] prepared (70.828 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:52:02,321 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Built script [/galaxy/server/database/jobs_directory/000/9/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/9/registry.xml' '/galaxy/server/database/jobs_directory/000/9/upload_params.json' '10:/galaxy/server/database/objects/8/f/a/dataset_8faa6cfe-41d6-4761-91ed-4863eaa406f4_files:/galaxy/server/database/objects/8/f/a/dataset_8faa6cfe-41d6-4761-91ed-4863eaa406f4.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:02,332 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] (9) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/9/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/9/galaxy_9.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:09,784 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 9: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:14,510 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 9 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:52:14,563 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 10
+galaxy.jobs INFO 2024-09-14 00:52:14,600 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 9 in /galaxy/server/database/jobs_directory/000/9
+galaxy.jobs DEBUG 2024-09-14 00:52:14,655 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 9 executed (120.513 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:15,465 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 10
+tpv.core.entities DEBUG 2024-09-14 00:52:15,503 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=default, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:52:15,504 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (10) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:15,511 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (10) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:52:15,528 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (10) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:52:15,548 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (10) Working directory for job is: /galaxy/server/database/jobs_directory/000/10
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:15,555 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [10] queued (44.314 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:52:15,558 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (10) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:52:15,562 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 10
+galaxy.jobs DEBUG 2024-09-14 00:52:15,632 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [10] prepared (52.371 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:15,632 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:52:15,632 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/iuc/datamash_reverse/datamash_reverse/1.8+galaxy0: datamash:1.8
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:15,799 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/datamash:1.8,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:52:15,820 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/10/tool_script.sh] for tool command [datamash reverse < /galaxy/server/database/objects/8/f/a/dataset_8faa6cfe-41d6-4761-91ed-4863eaa406f4.dat > /galaxy/server/database/objects/e/f/b/dataset_efb893ad-c264-445f-a823-63734f94805f.dat]
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:15,831 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (10) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/10/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/10/galaxy_10.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:15,842 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:52:15,842 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/iuc/datamash_reverse/datamash_reverse/1.8+galaxy0: datamash:1.8
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:15,858 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/datamash:1.8,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:22,992 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 10: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:27,819 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 10 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:52:27,873 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 11
+galaxy.jobs INFO 2024-09-14 00:52:27,909 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 10 in /galaxy/server/database/jobs_directory/000/10
+galaxy.jobs DEBUG 2024-09-14 00:52:27,958 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 10 executed (114.196 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:31,891 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 11
+tpv.core.entities DEBUG 2024-09-14 00:52:31,928 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:52:31,929 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (11) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:31,934 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (11) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:52:31,949 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (11) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:52:31,964 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (11) Working directory for job is: /galaxy/server/database/jobs_directory/000/11
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:31,970 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [11] queued (35.239 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:52:31,973 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (11) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:52:31,978 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 11
+galaxy.jobs DEBUG 2024-09-14 00:52:32,082 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [11] prepared (86.333 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:52:32,102 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/11/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/11/registry.xml' '/galaxy/server/database/jobs_directory/000/11/upload_params.json' '12:/galaxy/server/database/objects/7/3/e/dataset_73eedd44-ddbb-4f2d-8dde-5dfd5a60c948_files:/galaxy/server/database/objects/7/3/e/dataset_73eedd44-ddbb-4f2d-8dde-5dfd5a60c948.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:32,115 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (11) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/11/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/11/galaxy_11.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:40,210 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 11: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:45,055 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 11 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:52:45,108 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 12
+galaxy.jobs INFO 2024-09-14 00:52:45,144 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 11 in /galaxy/server/database/jobs_directory/000/11
+galaxy.jobs DEBUG 2024-09-14 00:52:45,198 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 11 executed (118.465 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:46,309 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 12
+tpv.core.entities DEBUG 2024-09-14 00:52:46,341 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:52:46,341 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (12) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:52:46,346 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (12) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:52:46,360 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (12) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:52:46,383 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (12) Working directory for job is: /galaxy/server/database/jobs_directory/000/12
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:46,391 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [12] queued (44.449 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:52:46,394 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (12) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:52:46,399 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 12
+galaxy.jobs DEBUG 2024-09-14 00:52:46,509 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [12] prepared (92.596 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:46,510 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:52:46,510 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:46,687 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:52:46,706 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/12/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/7/3/e/dataset_73eedd44-ddbb-4f2d-8dde-5dfd5a60c948.dat' fastq_in.'fastqsanger' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar SE -threads ${GALAXY_SLOTS:-6} fastq_in.'fastqsanger' fastq_out.'fastqsanger' SLIDINGWINDOW:4:20 -trimlog trimlog 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out.'fastqsanger' '/galaxy/server/database/objects/3/5/a/dataset_35ab1889-5e9b-4f85-984d-7ac85d5d87af.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:52:46,727 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (12) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/12/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/12/galaxy_12.ec;
+if [ -f "/galaxy/server/database/jobs_directory/000/12/working/trimlog" ] ; then cp "/galaxy/server/database/jobs_directory/000/12/working/trimlog" "/galaxy/server/database/objects/c/f/2/dataset_cf220079-4323-49eb-9ac7-0a9883096b20.dat" ; fi;
+if [ -f "/galaxy/server/database/jobs_directory/000/12/working/trimmomatic.log" ] ; then cp "/galaxy/server/database/jobs_directory/000/12/working/trimmomatic.log" "/galaxy/server/database/objects/a/a/9/dataset_aa92a19e-03d0-4938-a23c-88c25d7ae205.dat" ; fi; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:46,739 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:52:46,739 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:52:46,756 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:05,788 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] executing external set_meta script for job 12: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:10,666 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] execution of external set_meta for job 12 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:53:10,725 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] loading metadata from file for: HistoryDatasetAssociation 13
+galaxy.model.metadata DEBUG 2024-09-14 00:53:10,735 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] loading metadata from file for: HistoryDatasetAssociation 14
+galaxy.model.metadata DEBUG 2024-09-14 00:53:10,745 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] loading metadata from file for: HistoryDatasetAssociation 15
+galaxy.util WARNING 2024-09-14 00:53:10,754 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/3/5/a/dataset_35ab1889-5e9b-4f85-984d-7ac85d5d87af.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/3/5/a/dataset_35ab1889-5e9b-4f85-984d-7ac85d5d87af.dat'
+galaxy.jobs INFO 2024-09-14 00:53:10,787 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Collecting metrics for Job 12 in /galaxy/server/database/jobs_directory/000/12
+galaxy.objectstore CRITICAL 2024-09-14 00:53:10,837 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] None delete error [Errno 13] Permission denied: 'tmpuvdw9znp'
+galaxy.jobs DEBUG 2024-09-14 00:53:10,838 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] job_wrapper.finish for job 12 executed (145.422 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:15,040 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 13
+tpv.core.entities DEBUG 2024-09-14 00:53:15,078 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:53:15,078 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (13) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:15,083 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (13) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:53:15,098 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (13) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:53:15,121 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (13) Working directory for job is: /galaxy/server/database/jobs_directory/000/13
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:15,128 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [13] queued (45.058 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:53:15,132 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (13) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:15,137 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 13
+galaxy.jobs DEBUG 2024-09-14 00:53:15,235 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [13] prepared (80.032 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:53:15,255 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/13/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/13/registry.xml' '/galaxy/server/database/jobs_directory/000/13/upload_params.json' '16:/galaxy/server/database/objects/d/d/7/dataset_dd7cfbb0-f7a0-489a-868d-01808f00a8fd_files:/galaxy/server/database/objects/d/d/7/dataset_dd7cfbb0-f7a0-489a-868d-01808f00a8fd.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:15,266 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (13) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/13/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/13/galaxy_13.ec; sh -c "exit $return_code"
+galaxy.jobs.runners.kubernetes ERROR 2024-09-14 00:53:22,931 [pN:job_handler_0,p:8,tN:KubernetesRunner.monitor_thread] Could not clean up k8s batch job. Ignoring...
+Traceback (most recent call last):
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 437, in raise_for_status
+ resp.raise_for_status()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+ raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://10.108.80.1:443/apis/batch/v1/namespaces/prod-24-09-14-00-32-1/jobs/gxy-rzcrg
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 817, in _handle_job_failure
+ self.__cleanup_k8s_job(job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 824, in __cleanup_k8s_job
+ delete_job(job, k8s_cleanup_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/util/pykube_util.py", line 107, in delete_job
+ job.scale(replicas=0)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/mixins.py", line 30, in scale
+ self.update()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 165, in update
+ self.patch(self.obj, subresource=subresource)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 157, in patch
+ self.api.raise_for_status(r)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 444, in raise_for_status
+ raise HTTPError(resp.status_code, payload["message"])
+pykube.exceptions.HTTPError: Operation cannot be fulfilled on jobs.batch "gxy-rzcrg": the object has been modified; please apply your changes to the latest version and try again
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:22,975 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] (13/gxy-rzcrg) Terminated at user's request
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:24,322 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 14
+tpv.core.entities DEBUG 2024-09-14 00:53:24,357 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:53:24,357 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (14) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:24,363 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (14) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:53:24,377 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (14) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:53:24,393 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (14) Working directory for job is: /galaxy/server/database/jobs_directory/000/14
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:24,399 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [14] queued (36.192 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:53:24,402 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (14) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:24,407 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 14
+galaxy.jobs DEBUG 2024-09-14 00:53:24,506 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [14] prepared (80.328 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:53:24,528 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/14/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/14/registry.xml' '/galaxy/server/database/jobs_directory/000/14/upload_params.json' '17:/galaxy/server/database/objects/5/3/5/dataset_53530603-4b11-4771-b00f-55b679cdffef_files:/galaxy/server/database/objects/5/3/5/dataset_53530603-4b11-4771-b00f-55b679cdffef.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:24,542 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (14) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/14/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/14/galaxy_14.ec; sh -c "exit $return_code"
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:25,405 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 15
+tpv.core.entities DEBUG 2024-09-14 00:53:25,437 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:53:25,437 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (15) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:25,442 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (15) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:53:25,461 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (15) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:53:25,479 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (15) Working directory for job is: /galaxy/server/database/jobs_directory/000/15
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:25,485 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [15] queued (43.269 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:53:25,489 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (15) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:25,494 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 15
+galaxy.jobs DEBUG 2024-09-14 00:53:25,593 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [15] prepared (79.938 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:53:25,616 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/15/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/15/registry.xml' '/galaxy/server/database/jobs_directory/000/15/upload_params.json' '18:/galaxy/server/database/objects/3/7/6/dataset_376767a6-2c00-4bc6-9292-29286bacc97d_files:/galaxy/server/database/objects/3/7/6/dataset_376767a6-2c00-4bc6-9292-29286bacc97d.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:25,629 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (15) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/15/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/15/galaxy_15.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:32,701 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] executing external set_meta script for job 14: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:33,848 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 15: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:37,602 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] execution of external set_meta for job 14 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:53:37,655 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] loading metadata from file for: HistoryDatasetAssociation 17
+galaxy.jobs INFO 2024-09-14 00:53:37,690 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Collecting metrics for Job 14 in /galaxy/server/database/jobs_directory/000/14
+galaxy.jobs DEBUG 2024-09-14 00:53:37,738 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] job_wrapper.finish for job 14 executed (112.878 ms)
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:38,654 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 15 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:53:38,745 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 18
+galaxy.jobs INFO 2024-09-14 00:53:38,806 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 15 in /galaxy/server/database/jobs_directory/000/15
+galaxy.jobs DEBUG 2024-09-14 00:53:38,863 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 15 executed (150.214 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:39,927 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 16
+tpv.core.entities DEBUG 2024-09-14 00:53:39,969 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:53:39,970 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (16) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:39,977 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (16) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:53:39,993 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (16) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:53:40,020 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (16) Working directory for job is: /galaxy/server/database/jobs_directory/000/16
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:40,030 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [16] queued (52.133 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:53:40,035 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (16) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:40,037 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 16
+galaxy.jobs DEBUG 2024-09-14 00:53:40,124 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [16] prepared (71.079 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:53:40,125 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:53:40,125 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:53:40,144 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:53:40,165 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/16/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/5/3/5/dataset_53530603-4b11-4771-b00f-55b679cdffef.dat' fastq_r1.'fastqsanger.gz' && ln -s '/galaxy/server/database/objects/3/7/6/dataset_376767a6-2c00-4bc6-9292-29286bacc97d.dat' fastq_r2.'fastqsanger.gz' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar PE -threads ${GALAXY_SLOTS:-6} fastq_r1.'fastqsanger.gz' fastq_r2.'fastqsanger.gz' fastq_out_r1_paired.'fastqsanger.gz' fastq_out_r1_unpaired.'fastqsanger.gz' fastq_out_r2_paired.'fastqsanger.gz' fastq_out_r2_unpaired.'fastqsanger.gz' SLIDINGWINDOW:4:20 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out_r1_paired.'fastqsanger.gz' '/galaxy/server/database/objects/f/3/d/dataset_f3d1ff58-dc18-49db-b4e4-db6176e0218b.dat' && mv fastq_out_r1_unpaired.'fastqsanger.gz' '/galaxy/server/database/objects/f/5/0/dataset_f50f33ad-1d07-4f0d-b25a-a7f8da385242.dat' && mv fastq_out_r2_paired.'fastqsanger.gz' '/galaxy/server/database/objects/3/9/1/dataset_391d572f-5ab9-4e2e-9ee3-ab4992b86b14.dat' && mv fastq_out_r2_unpaired.'fastqsanger.gz' '/galaxy/server/database/objects/7/3/8/dataset_738b4cb6-8c7c-41cf-978d-c535b76e59a5.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:40,194 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (16) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/16/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/16/galaxy_16.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:53:40,207 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:53:40,207 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:53:40,224 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:45,203 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 16: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:50,069 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 16 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:53:50,130 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 19
+galaxy.model.metadata DEBUG 2024-09-14 00:53:50,141 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 20
+galaxy.model.metadata DEBUG 2024-09-14 00:53:50,153 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 21
+galaxy.model.metadata DEBUG 2024-09-14 00:53:50,165 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 22
+galaxy.util WARNING 2024-09-14 00:53:50,175 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/f/3/d/dataset_f3d1ff58-dc18-49db-b4e4-db6176e0218b.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/f/3/d/dataset_f3d1ff58-dc18-49db-b4e4-db6176e0218b.dat'
+galaxy.util WARNING 2024-09-14 00:53:50,175 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/3/9/1/dataset_391d572f-5ab9-4e2e-9ee3-ab4992b86b14.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/3/9/1/dataset_391d572f-5ab9-4e2e-9ee3-ab4992b86b14.dat'
+galaxy.util WARNING 2024-09-14 00:53:50,176 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/f/5/0/dataset_f50f33ad-1d07-4f0d-b25a-a7f8da385242.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/f/5/0/dataset_f50f33ad-1d07-4f0d-b25a-a7f8da385242.dat'
+galaxy.util WARNING 2024-09-14 00:53:50,177 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/7/3/8/dataset_738b4cb6-8c7c-41cf-978d-c535b76e59a5.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/7/3/8/dataset_738b4cb6-8c7c-41cf-978d-c535b76e59a5.dat'
+galaxy.jobs INFO 2024-09-14 00:53:50,211 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 16 in /galaxy/server/database/jobs_directory/000/16
+galaxy.objectstore CRITICAL 2024-09-14 00:53:50,266 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] None delete error [Errno 13] Permission denied: 'tmpuq3dymqu'
+galaxy.jobs DEBUG 2024-09-14 00:53:50,266 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 16 executed (171.682 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:55,394 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 17, 18
+tpv.core.entities DEBUG 2024-09-14 00:53:55,427 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:53:55,428 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (17) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:55,432 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (17) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:53:55,448 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (17) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:53:55,466 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (17) Working directory for job is: /galaxy/server/database/jobs_directory/000/17
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:55,472 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [17] queued (40.600 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:53:55,476 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (17) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:55,482 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 17
+tpv.core.entities DEBUG 2024-09-14 00:53:55,493 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:53:55,494 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (18) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:53:55,500 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (18) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:53:55,519 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (18) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:53:55,551 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (18) Working directory for job is: /galaxy/server/database/jobs_directory/000/18
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:55,562 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [18] queued (61.371 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:53:55,567 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (18) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:53:55,572 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Starting queue_job for job 18
+galaxy.jobs DEBUG 2024-09-14 00:53:55,598 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [17] prepared (96.389 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:53:55,624 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/17/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/17/registry.xml' '/galaxy/server/database/jobs_directory/000/17/upload_params.json' '23:/galaxy/server/database/objects/a/3/9/dataset_a399456d-910b-4145-8d59-78d9823e55eb_files:/galaxy/server/database/objects/a/3/9/dataset_a399456d-910b-4145-8d59-78d9823e55eb.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:55,646 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (17) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/17/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/17/galaxy_17.ec; sh -c "exit $return_code"
+galaxy.jobs DEBUG 2024-09-14 00:53:55,700 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Job wrapper for Job [18] prepared (108.063 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:53:55,721 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Built script [/galaxy/server/database/jobs_directory/000/18/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/18/registry.xml' '/galaxy/server/database/jobs_directory/000/18/upload_params.json' '24:/galaxy/server/database/objects/9/7/6/dataset_9761247d-c9ae-48df-900f-53bd8cc116f1_files:/galaxy/server/database/objects/9/7/6/dataset_9761247d-c9ae-48df-900f-53bd8cc116f1.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:53:55,735 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] (18) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/18/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/18/galaxy_18.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:03,546 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] executing external set_meta script for job 17: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:03,561 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] executing external set_meta script for job 18: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:08,345 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] execution of external set_meta for job 17 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:54:08,406 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 23
+galaxy.jobs INFO 2024-09-14 00:54:08,439 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Collecting metrics for Job 17 in /galaxy/server/database/jobs_directory/000/17
+galaxy.jobs DEBUG 2024-09-14 00:54:08,492 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] job_wrapper.finish for job 17 executed (120.618 ms)
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:08,494 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] execution of external set_meta for job 18 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:54:08,553 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] loading metadata from file for: HistoryDatasetAssociation 24
+galaxy.jobs INFO 2024-09-14 00:54:08,593 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Collecting metrics for Job 18 in /galaxy/server/database/jobs_directory/000/18
+galaxy.jobs DEBUG 2024-09-14 00:54:08,649 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] job_wrapper.finish for job 18 executed (127.231 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:09,856 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 19
+tpv.core.entities DEBUG 2024-09-14 00:54:09,887 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:54:09,887 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (19) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:09,891 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (19) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:54:09,908 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (19) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:54:09,935 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (19) Working directory for job is: /galaxy/server/database/jobs_directory/000/19
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:09,942 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [19] queued (50.152 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:54:09,945 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (19) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:54:09,950 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 19
+galaxy.jobs DEBUG 2024-09-14 00:54:10,045 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [19] prepared (74.758 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:10,046 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:54:10,046 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:10,227 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:54:10,248 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/19/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/a/3/9/dataset_a399456d-910b-4145-8d59-78d9823e55eb.dat' fastq_r1.'fastqsanger' && ln -s '/galaxy/server/database/objects/9/7/6/dataset_9761247d-c9ae-48df-900f-53bd8cc116f1.dat' fastq_r2.'fastqsanger' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar PE -threads ${GALAXY_SLOTS:-6} fastq_r1.'fastqsanger' fastq_r2.'fastqsanger' fastq_out_r1_paired.'fastqsanger' fastq_out_r1_unpaired.'fastqsanger' fastq_out_r2_paired.'fastqsanger' fastq_out_r2_unpaired.'fastqsanger' SLIDINGWINDOW:4:20 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out_r1_paired.'fastqsanger' '/galaxy/server/database/objects/f/4/9/dataset_f49461db-2aa7-4951-a7f8-5f1de529a7ba.dat' && mv fastq_out_r1_unpaired.'fastqsanger' '/galaxy/server/database/objects/e/e/8/dataset_ee808beb-2d67-4978-9f04-282d366f017a.dat' && mv fastq_out_r2_paired.'fastqsanger' '/galaxy/server/database/objects/9/c/e/dataset_9ce668de-b481-4e51-8d19-82fbcc93e8c9.dat' && mv fastq_out_r2_unpaired.'fastqsanger' '/galaxy/server/database/objects/c/d/e/dataset_cde3d066-4d48-4554-8895-ec108140037e.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:10,276 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (19) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/19/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/19/galaxy_19.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:10,289 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:54:10,289 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:10,307 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:14,748 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] executing external set_meta script for job 19: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:19,544 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] execution of external set_meta for job 19 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:54:19,593 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 25
+galaxy.model.metadata DEBUG 2024-09-14 00:54:19,603 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 26
+galaxy.model.metadata DEBUG 2024-09-14 00:54:19,632 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 27
+galaxy.model.metadata DEBUG 2024-09-14 00:54:19,663 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 28
+galaxy.util WARNING 2024-09-14 00:54:19,670 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/f/4/9/dataset_f49461db-2aa7-4951-a7f8-5f1de529a7ba.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/f/4/9/dataset_f49461db-2aa7-4951-a7f8-5f1de529a7ba.dat'
+galaxy.util WARNING 2024-09-14 00:54:19,671 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/9/c/e/dataset_9ce668de-b481-4e51-8d19-82fbcc93e8c9.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/9/c/e/dataset_9ce668de-b481-4e51-8d19-82fbcc93e8c9.dat'
+galaxy.util WARNING 2024-09-14 00:54:19,671 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/e/e/8/dataset_ee808beb-2d67-4978-9f04-282d366f017a.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/e/e/8/dataset_ee808beb-2d67-4978-9f04-282d366f017a.dat'
+galaxy.util WARNING 2024-09-14 00:54:19,671 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/c/d/e/dataset_cde3d066-4d48-4554-8895-ec108140037e.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/c/d/e/dataset_cde3d066-4d48-4554-8895-ec108140037e.dat'
+galaxy.jobs INFO 2024-09-14 00:54:19,699 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Collecting metrics for Job 19 in /galaxy/server/database/jobs_directory/000/19
+galaxy.objectstore CRITICAL 2024-09-14 00:54:19,739 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] None delete error [Errno 13] Permission denied: 'tmpjnc0o6y1'
+galaxy.jobs DEBUG 2024-09-14 00:54:19,740 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] job_wrapper.finish for job 19 executed (172.662 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:24,225 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 20
+tpv.core.entities DEBUG 2024-09-14 00:54:24,254 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:54:24,255 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (20) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:24,259 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (20) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:54:24,272 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (20) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:54:24,287 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (20) Working directory for job is: /galaxy/server/database/jobs_directory/000/20
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:24,293 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [20] queued (33.553 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:54:24,296 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (20) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:54:24,300 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 20
+galaxy.jobs DEBUG 2024-09-14 00:54:24,387 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [20] prepared (72.894 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:54:24,406 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/20/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/20/registry.xml' '/galaxy/server/database/jobs_directory/000/20/upload_params.json' '29:/galaxy/server/database/objects/a/7/5/dataset_a754df7e-7666-4963-9680-c1eb2b89c689_files:/galaxy/server/database/objects/a/7/5/dataset_a754df7e-7666-4963-9680-c1eb2b89c689.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:24,416 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (20) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/20/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/20/galaxy_20.ec; sh -c "exit $return_code"
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:25,299 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 21
+tpv.core.entities DEBUG 2024-09-14 00:54:25,331 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:54:25,332 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (21) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:25,336 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (21) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:54:25,350 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (21) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:54:25,367 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (21) Working directory for job is: /galaxy/server/database/jobs_directory/000/21
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:25,374 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [21] queued (37.956 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:54:25,378 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (21) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:54:25,382 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 21
+galaxy.jobs DEBUG 2024-09-14 00:54:25,473 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [21] prepared (76.823 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:54:25,492 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/21/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/21/registry.xml' '/galaxy/server/database/jobs_directory/000/21/upload_params.json' '30:/galaxy/server/database/objects/8/5/8/dataset_858a76de-2a92-4830-bf20-150923ad3cee_files:/galaxy/server/database/objects/8/5/8/dataset_858a76de-2a92-4830-bf20-150923ad3cee.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:25,502 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (21) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/21/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/21/galaxy_21.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:40,907 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 20: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:40,931 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] executing external set_meta script for job 21: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:45,691 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 20 finished
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:45,698 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] execution of external set_meta for job 21 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:54:45,746 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 29
+galaxy.model.metadata DEBUG 2024-09-14 00:54:45,747 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 30
+galaxy.jobs INFO 2024-09-14 00:54:45,790 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 20 in /galaxy/server/database/jobs_directory/000/20
+galaxy.jobs INFO 2024-09-14 00:54:45,793 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Collecting metrics for Job 21 in /galaxy/server/database/jobs_directory/000/21
+galaxy.jobs DEBUG 2024-09-14 00:54:45,841 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 20 executed (125.066 ms)
+galaxy.jobs DEBUG 2024-09-14 00:54:45,842 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] job_wrapper.finish for job 21 executed (124.093 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:46,787 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 22
+tpv.core.entities DEBUG 2024-09-14 00:54:46,824 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:54:46,824 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (22) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:54:46,829 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (22) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:54:46,842 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (22) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:54:46,866 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (22) Working directory for job is: /galaxy/server/database/jobs_directory/000/22
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:46,875 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [22] queued (46.392 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:54:46,879 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (22) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:54:46,883 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Starting queue_job for job 22
+galaxy.jobs DEBUG 2024-09-14 00:54:46,968 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Job wrapper for Job [22] prepared (67.704 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:46,969 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:54:46,969 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:46,989 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:54:47,012 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Built script [/galaxy/server/database/jobs_directory/000/22/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/a/7/5/dataset_a754df7e-7666-4963-9680-c1eb2b89c689.dat' fastq_r1.'fastqillumina' && ln -s '/galaxy/server/database/objects/8/5/8/dataset_858a76de-2a92-4830-bf20-150923ad3cee.dat' fastq_r2.'fastqillumina' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar PE -threads ${GALAXY_SLOTS:-6} fastq_r1.'fastqillumina' fastq_r2.'fastqillumina' fastq_out_r1_paired.'fastqillumina' fastq_out_r1_unpaired.'fastqillumina' fastq_out_r2_paired.'fastqillumina' fastq_out_r2_unpaired.'fastqillumina' SLIDINGWINDOW:4:20 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out_r1_paired.'fastqillumina' '/galaxy/server/database/objects/e/6/f/dataset_e6f68f70-9838-4b6c-9ab8-4d2f4ece8862.dat' && mv fastq_out_r1_unpaired.'fastqillumina' '/galaxy/server/database/objects/c/6/d/dataset_c6d8a8b3-4385-4bce-b0a1-1f6a777cbd6c.dat' && mv fastq_out_r2_paired.'fastqillumina' '/galaxy/server/database/objects/9/e/6/dataset_9e625eef-b2f0-4558-a53d-9ad9c770a633.dat' && mv fastq_out_r2_unpaired.'fastqillumina' '/galaxy/server/database/objects/d/1/d/dataset_d1de5762-2fca-4023-a70d-97380dd0d639.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:47,041 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] (22) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/22/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/22/galaxy_22.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:47,054 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:54:47,054 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:54:47,071 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:52,123 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 22: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:54:56,693 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 22 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:54:56,746 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 31
+galaxy.model.metadata DEBUG 2024-09-14 00:54:56,755 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 32
+galaxy.model.metadata DEBUG 2024-09-14 00:54:56,784 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 33
+galaxy.model.metadata DEBUG 2024-09-14 00:54:56,817 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 34
+galaxy.util WARNING 2024-09-14 00:54:56,824 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/e/6/f/dataset_e6f68f70-9838-4b6c-9ab8-4d2f4ece8862.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/e/6/f/dataset_e6f68f70-9838-4b6c-9ab8-4d2f4ece8862.dat'
+galaxy.util WARNING 2024-09-14 00:54:56,824 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/9/e/6/dataset_9e625eef-b2f0-4558-a53d-9ad9c770a633.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/9/e/6/dataset_9e625eef-b2f0-4558-a53d-9ad9c770a633.dat'
+galaxy.util WARNING 2024-09-14 00:54:56,825 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/c/6/d/dataset_c6d8a8b3-4385-4bce-b0a1-1f6a777cbd6c.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/c/6/d/dataset_c6d8a8b3-4385-4bce-b0a1-1f6a777cbd6c.dat'
+galaxy.util WARNING 2024-09-14 00:54:56,825 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/d/1/d/dataset_d1de5762-2fca-4023-a70d-97380dd0d639.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/d/1/d/dataset_d1de5762-2fca-4023-a70d-97380dd0d639.dat'
+galaxy.jobs INFO 2024-09-14 00:54:56,855 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 22 in /galaxy/server/database/jobs_directory/000/22
+galaxy.objectstore CRITICAL 2024-09-14 00:54:56,898 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] None delete error [Errno 13] Permission denied: 'tmpfqwz9v1b'
+galaxy.jobs DEBUG 2024-09-14 00:54:56,898 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 22 executed (182.165 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:01,248 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 23
+tpv.core.entities DEBUG 2024-09-14 00:55:01,274 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:01,275 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (23) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:01,278 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (23) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:01,290 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (23) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:01,304 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (23) Working directory for job is: /galaxy/server/database/jobs_directory/000/23
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:01,310 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [23] queued (31.302 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:01,313 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (23) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:01,317 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Starting queue_job for job 23
+galaxy.jobs DEBUG 2024-09-14 00:55:01,409 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Job wrapper for Job [23] prepared (77.384 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:01,429 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Built script [/galaxy/server/database/jobs_directory/000/23/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/23/registry.xml' '/galaxy/server/database/jobs_directory/000/23/upload_params.json' '35:/galaxy/server/database/objects/b/c/d/dataset_bcda9236-4541-4155-8ce6-4c842913f7b4_files:/galaxy/server/database/objects/b/c/d/dataset_bcda9236-4541-4155-8ce6-4c842913f7b4.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:01,440 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] (23) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/23/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/23/galaxy_23.ec; sh -c "exit $return_code"
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:02,315 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 24
+tpv.core.entities DEBUG 2024-09-14 00:55:02,342 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:02,342 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (24) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:02,346 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (24) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:02,357 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (24) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:02,369 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (24) Working directory for job is: /galaxy/server/database/jobs_directory/000/24
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:02,375 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [24] queued (29.491 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:02,379 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (24) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:02,381 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Starting queue_job for job 24
+galaxy.jobs DEBUG 2024-09-14 00:55:02,466 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Job wrapper for Job [24] prepared (70.988 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:02,483 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Built script [/galaxy/server/database/jobs_directory/000/24/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/24/registry.xml' '/galaxy/server/database/jobs_directory/000/24/upload_params.json' '36:/galaxy/server/database/objects/e/c/7/dataset_ec7b65f2-1093-405b-9cc9-3f438d77c704_files:/galaxy/server/database/objects/e/c/7/dataset_ec7b65f2-1093-405b-9cc9-3f438d77c704.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:02,495 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] (24) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/24/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/24/galaxy_24.ec; sh -c "exit $return_code"
+galaxy.jobs.runners.kubernetes ERROR 2024-09-14 00:55:09,343 [pN:job_handler_0,p:8,tN:KubernetesRunner.monitor_thread] Could not clean up k8s batch job. Ignoring...
+Traceback (most recent call last):
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 437, in raise_for_status
+ resp.raise_for_status()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+ raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://10.108.80.1:443/apis/batch/v1/namespaces/prod-24-09-14-00-32-1/jobs/gxy-sf4nz
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 817, in _handle_job_failure
+ self.__cleanup_k8s_job(job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 824, in __cleanup_k8s_job
+ delete_job(job, k8s_cleanup_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/util/pykube_util.py", line 107, in delete_job
+ job.scale(replicas=0)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/mixins.py", line 30, in scale
+ self.update()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 165, in update
+ self.patch(self.obj, subresource=subresource)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 157, in patch
+ self.api.raise_for_status(r)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 444, in raise_for_status
+ raise HTTPError(resp.status_code, payload["message"])
+pykube.exceptions.HTTPError: Operation cannot be fulfilled on jobs.batch "gxy-sf4nz": the object has been modified; please apply your changes to the latest version and try again
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:09,381 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] (23/gxy-sf4nz) Terminated at user's request
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:10,505 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 24: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:10,543 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 25
+tpv.core.entities DEBUG 2024-09-14 00:55:10,582 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:10,582 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (25) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:10,588 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (25) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:10,606 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (25) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:10,627 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (25) Working directory for job is: /galaxy/server/database/jobs_directory/000/25
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:10,634 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [25] queued (45.834 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:10,637 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (25) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:10,641 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 25
+galaxy.jobs DEBUG 2024-09-14 00:55:10,741 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [25] prepared (80.127 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:10,763 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/25/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/25/registry.xml' '/galaxy/server/database/jobs_directory/000/25/upload_params.json' '37:/galaxy/server/database/objects/5/5/f/dataset_55fff3a9-de72-43ff-9f76-a236277f93bd_files:/galaxy/server/database/objects/5/5/f/dataset_55fff3a9-de72-43ff-9f76-a236277f93bd.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:10,776 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (25) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/25/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/25/galaxy_25.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:15,175 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 24 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:55:15,230 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 36
+galaxy.jobs INFO 2024-09-14 00:55:15,261 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 24 in /galaxy/server/database/jobs_directory/000/24
+galaxy.jobs DEBUG 2024-09-14 00:55:15,304 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 24 executed (105.261 ms)
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:18,885 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 25: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:23,242 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 25 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:55:23,294 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 37
+galaxy.jobs INFO 2024-09-14 00:55:23,325 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 25 in /galaxy/server/database/jobs_directory/000/25
+galaxy.jobs DEBUG 2024-09-14 00:55:23,367 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 25 executed (101.237 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:23,910 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 26
+tpv.core.entities DEBUG 2024-09-14 00:55:23,943 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:23,943 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (26) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:23,947 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (26) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:23,960 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (26) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:23,973 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (26) Working directory for job is: /galaxy/server/database/jobs_directory/000/26
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:23,979 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [26] queued (31.853 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:23,982 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (26) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:23,986 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Starting queue_job for job 26
+galaxy.jobs DEBUG 2024-09-14 00:55:24,062 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Job wrapper for Job [26] prepared (58.965 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:55:24,063 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:55:24,063 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:55:24,081 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:24,099 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Built script [/galaxy/server/database/jobs_directory/000/26/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/5/5/f/dataset_55fff3a9-de72-43ff-9f76-a236277f93bd.dat' fastq_in.'fastqsanger' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar SE -threads ${GALAXY_SLOTS:-6} fastq_in.'fastqsanger' fastq_out.'fastqsanger' CROP:10 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out.'fastqsanger' '/galaxy/server/database/objects/a/7/4/dataset_a742330e-6930-4420-9551-f2011b494d34.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:24,109 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] (26) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/26/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/26/galaxy_26.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:55:24,120 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:55:24,121 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:55:24,136 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:29,032 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] executing external set_meta script for job 26: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:33,472 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] execution of external set_meta for job 26 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:55:33,528 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 38
+galaxy.util WARNING 2024-09-14 00:55:33,534 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/a/7/4/dataset_a742330e-6930-4420-9551-f2011b494d34.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/a/7/4/dataset_a742330e-6930-4420-9551-f2011b494d34.dat'
+galaxy.jobs INFO 2024-09-14 00:55:33,559 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Collecting metrics for Job 26 in /galaxy/server/database/jobs_directory/000/26
+galaxy.objectstore CRITICAL 2024-09-14 00:55:33,597 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] None delete error [Errno 13] Permission denied: 'tmp8nb3oq1v'
+galaxy.jobs DEBUG 2024-09-14 00:55:33,597 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] job_wrapper.finish for job 26 executed (101.026 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:36,214 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 27, 28
+tpv.core.entities DEBUG 2024-09-14 00:55:36,247 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:36,247 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (27) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:36,251 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (27) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:36,265 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (27) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:36,281 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (27) Working directory for job is: /galaxy/server/database/jobs_directory/000/27
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:36,288 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [27] queued (37.114 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:36,292 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (27) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:36,297 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Starting queue_job for job 27
+tpv.core.entities DEBUG 2024-09-14 00:55:36,308 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:36,308 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (28) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:36,314 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (28) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:36,330 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (28) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:36,355 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (28) Working directory for job is: /galaxy/server/database/jobs_directory/000/28
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:36,363 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [28] queued (49.105 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:36,368 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (28) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:36,373 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 28
+galaxy.jobs DEBUG 2024-09-14 00:55:36,405 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Job wrapper for Job [27] prepared (89.437 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:36,436 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Built script [/galaxy/server/database/jobs_directory/000/27/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/27/registry.xml' '/galaxy/server/database/jobs_directory/000/27/upload_params.json' '39:/galaxy/server/database/objects/f/d/6/dataset_fd66c5b9-106d-4919-aced-58fe1b8d5934_files:/galaxy/server/database/objects/f/d/6/dataset_fd66c5b9-106d-4919-aced-58fe1b8d5934.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:36,460 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] (27) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/27/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/27/galaxy_27.ec; sh -c "exit $return_code"
+galaxy.jobs DEBUG 2024-09-14 00:55:36,481 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [28] prepared (87.720 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:36,502 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/28/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/28/registry.xml' '/galaxy/server/database/jobs_directory/000/28/upload_params.json' '40:/galaxy/server/database/objects/d/3/9/dataset_d393355a-1c2b-44f2-8534-fa14c8a5a6c0_files:/galaxy/server/database/objects/d/3/9/dataset_d393355a-1c2b-44f2-8534-fa14c8a5a6c0.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:36,514 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (28) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/28/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/28/galaxy_28.ec; sh -c "exit $return_code"
+galaxy.jobs.runners.kubernetes ERROR 2024-09-14 00:55:45,093 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (28/gxy-ww4c7) User killed running job, but error encountered during termination: Operation cannot be fulfilled on jobs.batch "gxy-ww4c7": the object has been modified; please apply your changes to the latest version and try again
+Traceback (most recent call last):
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 437, in raise_for_status
+ resp.raise_for_status()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+ raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://10.108.80.1:443/apis/batch/v1/namespaces/prod-24-09-14-00-32-1/jobs/gxy-ww4c7
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 908, in stop_job
+ self.__cleanup_k8s_job(k8s_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 824, in __cleanup_k8s_job
+ delete_job(job, k8s_cleanup_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/util/pykube_util.py", line 107, in delete_job
+ job.scale(replicas=0)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/mixins.py", line 30, in scale
+ self.update()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 165, in update
+ self.patch(self.obj, subresource=subresource)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 157, in patch
+ self.api.raise_for_status(r)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 444, in raise_for_status
+ raise HTTPError(resp.status_code, payload["message"])
+pykube.exceptions.HTTPError: Operation cannot be fulfilled on jobs.batch "gxy-ww4c7": the object has been modified; please apply your changes to the latest version and try again
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:46,420 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 27: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:51,126 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 27 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:55:51,187 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 39
+galaxy.jobs INFO 2024-09-14 00:55:51,220 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 27 in /galaxy/server/database/jobs_directory/000/27
+galaxy.jobs DEBUG 2024-09-14 00:55:51,268 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 27 executed (115.324 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:52,670 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 29
+tpv.core.entities DEBUG 2024-09-14 00:55:52,699 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:52,700 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (29) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:52,704 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (29) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:52,718 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (29) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:52,732 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (29) Working directory for job is: /galaxy/server/database/jobs_directory/000/29
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:52,738 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [29] queued (34.069 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:52,741 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (29) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:52,744 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 29
+galaxy.jobs DEBUG 2024-09-14 00:55:52,833 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [29] prepared (70.905 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:52,853 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/29/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/29/registry.xml' '/galaxy/server/database/jobs_directory/000/29/upload_params.json' '41:/galaxy/server/database/objects/9/2/a/dataset_92a08b46-ac6f-476a-8cfa-16e186772b9c_files:/galaxy/server/database/objects/9/2/a/dataset_92a08b46-ac6f-476a-8cfa-16e186772b9c.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:52,866 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (29) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/29/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/29/galaxy_29.ec; sh -c "exit $return_code"
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:53,743 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 30
+tpv.core.entities DEBUG 2024-09-14 00:55:53,776 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:55:53,777 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (30) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:55:53,781 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (30) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:55:53,794 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (30) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:55:53,812 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (30) Working directory for job is: /galaxy/server/database/jobs_directory/000/30
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:53,818 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [30] queued (36.791 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:55:53,821 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (30) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:55:53,825 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 30
+galaxy.jobs DEBUG 2024-09-14 00:55:53,909 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [30] prepared (67.761 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:55:53,926 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/30/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/30/registry.xml' '/galaxy/server/database/jobs_directory/000/30/upload_params.json' '42:/galaxy/server/database/objects/6/c/4/dataset_6c491894-0ad3-4510-a9c4-8a10c6045ac5_files:/galaxy/server/database/objects/6/c/4/dataset_6c491894-0ad3-4510-a9c4-8a10c6045ac5.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:55:53,938 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (30) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/30/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/30/galaxy_30.ec; sh -c "exit $return_code"
+galaxy.jobs.runners.kubernetes ERROR 2024-09-14 00:56:01,122 [pN:job_handler_0,p:8,tN:KubernetesRunner.monitor_thread] Could not clean up k8s batch job. Ignoring...
+Traceback (most recent call last):
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 437, in raise_for_status
+ resp.raise_for_status()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+ raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://10.108.80.1:443/apis/batch/v1/namespaces/prod-24-09-14-00-32-1/jobs/gxy-bh495
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 817, in _handle_job_failure
+ self.__cleanup_k8s_job(job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 824, in __cleanup_k8s_job
+ delete_job(job, k8s_cleanup_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/util/pykube_util.py", line 107, in delete_job
+ job.scale(replicas=0)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/mixins.py", line 30, in scale
+ self.update()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 165, in update
+ self.patch(self.obj, subresource=subresource)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 157, in patch
+ self.api.raise_for_status(r)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 444, in raise_for_status
+ raise HTTPError(resp.status_code, payload["message"])
+pykube.exceptions.HTTPError: Operation cannot be fulfilled on jobs.batch "gxy-bh495": the object has been modified; please apply your changes to the latest version and try again
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:56:01,156 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (30/gxy-bh495) Terminated at user's request
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:01,165 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 29: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:05,688 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 29 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:56:05,733 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 41
+galaxy.jobs INFO 2024-09-14 00:56:05,767 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 29 in /galaxy/server/database/jobs_directory/000/29
+galaxy.jobs DEBUG 2024-09-14 00:56:05,812 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 29 executed (100.949 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:08,072 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 31
+tpv.core.entities DEBUG 2024-09-14 00:56:08,100 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:56:08,100 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (31) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:08,104 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (31) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:56:08,116 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (31) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:56:08,131 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (31) Working directory for job is: /galaxy/server/database/jobs_directory/000/31
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:08,138 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [31] queued (33.749 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:56:08,140 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (31) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:56:08,144 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 31
+galaxy.jobs DEBUG 2024-09-14 00:56:08,238 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [31] prepared (80.240 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:56:08,256 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/31/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/31/registry.xml' '/galaxy/server/database/jobs_directory/000/31/upload_params.json' '43:/galaxy/server/database/objects/8/1/3/dataset_813b5365-28a5-418e-a20e-6b04f7f61870_files:/galaxy/server/database/objects/8/1/3/dataset_813b5365-28a5-418e-a20e-6b04f7f61870.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:08,267 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (31) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/31/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/31/galaxy_31.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:16,419 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] executing external set_meta script for job 31: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:21,085 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] execution of external set_meta for job 31 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:56:21,135 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] loading metadata from file for: HistoryDatasetAssociation 43
+galaxy.jobs INFO 2024-09-14 00:56:21,167 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Collecting metrics for Job 31 in /galaxy/server/database/jobs_directory/000/31
+galaxy.jobs DEBUG 2024-09-14 00:56:21,212 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] job_wrapper.finish for job 31 executed (103.062 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:22,511 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 32
+tpv.core.entities DEBUG 2024-09-14 00:56:22,543 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:56:22,543 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (32) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:22,548 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (32) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:56:22,561 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (32) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:56:22,573 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (32) Working directory for job is: /galaxy/server/database/jobs_directory/000/32
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:22,580 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [32] queued (32.014 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:56:22,583 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (32) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:56:22,587 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Starting queue_job for job 32
+galaxy.jobs DEBUG 2024-09-14 00:56:22,649 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Job wrapper for Job [32] prepared (46.793 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:22,649 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:56:22,650 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:22,666 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:56:22,685 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Built script [/galaxy/server/database/jobs_directory/000/32/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/8/1/3/dataset_813b5365-28a5-418e-a20e-6b04f7f61870.dat' fastq_in.'fastqsanger' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar SE -threads ${GALAXY_SLOTS:-6} fastq_in.'fastqsanger' fastq_out.'fastqsanger' AVGQUAL:30 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out.'fastqsanger' '/galaxy/server/database/objects/5/5/0/dataset_550be6c4-6a57-46b0-918e-0aacda978dd7.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:22,697 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] (32) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/32/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/32/galaxy_32.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:22,708 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:56:22,708 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:22,723 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:27,539 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] executing external set_meta script for job 32: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:32,117 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] execution of external set_meta for job 32 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:56:32,176 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] loading metadata from file for: HistoryDatasetAssociation 44
+galaxy.util WARNING 2024-09-14 00:56:32,182 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/5/5/0/dataset_550be6c4-6a57-46b0-918e-0aacda978dd7.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/5/5/0/dataset_550be6c4-6a57-46b0-918e-0aacda978dd7.dat'
+galaxy.jobs INFO 2024-09-14 00:56:32,209 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] Collecting metrics for Job 32 in /galaxy/server/database/jobs_directory/000/32
+galaxy.objectstore CRITICAL 2024-09-14 00:56:32,245 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] None delete error [Errno 13] Permission denied: 'tmpn3y9yqye'
+galaxy.jobs DEBUG 2024-09-14 00:56:32,245 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-0] job_wrapper.finish for job 32 executed (99.914 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:34,820 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 33
+tpv.core.entities DEBUG 2024-09-14 00:56:34,850 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:56:34,851 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (33) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:34,854 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (33) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:56:34,869 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (33) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:56:34,883 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (33) Working directory for job is: /galaxy/server/database/jobs_directory/000/33
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:34,889 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [33] queued (34.680 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:56:34,892 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (33) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:56:34,897 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Starting queue_job for job 33
+galaxy.jobs DEBUG 2024-09-14 00:56:34,991 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Job wrapper for Job [33] prepared (77.466 ms)
+galaxy.jobs.command_factory INFO 2024-09-14 00:56:35,011 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Built script [/galaxy/server/database/jobs_directory/000/33/tool_script.sh] for tool command [python '/galaxy/server/database/tools/data_source/upload.py' '/galaxy/server' '/galaxy/server/database/jobs_directory/000/33/registry.xml' '/galaxy/server/database/jobs_directory/000/33/upload_params.json' '45:/galaxy/server/database/objects/3/7/7/dataset_37778b71-9669-4df5-880b-532f8c4b276b_files:/galaxy/server/database/objects/3/7/7/dataset_37778b71-9669-4df5-880b-532f8c4b276b.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:35,023 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (33) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/33/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/33/galaxy_33.ec; sh -c "exit $return_code"
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:42,789 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] executing external set_meta script for job 33: GALAXY_LIB="/galaxy/server/lib"; if [ "$GALAXY_LIB" != "None" ]; then if [ -n "$PYTHONPATH" ]; then PYTHONPATH="$GALAXY_LIB:$PYTHONPATH"; else PYTHONPATH="$GALAXY_LIB"; fi; export PYTHONPATH; fi; GALAXY_VIRTUAL_ENV="None"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi; python metadata/set.py
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:47,316 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] execution of external set_meta for job 33 finished
+galaxy.model.metadata DEBUG 2024-09-14 00:56:47,362 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] loading metadata from file for: HistoryDatasetAssociation 45
+galaxy.jobs INFO 2024-09-14 00:56:47,393 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] Collecting metrics for Job 33 in /galaxy/server/database/jobs_directory/000/33
+galaxy.jobs DEBUG 2024-09-14 00:56:47,437 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-2] job_wrapper.finish for job 33 executed (98.461 ms)
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:48,121 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 34
+tpv.core.entities DEBUG 2024-09-14 00:56:48,151 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/.*, abstract=False, cores=6, mem=12, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[{'name': '_JAVA_OPTIONS', 'value': '-Xmx{int(mem)}G -Xms1G'}], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:56:48,151 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (34) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:48,155 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (34) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:56:48,166 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (34) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:56:48,180 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (34) Working directory for job is: /galaxy/server/database/jobs_directory/000/34
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:48,187 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [34] queued (31.909 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:56:48,190 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (34) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:56:48,193 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 34
+galaxy.jobs DEBUG 2024-09-14 00:56:48,252 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Job wrapper for Job [34] prepared (46.499 ms)
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:48,252 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:56:48,252 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:48,270 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.command_factory INFO 2024-09-14 00:56:48,290 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Built script [/galaxy/server/database/jobs_directory/000/34/tool_script.sh] for tool command [if [ -z "$TRIMMOMATIC_JAR_PATH" ]; then export TRIMMOMATIC_JAR_PATH=$(dirname $(readlink -e $(which trimmomatic))); fi && if [ -z "$TRIMMOMATIC_ADAPTERS_PATH" ]; then export TRIMMOMATIC_ADAPTERS_PATH=$(dirname $(readlink -e $(which trimmomatic)))/adapters; fi && ln -s '/galaxy/server/database/objects/3/7/7/dataset_37778b71-9669-4df5-880b-532f8c4b276b.dat' fastq_in.'fastqsanger' && java ${_JAVA_OPTIONS:--Xmx8G} -jar $TRIMMOMATIC_JAR_PATH/trimmomatic.jar SE -threads ${GALAXY_SLOTS:-6} fastq_in.'fastqsanger' fastq_out.'fastqsanger' MAXINFO:75:0.8 2>&1 | tee trimmomatic.log && if [ -z "$(tail -1 trimmomatic.log | grep "Completed successfully")" ]; then echo "Trimmomatic did not finish successfully" >&2 ; exit 1 ; fi && mv fastq_out.'fastqsanger' '/galaxy/server/database/objects/e/6/a/dataset_e6ac6186-9f77-45ca-a14f-43c8894a7811.dat']
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:48,299 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] (34) command is: mkdir -p working outputs configs
+if [ -d _working ]; then
+ rm -rf working/ outputs/ configs/; cp -R _working working; cp -R _outputs outputs; cp -R _configs configs
+else
+ cp -R working _working; cp -R outputs _outputs; cp -R configs _configs
+fi
+cd working; __out="${TMPDIR:-.}/out.$$" __err="${TMPDIR:-.}/err.$$"
+mkfifo "$__out" "$__err"
+trap 'rm -f "$__out" "$__err"' EXIT
+tee -a '../outputs/tool_stdout' < "$__out" &
+tee -a '../outputs/tool_stderr' < "$__err" >&2 & /bin/bash /galaxy/server/database/jobs_directory/000/34/tool_script.sh > "$__out" 2> "$__err"; return_code=$?; echo $return_code > /galaxy/server/database/jobs_directory/000/34/galaxy_34.ec; sh -c "exit $return_code"
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:48,310 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [ExplicitContainerResolver[]] found description [None]
+galaxy.tool_util.deps.container_resolvers.mulled DEBUG 2024-09-14 00:56:48,310 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Image name for tool toolshed.g2.bx.psu.edu/repos/pjbriggs/trimmomatic/trimmomatic/0.38.1: mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e
+galaxy.tool_util.deps.containers INFO 2024-09-14 00:56:48,326 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Checking with container resolver [MulledDockerContainerResolver[namespace=biocontainers]] found description [ContainerDescription[identifier=quay.io/biocontainers/mulled-v2-e1a6ade46756da3bd8ceef31e6d2e7645f383a87:5cea15127895d50e80d4a0350938adabaf77375e-0,type=docker]]
+galaxy.jobs.runners.kubernetes ERROR 2024-09-14 00:56:51,996 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] (34/gxy-q74tk) User killed running job, but error encountered during termination: Operation cannot be fulfilled on jobs.batch "gxy-q74tk": the object has been modified; please apply your changes to the latest version and try again
+Traceback (most recent call last):
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 437, in raise_for_status
+ resp.raise_for_status()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+ raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://10.108.80.1:443/apis/batch/v1/namespaces/prod-24-09-14-00-32-1/jobs/gxy-q74tk
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 908, in stop_job
+ self.__cleanup_k8s_job(k8s_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 824, in __cleanup_k8s_job
+ delete_job(job, k8s_cleanup_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/util/pykube_util.py", line 107, in delete_job
+ job.scale(replicas=0)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/mixins.py", line 30, in scale
+ self.update()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 165, in update
+ self.patch(self.obj, subresource=subresource)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 157, in patch
+ self.api.raise_for_status(r)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 444, in raise_for_status
+ raise HTTPError(resp.status_code, payload["message"])
+pykube.exceptions.HTTPError: Operation cannot be fulfilled on jobs.batch "gxy-q74tk": the object has been modified; please apply your changes to the latest version and try again
+galaxy.jobs.runners.kubernetes ERROR 2024-09-14 00:56:52,006 [pN:job_handler_0,p:8,tN:KubernetesRunner.monitor_thread] Could not clean up k8s batch job. Ignoring...
+Traceback (most recent call last):
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 437, in raise_for_status
+ resp.raise_for_status()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/requests/models.py", line 1021, in raise_for_status
+ raise HTTPError(http_error_msg, response=self)
+requests.exceptions.HTTPError: 409 Client Error: Conflict for url: https://10.108.80.1:443/apis/batch/v1/namespaces/prod-24-09-14-00-32-1/jobs/gxy-q74tk
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 817, in _handle_job_failure
+ self.__cleanup_k8s_job(job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/kubernetes.py", line 824, in __cleanup_k8s_job
+ delete_job(job, k8s_cleanup_job)
+ File "/galaxy/server/lib/galaxy/jobs/runners/util/pykube_util.py", line 107, in delete_job
+ job.scale(replicas=0)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/mixins.py", line 30, in scale
+ self.update()
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 165, in update
+ self.patch(self.obj, subresource=subresource)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/objects.py", line 157, in patch
+ self.api.raise_for_status(r)
+ File "/galaxy/server/.venv/lib/python3.10/site-packages/pykube/http.py", line 444, in raise_for_status
+ raise HTTPError(resp.status_code, payload["message"])
+pykube.exceptions.HTTPError: Operation cannot be fulfilled on jobs.batch "gxy-q74tk": the object has been modified; please apply your changes to the latest version and try again
+galaxy.util WARNING 2024-09-14 00:56:52,052 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] Unable to honor primary group (grp.struct_group(gr_name='galaxy', gr_passwd='x', gr_gid=101, gr_mem=[])) for /galaxy/server/database/objects/e/6/a/dataset_e6ac6186-9f77-45ca-a14f-43c8894a7811.dat, group remains grp.struct_group(gr_name='root', gr_passwd='x', gr_gid=0, gr_mem=[]), error was: [Errno 1] Operation not permitted: '/galaxy/server/database/objects/e/6/a/dataset_e6ac6186-9f77-45ca-a14f-43c8894a7811.dat'
+galaxy.objectstore CRITICAL 2024-09-14 00:56:52,071 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-3] None delete error [Errno 13] Permission denied: 'tmpzel01lhj'
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:54,289 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Grabbed Job(s): 35, 36
+tpv.core.entities DEBUG 2024-09-14 00:56:54,318 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags= tags=[, ], rank=helpers.we, inherits=None, context={}, rules={} using custom function
+galaxy.jobs.mapper DEBUG 2024-09-14 00:56:54,318 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (35) Mapped job to destination id: k8s
+galaxy.jobs.handler DEBUG 2024-09-14 00:56:54,322 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (35) Dispatching to k8s runner
+galaxy.jobs DEBUG 2024-09-14 00:56:54,335 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (35) Persisting job destination (destination id: k8s)
+galaxy.jobs DEBUG 2024-09-14 00:56:54,349 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (35) Working directory for job is: /galaxy/server/database/jobs_directory/000/35
+galaxy.jobs.runners DEBUG 2024-09-14 00:56:54,354 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Job [35] queued (31.904 ms)
+galaxy.jobs.handler INFO 2024-09-14 00:56:54,357 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] (35) Job dispatched
+galaxy.jobs.runners.kubernetes DEBUG 2024-09-14 00:56:54,362 [pN:job_handler_0,p:8,tN:KubernetesRunner.work_thread-1] Starting queue_job for job 35
+tpv.core.entities DEBUG 2024-09-14 00:56:54,370 [pN:job_handler_0,p:8,tN:JobHandlerQueue.monitor_thread] Ranking destinations: [runner=k8s, dest_name=k8s, min_accepted_cores=None, min_accepted_mem=None, min_accepted_gpus=None, max_accepted_cores=100, max_accepted_mem=800, max_accepted_gpus=None, tpv_dest_tags= tags=[], handler_tags=None id=k8s, abstract=False, cores=None, mem=None, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = 8, max_mem = 48, max_gpus = None, env=[], params={'docker_enabled': 'true', 'limits_cpu': '{cores}', 'limits_memory': '{mem}Gi', 'requests_cpu': '{cores}', 'requests_memory': '{mem}Gi'}, resubmit={}, tags= tags=[], rank=, inherits=None, context={}, rules={}] for entity: id=force_default_container_for_built_in_tools, Rule: force_default_container_for_built_in_tools, abstract=False, cores=1, mem=cores * 3.8, gpus=None, min_cores = None, min_mem = None, min_gpus = None, max_cores = None, max_mem = None, max_gpus = None, env=[], params={'container_monitor': False, 'docker_default_container_id': 'quay.io/galaxyproject/galaxy-anvil:23.1', 'tmp_dir': 'true', 'docker_container_id_override': 'quay.io/galaxyproject/galaxy-anvil:23.1'}, resubmit={}, tags=