From 88dc031f772916e7165ca75acde1c320afdd933b Mon Sep 17 00:00:00 2001 From: realvudi Date: Mon, 29 Apr 2024 17:28:06 -0700 Subject: [PATCH 1/7] Update to DB qiita.slurm_resource_allocations --- notebooks/resource-allocation/upload_df.py | 72 ++ qiita_db/support_files/populate_test_db.sql | 929 ++++++++++++++++++++ 2 files changed, 1001 insertions(+) create mode 100644 notebooks/resource-allocation/upload_df.py diff --git a/notebooks/resource-allocation/upload_df.py b/notebooks/resource-allocation/upload_df.py new file mode 100644 index 000000000..e74bcce04 --- /dev/null +++ b/notebooks/resource-allocation/upload_df.py @@ -0,0 +1,72 @@ +import pandas as pd + +# Example data loading +filename = './data/jobs_2024-02-21.tsv.gz' +df = pd.read_csv(filename, sep='\t', dtype={'extra_info': str}) + +# Convert string to timedelta, then to total seconds +df['ElapsedRawTime'] = pd.to_timedelta( + df['ElapsedRawTime']).apply( + lambda x: x.total_seconds()) + +cname = "Validate" +sname = "Diversity types - alpha_vector" +df = df[(df.cName == cname) & (df.sName == sname)] + +df['samples'] = df['samples'].fillna(0).astype(int) +df['columns'] = df['columns'].fillna(0).astype(int) +df['input_size'] = df['input_size'].fillna(0).astype(int) +df['MaxRSSRaw'] = df['MaxRSSRaw'].fillna(0).astype(int) +df['ElapsedRawTime'] = df['ElapsedRawTime'].fillna(0).astype(int) + +COL_NAME = 'samples * columns' +df[COL_NAME] = df['samples'] * df['columns'] +columns = ["MaxRSSRaw", "ElapsedRawTime"] +max_rows = [] + +for curr in columns: + # Get the maximum value for 'curr' within each COL_NAME group + max_values = df.groupby(COL_NAME)[curr].transform(max) + # Filter rows where the current column's value + # is the maximum within its group + curr_rows = df[df[curr] == max_values] + max_rows.append(curr_rows) + +filtered_df = pd.concat(max_rows).drop_duplicates().reset_index(drop=True) + +# INSERT INTO qiita.processing_job(processing_job_id, email, command_id, +# command_parameters, processing_job_status_id) +# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', +# 'admin@foo.com', 1, '""'::json, 1); + +# INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, +# columns, input_size, extra_info, memory_used, walltime_used) +# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 39, 81, 2, 'nan', +# 327036000, 91); + +# processing_job_id uuid NOT NULL, +# samples integer, +# columns integer, +# input_size bigint, +# extra_info varchar DEFAULT NULL, +# memory_used bigint, +# walltime_used integer, + +res = "" + +for index, row in filtered_df.iterrows(): + res += f"""('{row['QiitaID']}', 'admin@foo.bar', 1, '""'::json, 1),\n""" +res += ";\n" +res += "Split\n" +for index, row in filtered_df.iterrows(): + res += ( + f"('{row['QiitaID']}', {int(row['samples'])}, " + f"{int(row['columns'])}, {int(row['input_size'])}, " + f"'{row['extra_info']}', {int(row['MaxRSSRaw'])}, " + f"{int(row['ElapsedRawTime'])}),\n" + ) + +res += ";\n" + +with open("sql.txt", 'w') as filename: + filename.write(res) diff --git a/qiita_db/support_files/populate_test_db.sql b/qiita_db/support_files/populate_test_db.sql index f8b5466b2..c0a8c7536 100644 --- a/qiita_db/support_files/populate_test_db.sql +++ b/qiita_db/support_files/populate_test_db.sql @@ -582,3 +582,932 @@ INSERT INTO qiita.processing_job_workflow_root (processing_job_workflow_id, proc INSERT INTO qiita.parent_processing_job (parent_id, child_id) VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'd19f76ee-274e-4c1b-b3a2-a12d73507c55'); + +-- SPLIT + +INSERT INTO qiita.processing_job(processing_job_id, email, command_id, command_parameters, processing_job_status_id) +VALUES ('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 'admin@foo.bar', 1, '""'::json, 1), +('b0f36550-d97c-4dd5-ba1b-910470062801', 'admin@foo.bar', 1, '""'::json, 1), +('b13eefd1-12b7-4c3e-a380-94ec0a3f9f91', 'admin@foo.bar', 1, '""'::json, 1), +('469c27b7-81c4-4f41-b09d-149260961227', 'admin@foo.bar', 1, '""'::json, 1), +('6f198e33-b4ae-47e9-973b-bd5e0b06c353', 'admin@foo.bar', 1, '""'::json, 1), +('045eb36c-5144-4d54-a621-23eb5dc4d6ef', 'admin@foo.bar', 1, '""'::json, 1), +('041f3278-2752-44f9-8caa-ac6ae80c98e6', 'admin@foo.bar', 1, '""'::json, 1), +('6325e683-b215-4c56-bf5a-cbbc92330067', 'admin@foo.bar', 1, '""'::json, 1), +('03dcca05-0d4f-48c5-bc7a-7ef884be8efc', 'admin@foo.bar', 1, '""'::json, 1), +('8d84685f-97f4-4ea7-8857-2dcec5a20775', 'admin@foo.bar', 1, '""'::json, 1), +('8b714fe7-51d9-492c-9b49-4fe09b595067', 'admin@foo.bar', 1, '""'::json, 1), +('490c0623-4ed7-4064-9ffe-ba5afc0f0aa0', 'admin@foo.bar', 1, '""'::json, 1), +('c8301f3d-a4fe-407a-9aec-b9dfee0cc2fd', 'admin@foo.bar', 1, '""'::json, 1), +('561b9676-604a-4cdb-833f-a4a9b3afef42', 'admin@foo.bar', 1, '""'::json, 1), +('8bc35697-f3df-42f6-8a34-830d42000bd6', 'admin@foo.bar', 1, '""'::json, 1), +('474a4a8d-0963-4fe6-9da4-74e520532be1', 'admin@foo.bar', 1, '""'::json, 1), +('d4d03b6f-12f1-4f47-9ad4-b5597a3e5a16', 'admin@foo.bar', 1, '""'::json, 1), +('27d95aba-9b36-40c8-8d17-7ade170172aa', 'admin@foo.bar', 1, '""'::json, 1), +('a8f00090-6981-4fc3-bc37-208f01e6dceb', 'admin@foo.bar', 1, '""'::json, 1), +('d7562d61-7886-45b5-9fed-90fef6636e6f', 'admin@foo.bar', 1, '""'::json, 1), +('e4071da6-550d-469b-91dc-f6cda658df80', 'admin@foo.bar', 1, '""'::json, 1), +('c126e507-5f13-485f-9166-775ee7228f12', 'admin@foo.bar', 1, '""'::json, 1), +('d73906a4-f2c7-4d03-bd55-d637365ec3bc', 'admin@foo.bar', 1, '""'::json, 1), +('167efba9-a88e-493c-a2c6-0647556ba6dd', 'admin@foo.bar', 1, '""'::json, 1), +('2662c176-d042-4254-bc7e-20c092fc3af1', 'admin@foo.bar', 1, '""'::json, 1), +('54b19d0d-fa47-4c37-a4ea-e2be33d54c5b', 'admin@foo.bar', 1, '""'::json, 1), +('8b3e969e-78c4-4a09-9767-4aa72d99ee8c', 'admin@foo.bar', 1, '""'::json, 1), +('bef26557-60db-4987-b0bf-e9e0be77da37', 'admin@foo.bar', 1, '""'::json, 1), +('adcf6cb4-780b-4a9d-bb7c-cd2bf12ac201', 'admin@foo.bar', 1, '""'::json, 1), +('54464698-5860-4543-9376-d3d89722848f', 'admin@foo.bar', 1, '""'::json, 1), +('9a8b3ea7-36d7-46f7-8602-1b39c89f5ee4', 'admin@foo.bar', 1, '""'::json, 1), +('b57c1bc6-a805-463d-87fc-b76a186e5a63', 'admin@foo.bar', 1, '""'::json, 1), +('0c0579c8-dc34-4211-ad87-c453e9cb10bf', 'admin@foo.bar', 1, '""'::json, 1), +('04357795-299a-4900-9fdf-debf3be2ccc7', 'admin@foo.bar', 1, '""'::json, 1), +('fa1617ad-daf8-496d-937f-5ad9526f57d2', 'admin@foo.bar', 1, '""'::json, 1), +('056dd777-1c93-424a-ad7a-fbac4ff01e6b', 'admin@foo.bar', 1, '""'::json, 1), +('1d056722-c08c-45e7-abbf-794abf85b2ba', 'admin@foo.bar', 1, '""'::json, 1), +('a4e4457e-e191-4951-81d2-a1bd8afa7bb9', 'admin@foo.bar', 1, '""'::json, 1), +('2bd5eedb-f4bd-4221-a31d-7a82ed1ed85c', 'admin@foo.bar', 1, '""'::json, 1), +('59e6279e-5a92-4ad5-81cd-d83892eb35e3', 'admin@foo.bar', 1, '""'::json, 1), +('594e5a37-e00e-44e3-8baf-a5d0fbd8d1c5', 'admin@foo.bar', 1, '""'::json, 1), +('40e16ce2-d280-423b-908e-f74877751bb8', 'admin@foo.bar', 1, '""'::json, 1), +('df964c5d-d204-475d-bf37-c4281c830e36', 'admin@foo.bar', 1, '""'::json, 1), +('f1ff0a3f-5004-49d4-a781-fedab076f682', 'admin@foo.bar', 1, '""'::json, 1), +('d5b6766a-5697-4283-b7f5-ded6295e539d', 'admin@foo.bar', 1, '""'::json, 1), +('6e98873c-0c99-47f0-ae6f-fc12092c1c5d', 'admin@foo.bar', 1, '""'::json, 1), +('b816de4b-42f6-4495-8a05-7cb7faefcf3e', 'admin@foo.bar', 1, '""'::json, 1), +('2e196649-1641-4ced-bf74-f24db6123d66', 'admin@foo.bar', 1, '""'::json, 1), +('ca8271ce-ff25-4a22-8309-149396d20d38', 'admin@foo.bar', 1, '""'::json, 1), +('38496999-f362-45d1-a9e6-6c2ed7ad1847', 'admin@foo.bar', 1, '""'::json, 1), +('dedeab8b-ed71-4d3d-87c1-37f607e19e5d', 'admin@foo.bar', 1, '""'::json, 1), +('997ad2aa-59ff-4346-805c-cbe1a41bac00', 'admin@foo.bar', 1, '""'::json, 1), +('ff866805-7add-432f-90a8-15ab4b793cdd', 'admin@foo.bar', 1, '""'::json, 1), +('ef8a5faa-fc7b-4a09-a45a-7aa4d3dbb1f8', 'admin@foo.bar', 1, '""'::json, 1), +('b0db6327-e4ff-4dd5-85ae-b8a060f8ed9f', 'admin@foo.bar', 1, '""'::json, 1), +('900537c6-a4bb-4f20-9674-e0fd4851c10a', 'admin@foo.bar', 1, '""'::json, 1), +('3fe82723-af0e-4df0-a3bf-32223b202dd5', 'admin@foo.bar', 1, '""'::json, 1), +('6b22c272-f72c-4102-9abb-afff9d1c7d42', 'admin@foo.bar', 1, '""'::json, 1), +('6738f416-1ce1-4cac-90ef-ead48b9b0ace', 'admin@foo.bar', 1, '""'::json, 1), +('6ed39d69-1c01-42c1-8249-b8abf83c3608', 'admin@foo.bar', 1, '""'::json, 1), +('d00fd8ee-7d65-469c-af2a-ca00681d224b', 'admin@foo.bar', 1, '""'::json, 1), +('42244303-b4a1-43d3-9041-f4f811691b41', 'admin@foo.bar', 1, '""'::json, 1), +('4ac590b9-5733-4e20-b657-3b13d9b30a4f', 'admin@foo.bar', 1, '""'::json, 1), +('1409a826-8176-49db-84f9-04d4cac0db89', 'admin@foo.bar', 1, '""'::json, 1), +('21917565-5c24-4b9e-947e-0a7fc18b89dc', 'admin@foo.bar', 1, '""'::json, 1), +('cc4db5ef-c3d3-4067-98be-ee90a584a95a', 'admin@foo.bar', 1, '""'::json, 1), +('12f627b7-19aa-4583-bce6-a7f7d4dd66a2', 'admin@foo.bar', 1, '""'::json, 1), +('0c6dc99d-d20b-4381-a405-8813129b81ee', 'admin@foo.bar', 1, '""'::json, 1), +('449668bf-3294-42eb-9cef-d0c007b9b27f', 'admin@foo.bar', 1, '""'::json, 1), +('ee4c2079-e187-4a2f-ae9a-a95795a9d5fe', 'admin@foo.bar', 1, '""'::json, 1), +('0f0d6f75-6ba3-4265-b53e-cc53a18e52b8', 'admin@foo.bar', 1, '""'::json, 1), +('d63bdad8-ace7-4d92-a8a7-add8620460d3', 'admin@foo.bar', 1, '""'::json, 1), +('5a856042-6a60-4f9d-b256-d03295e2d7f9', 'admin@foo.bar', 1, '""'::json, 1), +('dd9969e8-9efe-4781-af4a-6f2f8ff70808', 'admin@foo.bar', 1, '""'::json, 1), +('37cff1b2-c11e-4511-8625-84def4635d84', 'admin@foo.bar', 1, '""'::json, 1), +('36f312b9-5055-443a-b07a-2ab09e8f8cfa', 'admin@foo.bar', 1, '""'::json, 1), +('de207a0d-932f-4a81-9412-5ff8baa9cdd5', 'admin@foo.bar', 1, '""'::json, 1), +('b567274d-4631-4aee-85df-7d6603c20cfe', 'admin@foo.bar', 1, '""'::json, 1), +('94d07841-1b75-42a1-9407-af6a341769b1', 'admin@foo.bar', 1, '""'::json, 1), +('d83a1dee-1451-4ea3-bd42-3881f1fa4e3e', 'admin@foo.bar', 1, '""'::json, 1), +('3c736f54-7d2d-4cd6-9c45-8cce37ce5482', 'admin@foo.bar', 1, '""'::json, 1), +('e600feed-52cc-4d8f-b28d-c6283dcde944', 'admin@foo.bar', 1, '""'::json, 1), +('41624971-b40b-4e03-bb12-cce149cd6eac', 'admin@foo.bar', 1, '""'::json, 1), +('9f769164-d25c-4e7d-b637-79f3489798c5', 'admin@foo.bar', 1, '""'::json, 1), +('0f382217-32ed-4ea6-b7ab-c98c66e8305f', 'admin@foo.bar', 1, '""'::json, 1), +('24d2cf39-25a1-486e-a763-f2639674f4b4', 'admin@foo.bar', 1, '""'::json, 1), +('b303df36-67d8-4883-a109-294b61ccbe23', 'admin@foo.bar', 1, '""'::json, 1), +('b46d8a4d-b969-4226-8719-6708c3755775', 'admin@foo.bar', 1, '""'::json, 1), +('07b691e7-a940-4257-bea8-ebba6bc47637', 'admin@foo.bar', 1, '""'::json, 1), +('9d624992-01fe-4ece-a7ba-ebf54da0eb7b', 'admin@foo.bar', 1, '""'::json, 1), +('0aba4000-8b6d-47c1-a4fc-7b6a2b419b67', 'admin@foo.bar', 1, '""'::json, 1), +('4635cab1-b1a7-4d6a-bb75-d95f0e1fd682', 'admin@foo.bar', 1, '""'::json, 1), +('b2602d9e-7ce6-43e2-a4b3-3962f812a80a', 'admin@foo.bar', 1, '""'::json, 1), +('f8d9c8b3-1e4f-4371-9d86-74778f7145a7', 'admin@foo.bar', 1, '""'::json, 1), +('490bf28c-7f5b-428f-9127-c1ac19d611fc', 'admin@foo.bar', 1, '""'::json, 1), +('a69254f1-d5d4-4f3f-991f-c197364b8187', 'admin@foo.bar', 1, '""'::json, 1), +('702f4828-f604-44b8-a7b0-9684e77a0279', 'admin@foo.bar', 1, '""'::json, 1), +('0d793328-c4c5-4739-b2f1-40afb4de55c8', 'admin@foo.bar', 1, '""'::json, 1), +('7227a61e-59af-4b96-8919-efe13839889c', 'admin@foo.bar', 1, '""'::json, 1), +('57e351db-c504-42dd-a699-fe405c1b5a27', 'admin@foo.bar', 1, '""'::json, 1), +('88bbeb39-2fc5-4a46-8bd9-ca28a2fedef3', 'admin@foo.bar', 1, '""'::json, 1), +('63a0101c-13b0-4c73-b4d7-00a33ae0f9ab', 'admin@foo.bar', 1, '""'::json, 1), +('e3c17c93-5993-49fc-891a-9c9d5b9aa43c', 'admin@foo.bar', 1, '""'::json, 1), +('6fd1fc5f-fce2-43c6-909d-8041a075ac3a', 'admin@foo.bar', 1, '""'::json, 1), +('8ea8b437-819b-4624-b15c-a653b53c0a2e', 'admin@foo.bar', 1, '""'::json, 1), +('c4ee461b-8d52-4338-8fa2-8a22b538456b', 'admin@foo.bar', 1, '""'::json, 1), +('1eaa62de-c3ae-414d-9168-7fb5ece89949', 'admin@foo.bar', 1, '""'::json, 1), +('6a63a491-c4c1-4176-9827-9460526ed319', 'admin@foo.bar', 1, '""'::json, 1), +('733a21cf-e969-4a17-b888-820f2f9475a1', 'admin@foo.bar', 1, '""'::json, 1), +('9a1e07e0-bac0-46e1-8b72-86a730f47c12', 'admin@foo.bar', 1, '""'::json, 1), +('6ca8ca71-dbb7-4347-adb3-9ab02b69ef9f', 'admin@foo.bar', 1, '""'::json, 1), +('07272b4e-88bf-4adc-9953-4b1f4cb3764f', 'admin@foo.bar', 1, '""'::json, 1), +('8b9f9e1e-2e52-432b-8528-53f8b25ea81e', 'admin@foo.bar', 1, '""'::json, 1), +('6395bd2a-fd1b-448b-8771-f07a77958d84', 'admin@foo.bar', 1, '""'::json, 1), +('b05bb0d8-a8bd-4634-82f9-47c52ccd83e3', 'admin@foo.bar', 1, '""'::json, 1), +('38ee009a-7c6a-42a2-a05f-cd3f5286cd9d', 'admin@foo.bar', 1, '""'::json, 1), +('d0799585-1e37-4134-b71f-481dc0d2be72', 'admin@foo.bar', 1, '""'::json, 1), +('2d5a21fd-1cdb-4ee6-8e08-584bf99ef40a', 'admin@foo.bar', 1, '""'::json, 1), +('a4aee4f4-91d5-46e1-9280-87b8ba7eeef9', 'admin@foo.bar', 1, '""'::json, 1), +('7e361763-962b-4f8d-8c63-1a329428f89f', 'admin@foo.bar', 1, '""'::json, 1), +('0abb0be4-b58e-4318-8624-4fc055f9d82d', 'admin@foo.bar', 1, '""'::json, 1), +('3497cd39-99dc-429e-8e10-67ecd7b2774a', 'admin@foo.bar', 1, '""'::json, 1), +('0ca036c1-ceb5-4c33-993b-d8511a96208b', 'admin@foo.bar', 1, '""'::json, 1), +('a22718ce-6e93-420b-89d6-cf2ddf104164', 'admin@foo.bar', 1, '""'::json, 1), +('278179bb-f481-4d50-aced-5485993699a1', 'admin@foo.bar', 1, '""'::json, 1), +('80272c6a-4312-4c93-9905-390e62a3d201', 'admin@foo.bar', 1, '""'::json, 1), +('d5ab9552-64e1-43b2-bf46-5065612c2cda', 'admin@foo.bar', 1, '""'::json, 1), +('af467934-f4db-464e-8ef1-b6bbbbe88667', 'admin@foo.bar', 1, '""'::json, 1), +('cfa62dd9-c6e0-48a2-9633-ee4d813d4f51', 'admin@foo.bar', 1, '""'::json, 1), +('bd37012e-0b22-4e3a-b0ae-eba006d87749', 'admin@foo.bar', 1, '""'::json, 1), +('d11bb684-448c-4916-a0f8-cd5748061638', 'admin@foo.bar', 1, '""'::json, 1), +('46cb1988-f5b7-4a27-92be-dd8b7c7289db', 'admin@foo.bar', 1, '""'::json, 1), +('dd000d70-ee4b-46a6-9e1f-26fd85b6df3c', 'admin@foo.bar', 1, '""'::json, 1), +('a12d7860-89d0-4640-a4a6-73738b47408a', 'admin@foo.bar', 1, '""'::json, 1), +('aae1e730-254d-401b-884d-d9f6965203f6', 'admin@foo.bar', 1, '""'::json, 1), +('66e2dbf0-d223-48b6-bf1e-45a40f87d1b1', 'admin@foo.bar', 1, '""'::json, 1), +('be2e6e7d-7992-4407-a7c1-aced14e6fa12', 'admin@foo.bar', 1, '""'::json, 1), +('671071f2-380c-42a3-88f6-a1642c07dc36', 'admin@foo.bar', 1, '""'::json, 1), +('5db2e298-251c-409f-81a2-844b9a2b7697', 'admin@foo.bar', 1, '""'::json, 1), +('9b0e1a4a-97a6-47fe-aa2a-6800d72fa611', 'admin@foo.bar', 1, '""'::json, 1), +('9553c043-e844-4c9f-8367-eb5d7d51772c', 'admin@foo.bar', 1, '""'::json, 1), +('78c3db71-ed10-457d-b048-d8142c535e14', 'admin@foo.bar', 1, '""'::json, 1), +('60a72e37-28f8-4288-81eb-830cb2924a03', 'admin@foo.bar', 1, '""'::json, 1), +('520e37c3-6e64-4fce-8a88-cceac2b3b5b8', 'admin@foo.bar', 1, '""'::json, 1), +('596f24ae-49e1-4a22-bbee-0b53b1bc4b26', 'admin@foo.bar', 1, '""'::json, 1), +('7fce7f74-0c73-4fc2-b5bc-45f570014054', 'admin@foo.bar', 1, '""'::json, 1), +('0061bb8f-219c-4e53-8ea5-137799e5fc3c', 'admin@foo.bar', 1, '""'::json, 1), +('09c27409-4fb5-415c-93e3-7e2233e0109e', 'admin@foo.bar', 1, '""'::json, 1), +('1a95492b-e852-413b-8450-72eb0adcd577', 'admin@foo.bar', 1, '""'::json, 1), +('c6315300-f68e-4bf7-b7d2-778c3409f346', 'admin@foo.bar', 1, '""'::json, 1), +('517dd44b-61bc-47ba-a7c8-c189378f6d73', 'admin@foo.bar', 1, '""'::json, 1), +('c167f049-4d43-4409-84be-233b44981bb8', 'admin@foo.bar', 1, '""'::json, 1), +('2c4290f6-e59f-438f-9664-a22810b495a0', 'admin@foo.bar', 1, '""'::json, 1), +('e7a1816e-c98c-4ae9-854c-b034d4dc77cf', 'admin@foo.bar', 1, '""'::json, 1), +('e5a06530-b313-4b75-ad14-5bceeb690d5a', 'admin@foo.bar', 1, '""'::json, 1), +('8d671718-8686-4cc5-899f-178a2fba763e', 'admin@foo.bar', 1, '""'::json, 1), +('ef88e6d6-affc-4511-8842-af8f32c1b49e', 'admin@foo.bar', 1, '""'::json, 1), +('a1e85eec-0ed6-43ad-b0b0-d50fa4a26ed8', 'admin@foo.bar', 1, '""'::json, 1), +('79baff1d-1f54-429f-91b2-4bc4e6ed5c18', 'admin@foo.bar', 1, '""'::json, 1), +('a318ddfc-153d-4462-b7f9-963f7b3a11b2', 'admin@foo.bar', 1, '""'::json, 1), +('8d376621-fad2-4329-9a54-5cc8a5347e9b', 'admin@foo.bar', 1, '""'::json, 1), +('0a78d681-284e-4898-a80c-271edad7b1d3', 'admin@foo.bar', 1, '""'::json, 1), +('0cf2846c-a2cc-4b4c-bc53-b78bab20d678', 'admin@foo.bar', 1, '""'::json, 1), +('43ad2854-6f47-43ad-8b8f-4e77591598c0', 'admin@foo.bar', 1, '""'::json, 1), +('fc591b9e-f6df-457c-ac61-d3ddc8a97df6', 'admin@foo.bar', 1, '""'::json, 1), +('3b0ee8e6-d921-4c11-a19a-e482abff09da', 'admin@foo.bar', 1, '""'::json, 1), +('efda7c49-2865-4bf9-847a-46ec0ed48c9e', 'admin@foo.bar', 1, '""'::json, 1), +('1ef05ca4-74d6-4cb1-a0dd-64412a5e8f26', 'admin@foo.bar', 1, '""'::json, 1), +('f8a29cc6-a0cb-458e-938e-b30635035f74', 'admin@foo.bar', 1, '""'::json, 1), +('69187ffb-8155-4c06-baa9-69b3c53f901e', 'admin@foo.bar', 1, '""'::json, 1), +('d3d1f2f0-6462-453c-b346-21ae50813c3b', 'admin@foo.bar', 1, '""'::json, 1), +('f0b99e31-aa9d-4111-b2a5-0ddcb7a2c063', 'admin@foo.bar', 1, '""'::json, 1), +('449d5f69-50db-4c11-850e-ae578bdc68ad', 'admin@foo.bar', 1, '""'::json, 1), +('108c14b7-9be9-4107-be16-4e70ae0bdb8d', 'admin@foo.bar', 1, '""'::json, 1), +('3161c5b5-05c9-4513-bbc1-9b7cfbac6275', 'admin@foo.bar', 1, '""'::json, 1), +('69fcd88b-6edf-4cde-a610-0363effd0097', 'admin@foo.bar', 1, '""'::json, 1), +('a9b8ad78-8f0d-4352-aa0d-c9fb4eab8c2a', 'admin@foo.bar', 1, '""'::json, 1), +('619fa191-ec16-46f2-86bd-6d8ef583db5f', 'admin@foo.bar', 1, '""'::json, 1), +('6779287f-41eb-4733-977e-8bd8e3991a24', 'admin@foo.bar', 1, '""'::json, 1), +('ba770cd8-8667-40c9-b97e-c714cda2f80c', 'admin@foo.bar', 1, '""'::json, 1), +('58f871c7-830c-4d71-9300-e3779e203c5d', 'admin@foo.bar', 1, '""'::json, 1), +('15b23bef-7bdb-4361-891e-c8a259f22230', 'admin@foo.bar', 1, '""'::json, 1), +('246ebe66-68e4-4a9f-9a97-06f881c83cbd', 'admin@foo.bar', 1, '""'::json, 1), +('99529852-6a5c-4d5b-b029-3455aa3a8c8d', 'admin@foo.bar', 1, '""'::json, 1), +('1a8cffdc-13d8-4779-a38f-a9abeb854e44', 'admin@foo.bar', 1, '""'::json, 1), +('144380ec-debc-468a-9f7a-930197d9547b', 'admin@foo.bar', 1, '""'::json, 1), +('6365416f-e395-45d2-b558-270d0992fa0f', 'admin@foo.bar', 1, '""'::json, 1), +('eeb06d17-12d1-45c1-ac8d-4e7473983ebf', 'admin@foo.bar', 1, '""'::json, 1), +('4c38dfa2-b27b-4409-ba90-2c75eea4bba4', 'admin@foo.bar', 1, '""'::json, 1), +('a70578ed-87e9-4ef7-a1d2-760d76962e40', 'admin@foo.bar', 1, '""'::json, 1), +('eaffc15a-0d71-427b-b672-0487943d7da7', 'admin@foo.bar', 1, '""'::json, 1), +('c86955d9-980c-46be-af88-26d4ac2d2a7d', 'admin@foo.bar', 1, '""'::json, 1), +('7f7fd4fe-d7df-4e7d-a37c-936bc8377f5d', 'admin@foo.bar', 1, '""'::json, 1), +('ba61cf3b-6c70-456d-8e9e-c8541a2a6407', 'admin@foo.bar', 1, '""'::json, 1), +('ed986358-8a50-4b19-af8b-d5df11595183', 'admin@foo.bar', 1, '""'::json, 1), +('70f9bec4-9459-46f5-aeab-ac3ac7f55cf1', 'admin@foo.bar', 1, '""'::json, 1), +('38c42216-080e-413c-8c79-f3a27ab556bc', 'admin@foo.bar', 1, '""'::json, 1), +('d3c8f668-6393-4e41-8160-a17e516ff500', 'admin@foo.bar', 1, '""'::json, 1), +('692c4d20-72b3-4627-8dad-7379174604a1', 'admin@foo.bar', 1, '""'::json, 1), +('3382c6a9-b778-4a75-9092-6c76afeeb45e', 'admin@foo.bar', 1, '""'::json, 1), +('bbfa6b71-fd64-45a3-a66e-5d0d7f454e9d', 'admin@foo.bar', 1, '""'::json, 1), +('27b552f9-b3be-4fbf-93dc-323774590e1b', 'admin@foo.bar', 1, '""'::json, 1), +('4af523ba-0276-48a3-bfac-246ff361310b', 'admin@foo.bar', 1, '""'::json, 1), +('66630aa4-c73b-4dbd-9290-b5a6ff8d0129', 'admin@foo.bar', 1, '""'::json, 1), +('9e5aa6a4-23f4-4949-82df-748e4a651d62', 'admin@foo.bar', 1, '""'::json, 1), +('b6965426-898a-46e1-a40f-2fe34a9d7bb1', 'admin@foo.bar', 1, '""'::json, 1), +('052163a0-2807-40fc-8ae7-ecb8c4c1d421', 'admin@foo.bar', 1, '""'::json, 1), +('bd0b608a-05cd-4c67-a4d9-e3337075e2e4', 'admin@foo.bar', 1, '""'::json, 1), +('a5d9bb39-116b-49e3-97c4-831580f6729f', 'admin@foo.bar', 1, '""'::json, 1), +('cefcf3fb-26f2-4349-9fc8-d81c63e928ff', 'admin@foo.bar', 1, '""'::json, 1), +('c573d2c7-91bd-4dfe-a433-a6ef89a9656c', 'admin@foo.bar', 1, '""'::json, 1), +('869f9610-e808-4935-a57e-781abc9960af', 'admin@foo.bar', 1, '""'::json, 1), +('83c880c1-20e6-4134-aca4-ced663066543', 'admin@foo.bar', 1, '""'::json, 1), +('cd376889-4d87-40f8-8d3e-b9a4c0623f55', 'admin@foo.bar', 1, '""'::json, 1), +('d8d739f2-6d56-41b2-a5a7-38133f9d7bbd', 'admin@foo.bar', 1, '""'::json, 1), +('be224c35-461b-48a3-ab48-4e4838019c64', 'admin@foo.bar', 1, '""'::json, 1), +('8b99435f-643a-4347-b552-18a5aea558e0', 'admin@foo.bar', 1, '""'::json, 1), +('9d372971-7763-4731-a44e-cb87a3434cc0', 'admin@foo.bar', 1, '""'::json, 1), +('9a3a5d72-c2b0-449a-a157-1c088e0af56d', 'admin@foo.bar', 1, '""'::json, 1), +('a8afae6b-c328-48df-bce0-bab522a98f85', 'admin@foo.bar', 1, '""'::json, 1), +('1fa748d3-fa80-4af4-a8fa-7782a27fa585', 'admin@foo.bar', 1, '""'::json, 1), +('559da3f1-bc29-4c26-a6ac-06b52b054594', 'admin@foo.bar', 1, '""'::json, 1), +('cb41a3f9-0be7-4dad-b015-68016ef51022', 'admin@foo.bar', 1, '""'::json, 1), +('19203be8-1052-4c80-9bce-fff4e77f05c8', 'admin@foo.bar', 1, '""'::json, 1), +('a91d644d-9c86-411c-9ae2-4eeac7545d2b', 'admin@foo.bar', 1, '""'::json, 1), +('5e0bba68-4bfb-40e1-96d1-daf1a454fd46', 'admin@foo.bar', 1, '""'::json, 1), +('510e59c2-ba8e-4b1c-b4be-17dc59c80457', 'admin@foo.bar', 1, '""'::json, 1), +('a4b6745c-bad5-470a-8e32-2f75236c2fce', 'admin@foo.bar', 1, '""'::json, 1), +('f69d7684-607e-4661-a836-c09aba157c95', 'admin@foo.bar', 1, '""'::json, 1), +('5c679aad-68f7-4855-be14-8312110bc173', 'admin@foo.bar', 1, '""'::json, 1), +('e015a0a1-5c4f-4178-8ebc-2b9ac9e4e8b6', 'admin@foo.bar', 1, '""'::json, 1), +('cd2374bc-649c-45e2-8540-991fb36cb487', 'admin@foo.bar', 1, '""'::json, 1), +('7c707855-1640-42e9-82a7-d720e58874eb', 'admin@foo.bar', 1, '""'::json, 1), +('20af2ade-f786-4305-85b1-b5272ab356e0', 'admin@foo.bar', 1, '""'::json, 1), +('490b3500-dc03-4fa9-9cd2-78126b9333c2', 'admin@foo.bar', 1, '""'::json, 1), +('e5504565-cf0e-4f2d-b925-1c2ca7cb5fbb', 'admin@foo.bar', 1, '""'::json, 1), +('f9a9e2d7-b568-45d6-89b8-89cfe9991f77', 'admin@foo.bar', 1, '""'::json, 1), +('8500b61a-9f08-49c9-a897-2773f4f383c5', 'admin@foo.bar', 1, '""'::json, 1), +('7e8480aa-af3b-4df7-8d66-8f93a28ccb74', 'admin@foo.bar', 1, '""'::json, 1), +('f87a89b9-4b6e-4398-8e48-0b6ec238cc97', 'admin@foo.bar', 1, '""'::json, 1), +('92b6145c-1248-4024-b825-a8b6cb07c4b0', 'admin@foo.bar', 1, '""'::json, 1), +('04e67d15-1907-4fd1-9a4d-48b47f6c71f8', 'admin@foo.bar', 1, '""'::json, 1), +('f7466925-9a5c-40d8-9536-9c43ad30cc2d', 'admin@foo.bar', 1, '""'::json, 1), +('2f21b2ec-91b2-43e2-8d70-b35b64fcaf0c', 'admin@foo.bar', 1, '""'::json, 1), +('3df051d5-99a8-411a-b40f-bc269eeb5a6c', 'admin@foo.bar', 1, '""'::json, 1), +('6048e851-42a4-4b0b-b040-a05cfe9686b0', 'admin@foo.bar', 1, '""'::json, 1), +('06bbba17-3761-4e80-a7c0-56a01e92e7f2', 'admin@foo.bar', 1, '""'::json, 1), +('1d1f338a-0b0b-4597-a960-1ecf5019dcfe', 'admin@foo.bar', 1, '""'::json, 1), +('ba1c6fdd-bedd-45f1-a255-aa6b5e21fbd7', 'admin@foo.bar', 1, '""'::json, 1), +('1f0a1ce3-35b9-427a-b11d-096690fe4a27', 'admin@foo.bar', 1, '""'::json, 1), +('194ec434-bedd-4403-9e14-2351ea3c7720', 'admin@foo.bar', 1, '""'::json, 1), +('633b6978-0c6b-400a-b466-f3eb7e2035e9', 'admin@foo.bar', 1, '""'::json, 1), +('dd18384b-c038-48bf-b751-573163a4223d', 'admin@foo.bar', 1, '""'::json, 1), +('635c4f2a-8417-4fae-9078-e94846c8b7b1', 'admin@foo.bar', 1, '""'::json, 1), +('759be360-787b-410e-9e07-7db03e4df5c6', 'admin@foo.bar', 1, '""'::json, 1), +('37d514e8-ce44-4163-b47b-f0af05397aff', 'admin@foo.bar', 1, '""'::json, 1), +('7b0f4fc2-d772-4921-ad44-ba64f148a84d', 'admin@foo.bar', 1, '""'::json, 1), +('9a730dc2-23b7-4c41-8270-5fa7af22d468', 'admin@foo.bar', 1, '""'::json, 1), +('c033bd49-d4e9-4df3-9317-ce030d4b80d6', 'admin@foo.bar', 1, '""'::json, 1), +('09ee747d-638c-4933-bbe6-4e999d4b7480', 'admin@foo.bar', 1, '""'::json, 1), +('16060ef4-c199-493f-ab88-d4b4f06138a4', 'admin@foo.bar', 1, '""'::json, 1), +('8c1f6f46-75e6-4439-a5c0-c4e9bad71ec0', 'admin@foo.bar', 1, '""'::json, 1), +('22c6824e-2223-4a56-8686-dcbf3db8b2f8', 'admin@foo.bar', 1, '""'::json, 1), +('45c822ad-7817-48b6-8208-7b65922a5b01', 'admin@foo.bar', 1, '""'::json, 1), +('40e38e61-5426-4495-95a8-9bfd717b779e', 'admin@foo.bar', 1, '""'::json, 1), +('6b7f13b9-d019-4cd7-9913-6cc2a2d558c0', 'admin@foo.bar', 1, '""'::json, 1), +('643a6bdc-48a3-4624-ab93-1e70053c0f06', 'admin@foo.bar', 1, '""'::json, 1), +('d97b7b26-6c93-4d55-bd52-c4beae6fd9fc', 'admin@foo.bar', 1, '""'::json, 1), +('caab8faa-5ebd-4734-91d3-cc598fe332c4', 'admin@foo.bar', 1, '""'::json, 1), +('a005ae56-c6a3-4e0f-8280-49d85e32023e', 'admin@foo.bar', 1, '""'::json, 1), +('1a93bbf7-b118-4b67-9477-e5a067291f7e', 'admin@foo.bar', 1, '""'::json, 1), +('670fcf04-077d-41cd-b9b6-d5eb7b37b42a', 'admin@foo.bar', 1, '""'::json, 1), +('97704373-2097-4e0f-aa15-5a6087877b86', 'admin@foo.bar', 1, '""'::json, 1), +('3f1f3955-04a0-4554-ad87-55b4c2b13ee4', 'admin@foo.bar', 1, '""'::json, 1), +('97d9b7c0-b68d-41ff-954c-3755e6ff1cb9', 'admin@foo.bar', 1, '""'::json, 1), +('15f391a5-8c92-46e0-a321-5b981848d586', 'admin@foo.bar', 1, '""'::json, 1), +('e0d0e1ad-55f7-4486-a53a-7df8b44bd4ab', 'admin@foo.bar', 1, '""'::json, 1), +('b5415610-17a1-4ac7-a985-a060df556451', 'admin@foo.bar', 1, '""'::json, 1), +('80dc4574-8278-4492-9473-f6091c52b98c', 'admin@foo.bar', 1, '""'::json, 1), +('f838c5ba-bc20-4c5c-94cb-52973b522be4', 'admin@foo.bar', 1, '""'::json, 1), +('5a1d5e55-4fe4-4dba-9406-684ffb734c5f', 'admin@foo.bar', 1, '""'::json, 1), +('a5f23b08-8499-4e16-9b6c-69d089c9fd09', 'admin@foo.bar', 1, '""'::json, 1), +('2b365ffc-a96b-47a6-89c2-6d2aaae535df', 'admin@foo.bar', 1, '""'::json, 1), +('32a9bb7f-56c2-4bbc-9120-d66d4d8debbc', 'admin@foo.bar', 1, '""'::json, 1), +('2a1d211e-5b53-4121-93f6-49721523eda4', 'admin@foo.bar', 1, '""'::json, 1), +('bffc8bce-1f6f-4bff-ac7f-8dad7f43895a', 'admin@foo.bar', 1, '""'::json, 1), +('fc86707d-54ca-4eee-92d3-dbcde06ed75c', 'admin@foo.bar', 1, '""'::json, 1), +('0c7ddae6-83d6-4db6-9a79-9720c614142b', 'admin@foo.bar', 1, '""'::json, 1), +('9194b7f9-a420-426e-8073-78ad6607cd1d', 'admin@foo.bar', 1, '""'::json, 1), +('db9b1f49-f848-4a40-bbbb-ed9d3848965a', 'admin@foo.bar', 1, '""'::json, 1), +('84eff552-5723-4729-a86b-8f283874c289', 'admin@foo.bar', 1, '""'::json, 1), +('b251dfae-0bb8-4bee-ab47-bdcc17d25b62', 'admin@foo.bar', 1, '""'::json, 1), +('316b9754-2d5d-4822-bd72-f86f6ca9ae4b', 'admin@foo.bar', 1, '""'::json, 1), +('d509e593-4053-406a-8ab0-8af8b1b4c8a5', 'admin@foo.bar', 1, '""'::json, 1), +('e66faf00-0ffa-404f-ac08-7121c6337b70', 'admin@foo.bar', 1, '""'::json, 1), +('66782585-883b-4784-a443-a2dfff848d88', 'admin@foo.bar', 1, '""'::json, 1), +('9291cc5a-9ab4-49c0-b8a1-9fc9b6c46b44', 'admin@foo.bar', 1, '""'::json, 1), +('decafc1b-5c70-466b-a8cc-33f29c6db3df', 'admin@foo.bar', 1, '""'::json, 1), +('5b319607-4bec-4e36-ae54-5b7c21e91064', 'admin@foo.bar', 1, '""'::json, 1), +('2f3f309f-51bd-43b8-80d1-10d10a1c274e', 'admin@foo.bar', 1, '""'::json, 1), +('893621a8-d4dd-464b-8977-223470fc297e', 'admin@foo.bar', 1, '""'::json, 1), +('12e91952-b2b1-424d-884c-865e77474bf2', 'admin@foo.bar', 1, '""'::json, 1), +('d6946499-e2e7-4286-a95b-786b8ec633a9', 'admin@foo.bar', 1, '""'::json, 1), +('1fa39acc-83ef-409f-91d5-13ef1934635d', 'admin@foo.bar', 1, '""'::json, 1), +('694a13a5-9e5d-4f54-b34c-a63167a045b1', 'admin@foo.bar', 1, '""'::json, 1), +('c4ff14f9-3a89-40d6-aa85-1fbd8dd80c17', 'admin@foo.bar', 1, '""'::json, 1), +('d12d6d89-a8e9-4fd9-8e1c-16968035fb38', 'admin@foo.bar', 1, '""'::json, 1), +('9627e0a4-8f47-430e-b1d2-30c1295a722f', 'admin@foo.bar', 1, '""'::json, 1), +('f50accb5-3461-4a7f-9bff-761e4414a026', 'admin@foo.bar', 1, '""'::json, 1), +('e0d4954f-e964-4755-bac3-bc47610206aa', 'admin@foo.bar', 1, '""'::json, 1), +('2d60289d-1b7f-4945-8264-b01e2426c23e', 'admin@foo.bar', 1, '""'::json, 1), +('179d0c15-5af2-4671-b807-fb374fd3b842', 'admin@foo.bar', 1, '""'::json, 1), +('f6a8189a-caa7-4a41-bbc0-09ac46bf5a8f', 'admin@foo.bar', 1, '""'::json, 1), +('59ca6817-c388-47ad-ba64-47989313dd44', 'admin@foo.bar', 1, '""'::json, 1), +('61d6d576-362c-4261-859e-7209fa346f1e', 'admin@foo.bar', 1, '""'::json, 1), +('bd89bfe7-51b0-429b-9a88-2e71c8cb7545', 'admin@foo.bar', 1, '""'::json, 1), +('c5cc0f29-0fa0-4c44-b33c-103f0e3b6f32', 'admin@foo.bar', 1, '""'::json, 1), +('53db452b-9234-4830-94ef-cf9ab49ef0d1', 'admin@foo.bar', 1, '""'::json, 1), +('a268c541-fe8d-4521-8a49-e3a8439aef64', 'admin@foo.bar', 1, '""'::json, 1), +('387f71b9-3283-4923-860b-8065a62248fd', 'admin@foo.bar', 1, '""'::json, 1), +('8194c64a-1a43-499f-8f47-3a3935c557f4', 'admin@foo.bar', 1, '""'::json, 1), +('294d4a3f-f8ff-4644-ab15-3ddcb4390af1', 'admin@foo.bar', 1, '""'::json, 1), +('a373adcd-9b62-41e3-9ae4-a6f79ff45f45', 'admin@foo.bar', 1, '""'::json, 1), +('ef7406a1-a4fa-4af2-8e67-92607ebdd8f9', 'admin@foo.bar', 1, '""'::json, 1), +('59dbbe08-8701-4e48-95ab-3882a02ee1e7', 'admin@foo.bar', 1, '""'::json, 1), +('088a65e5-c848-4e34-b47d-4664f9f27f83', 'admin@foo.bar', 1, '""'::json, 1), +('45d538e2-97c5-4893-af08-f6bf04abb767', 'admin@foo.bar', 1, '""'::json, 1), +('0b9706a2-e0a6-438f-a5b3-a094161a15a7', 'admin@foo.bar', 1, '""'::json, 1), +('eb3650f0-7aea-4cab-a520-6e581e3bd978', 'admin@foo.bar', 1, '""'::json, 1), +('c6635e7e-93c0-403e-b615-3e8f7da29352', 'admin@foo.bar', 1, '""'::json, 1), +('2cb50802-1ac7-48a4-bebd-4d9be546cf69', 'admin@foo.bar', 1, '""'::json, 1), +('1c6d9feb-17fb-40ca-a3f4-d1e7834c5d53', 'admin@foo.bar', 1, '""'::json, 1), +('ed2a60cf-2164-4b01-a59c-07518613ef8d', 'admin@foo.bar', 1, '""'::json, 1), +('469e8fcc-37ef-4dfb-b4cd-2b109200ce4c', 'admin@foo.bar', 1, '""'::json, 1), +('52802b61-7df2-4e82-9ccf-5c8075c2da45', 'admin@foo.bar', 1, '""'::json, 1), +('0c715e72-47dd-48ac-b733-a845e440e86a', 'admin@foo.bar', 1, '""'::json, 1), +('13f6b03d-40e1-47c2-a7df-163c0e3e7d23', 'admin@foo.bar', 1, '""'::json, 1), +('9639186c-9084-47ed-8075-3f14798eceef', 'admin@foo.bar', 1, '""'::json, 1), +('6e54aa84-ffd1-4943-8ae2-7e75319a7e30', 'admin@foo.bar', 1, '""'::json, 1), +('dba3182c-80c0-46ed-a9e6-64e0cf699741', 'admin@foo.bar', 1, '""'::json, 1), +('b33244f5-30ce-4db3-94c7-4dcbe5cff4a7', 'admin@foo.bar', 1, '""'::json, 1), +('ba196947-c2d6-4ae1-ad23-fb768c087125', 'admin@foo.bar', 1, '""'::json, 1), +('08d92acd-992c-4500-a33a-094225dec9cb', 'admin@foo.bar', 1, '""'::json, 1), +('b0bfe252-9f89-4160-839e-738a649de0f1', 'admin@foo.bar', 1, '""'::json, 1), +('ea9e5e73-c141-4cf6-944a-6f43541ed747', 'admin@foo.bar', 1, '""'::json, 1), +('b6d61911-54b5-4917-957a-11f5931f1b6e', 'admin@foo.bar', 1, '""'::json, 1), +('3cf83989-1316-47f4-a291-03aa41da31f1', 'admin@foo.bar', 1, '""'::json, 1), +('531b6413-c293-4a87-9444-3939eebf79bd', 'admin@foo.bar', 1, '""'::json, 1), +('18919546-fade-4e83-b5dc-1708043bec82', 'admin@foo.bar', 1, '""'::json, 1), +('6e6eb332-9166-4059-8a7c-cc4dfa5417f3', 'admin@foo.bar', 1, '""'::json, 1), +('c36b0187-97f8-49c4-9f78-dfb75a04f41d', 'admin@foo.bar', 1, '""'::json, 1), +('f9552ba6-db05-4a74-acb7-f851c627777d', 'admin@foo.bar', 1, '""'::json, 1), +('4777ec99-b163-466f-81a8-a5e5f3c81b5d', 'admin@foo.bar', 1, '""'::json, 1), +('3429c0f3-9f05-4234-a3ec-ead9285d9c91', 'admin@foo.bar', 1, '""'::json, 1), +('a9e4fa1b-5f48-4143-a06e-20c6013c37f0', 'admin@foo.bar', 1, '""'::json, 1), +('eb734567-fa4f-4aff-8ac4-e119b7cf31cc', 'admin@foo.bar', 1, '""'::json, 1), +('52b3c3ce-0681-4992-9cd5-bdb0eddffff6', 'admin@foo.bar', 1, '""'::json, 1), +('ddb6e8c1-700b-4c2e-834b-26ceca5be035', 'admin@foo.bar', 1, '""'::json, 1), +('ed55d400-04c2-4aa7-be76-63eef565cbae', 'admin@foo.bar', 1, '""'::json, 1), +('23d23e6a-7e1e-41e6-aedb-5c997c6a31a6', 'admin@foo.bar', 1, '""'::json, 1), +('160160e6-ee83-47f7-8d59-1e6df6dd1a4d', 'admin@foo.bar', 1, '""'::json, 1), +('e3b07a6e-145b-4551-b27c-905a1601a1f3', 'admin@foo.bar', 1, '""'::json, 1), +('462b169a-acce-47a7-adf9-1daa0a465639', 'admin@foo.bar', 1, '""'::json, 1), +('a43beee6-aab4-4ddd-895e-b7f95e258d17', 'admin@foo.bar', 1, '""'::json, 1), +('4df02286-2ecd-4dd0-9d13-ec5d6b2e76d9', 'admin@foo.bar', 1, '""'::json, 1), +('50b68208-0bdc-4ecd-9f75-2036a2332d55', 'admin@foo.bar', 1, '""'::json, 1), +('9bba65d6-12be-4f4b-b40a-018489ebbc3b', 'admin@foo.bar', 1, '""'::json, 1), +('40fe74d9-7fa2-4ef1-a64d-c092b3e5128b', 'admin@foo.bar', 1, '""'::json, 1), +('716b59bd-d2f0-404e-b6c5-ad5b69e8db3a', 'admin@foo.bar', 1, '""'::json, 1), +('cd5299dd-f160-4b8e-9eac-ea2cb54c003a', 'admin@foo.bar', 1, '""'::json, 1), +('6140d69a-3c93-4378-bc99-fc0596aa6703', 'admin@foo.bar', 1, '""'::json, 1), +('c03cd275-e2c0-4e28-a0a1-e07be2c24c03', 'admin@foo.bar', 1, '""'::json, 1), +('9afa1b4a-244e-44a4-8b9c-69c229007667', 'admin@foo.bar', 1, '""'::json, 1), +('655e4674-a14d-485b-9294-18d8e76014ca', 'admin@foo.bar', 1, '""'::json, 1), +('2a009904-eb9e-49bd-bee0-da0bb1bfdba5', 'admin@foo.bar', 1, '""'::json, 1), +('186f76c0-c877-428c-ab08-46cbcf4524a1', 'admin@foo.bar', 1, '""'::json, 1), +('f5d2cdc0-0061-4a7b-9c86-bee6d7e08d29', 'admin@foo.bar', 1, '""'::json, 1), +('f147fe13-56a8-41db-b399-ecabba242ca9', 'admin@foo.bar', 1, '""'::json, 1), +('0bef5c46-6ad7-4a05-8c4b-8d227f0a0352', 'admin@foo.bar', 1, '""'::json, 1), +('fc0ebaac-1bbb-4fb6-b20d-2fe74f31b5ef', 'admin@foo.bar', 1, '""'::json, 1), +('fcd7e83f-804c-40bb-9f3d-8483cffe34c0', 'admin@foo.bar', 1, '""'::json, 1), +('284cc76f-487e-4d49-9c2f-d4d10e74385e', 'admin@foo.bar', 1, '""'::json, 1), +('b8e3c3ac-5b5b-4f6d-97c2-3296b0bab7e6', 'admin@foo.bar', 1, '""'::json, 1), +('9262c5ec-b92b-425b-8fd1-10ea4905d595', 'admin@foo.bar', 1, '""'::json, 1), +('7bae4416-a699-4985-9736-60bf2b4aa4d6', 'admin@foo.bar', 1, '""'::json, 1), +('7cb7e7e1-48c1-4259-9856-395911e88d4d', 'admin@foo.bar', 1, '""'::json, 1), +('b127f3dc-fc5b-4054-a20b-e9216caff4ef', 'admin@foo.bar', 1, '""'::json, 1), +('dfb12c24-d67a-4dc4-b221-b6fb1e49602b', 'admin@foo.bar', 1, '""'::json, 1), +('507da6a3-0df4-4e30-9763-f392eb62aac5', 'admin@foo.bar', 1, '""'::json, 1), +('6fe5c170-4774-4576-813f-7c164866a3bf', 'admin@foo.bar', 1, '""'::json, 1), +('3c31aad8-1e86-48be-b65c-4924388bd587', 'admin@foo.bar', 1, '""'::json, 1), +('2b2e2473-fbc6-4e6a-9072-19f6411b0817', 'admin@foo.bar', 1, '""'::json, 1), +('5994ad72-2f1a-416b-ba3c-9cc4a1c61c20', 'admin@foo.bar', 1, '""'::json, 1), +('ba111653-1c15-4425-8ab9-11a5422eff1d', 'admin@foo.bar', 1, '""'::json, 1), +('fdec2e67-e193-4185-991a-be136e63b2b0', 'admin@foo.bar', 1, '""'::json, 1), +('6218ce6d-5208-4ebc-9e48-633b3f54cdcc', 'admin@foo.bar', 1, '""'::json, 1), +('d6eef75a-4505-4111-9c4d-5a748d403dd2', 'admin@foo.bar', 1, '""'::json, 1), +('7a24dba0-f6b4-48cc-b1cb-4c9b884170e6', 'admin@foo.bar', 1, '""'::json, 1), +('ff077c77-a1ba-46ac-b5ca-d5c894242770', 'admin@foo.bar', 1, '""'::json, 1), +('65526acd-86e9-421b-9c91-b885cd36729a', 'admin@foo.bar', 1, '""'::json, 1), +('04d09a9f-2151-47df-8475-36fc8ec7aafc', 'admin@foo.bar', 1, '""'::json, 1), +('7e72e913-ce87-4bc1-adf7-c3cc7d5184e5', 'admin@foo.bar', 1, '""'::json, 1), +('9e6f2ff5-f749-4089-b828-ef29fcca1a7d', 'admin@foo.bar', 1, '""'::json, 1), +('335635a7-8153-4e4c-9b53-e1bc145497b0', 'admin@foo.bar', 1, '""'::json, 1), +('5452c17e-ae82-4b46-a85a-ece0501a5d5e', 'admin@foo.bar', 1, '""'::json, 1), +('f5a06f01-7405-4ff1-b047-64f9e7505d6c', 'admin@foo.bar', 1, '""'::json, 1), +('bf9fc8f3-39b7-4771-95e8-f858494022a5', 'admin@foo.bar', 1, '""'::json, 1), +('98270055-d47a-4cc4-b40c-ac006f7823f6', 'admin@foo.bar', 1, '""'::json, 1), +('009b95ec-18b6-4fa0-a0bb-0785cfe03409', 'admin@foo.bar', 1, '""'::json, 1), +('89ae9ba5-f920-46e6-9279-40d5be29f845', 'admin@foo.bar', 1, '""'::json, 1), +('0f24f638-13fe-4898-b3a9-b15aafb260f8', 'admin@foo.bar', 1, '""'::json, 1), +('37d12cdd-c03b-4921-a77b-851269d3e557', 'admin@foo.bar', 1, '""'::json, 1), +('9308c777-1c30-43ac-acf6-9454601601ba', 'admin@foo.bar', 1, '""'::json, 1), +('a1910b9b-2d93-4239-9bc1-1250fea56837', 'admin@foo.bar', 1, '""'::json, 1), +('1280dc5a-d797-42ca-a613-3fbce32f55a0', 'admin@foo.bar', 1, '""'::json, 1), +('0a329ce1-1b82-4faf-a1f7-0d17e4b25251', 'admin@foo.bar', 1, '""'::json, 1), +('3b212797-bb0b-422c-bba1-79921ce40c78', 'admin@foo.bar', 1, '""'::json, 1), +('c14e97a2-a083-4e49-a14b-0abd23483272', 'admin@foo.bar', 1, '""'::json, 1), +('7cd84e00-773a-4387-bcdb-37da819ce5f7', 'admin@foo.bar', 1, '""'::json, 1), +('8553c8e2-5bc9-41b6-9f5e-96a2abfb7055', 'admin@foo.bar', 1, '""'::json, 1), +('65212822-529e-4cf4-a427-78032b9e569b', 'admin@foo.bar', 1, '""'::json, 1), +('3a8d8b29-d408-45ab-8d63-043741de9679', 'admin@foo.bar', 1, '""'::json, 1), +('d68f1265-c669-4f2b-a3ab-ee0e547e0143', 'admin@foo.bar', 1, '""'::json, 1), +('f0b0c5aa-9f97-42e3-82e4-71968f28ad15', 'admin@foo.bar', 1, '""'::json, 1), +('1bf64e7e-4ab9-4dfa-9aeb-6402ad95c165', 'admin@foo.bar', 1, '""'::json, 1), +('605f6954-8697-42c4-97a4-7c3aa6b3cde9', 'admin@foo.bar', 1, '""'::json, 1), +('3688f502-6895-4041-b957-6487f6ba7c98', 'admin@foo.bar', 1, '""'::json, 1), +('013f8041-6f36-4dcb-8af9-67ac1fd78d9e', 'admin@foo.bar', 1, '""'::json, 1), +('58e69a18-ecef-4b61-a62f-7efc1f733731', 'admin@foo.bar', 1, '""'::json, 1), +('e7cb303d-ffe0-4a5e-8eb0-627dd0513f51', 'admin@foo.bar', 1, '""'::json, 1), +('7cc4a810-137c-468e-a249-60b0faf9a003', 'admin@foo.bar', 1, '""'::json, 1), +('056330e4-36eb-4c38-b87f-56484ac69623', 'admin@foo.bar', 1, '""'::json, 1), +('ea9e5c11-f7a7-4dcb-b8b5-20b499611b9f', 'admin@foo.bar', 1, '""'::json, 1), +('8df2b291-adbf-4def-a172-ece656410050', 'admin@foo.bar', 1, '""'::json, 1), +('ce47b98d-8a78-4e97-b20c-297f75394f28', 'admin@foo.bar', 1, '""'::json, 1), +('d8e90f5f-1b10-4f00-9c3b-6bc8daf2cc8a', 'admin@foo.bar', 1, '""'::json, 1), +('cec46141-0286-4430-ac41-ed830decb5e8', 'admin@foo.bar', 1, '""'::json, 1), +('fc9b54c3-0447-49db-a6d0-5b913d0e285a', 'admin@foo.bar', 1, '""'::json, 1), +('3e6d362e-3880-4b90-b693-868547ce0f5a', 'admin@foo.bar', 1, '""'::json, 1), +('d5891efa-0e7d-409b-9bb7-b6ecc081c836', 'admin@foo.bar', 1, '""'::json, 1), +('e4b960c8-33c6-4304-8355-9bafd70569ed', 'admin@foo.bar', 1, '""'::json, 1), +('35c4eb46-3560-4d17-a210-979ba3037c53', 'admin@foo.bar', 1, '""'::json, 1), +('2b85bd17-6d7e-48e0-8e02-477eb9ffe353', 'admin@foo.bar', 1, '""'::json, 1), +('8c7e4562-dcc2-4a31-ab20-b873182b334d', 'admin@foo.bar', 1, '""'::json, 1), +('49fda716-a469-4a6d-a3ef-3bcbd166e668', 'admin@foo.bar', 1, '""'::json, 1), +('f424b26a-840a-42d3-a61f-3bbffd8b194d', 'admin@foo.bar', 1, '""'::json, 1), +('7226d658-da67-4d7c-a898-e59cd77d977b', 'admin@foo.bar', 1, '""'::json, 1), +('50dc9288-e18c-41c2-a271-601289ff5685', 'admin@foo.bar', 1, '""'::json, 1), +('4883db35-3ca6-4b0c-9377-cf7d1d97245d', 'admin@foo.bar', 1, '""'::json, 1), +('3ff09f4e-ada5-45d6-b7e1-fa003e31bf77', 'admin@foo.bar', 1, '""'::json, 1), +('26c68022-6df2-4928-a4fb-220a156b86f7', 'admin@foo.bar', 1, '""'::json, 1), +('cdd878a9-e3f1-4478-b613-90f2b2e9ccbd', 'admin@foo.bar', 1, '""'::json, 1), +('d6e02749-45fb-4b92-8a8e-48b33b617f9c', 'admin@foo.bar', 1, '""'::json, 1), +('47900ec7-0fea-41db-a320-c78b9da36538', 'admin@foo.bar', 1, '""'::json, 1), +('1ef4d550-afaa-44b5-a06c-65286b9f3e00', 'admin@foo.bar', 1, '""'::json, 1), +('8533d2a8-1fcf-4842-8826-c5458c002332', 'admin@foo.bar', 1, '""'::json, 1), +('ab9095d1-af6d-49d5-9c87-bc2576f06135', 'admin@foo.bar', 1, '""'::json, 1), +('61da73ff-b4ff-49a1-b775-c6215cfbd291', 'admin@foo.bar', 1, '""'::json, 1), +('6c84dcf1-c5ea-4e69-b17f-d2d5b8d48bdf', 'admin@foo.bar', 1, '""'::json, 1), +('dcb12603-4142-44d1-9a52-3ca3511e380e', 'admin@foo.bar', 1, '""'::json, 1), +('a0dd0a4d-b73f-4e9d-87dd-d29efba25336', 'admin@foo.bar', 1, '""'::json, 1),; +-- SPLIT + +INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, columns, input_size, extra_info, memory_used, walltime_used) +VALUES ('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 39, 81, 2, 'nan', 327036000, 91), +('b0f36550-d97c-4dd5-ba1b-910470062801', 160, 212, 2, 'nan', 342204000, 219), +('b13eefd1-12b7-4c3e-a380-94ec0a3f9f91', 62, 83, 2, 'nan', 328716000, 165), +('469c27b7-81c4-4f41-b09d-149260961227', 16, 83, 2, 'nan', 326344000, 154), +('6f198e33-b4ae-47e9-973b-bd5e0b06c353', 96, 152, 2, 'nan', 332492000, 185), +('045eb36c-5144-4d54-a621-23eb5dc4d6ef', 242, 43, 2, 'nan', 369988000, 137), +('041f3278-2752-44f9-8caa-ac6ae80c98e6', 231, 97, 2, 'nan', 375820000, 151), +('6325e683-b215-4c56-bf5a-cbbc92330067', 120, 83, 2, 'nan', 373536000, 234), +('03dcca05-0d4f-48c5-bc7a-7ef884be8efc', 92, 83, 2, 'nan', 372188000, 407), +('8d84685f-97f4-4ea7-8857-2dcec5a20775', 40, 83, 2, 'nan', 368816000, 296), +('8b714fe7-51d9-492c-9b49-4fe09b595067', 96, 87, 2, 'nan', 371972000, 138), +('490c0623-4ed7-4064-9ffe-ba5afc0f0aa0', 28, 50, 2, 'nan', 369696000, 185), +('c8301f3d-a4fe-407a-9aec-b9dfee0cc2fd', 12, 83, 2, 'nan', 368052000, 97), +('561b9676-604a-4cdb-833f-a4a9b3afef42', 41, 50, 2, 'nan', 369580000, 101), +('8bc35697-f3df-42f6-8a34-830d42000bd6', 336, 22, 2, 'nan', 369828000, 86), +('474a4a8d-0963-4fe6-9da4-74e520532be1', 32, 80, 2, 'nan', 364956000, 105), +('d4d03b6f-12f1-4f47-9ad4-b5597a3e5a16', 152, 66, 2, 'nan', 373528000, 129), +('27d95aba-9b36-40c8-8d17-7ade170172aa', 36, 83, 2, 'nan', 367776000, 155), +('a8f00090-6981-4fc3-bc37-208f01e6dceb', 44, 83, 2, 'nan', 370620000, 93), +('d7562d61-7886-45b5-9fed-90fef6636e6f', 152, 83, 2, 'nan', 372888000, 100), +('e4071da6-550d-469b-91dc-f6cda658df80', 32, 64, 2, 'nan', 369900000, 155), +('c126e507-5f13-485f-9166-775ee7228f12', 28, 26, 2, 'nan', 368464000, 97), +('d73906a4-f2c7-4d03-bd55-d637365ec3bc', 138, 89, 2, 'nan', 375516000, 288), +('167efba9-a88e-493c-a2c6-0647556ba6dd', 35, 42, 2, 'nan', 369480000, 81), +('2662c176-d042-4254-bc7e-20c092fc3af1', 152, 72, 2, 'nan', 373376000, 135), +('54b19d0d-fa47-4c37-a4ea-e2be33d54c5b', 152, 80, 2, 'nan', 369868000, 92), +('8b3e969e-78c4-4a09-9767-4aa72d99ee8c', 90, 75, 2, 'nan', 368312000, 99), +('bef26557-60db-4987-b0bf-e9e0be77da37', 32, 83, 2, 'nan', 372748000, 100), +('adcf6cb4-780b-4a9d-bb7c-cd2bf12ac201', 28, 51, 2, 'nan', 370832000, 139), +('54464698-5860-4543-9376-d3d89722848f', 152, 81, 2, 'nan', 372652000, 95), +('9a8b3ea7-36d7-46f7-8602-1b39c89f5ee4', 28, 46, 2, 'nan', 369584000, 193), +('b57c1bc6-a805-463d-87fc-b76a186e5a63', 21302, 90, 2, 'nan', 30269928000, 187836), +('0c0579c8-dc34-4211-ad87-c453e9cb10bf', 4566, 178, 2, 'nan', 5477396000, 11596), +('04357795-299a-4900-9fdf-debf3be2ccc7', 56, 80, 2, 'nan', 370324000, 100), +('fa1617ad-daf8-496d-937f-5ad9526f57d2', 214, 84, 2, 'nan', 380764000, 139), +('056dd777-1c93-424a-ad7a-fbac4ff01e6b', 174, 81, 2, 'nan', 371052000, 105), +('1d056722-c08c-45e7-abbf-794abf85b2ba', 376, 94, 2, 'nan', 394040000, 215), +('a4e4457e-e191-4951-81d2-a1bd8afa7bb9', 189, 142, 2, 'nan', 382424000, 165), +('2bd5eedb-f4bd-4221-a31d-7a82ed1ed85c', 1067, 12, 2, 'nan', 372276000, 111), +('59e6279e-5a92-4ad5-81cd-d83892eb35e3', 6374, 49, 2, 'nan', 9936144000, 21930), +('594e5a37-e00e-44e3-8baf-a5d0fbd8d1c5', 1917, 102, 2, 'nan', 1292284000, 1367), +('40e16ce2-d280-423b-908e-f74877751bb8', 152, 85, 2, 'nan', 372464000, 159), +('df964c5d-d204-475d-bf37-c4281c830e36', 266, 67, 2, 'nan', 369540000, 98), +('f1ff0a3f-5004-49d4-a781-fedab076f682', 39, 93, 2, 'nan', 373044000, 152), +('d5b6766a-5697-4283-b7f5-ded6295e539d', 150, 63, 2, 'nan', 374136000, 444), +('6e98873c-0c99-47f0-ae6f-fc12092c1c5d', 856, 49, 2, 'nan', 584216000, 578), +('b816de4b-42f6-4495-8a05-7cb7faefcf3e', 856, 48, 2, 'nan', 634320000, 557), +('2e196649-1641-4ced-bf74-f24db6123d66', 394, 42, 2, 'nan', 380860000, 767), +('ca8271ce-ff25-4a22-8309-149396d20d38', 384, 22, 2, 'nan', 376636000, 374), +('38496999-f362-45d1-a9e6-6c2ed7ad1847', 69, 70, 2, 'nan', 373668000, 101), +('dedeab8b-ed71-4d3d-87c1-37f607e19e5d', 20, 47, 2, 'nan', 365196000, 280), +('997ad2aa-59ff-4346-805c-cbe1a41bac00', 413, 40, 2, 'nan', 412392000, 181), +('ff866805-7add-432f-90a8-15ab4b793cdd', 1005, 137, 2, 'nan', 425864000, 165), +('ef8a5faa-fc7b-4a09-a45a-7aa4d3dbb1f8', 177, 384, 2, 'nan', 380400000, 231), +('b0db6327-e4ff-4dd5-85ae-b8a060f8ed9f', 96, 99, 2, 'nan', 376124000, 234), +('900537c6-a4bb-4f20-9674-e0fd4851c10a', 22, 47, 2, 'nan', 368924000, 207), +('3fe82723-af0e-4df0-a3bf-32223b202dd5', 12, 50, 2, 'nan', 367552000, 108), +('6b22c272-f72c-4102-9abb-afff9d1c7d42', 14, 50, 2, 'nan', 371208000, 106), +('6738f416-1ce1-4cac-90ef-ead48b9b0ace', 150, 77, 2, 'nan', 375128000, 886), +('6ed39d69-1c01-42c1-8249-b8abf83c3608', 399, 844, 2, 'nan', 567664000, 508), +('d00fd8ee-7d65-469c-af2a-ca00681d224b', 1150, 54, 2, 'nan', 394760000, 160), +('42244303-b4a1-43d3-9041-f4f811691b41', 96, 81, 2, 'nan', 374352000, 118), +('4ac590b9-5733-4e20-b657-3b13d9b30a4f', 399, 843, 2, 'nan', 554660000, 491), +('1409a826-8176-49db-84f9-04d4cac0db89', 511, 146, 2, 'nan', 405100000, 228), +('21917565-5c24-4b9e-947e-0a7fc18b89dc', 1055, 102, 2, 'nan', 594684000, 424), +('cc4db5ef-c3d3-4067-98be-ee90a584a95a', 53, 71, 2, 'nan', 371640000, 1355), +('12f627b7-19aa-4583-bce6-a7f7d4dd66a2', 234, 45, 2, 'nan', 372792000, 119), +('0c6dc99d-d20b-4381-a405-8813129b81ee', 282, 54, 2, 'nan', 373592000, 255), +('449668bf-3294-42eb-9cef-d0c007b9b27f', 5924, 12, 2, 'nan', 400672000, 105), +('ee4c2079-e187-4a2f-ae9a-a95795a9d5fe', 476, 68, 2, 'nan', 446712000, 408), +('0f0d6f75-6ba3-4265-b53e-cc53a18e52b8', 3042, 13, 2, 'nan', 387236000, 117), +('d63bdad8-ace7-4d92-a8a7-add8620460d3', 353, 56, 2, 'nan', 376128000, 86), +('5a856042-6a60-4f9d-b256-d03295e2d7f9', 96, 82, 2, 'nan', 373728000, 100), +('dd9969e8-9efe-4781-af4a-6f2f8ff70808', 706, 56, 2, 'nan', 384544000, 202), +('37cff1b2-c11e-4511-8625-84def4635d84', 413, 42, 2, 'nan', 374920000, 112), +('36f312b9-5055-443a-b07a-2ab09e8f8cfa', 32, 14, 2, 'nan', 368144000, 119), +('de207a0d-932f-4a81-9412-5ff8baa9cdd5', 384, 87, 2, 'nan', 384584000, 122), +('b567274d-4631-4aee-85df-7d6603c20cfe', 287, 37, 2, 'nan', 391964000, 289), +('94d07841-1b75-42a1-9407-af6a341769b1', 5540, 32, 2, 'nan', 4107984000, 3302), +('d83a1dee-1451-4ea3-bd42-3881f1fa4e3e', 768, 87, 2, 'nan', 395492000, 126), +('3c736f54-7d2d-4cd6-9c45-8cce37ce5482', 112, 36, 2, 'nan', 372180000, 212), +('e600feed-52cc-4d8f-b28d-c6283dcde944', 46, 53, 2, 'nan', 368888000, 174), +('41624971-b40b-4e03-bb12-cce149cd6eac', 39, 73, 2, 'nan', 369812000, 108), +('9f769164-d25c-4e7d-b637-79f3489798c5', 30, 62, 2, 'nan', 368840000, 88), +('0f382217-32ed-4ea6-b7ab-c98c66e8305f', 291, 76, 2, 'nan', 378096000, 102), +('24d2cf39-25a1-486e-a763-f2639674f4b4', 116, 60, 2, 'nan', 367924000, 170), +('b303df36-67d8-4883-a109-294b61ccbe23', 113, 250, 2, 'nan', 381692000, 386), +('b46d8a4d-b969-4226-8719-6708c3755775', 391, 92, 2, 'nan', 379532000, 92), +('07b691e7-a940-4257-bea8-ebba6bc47637', 157, 67, 2, 'nan', 379060000, 126), +('9d624992-01fe-4ece-a7ba-ebf54da0eb7b', 33, 92, 2, 'nan', 369760000, 120), +('0aba4000-8b6d-47c1-a4fc-7b6a2b419b67', 384, 258, 2, 'nan', 421788000, 336), +('4635cab1-b1a7-4d6a-bb75-d95f0e1fd682', 24, 58, 2, 'nan', 333928000, 79), +('b2602d9e-7ce6-43e2-a4b3-3962f812a80a', 353, 149, 2, 'nan', 389096000, 219), +('f8d9c8b3-1e4f-4371-9d86-74778f7145a7', 150, 128, 2, 'nan', 376468000, 602), +('490bf28c-7f5b-428f-9127-c1ac19d611fc', 147, 23, 2, 'nan', 371728000, 92), +('a69254f1-d5d4-4f3f-991f-c197364b8187', 278, 35, 2, 'nan', 376404000, 116), +('702f4828-f604-44b8-a7b0-9684e77a0279', 64, 41, 2, 'nan', 366940000, 90), +('0d793328-c4c5-4739-b2f1-40afb4de55c8', 95, 37, 2, 'nan', 371568000, 249), +('7227a61e-59af-4b96-8919-efe13839889c', 4302, 62, 2, 'nan', 8104616000, 18008), +('57e351db-c504-42dd-a699-fe405c1b5a27', 1038, 37, 2, 'nan', 406872000, 245), +('88bbeb39-2fc5-4a46-8bd9-ca28a2fedef3', 399, 970, 2, 'nan', 612944000, 517), +('63a0101c-13b0-4c73-b4d7-00a33ae0f9ab', 119, 40, 2, 'nan', 320928000, 81), +('e3c17c93-5993-49fc-891a-9c9d5b9aa43c', 6, 11, 2, 'nan', 370792000, 194), +('6fd1fc5f-fce2-43c6-909d-8041a075ac3a', 537, 92, 2, 'nan', 418592000, 299), +('8ea8b437-819b-4624-b15c-a653b53c0a2e', 32, 41, 2, 'nan', 368692000, 89), +('c4ee461b-8d52-4338-8fa2-8a22b538456b', 12, 466, 2, 'nan', 374988000, 715), +('1eaa62de-c3ae-414d-9168-7fb5ece89949', 16, 41, 2, 'nan', 368992000, 239), +('6a63a491-c4c1-4176-9827-9460526ed319', 528, 76, 2, 'nan', 382696000, 99), +('733a21cf-e969-4a17-b888-820f2f9475a1', 307, 85, 2, 'nan', 384432000, 167), +('9a1e07e0-bac0-46e1-8b72-86a730f47c12', 862, 105, 2, 'nan', 456076000, 358), +('6ca8ca71-dbb7-4347-adb3-9ab02b69ef9f', 669, 95, 2, 'nan', 451828000, 523), +('07272b4e-88bf-4adc-9953-4b1f4cb3764f', 151, 15, 2, 'nan', 371468000, 83), +('8b9f9e1e-2e52-432b-8528-53f8b25ea81e', 2259, 34, 2, 'nan', 391664000, 104), +('6395bd2a-fd1b-448b-8771-f07a77958d84', 394, 48, 2, 'nan', 377592000, 265), +('b05bb0d8-a8bd-4634-82f9-47c52ccd83e3', 450, 140, 2, 'nan', 390568000, 125), +('38ee009a-7c6a-42a2-a05f-cd3f5286cd9d', 384, 92, 2, 'nan', 379920000, 107), +('d0799585-1e37-4134-b71f-481dc0d2be72', 384, 171, 2, 'nan', 402372000, 349), +('2d5a21fd-1cdb-4ee6-8e08-584bf99ef40a', 9, 12, 2, 'nan', 369064000, 152), +('a4aee4f4-91d5-46e1-9280-87b8ba7eeef9', 394, 50, 2, 'nan', 378484000, 120), +('7e361763-962b-4f8d-8c63-1a329428f89f', 384, 172, 2, 'nan', 406412000, 233), +('0abb0be4-b58e-4318-8624-4fc055f9d82d', 3938, 91, 2, 'nan', 532680000, 246), +('3497cd39-99dc-429e-8e10-67ecd7b2774a', 3, 79, 2, 'nan', 369232000, 193), +('0ca036c1-ceb5-4c33-993b-d8511a96208b', 78, 40, 2, 'nan', 369532000, 88), +('a22718ce-6e93-420b-89d6-cf2ddf104164', 57, 40, 2, 'nan', 369396000, 91), +('278179bb-f481-4d50-aced-5485993699a1', 95, 46, 2, 'nan', 369740000, 91), +('80272c6a-4312-4c93-9905-390e62a3d201', 384, 1747, 2, 'nan', 538556000, 2487), +('d5ab9552-64e1-43b2-bf46-5065612c2cda', 384, 1759, 2, 'nan', 1468608000, 2527), +('af467934-f4db-464e-8ef1-b6bbbbe88667', 394, 51, 2, 'nan', 376672000, 109), +('cfa62dd9-c6e0-48a2-9633-ee4d813d4f51', 148, 78, 2, 'nan', 371308000, 160), +('bd37012e-0b22-4e3a-b0ae-eba006d87749', 60, 34, 2, 'nan', 369272000, 119), +('d11bb684-448c-4916-a0f8-cd5748061638', 48, 15, 2, 'nan', 369052000, 163), +('46cb1988-f5b7-4a27-92be-dd8b7c7289db', 48, 15, 2, 'nan', 369052000, 147), +('dd000d70-ee4b-46a6-9e1f-26fd85b6df3c', 412, 11, 2, 'nan', 370240000, 122), +('a12d7860-89d0-4640-a4a6-73738b47408a', 11, 15, 2, 'nan', 368712000, 92), +('aae1e730-254d-401b-884d-d9f6965203f6', 11, 22, 2, 'nan', 369712000, 200), +('66e2dbf0-d223-48b6-bf1e-45a40f87d1b1', 45, 160, 2, 'nan', 363000000, 152), +('be2e6e7d-7992-4407-a7c1-aced14e6fa12', 112, 13, 2, 'nan', 370968000, 90), +('671071f2-380c-42a3-88f6-a1642c07dc36', 220, 68, 2, 'nan', 373128000, 233), +('5db2e298-251c-409f-81a2-844b9a2b7697', 1232, 61, 2, 'nan', 391532000, 206), +('9b0e1a4a-97a6-47fe-aa2a-6800d72fa611', 602, 72, 2, 'nan', 399216000, 309), +('9553c043-e844-4c9f-8367-eb5d7d51772c', 54, 64, 2, 'nan', 371940000, 141), +('78c3db71-ed10-457d-b048-d8142c535e14', 78, 28, 2, 'nan', 369532000, 96), +('60a72e37-28f8-4288-81eb-830cb2924a03', 76, 33, 2, 'nan', 370068000, 198), +('520e37c3-6e64-4fce-8a88-cceac2b3b5b8', 80, 68, 2, 'nan', 372164000, 139), +('596f24ae-49e1-4a22-bbee-0b53b1bc4b26', 921, 486, 2, 'nan', 1131140000, 2776), +('7fce7f74-0c73-4fc2-b5bc-45f570014054', 192, 32, 2, 'nan', 400300000, 270), +('0061bb8f-219c-4e53-8ea5-137799e5fc3c', 95, 32, 2, 'nan', 398676000, 134), +('09c27409-4fb5-415c-93e3-7e2233e0109e', 115, 61, 2, 'nan', 398460000, 111), +('1a95492b-e852-413b-8450-72eb0adcd577', 460, 61, 2, 'nan', 401832000, 98), +('c6315300-f68e-4bf7-b7d2-778c3409f346', 939, 174, 2, 'nan', 717876000, 917), +('517dd44b-61bc-47ba-a7c8-c189378f6d73', 2591, 206, 2, 'nan', 1137112000, 2656), +('c167f049-4d43-4409-84be-233b44981bb8', 286, 81, 2, 'nan', 400352000, 111), +('2c4290f6-e59f-438f-9664-a22810b495a0', 22, 67, 2, 'nan', 395520000, 114), +('e7a1816e-c98c-4ae9-854c-b034d4dc77cf', 16, 77, 2, 'nan', 394388000, 99), +('e5a06530-b313-4b75-ad14-5bceeb690d5a', 6, 67, 2, 'nan', 394636000, 110), +('8d671718-8686-4cc5-899f-178a2fba763e', 168, 59, 2, 'nan', 399172000, 184), +('ef88e6d6-affc-4511-8842-af8f32c1b49e', 40, 56, 2, 'nan', 395716000, 109), +('a1e85eec-0ed6-43ad-b0b0-d50fa4a26ed8', 24, 56, 2, 'nan', 393124000, 115), +('79baff1d-1f54-429f-91b2-4bc4e6ed5c18', 294, 64, 2, 'nan', 401648000, 253), +('a318ddfc-153d-4462-b7f9-963f7b3a11b2', 8, 41, 2, 'nan', 394628000, 257), +('8d376621-fad2-4329-9a54-5cc8a5347e9b', 287, 19, 2, 'nan', 416688000, 208), +('0a78d681-284e-4898-a80c-271edad7b1d3', 44, 43, 2, 'nan', 395092000, 153), +('0cf2846c-a2cc-4b4c-bc53-b78bab20d678', 168, 65, 2, 'nan', 397236000, 305), +('43ad2854-6f47-43ad-8b8f-4e77591598c0', 143, 33, 2, 'nan', 397416000, 439), +('fc591b9e-f6df-457c-ac61-d3ddc8a97df6', 30, 43, 2, 'nan', 397724000, 105), +('3b0ee8e6-d921-4c11-a19a-e482abff09da', 168, 67, 2, 'nan', 398316000, 141), +('efda7c49-2865-4bf9-847a-46ec0ed48c9e', 192, 486, 2, 'nan', 471312000, 466), +('1ef05ca4-74d6-4cb1-a0dd-64412a5e8f26', 88, 1758, 2, 'nan', 710156000, 216), +('f8a29cc6-a0cb-458e-938e-b30635035f74', 192, 485, 2, 'nan', 473932000, 486), +('69187ffb-8155-4c06-baa9-69b3c53f901e', 60, 57, 2, 'nan', 396240000, 519), +('d3d1f2f0-6462-453c-b346-21ae50813c3b', 192, 125, 2, 'nan', 407884000, 128), +('f0b99e31-aa9d-4111-b2a5-0ddcb7a2c063', 192, 67, 2, 'nan', 400544000, 166), +('449d5f69-50db-4c11-850e-ae578bdc68ad', 288, 97, 2, 'nan', 404500000, 286), +('108c14b7-9be9-4107-be16-4e70ae0bdb8d', 778, 68, 2, 'nan', 586496000, 225), +('3161c5b5-05c9-4513-bbc1-9b7cfbac6275', 96, 92, 2, 'nan', 398024000, 109), +('69fcd88b-6edf-4cde-a610-0363effd0097', 158, 66, 2, 'nan', 398412000, 113), +('a9b8ad78-8f0d-4352-aa0d-c9fb4eab8c2a', 60, 56, 2, 'nan', 395452000, 393), +('619fa191-ec16-46f2-86bd-6d8ef583db5f', 168, 66, 2, 'nan', 398248000, 93), +('6779287f-41eb-4733-977e-8bd8e3991a24', 24, 10, 2, 'nan', 393820000, 174), +('ba770cd8-8667-40c9-b97e-c714cda2f80c', 30, 40, 2, 'nan', 395556000, 129), +('58f871c7-830c-4d71-9300-e3779e203c5d', 80, 41, 2, 'nan', 396468000, 104), +('15b23bef-7bdb-4361-891e-c8a259f22230', 80, 34, 2, 'nan', 397080000, 216), +('246ebe66-68e4-4a9f-9a97-06f881c83cbd', 19, 17, 2, 'nan', 395288000, 105), +('99529852-6a5c-4d5b-b029-3455aa3a8c8d', 862, 79, 2, 'nan', 425444000, 261), +('1a8cffdc-13d8-4779-a38f-a9abeb854e44', 173, 80, 2, 'nan', 399860000, 894), +('144380ec-debc-468a-9f7a-930197d9547b', 233, 50, 2, 'nan', 397660000, 306), +('6365416f-e395-45d2-b558-270d0992fa0f', 18, 10, 2, 'nan', 392916000, 143), +('eeb06d17-12d1-45c1-ac8d-4e7473983ebf', 80, 60, 2, 'nan', 395456000, 183), +('4c38dfa2-b27b-4409-ba90-2c75eea4bba4', 233, 53, 2, 'nan', 397488000, 93), +('a70578ed-87e9-4ef7-a1d2-760d76962e40', 216, 58, 2, 'nan', 398032000, 193), +('eaffc15a-0d71-427b-b672-0487943d7da7', 16, 57, 2, 'nan', 394104000, 131), +('c86955d9-980c-46be-af88-26d4ac2d2a7d', 25, 11, 2, 'nan', 394264000, 313), +('7f7fd4fe-d7df-4e7d-a37c-936bc8377f5d', 20, 41, 2, 'nan', 394172000, 142), +('ba61cf3b-6c70-456d-8e9e-c8541a2a6407', 579, 42, 2, 'nan', 473148000, 321), +('ed986358-8a50-4b19-af8b-d5df11595183', 272, 38, 2, 'nan', 397624000, 165), +('70f9bec4-9459-46f5-aeab-ac3ac7f55cf1', 10, 36, 2, 'nan', 390764000, 78), +('38c42216-080e-413c-8c79-f3a27ab556bc', 32, 19, 2, 'nan', 391968000, 82), +('d3c8f668-6393-4e41-8160-a17e516ff500', 24, 466, 2, 'nan', 400796000, 222), +('692c4d20-72b3-4627-8dad-7379174604a1', 304, 26, 2, 'nan', 397456000, 209), +('3382c6a9-b778-4a75-9092-6c76afeeb45e', 16, 56, 2, 'nan', 390720000, 105), +('bbfa6b71-fd64-45a3-a66e-5d0d7f454e9d', 96, 125, 2, 'nan', 400764000, 170), +('27b552f9-b3be-4fbf-93dc-323774590e1b', 768, 94, 2, 'nan', 534576000, 860), +('4af523ba-0276-48a3-bfac-246ff361310b', 384, 105, 2, 'nan', 413812000, 160), +('66630aa4-c73b-4dbd-9290-b5a6ff8d0129', 137, 121, 2, 'nan', 399664000, 149), +('9e5aa6a4-23f4-4949-82df-748e4a651d62', 137, 121, 2, 'nan', 399664000, 162), +('b6965426-898a-46e1-a40f-2fe34a9d7bb1', 24, 41, 2, 'nan', 394588000, 95), +('052163a0-2807-40fc-8ae7-ecb8c4c1d421', 296, 108, 2, 'nan', 405108000, 118), +('bd0b608a-05cd-4c67-a4d9-e3337075e2e4', 36, 19, 2, 'nan', 394524000, 107), +('a5d9bb39-116b-49e3-97c4-831580f6729f', 399, 50, 2, 'nan', 406772000, 168), +('cefcf3fb-26f2-4349-9fc8-d81c63e928ff', 146, 60, 2, 'nan', 397560000, 115), +('c573d2c7-91bd-4dfe-a433-a6ef89a9656c', 1093, 114, 2, 'nan', 526824000, 646), +('869f9610-e808-4935-a57e-781abc9960af', 286, 14, 2, 'nan', 329000000, 201), +('83c880c1-20e6-4134-aca4-ced663066543', 96, 146, 2, 'nan', 332600000, 178), +('cd376889-4d87-40f8-8d3e-b9a4c0623f55', 183, 20, 2, 'nan', 301320000, 76), +('d8d739f2-6d56-41b2-a5a7-38133f9d7bbd', 192, 34, 2, 'nan', 328240000, 251), +('be224c35-461b-48a3-ab48-4e4838019c64', 10, 53, 2, 'nan', 323908000, 171), +('8b99435f-643a-4347-b552-18a5aea558e0', 576, 104, 2, 'nan', 372232000, 257), +('9d372971-7763-4731-a44e-cb87a3434cc0', 355, 42, 2, 'nan', 333436000, 125), +('9a3a5d72-c2b0-449a-a157-1c088e0af56d', 511, 140, 2, 'nan', 408944000, 348), +('a8afae6b-c328-48df-bce0-bab522a98f85', 168, 31, 2, 'nan', 327776000, 130), +('1fa748d3-fa80-4af4-a8fa-7782a27fa585', 1728, 121, 2, 'nan', 446964000, 371), +('559da3f1-bc29-4c26-a6ac-06b52b054594', 985, 27, 2, 'nan', 349308000, 149), +('cb41a3f9-0be7-4dad-b015-68016ef51022', 94, 34, 2, 'nan', 326732000, 155), +('19203be8-1052-4c80-9bce-fff4e77f05c8', 192, 86, 2, 'nan', 332644000, 88), +('a91d644d-9c86-411c-9ae2-4eeac7545d2b', 96, 78, 2, 'nan', 329440000, 118), +('5e0bba68-4bfb-40e1-96d1-daf1a454fd46', 355, 41, 2, 'nan', 333388000, 81), +('510e59c2-ba8e-4b1c-b4be-17dc59c80457', 192, 103, 2, 'nan', 333004000, 94), +('a4b6745c-bad5-470a-8e32-2f75236c2fce', 19, 80, 2, 'nan', 326460000, 180), +('f69d7684-607e-4661-a836-c09aba157c95', 0, 0, 2, 'nan', 101468500000, 275814), +('5c679aad-68f7-4855-be14-8312110bc173', 273, 46, 2, 'nan', 332156000, 151), +('e015a0a1-5c4f-4178-8ebc-2b9ac9e4e8b6', 108, 22, 2, 'nan', 326384000, 94), +('cd2374bc-649c-45e2-8540-991fb36cb487', 198, 72, 2, 'nan', 329712000, 95), +('7c707855-1640-42e9-82a7-d720e58874eb', 4952, 780, 2, 'nan', 8902036000, 17603), +('20af2ade-f786-4305-85b1-b5272ab356e0', 4566, 177, 2, 'nan', 9945200000, 68222), +('490b3500-dc03-4fa9-9cd2-78126b9333c2', 372, 131, 2, 'nan', 336100000, 75), +('e5504565-cf0e-4f2d-b925-1c2ca7cb5fbb', 353, 143, 2, 'nan', 343548000, 145), +('f9a9e2d7-b568-45d6-89b8-89cfe9991f77', 96, 27, 2, 'nan', 326760000, 83), +('8500b61a-9f08-49c9-a897-2773f4f383c5', 12366, 782, 2, 'nan', 54234712000, 114505), +('7e8480aa-af3b-4df7-8d66-8f93a28ccb74', 27079, 17, 2, 'nan', 1158688000, 635), +('f87a89b9-4b6e-4398-8e48-0b6ec238cc97', 4, 12, 2, 'nan', 325740000, 862), +('92b6145c-1248-4024-b825-a8b6cb07c4b0', 4, 13, 2, 'nan', 323444000, 228), +('04e67d15-1907-4fd1-9a4d-48b47f6c71f8', 18, 71, 2, 'nan', 326436000, 128), +('f7466925-9a5c-40d8-9536-9c43ad30cc2d', 80, 63, 2, 'nan', 327592000, 96), +('2f21b2ec-91b2-43e2-8d70-b35b64fcaf0c', 768, 224, 2, 'nan', 543720000, 2123), +('3df051d5-99a8-411a-b40f-bc269eeb5a6c', 1536, 224, 2, 'nan', 587236000, 1428), +('6048e851-42a4-4b0b-b040-a05cfe9686b0', 560, 48, 2, 'nan', 339844000, 120), +('06bbba17-3761-4e80-a7c0-56a01e92e7f2', 8, 20, 2, 'nan', 326012000, 82), +('1d1f338a-0b0b-4597-a960-1ecf5019dcfe', 92, 63, 2, 'nan', 327720000, 85), +('ba1c6fdd-bedd-45f1-a255-aa6b5e21fbd7', 44, 45, 2, 'nan', 326472000, 83), +('1f0a1ce3-35b9-427a-b11d-096690fe4a27', 30, 42, 2, 'nan', 326428000, 268), +('194ec434-bedd-4403-9e14-2351ea3c7720', 1392, 189, 2, 'nan', 576900000, 825), +('633b6978-0c6b-400a-b466-f3eb7e2035e9', 80, 65, 2, 'nan', 326916000, 129), +('dd18384b-c038-48bf-b751-573163a4223d', 2591, 200, 2, 'nan', 1092796000, 3246), +('635c4f2a-8417-4fae-9078-e94846c8b7b1', 3822, 78, 2, 'nan', 5082892000, 6137), +('759be360-787b-410e-9e07-7db03e4df5c6', 14, 11, 2, 'nan', 325624000, 83), +('37d514e8-ce44-4163-b47b-f0af05397aff', 264, 70, 2, 'nan', 332012000, 95), +('7b0f4fc2-d772-4921-ad44-ba64f148a84d', 792, 70, 2, 'nan', 341216000, 100), +('9a730dc2-23b7-4c41-8270-5fa7af22d468', 5, 51, 2, 'nan', 326028000, 126), +('c033bd49-d4e9-4df3-9317-ce030d4b80d6', 320, 41, 2, 'nan', 331744000, 83), +('09ee747d-638c-4933-bbe6-4e999d4b7480', 960, 155, 2, 'nan', 606116000, 489), +('16060ef4-c199-493f-ab88-d4b4f06138a4', 155, 20, 2, 'nan', 330572000, 133), +('8c1f6f46-75e6-4439-a5c0-c4e9bad71ec0', 113, 36, 2, 'nan', 331016000, 116), +('22c6824e-2223-4a56-8686-dcbf3db8b2f8', 306, 75, 2, 'nan', 333324000, 83), +('45c822ad-7817-48b6-8208-7b65922a5b01', 263, 72, 2, 'nan', 331556000, 96), +('40e38e61-5426-4495-95a8-9bfd717b779e', 89, 68, 2, 'nan', 327840000, 122), +('6b7f13b9-d019-4cd7-9913-6cc2a2d558c0', 155, 37, 2, 'nan', 331360000, 142), +('643a6bdc-48a3-4624-ab93-1e70053c0f06', 155, 38, 2, 'nan', 332204000, 250), +('d97b7b26-6c93-4d55-bd52-c4beae6fd9fc', 14, 46, 2, 'nan', 326240000, 84), +('caab8faa-5ebd-4734-91d3-cc598fe332c4', 38, 31, 2, 'nan', 325800000, 169), +('a005ae56-c6a3-4e0f-8280-49d85e32023e', 384, 85, 2, 'nan', 356464000, 166), +('1a93bbf7-b118-4b67-9477-e5a067291f7e', 1359, 85, 2, 'nan', 1048300000, 1971), +('670fcf04-077d-41cd-b9b6-d5eb7b37b42a', 32, 38, 2, 'nan', 326332000, 85), +('97704373-2097-4e0f-aa15-5a6087877b86', 1216, 88, 2, 'nan', 612756000, 600), +('3f1f3955-04a0-4554-ad87-55b4c2b13ee4', 27, 38, 2, 'nan', 300956000, 80), +('97d9b7c0-b68d-41ff-954c-3755e6ff1cb9', 14, 31, 2, 'nan', 325888000, 161), +('15f391a5-8c92-46e0-a321-5b981848d586', 231, 101, 2, 'nan', 341364000, 154), +('e0d0e1ad-55f7-4486-a53a-7df8b44bd4ab', 320, 44, 2, 'nan', 352328000, 207), +('b5415610-17a1-4ac7-a985-a060df556451', 68, 43, 2, 'nan', 329320000, 97), +('80dc4574-8278-4492-9473-f6091c52b98c', 158, 85, 2, 'nan', 332248000, 527), +('f838c5ba-bc20-4c5c-94cb-52973b522be4', 122, 264, 2, 'nan', 337212000, 284), +('5a1d5e55-4fe4-4dba-9406-684ffb734c5f', 382, 94, 2, 'nan', 353788000, 178), +('a5f23b08-8499-4e16-9b6c-69d089c9fd09', 307, 80, 2, 'nan', 350960000, 186), +('2b365ffc-a96b-47a6-89c2-6d2aaae535df', 31, 80, 2, 'nan', 326692000, 121), +('32a9bb7f-56c2-4bbc-9120-d66d4d8debbc', 862, 94, 2, 'nan', 422736000, 397), +('2a1d211e-5b53-4121-93f6-49721523eda4', 2203, 76, 2, 'nan', 2450852000, 6303), +('bffc8bce-1f6f-4bff-ac7f-8dad7f43895a', 231, 107, 2, 'nan', 339048000, 141), +('fc86707d-54ca-4eee-92d3-dbcde06ed75c', 123, 50, 2, 'nan', 327536000, 81), +('0c7ddae6-83d6-4db6-9a79-9720c614142b', 424, 201, 2, 'nan', 425668000, 587), +('9194b7f9-a420-426e-8073-78ad6607cd1d', 39, 81, 2, 'nan', 301268000, 150), +('db9b1f49-f848-4a40-bbbb-ed9d3848965a', 39, 81, 2, 'nan', 301236000, 150), +('84eff552-5723-4729-a86b-8f283874c289', 62, 83, 2, 'nan', 328500000, 169), +('b251dfae-0bb8-4bee-ab47-bdcc17d25b62', 16, 83, 2, 'nan', 325920000, 211), +('316b9754-2d5d-4822-bd72-f86f6ca9ae4b', 96, 152, 2, 'nan', 331648000, 186), +('d509e593-4053-406a-8ab0-8af8b1b4c8a5', 231, 97, 2, 'nan', 372816000, 612), +('e66faf00-0ffa-404f-ac08-7121c6337b70', 242, 43, 2, 'nan', 368180000, 297), +('66782585-883b-4784-a443-a2dfff848d88', 152, 83, 2, 'nan', 370720000, 348), +('9291cc5a-9ab4-49c0-b8a1-9fc9b6c46b44', 92, 83, 2, 'nan', 371068000, 420), +('decafc1b-5c70-466b-a8cc-33f29c6db3df', 12, 83, 2, 'nan', 366976000, 127), +('5b319607-4bec-4e36-ae54-5b7c21e91064', 336, 22, 2, 'nan', 366464000, 97), +('2f3f309f-51bd-43b8-80d1-10d10a1c274e', 152, 66, 2, 'nan', 371584000, 210), +('893621a8-d4dd-464b-8977-223470fc297e', 152, 81, 2, 'nan', 365052000, 234), +('12e91952-b2b1-424d-884c-865e77474bf2', 28, 52, 2, 'nan', 367340000, 316), +('d6946499-e2e7-4286-a95b-786b8ec633a9', 44, 83, 2, 'nan', 367956000, 265), +('1fa39acc-83ef-409f-91d5-13ef1934635d', 32, 64, 2, 'nan', 369696000, 240), +('694a13a5-9e5d-4f54-b34c-a63167a045b1', 56, 80, 2, 'nan', 370264000, 251), +('c4ff14f9-3a89-40d6-aa85-1fbd8dd80c17', 138, 89, 2, 'nan', 371388000, 347), +('d12d6d89-a8e9-4fd9-8e1c-16968035fb38', 152, 72, 2, 'nan', 372660000, 201), +('9627e0a4-8f47-430e-b1d2-30c1295a722f', 32, 83, 2, 'nan', 367744000, 237), +('f50accb5-3461-4a7f-9bff-761e4414a026', 28, 51, 2, 'nan', 367140000, 156), +('e0d4954f-e964-4755-bac3-bc47610206aa', 21302, 90, 2, 'nan', 30206900000, 193946), +('2d60289d-1b7f-4945-8264-b01e2426c23e', 4566, 178, 2, 'nan', 5398704000, 12040), +('179d0c15-5af2-4671-b807-fb374fd3b842', 307, 85, 2, 'nan', 373952000, 195), +('f6a8189a-caa7-4a41-bbc0-09ac46bf5a8f', 1067, 12, 2, 'nan', 370924000, 112), +('59ca6817-c388-47ad-ba64-47989313dd44', 6374, 49, 2, 'nan', 9769896000, 22148), +('61d6d576-362c-4261-859e-7209fa346f1e', 152, 85, 2, 'nan', 370008000, 194), +('bd89bfe7-51b0-429b-9a88-2e71c8cb7545', 856, 49, 2, 'nan', 481572000, 597), +('c5cc0f29-0fa0-4c44-b33c-103f0e3b6f32', 39, 93, 2, 'nan', 371980000, 199), +('53db452b-9234-4830-94ef-cf9ab49ef0d1', 150, 63, 2, 'nan', 372480000, 494), +('a268c541-fe8d-4521-8a49-e3a8439aef64', 394, 42, 2, 'nan', 378392000, 1151), +('387f71b9-3283-4923-860b-8065a62248fd', 189, 142, 2, 'nan', 380964000, 373), +('8194c64a-1a43-499f-8f47-3a3935c557f4', 856, 48, 2, 'nan', 521784000, 784), +('294d4a3f-f8ff-4644-ab15-3ddcb4390af1', 177, 384, 2, 'nan', 379924000, 284), +('a373adcd-9b62-41e3-9ae4-a6f79ff45f45', 96, 99, 2, 'nan', 372380000, 265), +('ef7406a1-a4fa-4af2-8e67-92607ebdd8f9', 1005, 137, 2, 'nan', 410232000, 168), +('59dbbe08-8701-4e48-95ab-3882a02ee1e7', 413, 40, 2, 'nan', 405220000, 243), +('088a65e5-c848-4e34-b47d-4664f9f27f83', 150, 77, 2, 'nan', 374532000, 1131), +('45d538e2-97c5-4893-af08-f6bf04abb767', 1150, 54, 2, 'nan', 390968000, 368), +('0b9706a2-e0a6-438f-a5b3-a094161a15a7', 399, 844, 2, 'nan', 539092000, 751), +('eb3650f0-7aea-4cab-a520-6e581e3bd978', 399, 843, 2, 'nan', 539232000, 1134), +('c6635e7e-93c0-403e-b615-3e8f7da29352', 1055, 102, 2, 'nan', 554628000, 434), +('2cb50802-1ac7-48a4-bebd-4d9be546cf69', 413, 42, 2, 'nan', 374312000, 681), +('1c6d9feb-17fb-40ca-a3f4-d1e7834c5d53', 96, 82, 2, 'nan', 370640000, 197), +('ed2a60cf-2164-4b01-a59c-07518613ef8d', 282, 54, 2, 'nan', 373164000, 346), +('469e8fcc-37ef-4dfb-b4cd-2b109200ce4c', 5924, 12, 2, 'nan', 394020000, 112), +('52802b61-7df2-4e82-9ccf-5c8075c2da45', 96, 81, 2, 'nan', 371540000, 130), +('0c715e72-47dd-48ac-b733-a845e440e86a', 476, 68, 2, 'nan', 426996000, 469), +('13f6b03d-40e1-47c2-a7df-163c0e3e7d23', 353, 56, 2, 'nan', 225308000, 475), +('9639186c-9084-47ed-8075-3f14798eceef', 384, 87, 2, 'nan', 384048000, 1021), +('6e54aa84-ffd1-4943-8ae2-7e75319a7e30', 287, 37, 2, 'nan', 371328000, 585), +('dba3182c-80c0-46ed-a9e6-64e0cf699741', 39, 73, 2, 'nan', 369720000, 195), +('b33244f5-30ce-4db3-94c7-4dcbe5cff4a7', 353, 149, 2, 'nan', 388952000, 263), +('ba196947-c2d6-4ae1-ad23-fb768c087125', 511, 146, 2, 'nan', 388392000, 235), +('08d92acd-992c-4500-a33a-094225dec9cb', 147, 23, 2, 'nan', 371064000, 93), +('b0bfe252-9f89-4160-839e-738a649de0f1', 95, 37, 2, 'nan', 370196000, 252), +('ea9e5e73-c141-4cf6-944a-6f43541ed747', 768, 87, 2, 'nan', 392580000, 269), +('b6d61911-54b5-4917-957a-11f5931f1b6e', 399, 970, 2, 'nan', 600196000, 527), +('3cf83989-1316-47f4-a291-03aa41da31f1', 32, 41, 2, 'nan', 367032000, 92), +('531b6413-c293-4a87-9444-3939eebf79bd', 528, 76, 2, 'nan', 382568000, 104), +('18919546-fade-4e83-b5dc-1708043bec82', 151, 15, 2, 'nan', 367920000, 90), +('6e6eb332-9166-4059-8a7c-cc4dfa5417f3', 394, 48, 2, 'nan', 377028000, 283), +('c36b0187-97f8-49c4-9f78-dfb75a04f41d', 450, 140, 2, 'nan', 389624000, 126), +('f9552ba6-db05-4a74-acb7-f851c627777d', 384, 92, 2, 'nan', 379340000, 200), +('4777ec99-b163-466f-81a8-a5e5f3c81b5d', 862, 79, 2, 'nan', 398504000, 301), +('3429c0f3-9f05-4234-a3ec-ead9285d9c91', 384, 171, 2, 'nan', 400700000, 448), +('a9e4fa1b-5f48-4143-a06e-20c6013c37f0', 9, 12, 2, 'nan', 368400000, 193), +('eb734567-fa4f-4aff-8ac4-e119b7cf31cc', 384, 172, 2, 'nan', 403540000, 337), +('52b3c3ce-0681-4992-9cd5-bdb0eddffff6', 3938, 91, 2, 'nan', 486604000, 379), +('ddb6e8c1-700b-4c2e-834b-26ceca5be035', 3, 79, 2, 'nan', 368916000, 219), +('ed55d400-04c2-4aa7-be76-63eef565cbae', 3, 80, 2, 'nan', 368524000, 193), +('23d23e6a-7e1e-41e6-aedb-5c997c6a31a6', 384, 1759, 2, 'nan', 1362084000, 2677), +('160160e6-ee83-47f7-8d59-1e6df6dd1a4d', 394, 50, 2, 'nan', 376560000, 377), +('e3b07a6e-145b-4551-b27c-905a1601a1f3', 60, 34, 2, 'nan', 368544000, 198), +('462b169a-acce-47a7-adf9-1daa0a465639', 233, 50, 2, 'nan', 373160000, 382), +('a43beee6-aab4-4ddd-895e-b7f95e258d17', 54, 64, 2, 'nan', 370380000, 308), +('4df02286-2ecd-4dd0-9d13-ec5d6b2e76d9', 11, 15, 2, 'nan', 344776000, 108), +('50b68208-0bdc-4ecd-9f75-2036a2332d55', 11, 22, 2, 'nan', 369672000, 201), +('9bba65d6-12be-4f4b-b40a-018489ebbc3b', 35, 42, 2, 'nan', 368940000, 256), +('40fe74d9-7fa2-4ef1-a64d-c092b3e5128b', 1232, 61, 2, 'nan', 391368000, 967), +('716b59bd-d2f0-404e-b6c5-ad5b69e8db3a', 78, 28, 2, 'nan', 365844000, 152), +('cd5299dd-f160-4b8e-9eac-ea2cb54c003a', 78, 28, 2, 'nan', 362160000, 152), +('6140d69a-3c93-4378-bc99-fc0596aa6703', 76, 33, 2, 'nan', 369936000, 299), +('c03cd275-e2c0-4e28-a0a1-e07be2c24c03', 602, 72, 2, 'nan', 398576000, 1664), +('9afa1b4a-244e-44a4-8b9c-69c229007667', 80, 68, 2, 'nan', 370328000, 565), +('655e4674-a14d-485b-9294-18d8e76014ca', 921, 486, 2, 'nan', 636868000, 2955), +('2a009904-eb9e-49bd-bee0-da0bb1bfdba5', 192, 32, 2, 'nan', 399972000, 280), +('186f76c0-c877-428c-ab08-46cbcf4524a1', 115, 61, 2, 'nan', 387000000, 210), +('f5d2cdc0-0061-4a7b-9c86-bee6d7e08d29', 460, 61, 2, 'nan', 400424000, 114), +('f147fe13-56a8-41db-b399-ecabba242ca9', 2591, 206, 2, 'nan', 896908000, 2792), +('0bef5c46-6ad7-4a05-8c4b-8d227f0a0352', 22, 67, 2, 'nan', 392356000, 193), +('fc0ebaac-1bbb-4fb6-b20d-2fe74f31b5ef', 16, 77, 2, 'nan', 393268000, 112), +('fcd7e83f-804c-40bb-9f3d-8483cffe34c0', 95, 32, 2, 'nan', 395920000, 497), +('284cc76f-487e-4d49-9c2f-d4d10e74385e', 168, 59, 2, 'nan', 398028000, 416), +('b8e3c3ac-5b5b-4f6d-97c2-3296b0bab7e6', 40, 56, 2, 'nan', 395220000, 114), +('9262c5ec-b92b-425b-8fd1-10ea4905d595', 24, 56, 2, 'nan', 392936000, 117), +('7bae4416-a699-4985-9736-60bf2b4aa4d6', 8, 41, 2, 'nan', 394360000, 276), +('7cb7e7e1-48c1-4259-9856-395911e88d4d', 143, 33, 2, 'nan', 395756000, 679), +('b127f3dc-fc5b-4054-a20b-e9216caff4ef', 192, 485, 2, 'nan', 455936000, 698), +('dfb12c24-d67a-4dc4-b221-b6fb1e49602b', 287, 19, 2, 'nan', 393648000, 28278), +('507da6a3-0df4-4e30-9763-f392eb62aac5', 168, 67, 2, 'nan', 398216000, 371), +('6fe5c170-4774-4576-813f-7c164866a3bf', 88, 1758, 2, 'nan', 709548000, 216), +('3c31aad8-1e86-48be-b65c-4924388bd587', 192, 486, 2, 'nan', 418000000, 1123), +('2b2e2473-fbc6-4e6a-9072-19f6411b0817', 192, 67, 2, 'nan', 400156000, 170), +('5994ad72-2f1a-416b-ba3c-9cc4a1c61c20', 778, 68, 2, 'nan', 539272000, 227), +('ba111653-1c15-4425-8ab9-11a5422eff1d', 96, 92, 2, 'nan', 397212000, 188), +('fdec2e67-e193-4185-991a-be136e63b2b0', 158, 66, 2, 'nan', 396860000, 318), +('6218ce6d-5208-4ebc-9e48-633b3f54cdcc', 60, 57, 2, 'nan', 388668000, 26827), +('d6eef75a-4505-4111-9c4d-5a748d403dd2', 60, 56, 2, 'nan', 395448000, 561), +('7a24dba0-f6b4-48cc-b1cb-4c9b884170e6', 168, 66, 2, 'nan', 398184000, 153), +('ff077c77-a1ba-46ac-b5ca-d5c894242770', 80, 34, 2, 'nan', 395212000, 228), +('65526acd-86e9-421b-9c91-b885cd36729a', 19, 17, 2, 'nan', 395088000, 166), +('04d09a9f-2151-47df-8475-36fc8ec7aafc', 146, 60, 2, 'nan', 396424000, 784), +('7e72e913-ce87-4bc1-adf7-c3cc7d5184e5', 233, 53, 2, 'nan', 397100000, 395), +('9e6f2ff5-f749-4089-b828-ef29fcca1a7d', 216, 58, 2, 'nan', 396236000, 227), +('335635a7-8153-4e4c-9b53-e1bc145497b0', 16, 57, 2, 'nan', 373140000, 156), +('5452c17e-ae82-4b46-a85a-ece0501a5d5e', 20, 41, 2, 'nan', 393884000, 214), +('f5a06f01-7405-4ff1-b047-64f9e7505d6c', 579, 42, 2, 'nan', 442784000, 383), +('bf9fc8f3-39b7-4771-95e8-f858494022a5', 272, 38, 2, 'nan', 397296000, 30202), +('98270055-d47a-4cc4-b40c-ac006f7823f6', 24, 466, 2, 'nan', 400612000, 234), +('009b95ec-18b6-4fa0-a0bb-0785cfe03409', 304, 26, 2, 'nan', 396920000, 211), +('89ae9ba5-f920-46e6-9279-40d5be29f845', 36, 19, 2, 'nan', 394500000, 198), +('0f24f638-13fe-4898-b3a9-b15aafb260f8', 399, 50, 2, 'nan', 404516000, 174), +('37d12cdd-c03b-4921-a77b-851269d3e557', 1093, 114, 2, 'nan', 482504000, 654), +('9308c777-1c30-43ac-acf6-9454601601ba', 96, 146, 2, 'nan', 332352000, 245), +('a1910b9b-2d93-4239-9bc1-1250fea56837', 96, 146, 2, 'nan', 331548000, 245), +('1280dc5a-d797-42ca-a613-3fbce32f55a0', 14, 52, 2, 'nan', 300932000, 166), +('0a329ce1-1b82-4faf-a1f7-0d17e4b25251', 576, 104, 2, 'nan', 371176000, 260), +('3b212797-bb0b-422c-bba1-79921ce40c78', 168, 31, 2, 'nan', 327104000, 142), +('c14e97a2-a083-4e49-a14b-0abd23483272', 1728, 121, 2, 'nan', 446280000, 376), +('7cd84e00-773a-4387-bcdb-37da819ce5f7', 108, 22, 2, 'nan', 315832000, 140), +('8553c8e2-5bc9-41b6-9f5e-96a2abfb7055', 192, 34, 2, 'nan', 302704000, 255), +('65212822-529e-4cf4-a427-78032b9e569b', 198, 72, 2, 'nan', 329532000, 209), +('3a8d8b29-d408-45ab-8d63-043741de9679', 273, 46, 2, 'nan', 331556000, 181), +('d68f1265-c669-4f2b-a3ab-ee0e547e0143', 4566, 177, 2, 'nan', 9765660000, 70248), +('f0b0c5aa-9f97-42e3-82e4-71968f28ad15', 511, 140, 2, 'nan', 401220000, 585), +('1bf64e7e-4ab9-4dfa-9aeb-6402ad95c165', 353, 143, 2, 'nan', 343172000, 439), +('605f6954-8697-42c4-97a4-7c3aa6b3cde9', 96, 27, 2, 'nan', 326164000, 182), +('3688f502-6895-4041-b957-6487f6ba7c98', 80, 63, 2, 'nan', 325924000, 123), +('013f8041-6f36-4dcb-8af9-67ac1fd78d9e', 8, 20, 2, 'nan', 312564000, 94), +('58e69a18-ecef-4b61-a62f-7efc1f733731', 8, 20, 2, 'nan', 312896000, 94), +('e7cb303d-ffe0-4a5e-8eb0-627dd0513f51', 1536, 224, 2, 'nan', 562312000, 2121), +('7cc4a810-137c-468e-a249-60b0faf9a003', 560, 48, 2, 'nan', 339180000, 120), +('056330e4-36eb-4c38-b87f-56484ac69623', 92, 63, 2, 'nan', 326952000, 88), +('ea9e5c11-f7a7-4dcb-b8b5-20b499611b9f', 30, 43, 2, 'nan', 326568000, 2029), +('8df2b291-adbf-4def-a172-ece656410050', 1392, 189, 2, 'nan', 542424000, 825), +('ce47b98d-8a78-4e97-b20c-297f75394f28', 80, 65, 2, 'nan', 326612000, 184), +('d8e90f5f-1b10-4f00-9c3b-6bc8daf2cc8a', 44, 45, 2, 'nan', 325972000, 95), +('cec46141-0286-4430-ac41-ed830decb5e8', 12366, 782, 2, 'nan', 54233492000, 164399), +('fc9b54c3-0447-49db-a6d0-5b913d0e285a', 264, 70, 2, 'nan', 331724000, 147), +('3e6d362e-3880-4b90-b693-868547ce0f5a', 792, 70, 2, 'nan', 341096000, 110), +('d5891efa-0e7d-409b-9bb7-b6ecc081c836', 960, 155, 2, 'nan', 522896000, 589), +('e4b960c8-33c6-4304-8355-9bafd70569ed', 113, 36, 2, 'nan', 330756000, 162), +('35c4eb46-3560-4d17-a210-979ba3037c53', 306, 75, 2, 'nan', 332952000, 93), +('2b85bd17-6d7e-48e0-8e02-477eb9ffe353', 263, 72, 2, 'nan', 328812000, 153), +('8c7e4562-dcc2-4a31-ab20-b873182b334d', 266, 67, 2, 'nan', 330156000, 124), +('49fda716-a469-4a6d-a3ef-3bcbd166e668', 155, 38, 2, 'nan', 331720000, 609), +('f424b26a-840a-42d3-a61f-3bbffd8b194d', 32, 38, 2, 'nan', 325684000, 216), +('7226d658-da67-4d7c-a898-e59cd77d977b', 1216, 88, 2, 'nan', 558752000, 625), +('50dc9288-e18c-41c2-a271-601289ff5685', 231, 101, 2, 'nan', 340116000, 158), +('4883db35-3ca6-4b0c-9377-cf7d1d97245d', 158, 85, 2, 'nan', 332072000, 1464), +('3ff09f4e-ada5-45d6-b7e1-fa003e31bf77', 122, 264, 2, 'nan', 337104000, 286), +('26c68022-6df2-4928-a4fb-220a156b86f7', 382, 94, 2, 'nan', 353068000, 200), +('cdd878a9-e3f1-4478-b613-90f2b2e9ccbd', 424, 201, 2, 'nan', 397292000, 647), +('d6e02749-45fb-4b92-8a8e-48b33b617f9c', 307, 80, 2, 'nan', 348920000, 206), +('47900ec7-0fea-41db-a320-c78b9da36538', 32, 80, 2, 'nan', 324068000, 240), +('1ef4d550-afaa-44b5-a06c-65286b9f3e00', 152, 80, 2, 'nan', 325820000, 278), +('8533d2a8-1fcf-4842-8826-c5458c002332', 31, 80, 2, 'nan', 319000000, 245), +('ab9095d1-af6d-49d5-9c87-bc2576f06135', 68, 43, 2, 'nan', 326896000, 235), +('61da73ff-b4ff-49a1-b775-c6215cfbd291', 231, 107, 2, 'nan', 333544000, 200), +('6c84dcf1-c5ea-4e69-b17f-d2d5b8d48bdf', 123, 50, 2, 'nan', 327520000, 82), +('dcb12603-4142-44d1-9a52-3ca3511e380e', 320, 44, 2, 'nan', 329448000, 475), +('a0dd0a4d-b73f-4e9d-87dd-d29efba25336', 41, 50, 2, 'nan', 301108000, 144); From ddff1c457da4f23d7ed29150d29e8a116017b5e8 Mon Sep 17 00:00:00 2001 From: gossty Date: Thu, 11 Jul 2024 22:16:19 -0700 Subject: [PATCH 2/7] qiita-cron-job initialize-resource-allocations-redis --- qiita_db/meta_util.py | 47 ++++++++++++++++++++- qiita_db/test/test_meta_util.py | 17 ++++++++ qiita_db/test/test_util.py | 17 ++++---- qiita_db/util.py | 51 +++++++++++++++++++---- qiita_pet/handlers/resources.py | 67 ++++++++++++++++++++++++++++++ qiita_pet/templates/resources.html | 52 +++++++++++++++++++++++ qiita_pet/templates/sitebase.html | 1 + qiita_pet/webserver.py | 3 ++ scripts/all-qiita-cron-job | 1 + scripts/qiita-cron-job | 6 +++ 10 files changed, 246 insertions(+), 16 deletions(-) create mode 100644 qiita_pet/handlers/resources.py create mode 100644 qiita_pet/templates/resources.html diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 79b049e37..baf29d97f 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -37,11 +37,17 @@ from re import sub from json import loads, dump, dumps -from qiita_db.util import create_nested_path +from qiita_db.util import create_nested_path, _retrieve_resource_data +from qiita_db.util import resource_allocation_plot, get_software_commands from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.configuration_manager import ConfigurationManager import qiita_db as qdb +# global constant list used in resource_allocation_page +columns = [ + "sName", "sVersion", "cID", "cName", "processing_job_id", + "parameters", "samples", "columns", "input_size", "extra_info", + "MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"] def _get_data_fpids(constructor, object_id): """Small function for getting filepath IDS associated with data object @@ -546,3 +552,42 @@ def generate_plugin_releases(): # important to "flush" variables to avoid errors r_client.delete(redis_key) f(redis_key, v) + + +def initialize_resource_allocations_redis(): + time = datetime.now().strftime('%m-%d-%y %H:%M:%S') + scommands = get_software_commands() + for software, versions in scommands.items(): + for version, commands in versions.items(): + for command in commands: + print("Generating plot for:", software, version, command) + update_resource_allocation_redis(command, software, + version, time) + redis_key = 'resources:commands' + r_client.set(redis_key, str(scommands)) + + +def update_resource_allocation_redis(cname, sname, version, time): + col_name = "samples * columns" + + df = _retrieve_resource_data(cname, sname, version, columns) + fig, axs = resource_allocation_plot(df, cname, sname, col_name) + + fig.tight_layout() + plot = BytesIO() + fig.savefig(plot, format='png') + plot.seek(0) + + img = 'data:image/png;base64,' + quote(b64encode(plot.getbuffer()).decode('ascii')) + plt.close(fig) + + # SID, CID, col_name + values = [ + ("img", img, r_client.set), + ('time', time, r_client.set) + ] + + for k, v, f in values: + redis_key = 'resources$#%s$#%s$#%s:%s' % (cname, sname, col_name, k) + r_client.delete(redis_key) + f(redis_key, v) diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index e8fe4c0de..98b11509d 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -519,6 +519,23 @@ def test_generate_plugin_releases(self): '-', '').replace(':', '').replace(' ', '-') self.assertEqual(tgz_obs, [time]) + def test_update_resource_allocation_redis(self): + cname = "Split libraries FASTQ" + sname = "QIIMEq2" + col_name = "samples * columns" + qdb.meta_util.update_resource_allocation_redis(cname, sname) + + vals = [ + ("img", b's', r_client.get), + ('time', b'2024-11-11', r_client.get) + ] + + for k, exp, f in vals: + redis_key = 'resources$#%s$#%s$#%s:%s' % (cname, sname, col_name, k) + # checking redis values + print(f(redis_key)) + # self.assertEqual(f(redis_key), exp) + if __name__ == '__main__': main() diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index 2dfc31bfa..a71be3e49 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1311,8 +1311,9 @@ def test_quick_mounts_purge(self): class ResourceAllocationPlotTests(TestCase): def setUp(self): - self.CNAME = "Split libraries FASTQ" - self.SNAME = "QIIMEq2" + self.cname = "Split libraries FASTQ" + self.sname = "QIIMEq2" + self.version = "1.9.1" self.col_name = 'samples * columns' self.columns = [ "sName", "sVersion", "cID", "cName", "processing_job_id", @@ -1322,12 +1323,12 @@ def setUp(self): # df is a dataframe that represents a table with columns specified in # self.columns self.df = qdb.util._retrieve_resource_data( - self.CNAME, self.SNAME, self.columns) + self.cname, self.sname, self.version, self.columns) def test_plot_return(self): # check the plot returns correct objects fig1, axs1 = qdb.util.resource_allocation_plot( - self.df, self.CNAME, self.SNAME, self.col_name) + self.df, self.cname, self.sname, self.col_name) self.assertIsInstance( fig1, Figure, "Returned object fig1 is not a Matplotlib Figure") @@ -1338,13 +1339,13 @@ def test_plot_return(self): def test_minimize_const(self): self.df = self.df[ - (self.df.cName == self.CNAME) & (self.df.sName == self.SNAME)] + (self.df.cName == self.cname) & (self.df.sName == self.sname)] self.df.dropna(subset=['samples', 'columns'], inplace=True) self.df[self.col_name] = self.df.samples * self.df['columns'] fig, axs = plt.subplots(ncols=2, figsize=(10, 4), sharey=False) bm, options = qdb.util._resource_allocation_plot_helper( - self.df, axs[0], self.CNAME, self.SNAME, 'MaxRSSRaw', + self.df, axs[0], self.cname, self.sname, 'MaxRSSRaw', qdb.util.MODELS_MEM, self.col_name) # check that the algorithm chooses correct model for MaxRSSRaw and # has 0 failures @@ -1366,7 +1367,7 @@ def test_minimize_const(self): # check that the algorithm chooses correct model for ElapsedRaw and # has 1 failure bm, options = qdb.util._resource_allocation_plot_helper( - self.df, axs[1], self.CNAME, self.SNAME, 'ElapsedRaw', + self.df, axs[1], self.cname, self.sname, 'ElapsedRaw', qdb.util.MODELS_TIME, self.col_name) k, a, b = options.x failures_df = qdb.util._resource_allocation_failures( @@ -1423,7 +1424,7 @@ def test_db_update(self): for curr_cname, ids in types.items(): updated_df = qdb.util._retrieve_resource_data( - curr_cname, self.SNAME, self.columns) + curr_cname, self.sname, self.version, self.columns) updated_ids_set = set(updated_df['processing_job_id']) previous_ids_set = set(self.df['processing_job_id']) for id in ids: diff --git a/qiita_db/util.py b/qiita_db/util.py index e77648cb0..184f9b458 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2381,7 +2381,7 @@ def resource_allocation_plot(df, cname, sname, col_name): return fig, axs -def _retrieve_resource_data(cname, sname, columns): +def _retrieve_resource_data(cname, sname, version, columns): with qdb.sql_connection.TRN: sql = """ SELECT @@ -2411,9 +2411,10 @@ def _retrieve_resource_data(cname, sname, columns): ON pr.processing_job_id = sra.processing_job_id WHERE sc.name = %s - AND s.name = %s; + AND s.name = %s + AND s.version = %s """ - qdb.sql_connection.TRN.add(sql, sql_args=[cname, sname]) + qdb.sql_connection.TRN.add(sql, sql_args=[cname, sname, version]) res = qdb.sql_connection.TRN.execute_fetchindex() df = pd.DataFrame(res, columns=columns) return df @@ -2482,15 +2483,19 @@ def _resource_allocation_plot_helper( y_plot = best_model(x_plot, k, a, b) ax.plot(x_plot, y_plot, linewidth=1, color='orange') + cmin_value = min(y_plot) + cmax_value = max(y_plot) + maxi = naturalsize(df[curr].max(), gnu=True) if curr == "MaxRSSRaw" else \ timedelta(seconds=float(df[curr].max())) - cmax = naturalsize(max(y_plot), gnu=True) if curr == "MaxRSSRaw" else \ - timedelta(seconds=float(max(y_plot))) + cmax = naturalsize(cmax_value, gnu=True) if curr == "MaxRSSRaw" else \ + str(timedelta(seconds=round(cmax_value, 2))).rstrip('0').rstrip('.') mini = naturalsize(df[curr].min(), gnu=True) if curr == "MaxRSSRaw" else \ timedelta(seconds=float(df[curr].min())) - cmin = naturalsize(min(y_plot), gnu=True) if curr == "MaxRSSRaw" else \ - timedelta(seconds=float(min(y_plot))) + cmin = naturalsize(cmin_value, gnu=True) if curr == "MaxRSSRaw" else \ + str(timedelta(seconds=round(cmin_value, 2))).rstrip('0').rstrip('.') + x_plot = np.array(df[col_name]) failures_df = _resource_allocation_failures( @@ -2919,3 +2924,35 @@ def merge_rows(rows): row['node_model']] qdb.sql_connection.TRN.add(sql, sql_args=to_insert) qdb.sql_connection.TRN.execute() + + +def get_software_commands(): + res = [] + with qdb.sql_connection.TRN: + sql_command = """ + SELECT DISTINCT + s.name AS sName, + s.version AS sVersion, + sc.name AS cName + FROM + qiita.slurm_resource_allocations sra + JOIN + qiita.processing_job pr + ON sra.processing_job_id = pr.processing_job_id + JOIN + qiita.software_command sc on pr.command_id = sc.command_id + JOIN + qiita.software s ON sc.software_id = s.software_id + """ + qdb.sql_connection.TRN.add(sql_command) + res = qdb.sql_connection.TRN.execute_fetchindex() + + software_commands = dict() + for s_name, s_version, c_name in res: + if s_name not in software_commands: + software_commands[s_name] = {} + if s_version not in software_commands[s_name]: + software_commands[s_name][s_version] = set() + software_commands[s_name][s_version].add(c_name) + + return software_commands diff --git a/qiita_pet/handlers/resources.py b/qiita_pet/handlers/resources.py new file mode 100644 index 000000000..0bb0865b1 --- /dev/null +++ b/qiita_pet/handlers/resources.py @@ -0,0 +1,67 @@ + +from tornado.gen import coroutine, Task +from tornado.web import authenticated, HTTPError + + +from .base_handlers import BaseHandler +from qiita_core.qiita_settings import qiita_config, r_client +from qiita_core.util import execute_as_transaction + +default_cname = "Split libraries FASTQ" +default_sname = "QIIMEq2" +default_col_name = "samples * columns" +commands = 'resources:commands' + +class ResourcesHandler(BaseHandler): + def check_admin(self): + if self.current_user.level != "admin": + raise HTTPError(403, reason="%s does not have access to portal " + "editing!" % self.current_user.id) + + + @execute_as_transaction + def _get_resources(self, callback): + resources = {} + vals = [ + ('img', r_client.get), + ('time', r_client.get) + ] + for k, f in vals: + redis_key = 'resources$#%s$#%s$#%s:%s' % (default_cname, + default_sname, default_col_name, k) + resources[k] = f(redis_key) + callback(resources) + + + def _get_commands(self, callback): + r_client.get(commands) + callback(r_client) + + + @authenticated + @coroutine + @execute_as_transaction + def get(self): + self.check_admin() + resources = yield Task(self._get_resources) + commands = yield Task(self._get_commands) # TODO: it would make more sense to have this rendered once instead of everytime we make a get request, but i'm not sure how to do that right now + self.render('resources.html', + img=resources['img'], time=resources['time'], + commands=commands, + software=default_sname, command=default_cname, + col_name=default_col_name + ) + + + @authenticated + @execute_as_transaction + def post(self): + self.check_admin() + software = self.get_argument("software", "") + command = self.get_argument("command", "") + column_type = self.get_argument("column_type", "") + + + self.get() + + diff --git a/qiita_pet/templates/resources.html b/qiita_pet/templates/resources.html new file mode 100644 index 000000000..097a19ca6 --- /dev/null +++ b/qiita_pet/templates/resources.html @@ -0,0 +1,52 @@ +{% extends sitebase.html %} +{% block head %} + + + +{% end %} + +{% block content %} +
+
+ Generated on: {{time}} + Software: {{software}} + Command: {{command}} + Column name: {{col_name}} +
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+ + +
+ {% if img and img is not None %} +
+ +
+ {% end %} +
+{% end %} \ No newline at end of file diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 7165f33a2..5e2e0633f 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -384,6 +384,7 @@
  • View Studies awaiting approval
  • Edit study portal connections
  • Purge non-validated users
    • +
  • View Resource Allocation Plots
    • {% end %}
  • Sample Validation
  • Processing Jobs
    • diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index 50b34fd2b..d52549c42 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -50,6 +50,8 @@ from qiita_pet.handlers.upload import ( UploadFileHandler, StudyUploadFileHandler, StudyUploadViaRemote) from qiita_pet.handlers.stats import StatsHandler +from qiita_pet.handlers.resources import ResourcesHandler + from qiita_pet.handlers.download import ( DownloadHandler, DownloadStudyBIOMSHandler, DownloadRelease, DownloadRawData, DownloadEBISampleAccessions, DownloadEBIPrepAccessions, @@ -136,6 +138,7 @@ def __init__(self): (r"/admin/sample_validation/", SampleValidation), (r"/admin/purge_users/", PurgeUsersHandler), (r"/admin/purge_usersAjax/", PurgeUsersAJAXHandler), + (r"/admin/resources/", ResourcesHandler), (r"/ebi_submission/(.*)", EBISubmitHandler), # Study handlers (r"/study/create/", StudyEditHandler), diff --git a/scripts/all-qiita-cron-job b/scripts/all-qiita-cron-job index 0639a34c8..b414da5f2 100644 --- a/scripts/all-qiita-cron-job +++ b/scripts/all-qiita-cron-job @@ -4,5 +4,6 @@ qiita-cron-job empty-trash-upload-folder qiita-cron-job generate-biom-and-metadata-release qiita-cron-job purge-filepaths qiita-cron-job update-redis-stats +qiita-cron-job initialize-resource-allocations-redis qiita-cron-job generate-plugin-releases qiita-cron-job purge-json-web-tokens diff --git a/scripts/qiita-cron-job b/scripts/qiita-cron-job index 80d75d479..17ff09002 100755 --- a/scripts/qiita-cron-job +++ b/scripts/qiita-cron-job @@ -16,6 +16,7 @@ from qiita_db.util import ( quick_mounts_purge as qiita_quick_mounts_purge) from qiita_db.meta_util import ( update_redis_stats as qiita_update_redis_stats, + initialize_resource_allocations_redis as qiita_initialize_resource_allocations_redis, generate_biom_and_metadata_release as qiita_generate_biom_and_metadata_release, generate_plugin_releases as qiita_generate_plugin_releases) @@ -48,6 +49,11 @@ def update_redis_stats(): qiita_update_redis_stats() +@commands.command() +def initialize_resource_allocations_redis(): + qiita_initialize_resource_allocations_redis() + + @commands.command() def generate_biom_and_metadata_release(): qiita_generate_biom_and_metadata_release('public') From 8fc1b640eafc2bf30de056da7f59afcdfc9a9b47 Mon Sep 17 00:00:00 2001 From: gossty Date: Wed, 28 Aug 2024 08:17:44 -0700 Subject: [PATCH 3/7] Populate redis with resource-allocation data --- qiita_db/meta_util.py | 123 +++++++++++++++++++++++--------- qiita_db/test/test_meta_util.py | 36 ++++++---- qiita_db/util.py | 59 +++++++-------- scripts/all-qiita-cron-job | 2 +- scripts/qiita-cron-job | 6 +- 5 files changed, 143 insertions(+), 83 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index baf29d97f..12f40b057 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -38,7 +38,7 @@ from json import loads, dump, dumps from qiita_db.util import create_nested_path, _retrieve_resource_data -from qiita_db.util import resource_allocation_plot, get_software_commands +from qiita_db.util import resource_allocation_plot from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.configuration_manager import ConfigurationManager import qiita_db as qdb @@ -49,6 +49,7 @@ "parameters", "samples", "columns", "input_size", "extra_info", "MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"] + def _get_data_fpids(constructor, object_id): """Small function for getting filepath IDS associated with data object @@ -554,40 +555,96 @@ def generate_plugin_releases(): f(redis_key, v) -def initialize_resource_allocations_redis(): - time = datetime.now().strftime('%m-%d-%y %H:%M:%S') +def get_software_commands(): + # TODO change active=False to True. In test, the data is active=False. + software_list = [s for s in qdb.software.Software.iter(active=False)] + software_commands = dict() + for software in software_list: + sname = software.name + sversion = software.version + commands = software.commands + if sname not in software_commands: + software_commands[sname] = {} + if sversion not in software_commands[sname]: + software_commands[sname][sversion] = [] + for command in commands: + software_commands[sname][sversion].append(command.name) + return software_commands + + +def update_resource_allocation_redis(): + time = datetime.now().strftime('%m-%d-%y') scommands = get_software_commands() - for software, versions in scommands.items(): - for version, commands in versions.items(): - for command in commands: - print("Generating plot for:", software, version, command) - update_resource_allocation_redis(command, software, - version, time) redis_key = 'resources:commands' r_client.set(redis_key, str(scommands)) + for sname, versions in scommands.items(): + for version, commands in versions.items(): + for cname in commands: -def update_resource_allocation_redis(cname, sname, version, time): - col_name = "samples * columns" - - df = _retrieve_resource_data(cname, sname, version, columns) - fig, axs = resource_allocation_plot(df, cname, sname, col_name) - - fig.tight_layout() - plot = BytesIO() - fig.savefig(plot, format='png') - plot.seek(0) - - img = 'data:image/png;base64,' + quote(b64encode(plot.getbuffer()).decode('ascii')) - plt.close(fig) - - # SID, CID, col_name - values = [ - ("img", img, r_client.set), - ('time', time, r_client.set) - ] - - for k, v, f in values: - redis_key = 'resources$#%s$#%s$#%s:%s' % (cname, sname, col_name, k) - r_client.delete(redis_key) - f(redis_key, v) + print("Generating plot for:", sname, version, cname) + col_name = "samples * columns" + df = _retrieve_resource_data(cname, sname, version, columns) + if len(df) == 0: + print("No available data for", sname, version, cname) + continue + + fig, axs = resource_allocation_plot(df, cname, sname, col_name) + titles = [0, 0] + images = [0, 0] + + # Splitting 1 image plot into 2 separate for better layout. + for i, ax in enumerate(axs): + titles[i] = ax.get_title() + ax.set_title("") + # new_fig, new_ax – copy with either only memory plot or + # only time + new_fig = plt.figure() + new_ax = new_fig.add_subplot(111) + + scatter_data = ax.collections[0] + new_ax.scatter(scatter_data.get_offsets()[:, 0], + scatter_data.get_offsets()[:, 1], + s=scatter_data.get_sizes(), label="data") + + line = ax.lines[0] + new_ax.plot(line.get_xdata(), line.get_ydata(), + linewidth=1, color='orange') + + if len(ax.collections) > 1: + failure_data = ax.collections[1] + new_ax.scatter(failure_data.get_offsets()[:, 0], + failure_data.get_offsets()[:, 1], + color='red', s=3, label="failures") + + new_ax.set_xscale('log') + new_ax.set_yscale('log') + new_ax.set_xlabel(ax.get_xlabel()) + new_ax.set_ylabel(ax.get_ylabel()) + new_ax.legend(loc='upper left') + + new_fig.tight_layout() + plot = BytesIO() + new_fig.savefig(plot, format='png') + plot.seek(0) + img = 'data:image/png;base64,' + quote( + b64encode(plot.getvalue()).decode('ascii')) + images[i] = img + plt.close(new_fig) + plt.close(fig) + + # SID, CID, col_name + values = [ + ("img_mem", images[0], r_client.set), + ("img_time", images[1], r_client.set), + ('time', time, r_client.set), + ("title_mem", titles[0], r_client.set), + ("title_time", titles[1], r_client.set) + ] + print(time, titles[0], titles[1]) + + for k, v, f in values: + redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( + cname, sname, version, col_name, k) + r_client.delete(redis_key) + f(redis_key, v) diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index 98b11509d..f57109322 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -13,6 +13,7 @@ from os.path import exists, join import pandas as pd +from datetime import datetime from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.util import qiita_test_checker @@ -523,18 +524,29 @@ def test_update_resource_allocation_redis(self): cname = "Split libraries FASTQ" sname = "QIIMEq2" col_name = "samples * columns" - qdb.meta_util.update_resource_allocation_redis(cname, sname) - - vals = [ - ("img", b's', r_client.get), - ('time', b'2024-11-11', r_client.get) - ] - - for k, exp, f in vals: - redis_key = 'resources$#%s$#%s$#%s:%s' % (cname, sname, col_name, k) - # checking redis values - print(f(redis_key)) - # self.assertEqual(f(redis_key), exp) + version = "1.9.1" + qdb.meta_util.update_resource_allocation_redis() + # since time is month, day, year, it should be equal unless test is ran + # at midnight. + time = datetime.now().strftime('%m-%d-%y') + title_mem_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( + cname, sname, version, col_name, 'title_mem') + title_mem = str(r_client.get(title_mem_str)) + self.assertTrue( + "model_chosen: " + "k * log(x) + " + "b * log(x)^2 + " + "a * log(x)^2.5" in title_mem + ) + + title_time_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( + cname, sname, version, col_name, 'title_time') + title_time = str(r_client.get(title_time_str)) + self.assertTrue("model_chosen: a + b + log(x) * k" in title_time) + time_create_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( + cname, sname, version, col_name, 'time') + time_create = str(r_client.get(time_create_str)) + self.assertTrue(time in time_create) if __name__ == '__main__': diff --git a/qiita_db/util.py b/qiita_db/util.py index 184f9b458..1b4f232bf 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -99,6 +99,27 @@ MODELS_TIME = [time_model1, time_model2, time_model3, time_model4] +def get_model_name(model): + if model == mem_model1: + return "k * log(x) + x * a + b" + elif model == mem_model2: + return "k * log(x) + b * log(x)^2 + a" + elif model == mem_model3: + return "k * log(x) + b * log(x)^2 + a * log(x)^3" + elif model == mem_model4: + return "k * log(x) + b * log(x)^2 + a * log(x)^2.5" + elif model == time_model1: + return "a + b + log(x) * k" + elif model == time_model2: + return "a + b * x + log(x) * k" + elif model == time_model3: + return "a + b * log(x)^2 + log(x) * k" + elif model == time_model4: + return "a * log(x)^3 + b * log(x)^2 + log(x) * k" + else: + return "Unknown model" + + def scrub_data(s): r"""Scrubs data fields of characters not allowed by PostgreSQL @@ -2496,7 +2517,6 @@ def _resource_allocation_plot_helper( cmin = naturalsize(cmin_value, gnu=True) if curr == "MaxRSSRaw" else \ str(timedelta(seconds=round(cmin_value, 2))).rstrip('0').rstrip('.') - x_plot = np.array(df[col_name]) failures_df = _resource_allocation_failures( df, k, a, b, best_model, col_name, curr) @@ -2505,7 +2525,10 @@ def _resource_allocation_plot_helper( ax.scatter(failures_df[col_name], failures_df[curr], color='red', s=3, label="failures") - ax.set_title(f'{cname}: {sname}\n real: {mini} || {maxi}\n' + ax.set_title( + f'k||a||b: {k}||{a}||{b}\n' + f'model_chosen: {get_model_name(best_model)}\n' + f'real: {mini} || {maxi}\n' f'calculated: {cmin} || {cmax}\n' f'failures: {failures}') ax.legend(loc='upper left') @@ -2924,35 +2947,3 @@ def merge_rows(rows): row['node_model']] qdb.sql_connection.TRN.add(sql, sql_args=to_insert) qdb.sql_connection.TRN.execute() - - -def get_software_commands(): - res = [] - with qdb.sql_connection.TRN: - sql_command = """ - SELECT DISTINCT - s.name AS sName, - s.version AS sVersion, - sc.name AS cName - FROM - qiita.slurm_resource_allocations sra - JOIN - qiita.processing_job pr - ON sra.processing_job_id = pr.processing_job_id - JOIN - qiita.software_command sc on pr.command_id = sc.command_id - JOIN - qiita.software s ON sc.software_id = s.software_id - """ - qdb.sql_connection.TRN.add(sql_command) - res = qdb.sql_connection.TRN.execute_fetchindex() - - software_commands = dict() - for s_name, s_version, c_name in res: - if s_name not in software_commands: - software_commands[s_name] = {} - if s_version not in software_commands[s_name]: - software_commands[s_name][s_version] = set() - software_commands[s_name][s_version].add(c_name) - - return software_commands diff --git a/scripts/all-qiita-cron-job b/scripts/all-qiita-cron-job index b414da5f2..7534fda51 100644 --- a/scripts/all-qiita-cron-job +++ b/scripts/all-qiita-cron-job @@ -4,6 +4,6 @@ qiita-cron-job empty-trash-upload-folder qiita-cron-job generate-biom-and-metadata-release qiita-cron-job purge-filepaths qiita-cron-job update-redis-stats -qiita-cron-job initialize-resource-allocations-redis +qiita-cron-job update-resource-allocation-redis qiita-cron-job generate-plugin-releases qiita-cron-job purge-json-web-tokens diff --git a/scripts/qiita-cron-job b/scripts/qiita-cron-job index 17ff09002..252252a38 100755 --- a/scripts/qiita-cron-job +++ b/scripts/qiita-cron-job @@ -16,7 +16,7 @@ from qiita_db.util import ( quick_mounts_purge as qiita_quick_mounts_purge) from qiita_db.meta_util import ( update_redis_stats as qiita_update_redis_stats, - initialize_resource_allocations_redis as qiita_initialize_resource_allocations_redis, + update_resource_allocation_redis as qiita_update_resource_allocation_redis, generate_biom_and_metadata_release as qiita_generate_biom_and_metadata_release, generate_plugin_releases as qiita_generate_plugin_releases) @@ -50,8 +50,8 @@ def update_redis_stats(): @commands.command() -def initialize_resource_allocations_redis(): - qiita_initialize_resource_allocations_redis() +def update_resource_allocation_redis(): + qiita_update_resource_allocation_redis() @commands.command() From 7241754af42ec5f3287f96623eebefb5796e779f Mon Sep 17 00:00:00 2001 From: gossty Date: Wed, 28 Aug 2024 08:33:44 -0700 Subject: [PATCH 4/7] Removed changes to qiita_pet This pull request should only contain changes with uploading data to redis. I accidentally commited some changes to qiita_pet here. --- notebooks/resource-allocation/upload_df.py | 72 ---------------------- qiita_db/meta_util.py | 2 +- qiita_pet/handlers/resources.py | 67 -------------------- qiita_pet/templates/resources.html | 52 ---------------- qiita_pet/webserver.py | 3 - 5 files changed, 1 insertion(+), 195 deletions(-) delete mode 100644 notebooks/resource-allocation/upload_df.py delete mode 100644 qiita_pet/handlers/resources.py delete mode 100644 qiita_pet/templates/resources.html diff --git a/notebooks/resource-allocation/upload_df.py b/notebooks/resource-allocation/upload_df.py deleted file mode 100644 index e74bcce04..000000000 --- a/notebooks/resource-allocation/upload_df.py +++ /dev/null @@ -1,72 +0,0 @@ -import pandas as pd - -# Example data loading -filename = './data/jobs_2024-02-21.tsv.gz' -df = pd.read_csv(filename, sep='\t', dtype={'extra_info': str}) - -# Convert string to timedelta, then to total seconds -df['ElapsedRawTime'] = pd.to_timedelta( - df['ElapsedRawTime']).apply( - lambda x: x.total_seconds()) - -cname = "Validate" -sname = "Diversity types - alpha_vector" -df = df[(df.cName == cname) & (df.sName == sname)] - -df['samples'] = df['samples'].fillna(0).astype(int) -df['columns'] = df['columns'].fillna(0).astype(int) -df['input_size'] = df['input_size'].fillna(0).astype(int) -df['MaxRSSRaw'] = df['MaxRSSRaw'].fillna(0).astype(int) -df['ElapsedRawTime'] = df['ElapsedRawTime'].fillna(0).astype(int) - -COL_NAME = 'samples * columns' -df[COL_NAME] = df['samples'] * df['columns'] -columns = ["MaxRSSRaw", "ElapsedRawTime"] -max_rows = [] - -for curr in columns: - # Get the maximum value for 'curr' within each COL_NAME group - max_values = df.groupby(COL_NAME)[curr].transform(max) - # Filter rows where the current column's value - # is the maximum within its group - curr_rows = df[df[curr] == max_values] - max_rows.append(curr_rows) - -filtered_df = pd.concat(max_rows).drop_duplicates().reset_index(drop=True) - -# INSERT INTO qiita.processing_job(processing_job_id, email, command_id, -# command_parameters, processing_job_status_id) -# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', -# 'admin@foo.com', 1, '""'::json, 1); - -# INSERT INTO qiita.slurm_resource_allocations(processing_job_id, samples, -# columns, input_size, extra_info, memory_used, walltime_used) -# VALUES('ca27ddbc-a678-4b09-8a1d-b65f52f8eb49', 39, 81, 2, 'nan', -# 327036000, 91); - -# processing_job_id uuid NOT NULL, -# samples integer, -# columns integer, -# input_size bigint, -# extra_info varchar DEFAULT NULL, -# memory_used bigint, -# walltime_used integer, - -res = "" - -for index, row in filtered_df.iterrows(): - res += f"""('{row['QiitaID']}', 'admin@foo.bar', 1, '""'::json, 1),\n""" -res += ";\n" -res += "Split\n" -for index, row in filtered_df.iterrows(): - res += ( - f"('{row['QiitaID']}', {int(row['samples'])}, " - f"{int(row['columns'])}, {int(row['input_size'])}, " - f"'{row['extra_info']}', {int(row['MaxRSSRaw'])}, " - f"{int(row['ElapsedRawTime'])}),\n" - ) - -res += ";\n" - -with open("sql.txt", 'w') as filename: - filename.write(res) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 12f40b057..ea5614b06 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -588,7 +588,7 @@ def update_resource_allocation_redis(): if len(df) == 0: print("No available data for", sname, version, cname) continue - + fig, axs = resource_allocation_plot(df, cname, sname, col_name) titles = [0, 0] images = [0, 0] diff --git a/qiita_pet/handlers/resources.py b/qiita_pet/handlers/resources.py deleted file mode 100644 index 0bb0865b1..000000000 --- a/qiita_pet/handlers/resources.py +++ /dev/null @@ -1,67 +0,0 @@ - -from tornado.gen import coroutine, Task -from tornado.web import authenticated, HTTPError - - -from .base_handlers import BaseHandler -from qiita_core.qiita_settings import qiita_config, r_client -from qiita_core.util import execute_as_transaction - -default_cname = "Split libraries FASTQ" -default_sname = "QIIMEq2" -default_col_name = "samples * columns" -commands = 'resources:commands' - -class ResourcesHandler(BaseHandler): - def check_admin(self): - if self.current_user.level != "admin": - raise HTTPError(403, reason="%s does not have access to portal " - "editing!" % self.current_user.id) - - - @execute_as_transaction - def _get_resources(self, callback): - resources = {} - vals = [ - ('img', r_client.get), - ('time', r_client.get) - ] - for k, f in vals: - redis_key = 'resources$#%s$#%s$#%s:%s' % (default_cname, - default_sname, default_col_name, k) - resources[k] = f(redis_key) - callback(resources) - - - def _get_commands(self, callback): - r_client.get(commands) - callback(r_client) - - - @authenticated - @coroutine - @execute_as_transaction - def get(self): - self.check_admin() - resources = yield Task(self._get_resources) - commands = yield Task(self._get_commands) # TODO: it would make more sense to have this rendered once instead of everytime we make a get request, but i'm not sure how to do that right now - self.render('resources.html', - img=resources['img'], time=resources['time'], - commands=commands, - software=default_sname, command=default_cname, - col_name=default_col_name - ) - - - @authenticated - @execute_as_transaction - def post(self): - self.check_admin() - software = self.get_argument("software", "") - command = self.get_argument("command", "") - column_type = self.get_argument("column_type", "") - - - self.get() - - diff --git a/qiita_pet/templates/resources.html b/qiita_pet/templates/resources.html deleted file mode 100644 index 097a19ca6..000000000 --- a/qiita_pet/templates/resources.html +++ /dev/null @@ -1,52 +0,0 @@ -{% extends sitebase.html %} -{% block head %} - - - -{% end %} - -{% block content %} -
    -
    - Generated on: {{time}} - Software: {{software}} - Command: {{command}} - Column name: {{col_name}} -
    - -
    -
    - - -
    - -
    - - -
    - -
    - - -
    - - -
    - {% if img and img is not None %} -
    - -
    - {% end %} -
    -{% end %} \ No newline at end of file diff --git a/qiita_pet/webserver.py b/qiita_pet/webserver.py index d52549c42..50b34fd2b 100644 --- a/qiita_pet/webserver.py +++ b/qiita_pet/webserver.py @@ -50,8 +50,6 @@ from qiita_pet.handlers.upload import ( UploadFileHandler, StudyUploadFileHandler, StudyUploadViaRemote) from qiita_pet.handlers.stats import StatsHandler -from qiita_pet.handlers.resources import ResourcesHandler - from qiita_pet.handlers.download import ( DownloadHandler, DownloadStudyBIOMSHandler, DownloadRelease, DownloadRawData, DownloadEBISampleAccessions, DownloadEBIPrepAccessions, @@ -138,7 +136,6 @@ def __init__(self): (r"/admin/sample_validation/", SampleValidation), (r"/admin/purge_users/", PurgeUsersHandler), (r"/admin/purge_usersAjax/", PurgeUsersAJAXHandler), - (r"/admin/resources/", ResourcesHandler), (r"/ebi_submission/(.*)", EBISubmitHandler), # Study handlers (r"/study/create/", StudyEditHandler), From 437b8c840ed65b62ee657a7124ea5ec3d6cc0d94 Mon Sep 17 00:00:00 2001 From: gossty Date: Wed, 28 Aug 2024 10:58:52 -0700 Subject: [PATCH 5/7] Minor changes --- qiita_db/meta_util.py | 20 ++++++++++++-------- qiita_db/test/test_meta_util.py | 16 ++++------------ qiita_db/util.py | 2 +- qiita_pet/templates/sitebase.html | 1 - 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index ea5614b06..71a8df353 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -555,9 +555,8 @@ def generate_plugin_releases(): f(redis_key, v) -def get_software_commands(): - # TODO change active=False to True. In test, the data is active=False. - software_list = [s for s in qdb.software.Software.iter(active=False)] +def get_software_commands(active): + software_list = [s for s in qdb.software.Software.iter(active=active)] software_commands = dict() for software in software_list: sname = software.name @@ -572,9 +571,17 @@ def get_software_commands(): return software_commands -def update_resource_allocation_redis(): +def update_resource_allocation_redis(active=True): + """Updates redis with plots and information about current software. + + Parameters + ---------- + active: boolean, optional + Defaults to True. Should only be False when testing. + + """ time = datetime.now().strftime('%m-%d-%y') - scommands = get_software_commands() + scommands = get_software_commands(active) redis_key = 'resources:commands' r_client.set(redis_key, str(scommands)) @@ -582,11 +589,9 @@ def update_resource_allocation_redis(): for version, commands in versions.items(): for cname in commands: - print("Generating plot for:", sname, version, cname) col_name = "samples * columns" df = _retrieve_resource_data(cname, sname, version, columns) if len(df) == 0: - print("No available data for", sname, version, cname) continue fig, axs = resource_allocation_plot(df, cname, sname, col_name) @@ -641,7 +646,6 @@ def update_resource_allocation_redis(): ("title_mem", titles[0], r_client.set), ("title_time", titles[1], r_client.set) ] - print(time, titles[0], titles[1]) for k, v, f in values: redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % ( diff --git a/qiita_db/test/test_meta_util.py b/qiita_db/test/test_meta_util.py index f57109322..d1e409a7d 100644 --- a/qiita_db/test/test_meta_util.py +++ b/qiita_db/test/test_meta_util.py @@ -13,7 +13,6 @@ from os.path import exists, join import pandas as pd -from datetime import datetime from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.util import qiita_test_checker @@ -525,28 +524,21 @@ def test_update_resource_allocation_redis(self): sname = "QIIMEq2" col_name = "samples * columns" version = "1.9.1" - qdb.meta_util.update_resource_allocation_redis() - # since time is month, day, year, it should be equal unless test is ran - # at midnight. - time = datetime.now().strftime('%m-%d-%y') + qdb.meta_util.update_resource_allocation_redis(False) title_mem_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( cname, sname, version, col_name, 'title_mem') title_mem = str(r_client.get(title_mem_str)) self.assertTrue( - "model_chosen: " + "model: " "k * log(x) + " "b * log(x)^2 + " - "a * log(x)^2.5" in title_mem + "a * log(x)^3" in title_mem ) title_time_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( cname, sname, version, col_name, 'title_time') title_time = str(r_client.get(title_time_str)) - self.assertTrue("model_chosen: a + b + log(x) * k" in title_time) - time_create_str = 'resources$#%s$#%s$#%s$#%s:%s' % ( - cname, sname, version, col_name, 'time') - time_create = str(r_client.get(time_create_str)) - self.assertTrue(time in time_create) + self.assertTrue("model: a + b + log(x) * k" in title_time) if __name__ == '__main__': diff --git a/qiita_db/util.py b/qiita_db/util.py index 1b4f232bf..1007298e5 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2527,7 +2527,7 @@ def _resource_allocation_plot_helper( ax.set_title( f'k||a||b: {k}||{a}||{b}\n' - f'model_chosen: {get_model_name(best_model)}\n' + f'model: {get_model_name(best_model)}\n' f'real: {mini} || {maxi}\n' f'calculated: {cmin} || {cmax}\n' f'failures: {failures}') diff --git a/qiita_pet/templates/sitebase.html b/qiita_pet/templates/sitebase.html index 5e2e0633f..7165f33a2 100644 --- a/qiita_pet/templates/sitebase.html +++ b/qiita_pet/templates/sitebase.html @@ -384,7 +384,6 @@
  • View Studies awaiting approval
  • Edit study portal connections
  • Purge non-validated users
    • -
  • View Resource Allocation Plots
    • {% end %}
  • Sample Validation
  • Processing Jobs
    • From 4fa0c706c00255053281b7fac28155962f5b80e3 Mon Sep 17 00:00:00 2001 From: gossty Date: Fri, 30 Aug 2024 12:34:00 -0700 Subject: [PATCH 6/7] =?UTF-8?q?Updates=20to=20Antonio=E2=80=99s=20coments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiita_db/meta_util.py | 17 ++++++++--------- qiita_db/test/test_util.py | 4 ++-- qiita_db/util.py | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index 71a8df353..c36e8764a 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -37,14 +37,14 @@ from re import sub from json import loads, dump, dumps -from qiita_db.util import create_nested_path, _retrieve_resource_data +from qiita_db.util import create_nested_path, retrieve_resource_data from qiita_db.util import resource_allocation_plot from qiita_core.qiita_settings import qiita_config, r_client from qiita_core.configuration_manager import ConfigurationManager import qiita_db as qdb # global constant list used in resource_allocation_page -columns = [ +COLUMNS = [ "sName", "sVersion", "cID", "cName", "processing_job_id", "parameters", "samples", "columns", "input_size", "extra_info", "MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"] @@ -557,18 +557,17 @@ def generate_plugin_releases(): def get_software_commands(active): software_list = [s for s in qdb.software.Software.iter(active=active)] - software_commands = dict() + software_commands = defaultdict(lambda: defaultdict(list)) + for software in software_list: sname = software.name sversion = software.version commands = software.commands - if sname not in software_commands: - software_commands[sname] = {} - if sversion not in software_commands[sname]: - software_commands[sname][sversion] = [] + for command in commands: software_commands[sname][sversion].append(command.name) - return software_commands + + return dict(software_commands) def update_resource_allocation_redis(active=True): @@ -590,7 +589,7 @@ def update_resource_allocation_redis(active=True): for cname in commands: col_name = "samples * columns" - df = _retrieve_resource_data(cname, sname, version, columns) + df = retrieve_resource_data(cname, sname, version, COLUMNS) if len(df) == 0: continue diff --git a/qiita_db/test/test_util.py b/qiita_db/test/test_util.py index a71be3e49..cad800752 100644 --- a/qiita_db/test/test_util.py +++ b/qiita_db/test/test_util.py @@ -1322,7 +1322,7 @@ def setUp(self): # df is a dataframe that represents a table with columns specified in # self.columns - self.df = qdb.util._retrieve_resource_data( + self.df = qdb.util.retrieve_resource_data( self.cname, self.sname, self.version, self.columns) def test_plot_return(self): @@ -1423,7 +1423,7 @@ def test_db_update(self): qdb.util.update_resource_allocation_table(test=test_data) for curr_cname, ids in types.items(): - updated_df = qdb.util._retrieve_resource_data( + updated_df = qdb.util.retrieve_resource_data( curr_cname, self.sname, self.version, self.columns) updated_ids_set = set(updated_df['processing_job_id']) previous_ids_set = set(self.df['processing_job_id']) diff --git a/qiita_db/util.py b/qiita_db/util.py index 1007298e5..35a25cee9 100644 --- a/qiita_db/util.py +++ b/qiita_db/util.py @@ -2402,7 +2402,7 @@ def resource_allocation_plot(df, cname, sname, col_name): return fig, axs -def _retrieve_resource_data(cname, sname, version, columns): +def retrieve_resource_data(cname, sname, version, columns): with qdb.sql_connection.TRN: sql = """ SELECT From f5ab03575bcfc9c253bc6d4d00bfdc3a02493ddf Mon Sep 17 00:00:00 2001 From: gossty Date: Fri, 30 Aug 2024 12:37:45 -0700 Subject: [PATCH 7/7] Update meta_util.py --- qiita_db/meta_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiita_db/meta_util.py b/qiita_db/meta_util.py index c36e8764a..86ad7db75 100644 --- a/qiita_db/meta_util.py +++ b/qiita_db/meta_util.py @@ -558,15 +558,15 @@ def generate_plugin_releases(): def get_software_commands(active): software_list = [s for s in qdb.software.Software.iter(active=active)] software_commands = defaultdict(lambda: defaultdict(list)) - + for software in software_list: sname = software.name sversion = software.version commands = software.commands - + for command in commands: software_commands[sname][sversion].append(command.name) - + return dict(software_commands)