Remove manifest entries for links.json (#4565)

This makes #4565 obsolete/invalid
DataBiosphere · Aug 27, 2024 · cb1ec10 · cb1ec10
1 parent 7754836
commit cb1ec10
Show file tree

Hide file tree

Showing 55 changed files with 4 additions and 647 deletions.
diff --git a/src/azul/plugins/repository/tdr_hca/__init__.py b/src/azul/plugins/repository/tdr_hca/__init__.py
@@ -343,10 +343,7 @@ def _emulate_bundle(self, bundle_fqid: TDRBundleFQID) -> TDRHCABundle:
                               manifest=[],
                               metadata_files={})
         entities, root_entities, links_jsons = self._stitch_bundles(bundle)
-        bundle.add_entity(entity_key='links.json',
-                          entity_type='links',
-                          entity_row=self._merge_links(links_jsons),
-                          is_stitched=False)
+        bundle.metadata_files['links.json'] = self._merge_links(links_jsons)
 
         with ThreadPoolExecutor(max_workers=config.num_tdr_workers) as executor:
             futures = {
@@ -555,12 +552,6 @@ def _merge_links(self, links_jsons: JSONs) -> JSON:
         """
         root, *stitched = links_jsons
         if stitched:
-            merged = {
-                'links_id': root['links_id'],
-                'version': root['version']
-            }
-            for common_key in ('project_id', 'schema_type'):
-                merged[common_key] = one({row[common_key] for row in links_jsons})
             source_contents = [row['content'] for row in links_jsons]
             # FIXME: Explicitly verify compatible schema versions for stitched subgraphs
             #        https://github.com/DataBiosphere/azul/issues/3215
@@ -574,14 +565,9 @@ def _merge_links(self, links_jsons: JSONs) -> JSON:
                 'describedBy': str(schema_url),
                 'links': sum((sc['links'] for sc in source_contents), start=[])
             }
-            merged['content'] = merged_content  # Keep result of parsed JSON for reuse
-            merged['content_size'] = len(json.dumps(merged_content))
-            assert merged.keys() == one({
-                frozenset(row.keys()) for row in links_jsons
-            }), merged
             assert merged_content.keys() == one({
                 frozenset(sc.keys()) for sc in source_contents
             }), merged_content
-            return merged
+            return merged_content
         else:
-            return root
+            return root['content']
diff --git a/...mples/1M Immune Cells/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...mples/1M Immune Cells/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -48,18 +48,6 @@
             "uuid": "821601f1-64e8-4427-8680-34a743b35b69",
             "version": "1"
         },
-        {
-            "crc32c": "DDAABBB6",
-            "sha1": "82c4c5273e31923bf6e12139804c3a4cf10b338d",
-            "sha256": "3de258c171a757d80e53a617ce1834e14710c87db0289c9b2012d0a14a3ea705",
-            "s3_etag": "4dc6069c891063f28a0ee155c6cd7b65",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 1856,
-            "uuid": "04e79979-489e-4cd8-93c3-1d8e99d09ec8",
-            "version": "1"
-        },
         {
             "crc32c": "522D96A7",
             "sha1": "9876adb67827ea0328566811a1304bcf49a31b02",

diff --git a/...totoxic T lymphocytes/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...totoxic T lymphocytes/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -84,18 +84,6 @@
             "uuid": "5910c207-1093-4ef1-9537-dc614d128d14",
             "version": "1"
         },
-        {
-            "crc32c": "1CAA5E57",
-            "sha1": "e3d6064f20fa85351d9446be179ebb15458324e6",
-            "sha256": "e4529c8501160b286824388a51f50b6e94110d3f9a55998a3b088f918e92ae53",
-            "s3_etag": "1e11ded031de78c2c7cec5e5fb3b48a7",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 2381,
-            "uuid": "2aea848d-fbe7-472a-953d-2cd77716b939",
-            "version": "1"
-        },
         {
             "crc32c": "17049723",
             "sha1": "df812b18c751efd55e21318be9411200f3066513",

diff --git a/...an_cerebral_organoids/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...an_cerebral_organoids/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -156,18 +156,6 @@
             "uuid": "597f5fae-db6a-4ebc-b2f0-b9c527f5b3ed",
             "version": "1"
         },
-        {
-            "crc32c": "E8D34C5D",
-            "sha1": "e3336e075787d319a239533d8620a4a60a793bac",
-            "sha256": "062e56b2f314b8fd81b916985767abaedc4e22923e80607b023a3bf68554d4e0",
-            "s3_etag": "3ae64d0365c9c2d857c4194f41b471a6",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 7860,
-            "uuid": "19f9d2c2-2d30-4262-8d19-a5de9ca13dd7",
-            "version": "1"
-        },
         {
             "crc32c": "27917E6E",
             "sha1": "6be5be8c979bf95b9be25df1f16c9856bb55bff3",

diff --git a/...e 2 diabetes pancreas/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...e 2 diabetes pancreas/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -48,18 +48,6 @@
             "uuid": "3b8ffd02-7a9e-44ff-bec5-1387b87b0e34",
             "version": "1"
         },
-        {
-            "crc32c": "5AE674D4",
-            "sha1": "a2c43b27d4cdcd769ea9aa5fb6dc38d144a19448",
-            "sha256": "5444d86b7f7dde2c7521dde5821b12d04cd544ecae07464b45ee75ad98c9becc",
-            "s3_etag": "f2fb4a3a21e54e7b6455fcd9fac1e1b1",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 1856,
-            "uuid": "d6284ed3-ff4c-4153-90b5-a3f5fb10c799",
-            "version": "1"
-        },
         {
             "crc32c": "22B1F2B5",
             "sha1": "24335fec0fbe49e182b9a25d9953a7ef34f797fa",

diff --git a/...amples/Mouse Melanoma/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...amples/Mouse Melanoma/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -60,18 +60,6 @@
             "uuid": "6aa1f675-a3be-4def-8da9-259ae0fcad68",
             "version": "1"
         },
-        {
-            "crc32c": "837B6FBF",
-            "sha1": "4534daa77cd865f34273f6c8a8d0fd3a96361bfe",
-            "sha256": "26c92efa3719bda131663c452d7b385925d927eb5a49f6b2e5ebb1497e871c64",
-            "s3_etag": "dc518a7c1b8faea79320eb70e7c239e6",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 2027,
-            "uuid": "447bd719-baa7-4f32-b4d6-7f64fabfe0f1",
-            "version": "1"
-        },
         {
             "crc32c": "93A97712",
             "sha1": "4365e3dbeeff6b4c23c9057e5b30aab65807a405",

diff --git a/...sis of human pancreas/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...sis of human pancreas/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -36,18 +36,6 @@
             "uuid": "6ce66ac8-1e0f-49b4-a278-468f7b677ffe",
             "version": "1"
         },
-        {
-            "crc32c": "C0EF71FC",
-            "sha1": "79e8137293c18fc1ef1a4b2a532e59f6223fbec3",
-            "sha256": "d333cfe228457aaa476b4ccf39bdd4c620bfcf0f2c11ccfff83c9aacb7a17c13",
-            "s3_etag": "cbacd6ba075487820bcd49e2667581b6",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 1727,
-            "uuid": "169acedb-1e56-4b6d-a3d7-0d3e56d74098",
-            "version": "1"
-        },
         {
             "crc32c": "8D88697D",
             "sha1": "cfbab7537d593e0b1765b65b755e88244bd88ebc",

diff --git a/...ples/Tissue stability/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json b/...ples/Tissue stability/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-08-03T082009.272868Z.json
@@ -60,18 +60,6 @@
             "uuid": "1d64e884-84c1-438e-8f1b-7d42698942ad",
             "version": "1"
         },
-        {
-            "crc32c": "C1574314",
-            "sha1": "086e0e40e88c2ed0b4a1c68a3f2fd88e66454e6e",
-            "sha256": "88fa107073946ff3a01455ebeca62cb7a8c8253a10f90c68f8c37078f00be627",
-            "s3_etag": "151fe404519e0934fa5e24772d8002d6",
-            "content-type": "application/json; dcp-type=\"metadata/json\"",
-            "indexed": true,
-            "name": "links.json",
-            "size": 1643,
-            "uuid": "62f4b747-7bf2-4af5-b710-723a060f0d35",
-            "version": "1"
-        },
         {
             "crc32c": "D044B620",
             "sha1": "15c4e8085308b8f297b9b682c8aa892514e476de",

diff --git a/...etadata_api/cans/prod/6b498499-c5b4-452f-9ff9-2318dbb86000.2019-01-03T163633.780215Z.json b/...etadata_api/cans/prod/6b498499-c5b4-452f-9ff9-2318dbb86000.2019-01-03T163633.780215Z.json
@@ -512,18 +512,6 @@
             "uuid": "baa3a061-534c-4b6e-8c51-a11a8307f537",
             "version": "2019-01-03T120758.911000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "8296c70e",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "dcda06d2eb79d368ac84a47a06689773",
-            "sha1": "3d4d918d7deae23cade8abb3269ca957c315ad0d",
-            "sha256": "a0c53628a804b864567b1c6a87879a040dba1d24b539418710dc3979599301e1",
-            "size": 7860,
-            "uuid": "5b71a545-1f04-477b-98b5-f15664311970",
-            "version": "2019-01-03T163557.511721Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "dec968a1",

diff --git a/...etadata_api/cans/prod/86e7b58e-b9f0-4020-8b34-c61d6da02d44.2019-09-20T103932.395795Z.json b/...etadata_api/cans/prod/86e7b58e-b9f0-4020-8b34-c61d6da02d44.2019-09-20T103932.395795Z.json
@@ -228,18 +228,6 @@
             "uuid": "b3cddf6f-65ee-4aa8-8682-60f00c55be73",
             "version": "2019-09-20T082959.366000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "afe1de7d",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "e00d35caa8374da25b82de4a4358611d",
-            "sha1": "769e482e1902871370aa269dc264379a7bdb64b9",
-            "sha256": "e237b747a7f00824d9e7fca8285500dcf26c335c31a7169b2b5a655f1145776b",
-            "size": 2322,
-            "uuid": "774029ec-ba28-49e6-ae47-6403f5a9555e",
-            "version": "2019-09-20T104007.387625Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "759039c3",

diff --git a/...etadata_api/cans/prod/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-03-29T142048.835519Z.json b/...etadata_api/cans/prod/b2216048-7eaa-45f4-8077-5a3fb4204953.2018-03-29T142048.835519Z.json
@@ -83,18 +83,6 @@
             "size": 3174,
             "uuid": "7db5a820-e612-402c-aaf3-d2676e19d862",
             "version": "2018-03-29T142047.303889Z"
-        },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "4e6ce5dd",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "14c01392fd55df2b2cdbd08994009e24",
-            "sha1": "6ab7fdb229d9f0971a05e781ef52cbcfbf680e00",
-            "sha256": "be6a5c420109f6208e2a663c7d9fdc66de6dac9b08180c4aa0993ada30754645",
-            "size": 5963,
-            "uuid": "8c45f35c-c070-4bf4-999a-1de1676ad598",
-            "version": "2018-03-29T142047.912393Z"
         }
     ],
     "metadata": {

diff --git a/...etadata_api/cans/prod/cc0b5aa4-9f66-48d2-aa4f-ed019d1c9439.2019-05-15T222432.561000Z.json b/...etadata_api/cans/prod/cc0b5aa4-9f66-48d2-aa4f-ed019d1c9439.2019-05-15T222432.561000Z.json
@@ -120,18 +120,6 @@
             "uuid": "453a352c-94fb-4d3b-b609-df1e7abf8c09",
             "version": "2019-05-14T122652.545000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "2c25a3c2",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "1f7c2cbccc797da2d885bb52de52232d",
-            "sha1": "39ba144902ba5d1095af5b8d5dace8dd19a4b08d",
-            "sha256": "33052247612f39f6ea48568a12c761842af30030e1242943bb2c5eb238722488",
-            "size": 2083,
-            "uuid": "51168054-6dad-45aa-916a-ef71135651b2",
-            "version": "2019-05-16T015324.197421Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "86192092",

diff --git a/...etadata_api/cans/prod/ffee3a9b-14de-4dda-980f-c08092b2dabe.2019-04-17T175706.867000Z.json b/...etadata_api/cans/prod/ffee3a9b-14de-4dda-980f-c08092b2dabe.2019-04-17T175706.867000Z.json
@@ -192,18 +192,6 @@
             "uuid": "4276c1dd-50aa-4ffa-a618-dee757bff479",
             "version": "2019-04-17T165959.680000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "4392f0e2",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "ddce74cad640a9a30b2b3b6ac6c4f0c8",
-            "sha1": "8e596d873bf15ebc9f1ff405e17ec633248ebe0a",
-            "sha256": "024b37f1635851b6100974e5318170fbbca1875a02a43c25ce65cb4ac0e5e1e9",
-            "size": 2643,
-            "uuid": "468bd567-e319-4dcc-a636-daf90a534f6d",
-            "version": "2019-04-17T185612.241223Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "9e612174",

diff --git a/...etadata_api/cans/prod/ffee7f29-5c38-461a-8771-a68e20ec4a2e.2019-02-02T065454.662896Z.json b/...etadata_api/cans/prod/ffee7f29-5c38-461a-8771-a68e20ec4a2e.2019-02-02T065454.662896Z.json
@@ -612,18 +612,6 @@
             "uuid": "832b39e3-ebba-4686-a724-52babb96056f",
             "version": "2019-01-23T110544.002000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "290d41eb",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "0d6d83e234cf9a884a96403bbfc56d67",
-            "sha1": "3fee9537a1391d0ba0060e53d42e0a2186f6344d",
-            "sha256": "792cc5459ebf45cef1ef1fb5f62f44fbd79e6f73281d1f4622bb2b1b35abcca3",
-            "size": 7448,
-            "uuid": "700f28bc-8f57-4276-9fce-6b7774bf5210",
-            "version": "2019-02-02T065434.323908Z"
-        },
         {
             "content-type": "text/plain; dcp-type=data",
             "crc32c": "82ad16f4",

diff --git a/...data_api/cans/staging/68bdc676-c442-4581-923e-319c1c2d9018.2018-10-07T130111.835234Z.json b/...data_api/cans/staging/68bdc676-c442-4581-923e-319c1c2d9018.2018-10-07T130111.835234Z.json
@@ -168,18 +168,6 @@
             "uuid": "5a15ed38-5e37-44ff-ba35-0d17bb8564bd",
             "version": "2018-10-07T120517.018000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "56d56718",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "70623328382d0927e3242efc726fe29d",
-            "sha1": "a947b0283b948cf08e1197064506bfa2491d8b3a",
-            "sha256": "0bc6c6fa6a9093caddd8a4921826cd47c6a0773312cc62f51abcd55f93e52ecb",
-            "size": 2393,
-            "uuid": "a1681a39-e8d2-43f7-8781-ac4edab527fe",
-            "version": "2018-10-07T130110.624725Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "a3f89844",

diff --git a/...data_api/cans/staging/70184761-70fc-4b80-8c48-f406a478d5ab.2018-09-05T182535.846470Z.json b/...data_api/cans/staging/70184761-70fc-4b80-8c48-f406a478d5ab.2018-09-05T182535.846470Z.json
@@ -156,18 +156,6 @@
             "uuid": "6284381f-de72-4015-ae51-fa17e13677f8",
             "version": "2018-09-05T182534.886667Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "ac18a06b",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "46cce577c8cfc281c8e33233f0366f99",
-            "sha1": "da4381b79641d4de4fe15ff2e7385aa75b79d79f",
-            "sha256": "0d881b169fbc809219d9eaf6b1c77349b9bbd298ac5c7ce9714517f01ca47a0b",
-            "size": 2095,
-            "uuid": "d33848ba-a9f8-4aa9-a72c-26755f0273d1",
-            "version": "2018-09-05T182535.148513Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "4ef74578",

diff --git a/...data_api/cans/staging/94f2ba52-30c8-4de0-a78e-f95a3f8deb9c.2019-04-03T103426.471000Z.json b/...data_api/cans/staging/94f2ba52-30c8-4de0-a78e-f95a3f8deb9c.2019-04-03T103426.471000Z.json
@@ -2844,18 +2844,6 @@
             "uuid": "fee8a188-5783-44cd-a1c1-ed9d5d7c32a9",
             "version": "2019-04-03T101353.834000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "70bbc6b8",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "7385add25e1c482b4d9b4881fb3e196f",
-            "sha1": "df323c0304644ec6c9cc94694198f568fa43ca86",
-            "sha256": "3cec4e10f26331a01ccea9741267a9d5a59a00a39b5a67059cd3f18eb3030663",
-            "size": 14532,
-            "uuid": "48f0d8a4-fd04-4fee-8301-7441b6efcf48",
-            "version": "2019-04-03T105534.952135Z"
-        },
         {
             "content-type": "application/json; dcp-type=data",
             "crc32c": "d1fbf93a",

diff --git a/...data_api/cans/staging/eca05046-3dad-4e45-b86c-8720f33a5dde.2019-03-17T220646.332108Z.json b/...data_api/cans/staging/eca05046-3dad-4e45-b86c-8720f33a5dde.2019-03-17T220646.332108Z.json
@@ -180,18 +180,6 @@
             "uuid": "b147df84-443b-4dd2-8327-7ea42adcd151",
             "version": "2019-03-17T220103.254000Z"
         },
-        {
-            "content-type": "application/json; dcp-type=\"metadata/links\"",
-            "crc32c": "5356a6ba",
-            "indexed": true,
-            "name": "links.json",
-            "s3_etag": "9b3b812e32f35e5406449b2b4221da39",
-            "sha1": "0881415da33135bb54802129c3fec9b462035ab4",
-            "sha256": "82606b3dc782eeacdcc0d5a670d4e6bf5fc453998c63fc40a2f0318ac5dfc1ef",
-            "size": 2335,
-            "uuid": "feb61912-5a3c-4e12-a4ba-28a0207835fd",
-            "version": "2019-03-17T220643.786686Z"
-        },
         {
             "content-type": "application/gzip; dcp-type=data",
             "crc32c": "da1a6040",

diff --git a/test/indexer/data/00f48893-5e9d-52cd-b32d-af88edccabfa.dss.hca.json b/test/indexer/data/00f48893-5e9d-52cd-b32d-af88edccabfa.dss.hca.json
diff --git a/test/indexer/data/02e69c25-71e2-48ca-a87b-e256938c6a98.dss.hca.json b/test/indexer/data/02e69c25-71e2-48ca-a87b-e256938c6a98.dss.hca.json