[r a] Support for HCA data_use_restriction (#6131)

DataBiosphere · Sep 4, 2024 · 21edc29 · 21edc29
1 parent 9515744
commit 21edc29
Show file tree

Hide file tree

Showing 11 changed files with 184 additions and 28 deletions.
diff --git a/lambdas/service/app.py b/lambdas/service/app.py
@@ -232,7 +232,7 @@
         # changes and reset the minor version to zero. Otherwise, increment only
         # the minor version for backwards compatible changes. A backwards
         # compatible change is one that does not require updates to clients.
-        'version': '9.1'
+        'version': '9.2'
     },
     'tags': [
         {

diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json
diff --git a/src/azul/plugins/metadata/hca/__init__.py b/src/azul/plugins/metadata/hca/__init__.py
@@ -228,7 +228,8 @@ def _field_mapping(self) -> MetadataPlugin._FieldMapping:
                     'estimated_cell_count': 'projectEstimatedCellCount',
                     'is_tissue_atlas_project': 'isTissueAtlasProject',
                     'tissue_atlas': 'tissueAtlas',
-                    'bionetwork_name': 'bionetworkName'
+                    'bionetwork_name': 'bionetworkName',
+                    'data_use_restriction': 'dataUseRestriction'
                 },
                 'sequencing_protocols': {
                     'instrument_manufacturer_model': 'instrumentManufacturerModel',
@@ -332,7 +333,8 @@ def facets(self) -> Sequence[str]:
             'publicationTitle',
             'isTissueAtlasProject',
             'tissueAtlas',
-            'bionetworkName'
+            'bionetworkName',
+            'dataUseRestriction'
         ]
 
     @property

diff --git a/src/azul/plugins/metadata/hca/indexer/transform.py b/src/azul/plugins/metadata/hca/indexer/transform.py
@@ -686,7 +686,8 @@ def _project_types(cls) -> FieldTypes:
             'is_tissue_atlas_project': null_bool,
             'tissue_atlas': [tissue_atlas],
             'bionetwork_name': [null_str],
-            'estimated_cell_count': null_int
+            'estimated_cell_count': null_int,
+            'data_use_restriction': null_str
         }
 
     def _project(self, project: api.Project) -> MutableJSON:
@@ -733,7 +734,8 @@ def _project(self, project: api.Project) -> MutableJSON:
                                            for bionetwork in project.bionetworks),
             'tissue_atlas': list(map(self._tissue_atlas, project.bionetworks)),
             'bionetwork_name': sorted(bionetwork.name for bionetwork in project.bionetworks),
-            'estimated_cell_count': project.estimated_cell_count
+            'estimated_cell_count': project.estimated_cell_count,
+            'data_use_restriction': project.data_use_restriction
         }
 
     @classmethod

diff --git a/src/azul/plugins/metadata/hca/service/response.py b/src/azul/plugins/metadata/hca/service/response.py
@@ -340,7 +340,8 @@ def make_projects(self, entry) -> MutableJSONs:
                 'estimatedCellCount': project['estimated_cell_count'],
                 'isTissueAtlasProject': project['is_tissue_atlas_project'],
                 'tissueAtlas': project.get('tissue_atlas'),
-                'bionetworkName': project['bionetwork_name']
+                'bionetworkName': project['bionetwork_name'],
+                'dataUseRestriction': project.get('data_use_restriction')
             }
             if self.entity_type == 'projects':
                 translated_project['projectDescription'] = project.get('project_description', [])

diff --git a/src/humancellatlas/data/metadata/api.py b/src/humancellatlas/data/metadata/api.py
@@ -22,6 +22,7 @@
     Iterable,
     Mapping,
     MutableMapping,
+    Optional,
     TypeVar,
 )
 from uuid import (
@@ -292,6 +293,7 @@ class Project(Entity):
     supplementary_links: OrderedSet[str]
     estimated_cell_count: int | None
     bionetworks: OrderedSet[Bionetwork]
+    data_use_restriction: Optional[str]
 
     def __init__(self, json: JSON) -> None:
         super().__init__(json)
@@ -317,6 +319,7 @@ def __init__(self, json: JSON) -> None:
         self.bionetworks = OrderedSet(Bionetwork(**bionetwork)
                                       for bionetwork in content.get('hca_bionetworks', ())
                                       if bionetwork)
+        self.data_use_restriction = content.get('data_use_restriction')
 
     def _accessions(self, namespace: str) -> set[str]:
         return {a.accession for a in self.accessions if a.namespace == namespace}

diff --git a/...ndexer/data/aaa96233-bf27-44c7-82df-b4dc15ad4d9d.2018-11-02T11:33:44.698028Z.results.json b/...ndexer/data/aaa96233-bf27-44c7-82df-b4dc15ad4d9d.2018-11-02T11:33:44.698028Z.results.json
diff --git a/test/service/data/pfb_manifest.results.json b/test/service/data/pfb_manifest.results.json
@@ -505,6 +505,9 @@
                 null
             ],
             "contributors": [],
+            "data_use_restriction": [
+                null
+            ],
             "document_id": [
                 "6615efae-fca8-4dd2-a223-9cfcf30fe94d"
             ],
@@ -848,6 +851,9 @@
                 null
             ],
             "contributors": [],
+            "data_use_restriction": [
+                null
+            ],
             "document_id": [
                 "e8642221-4c2c-4fd7-b926-a68bce363c88"
             ],
@@ -3282,6 +3288,9 @@
                 null
             ],
             "contributors": [],
+            "data_use_restriction": [
+                null
+            ],
             "document_id": [
                 "90bf705c-d891-5ce2-aa54-094488b445c6"
             ],

diff --git a/test/service/data/pfb_manifest.schema.json b/test/service/data/pfb_manifest.schema.json
@@ -1976,6 +1976,14 @@
                                 "null",
                                 "long"
                             ]
+                        },
+                        {
+                            "name": "data_use_restriction",
+                            "namespace": "projects",
+                            "type": {
+                                "items": ["null", "string"],
+                                "type": "array"
+                            }
                         }
                     ],
                     "name": "projects",

diff --git a/test/service/test_index_projects.py b/test/service/test_index_projects.py
@@ -84,7 +84,8 @@ def assert_file_type_summaries(hit):
             'tissueAtlas',
             'isTissueAtlasProject',
             'bionetworkName',
-            'estimatedCellCount'
+            'estimatedCellCount',
+            'dataUseRestriction'
         }
         response_json = get_response_json()
         self.assertIn('hits', response_json)

diff --git a/test/service/test_response.py b/test/service/test_response.py
@@ -259,6 +259,7 @@ def test_response_stage_files(self):
                         "isTissueAtlasProject": [False],
                         "tissueAtlas": [],
                         "estimatedCellCount": None,
+                        "dataUseRestriction": [None],
                     }
                 ],
                 "protocols": [
@@ -590,7 +591,8 @@ def test_response_stage_projects(self):
                             "bionetworkName": [None],
                             "tissueAtlas": [],
                             "isTissueAtlasProject": False,
-                            "accessions": []
+                            "accessions": [],
+                            "dataUseRestriction": None,
                         }
                     ],
                     "protocols": [
@@ -854,6 +856,7 @@ def test_response_stage_projects_accessions(self):
                             {"namespace": "insdc_project", "accession": "SRP000001"},
                             {"namespace": "insdc_study", "accession": "PRJNA000000"},
                         ],
+                        "dataUseRestriction": "NRES",
                     }
                 ],
                 "protocols": [