Promotion 2024-08-27 prod (#6519)

Dockerfile

@@ -10,7 +10,9 @@ SHELL ["/bin/bash", "-c"]
 # are updated.
 #
 ARG azul_image_version=1
-RUN apt-get update && apt-get upgrade -y
+RUN apt-get update \
+    && apt-get upgrade -y \
+    && apt-get -y install build-essential curl unzip
 
 # Install helper for access to ECR with credendtials from EC2 metadata service
 #

deployments/prod/environment.py

@@ -1093,6 +1093,18 @@ def mkdict(previous_catalog: dict[str, str],
     mksrc('datarepo-bde87024', 'hca_prod_dc0b65b0771346f0a3390b03ea786046__20230427_dcp2_20240711_dcp40', 5)
 ]))
 
+dcp41_sources = mkdict(dcp40_sources, 463, mkdelta([
+    mksrc('datarepo-ed01025c', 'hca_prod_0cc58d0b17344e1d9113b32e52f75e36__20240531_dcp2_20240604_dcp39', 990),
+    mksrc('datarepo-50b00aaf', 'hca_prod_2079bb2e676e4bbf8c68f9c6459edcbb__20240327_dcp2_20240807_dcp41', 232),
+    mksrc('datarepo-92ae9f34', 'hca_prod_248c5dc36b754fb4ad8acc771968483f__20240806_dcp2_20240807_dcp41', 140),
+    mksrc('datarepo-32cb91ae', 'hca_prod_4bcc16b57a4745bbb9c0be9d5336df2d__20240327_dcp2_20240807_dcp41', 613),
+    mksrc('datarepo-17cfd151', 'hca_prod_660fc8b58fb840508c57e6313195bc81__20240806_dcp2_20240807_dcp41', 240),
+    mksrc('datarepo-cc9e8ac9', 'hca_prod_815c5ef50fb14eb798821d160362468e__20240806_dcp2_20240807_dcp41', 50),
+    mksrc('datarepo-c2886bdd', 'hca_prod_838d46603d624b08b32ddc5cbd93919d__20240531_dcp2_20240807_dcp41', 15),
+    mksrc('datarepo-832dbfa1', 'hca_prod_c16a754f5da346ed8c1e6426af2ef625__20220519_dcp2_20240807_dcp41', 220),
+    mksrc('datarepo-65bb12f3', 'hca_prod_e870ab5635374b6da66f534fbf8cc57f__20240806_dcp2_20240807_dcp41', 2)
+]))
+
 pilot1_sources = mkdict({}, 4, mkdelta([
     mksrc('datarepo-11e4dc06', 'hca_prod_59b3bfd9cf454d538c8ee240273cba71__20240410_dcp2_20240410_dcpPilot', 3),
     mksrc('datarepo-9ebf5be4', 'hca_prod_5bbd9f925bf447cb91999a9750d3fbcd__20240410_dcp2_20240410_dcpPilot', 3),
@@ -1176,6 +1188,7 @@ def env() -> Mapping[str, Optional[str]]:
                                        sources=mklist(sources))
             for atlas, catalog, sources in [
                 ('hca', 'dcp40', dcp40_sources),
+                ('hca', 'dcp41', dcp41_sources),
                 ('hca', 'pilot1', pilot1_sources),
                 ('lungmap', 'lm7', lm7_sources)
             ] for suffix, internal in [

environment.boot

@@ -1,4 +1,4 @@
 azul_python_version=3.11.9
-azul_python_image=docker.io/library/python:3.11.9-bullseye
+azul_python_image=docker.io/library/python:3.11.9-slim-bullseye
 azul_docker_version=27.1.1
 azul_terraform_version=1.9.4

environment.py

@@ -279,7 +279,7 @@ def env() -> Mapping[str, Optional[str]]:
             # Run `_refresh && make environment.boot` after modifying the Python
             # image reference.
             'python': {
-                'ref': 'docker.io/library/python:{azul_python_version}-bullseye',
+                'ref': 'docker.io/library/python:{azul_python_version}-slim-bullseye',
                 'url': 'https://hub.docker.com/_/python',
             },
             'pycharm': {

image_manifests.json

@@ -14,17 +14,17 @@
             }
         }
     },
-    "docker.io/library/python:3.11.9-bullseye": {
-        "digest": "sha256:c927121ba6f075a139438db25b22e0c21df8ecae790958f5f3764c32e3c25834",
+    "docker.io/library/python:3.11.9-slim-bullseye": {
+        "digest": "sha256:98d223d4801f1237560234274e23f7ed8571bdd5988879d37e5a1ff5f5764ecb",
         "manifests": {
             "linux/amd64": {
-                "digest": "sha256:a4639c0d6f9d27bd65c73159af557b66d22c512ad89c55198c9d812cb0d1fee0",
-                "id": "sha256:d893d69a12eec199d457c36f0944e933dd7f92910b462f651bdb1ec6014c394d",
+                "digest": "sha256:2fe85c25b98948eb8e9aa501c16e7a61324a03ad4ced3e54b1ae9e71c332ec8a",
+                "id": "sha256:41a460e44cd0023578a0fbe7155499a9f580a53c5854ce7871cf82d7eff57460",
                 "platform": "linux/amd64"
             },
             "linux/arm64": {
-                "digest": "sha256:8d89f62d1080cc609cd764b2bd06d3ce83e8da08fcc3ad0be0117c2baa593564",
-                "id": "sha256:1323e85d75bb4e77c548c25ed5c1aacba3d356c7330fc3ce427a957956dcd6f4",
+                "digest": "sha256:9e9102b040bf6e057e1080b0bb37081d42948aa140d8125aef8301dc6dbbb130",
+                "id": "sha256:3d6dd5e231cced1c7043e56c4dfd57102e056f0fd3cad2d71f4e0a695114c82c",
                 "platform": "linux/arm64"
             }
         }

lambdas/indexer/.chalice/config.json.template.py

@@ -25,7 +25,6 @@
     "manage_iam_role": False,
     "iam_role_arn": "${aws_iam_role.%s.arn}" % app_name,
     "environment_variables": config.lambda_env,
-    "minimum_compression_size": config.minimum_compression_size,
     "lambda_timeout": config.api_gateway_lambda_timeout,
     "lambda_memory_size": 128,
     **chalice.vpc_lambda_config(app_name),

lambdas/service/.chalice/config.json.template.py

@@ -25,7 +25,6 @@
     "manage_iam_role": False,
     "iam_role_arn": "${aws_iam_role.%s.arn}" % app_name,
     "environment_variables": config.lambda_env,
-    "minimum_compression_size": config.minimum_compression_size,
     "lambda_timeout": config.api_gateway_lambda_timeout,
     "lambda_memory_size": 2048,
     **chalice.vpc_lambda_config(app_name),

requirements.all.txt

@@ -10,7 +10,7 @@ blinker==1.8.2
 boto3==1.28.63
 boto3-stubs==1.28.63
 botocore==1.31.63
-botocore-stubs==1.34.160
+botocore-stubs==1.34.162
 brotli==1.1.0
 cachetools==5.4.0
 certifi==2024.7.4

requirements.dev.trans.txt

@@ -1,6 +1,6 @@
 blessed==1.20.0
 blinker==1.8.2
-botocore-stubs==1.34.160
+botocore-stubs==1.34.162
 brotli==1.1.0
 click==8.1.7
 colorama==0.4.4

src/azul/plugins/metadata/hca/indexer/transform.py

@@ -462,8 +462,8 @@ class BaseTransformer(Transformer, metaclass=ABCMeta):
     api_bundle: api.Bundle
 
     def replica_type(self, entity: EntityReference) -> str:
-        assert entity.entity_type == self.entity_type(), entity
-        return entity.entity_type.removesuffix('s')
+        api_entity = self.api_bundle.entities[UUID(entity.entity_id)]
+        return api_entity.schema_name
 
     @classmethod
     def aggregator(cls, entity_type: EntityType) -> Optional[EntityAggregator]:

src/azul/service/elasticsearch_service.py

@@ -336,7 +336,7 @@ def _prepare_aggregation(self, *, facet: str, facet_path: FieldPath) -> Agg:
                           agg_type='terms',
                           field=path,
                           size=config.terms_aggregation_size)
-        agg.bucket('untagged', 'missing', field=path)
+        nested_agg.bucket('untagged', 'missing', field=path)
         return agg
 
     def _annotate_aggs_for_translation(self, request: Search):
@@ -369,7 +369,7 @@ def _flatten_nested_aggs(self, aggs: MutableJSON):
             except KeyError:
                 pass
             else:
-                agg['myTerms'] = nested_agg['myTerms']
+                agg.update(nested_agg)
 
     def _translate_response_aggs(self, aggs: MutableJSON):
         """

src/azul/service/manifest_service.py

@@ -2081,7 +2081,7 @@ def create_file(self) -> tuple[str, Optional[str]]:
         with open(path, 'w') as f:
             for replica in self._all_replicas():
                 entry = {
-                    'contents': replica['contents'],
+                    'value': replica['contents'],
                     'type': replica['replica_type']
                 }
                 json.dump(entry, f)

src/azul/terraform.py

@@ -793,6 +793,20 @@ def tf_config(self, app_name):
         del deployment['lifecycle']
         deployment['triggers'] = {'redeployment': deployment.pop('stage_description')}
 
+        # Using Terraform to specify the REST API minimum compression size
+        # proved to be problematic as it would first make an UpdateRestApi call
+        # to set the property, followed by a PutRestApi call with mode=overwrite
+        # which would reset the property back to its default value (disabled).
+        # Setting this property using AWS API Gateway extensions to the OpenAPI
+        # specification works around this issue.
+        #
+        rest_api = resources['aws_api_gateway_rest_api'][app_name]
+        assert 'minimum_compression_size' not in rest_api, rest_api
+        openapi_spec = json.loads(locals[app_name])
+        key = 'x-amazon-apigateway-minimum-compression-size'
+        openapi_spec[key] = config.minimum_compression_size
+        locals[app_name] = json.dumps(openapi_spec)
+
         return {
             'resource': resources,
             'data': data,

terraform/shared/shared.tf.json.template.py

@@ -857,6 +857,9 @@ def conformance_pack(name: str) -> str:
             'shared': {
                 'name': config.qualified_resource_name(config.aws_config_term),
                 's3_bucket_name': '${aws_s3_bucket.aws_config.bucket}',
+                'snapshot_delivery_properties': {
+                    'delivery_frequency': 'TwentyFour_Hours'
+                },
                 'depends_on': [
                     'aws_config_configuration_recorder.shared'
                 ]

test/service/data/verbatim/hca/pfb_entities.json

@@ -14,28 +14,28 @@
                 },
                 {
                     "links": [],
-                    "name": "file",
+                    "name": "links",
                     "ontology_reference": "",
                     "properties": [],
                     "values": {}
                 },
                 {
                     "links": [],
-                    "name": "links",
+                    "name": "project",
                     "ontology_reference": "",
                     "properties": [],
                     "values": {}
                 },
                 {
                     "links": [],
-                    "name": "project",
+                    "name": "sequence_file",
                     "ontology_reference": "",
                     "properties": [],
                     "values": {}
                 },
                 {
                     "links": [],
-                    "name": "sample",
+                    "name": "specimen_from_organism",
                     "ontology_reference": "",
                     "properties": [],
                     "values": {}
@@ -46,7 +46,7 @@
     },
     {
         "id": "70d1af4a-82c8-478a-8960-e9028b3616ca",
-        "name": "file",
+        "name": "sequence_file",
         "object": {
             "describedBy": "https://schema.humancellatlas.org/type/file/6.5.2/sequence_file",
             "file_core": {
@@ -69,7 +69,7 @@
     },
     {
         "id": "0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb",
-        "name": "file",
+        "name": "sequence_file",
         "object": {
             "describedBy": "https://schema.humancellatlas.org/type/file/6.5.2/sequence_file",
             "file_core": {
@@ -122,7 +122,7 @@
     },
     {
         "id": "a21dc760-a500-4236-bcff-da34a0e873d2",
-        "name": "sample",
+        "name": "specimen_from_organism",
         "object": {
             "biomaterial_core": {
                 "biomaterial_id": "DID_scRSq06_pancreas",