From db4a9b62318b7208cc7990d19124a7e653bf8774 Mon Sep 17 00:00:00 2001 From: Noa Aviel Dove Date: Tue, 30 Jul 2024 13:33:31 -0700 Subject: [PATCH 01/11] Rename `contents` to `value` in JSONL manifest (#6453) --- src/azul/service/manifest_service.py | 2 +- test/service/test_manifest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/azul/service/manifest_service.py b/src/azul/service/manifest_service.py index 83b82c8424..953c58ece7 100644 --- a/src/azul/service/manifest_service.py +++ b/src/azul/service/manifest_service.py @@ -2081,7 +2081,7 @@ def create_file(self) -> tuple[str, Optional[str]]: with open(path, 'w') as f: for replica in self._all_replicas(): entry = { - 'contents': replica['contents'], + 'value': replica['contents'], 'type': replica['replica_type'] } json.dump(entry, f) diff --git a/test/service/test_manifest.py b/test/service/test_manifest.py index ff10c774bd..f1f955f18a 100644 --- a/test/service/test_manifest.py +++ b/test/service/test_manifest.py @@ -1333,7 +1333,7 @@ def test_verbatim_jsonl_manifest(self): expected = [ { 'type': replica_type, - 'contents': bundle.metadata_files[key], + 'value': bundle.metadata_files[key], } for bundle in map(self._load_canned_bundle, self.bundles()) for replica_type, key in [ @@ -2082,7 +2082,7 @@ def test_verbatim_jsonl_manifest(self): # Consolidate entities with the same replica (i.e. datasets) json_hash(entity).digest(): { 'type': 'anvil_' + entity_ref.entity_type, - 'contents': entity, + 'value': entity, } for bundle in self.bundles() for entity_ref, entity in self._load_canned_bundle(bundle).entities.items() From 7766c1d44bd51a8604d35405d79ca59470884d52 Mon Sep 17 00:00:00 2001 From: Noa Aviel Dove Date: Wed, 31 Jul 2024 12:46:27 -0700 Subject: [PATCH 02/11] [r] Use DCP2 schema name for HCA replica_type (#6453) --- .../plugins/metadata/hca/indexer/transform.py | 4 +- ...d.2018-11-02T11:33:44.698028Z.results.json | 6 +- .../data/verbatim/hca/pfb_entities.json | 14 +- .../service/data/verbatim/hca/pfb_schema.json | 222 +++++++++--------- test/service/test_manifest.py | 6 +- 5 files changed, 126 insertions(+), 126 deletions(-) diff --git a/src/azul/plugins/metadata/hca/indexer/transform.py b/src/azul/plugins/metadata/hca/indexer/transform.py index 62803e2998..8c6a3829f6 100644 --- a/src/azul/plugins/metadata/hca/indexer/transform.py +++ b/src/azul/plugins/metadata/hca/indexer/transform.py @@ -462,8 +462,8 @@ class BaseTransformer(Transformer, metaclass=ABCMeta): api_bundle: api.Bundle def replica_type(self, entity: EntityReference) -> str: - assert entity.entity_type == self.entity_type(), entity - return entity.entity_type.removesuffix('s') + api_entity = self.api_bundle.entities[UUID(entity.entity_id)] + return api_entity.schema_name @classmethod def aggregator(cls, entity_type: EntityType) -> Optional[EntityAggregator]: diff --git a/test/indexer/data/aaa96233-bf27-44c7-82df-b4dc15ad4d9d.2018-11-02T11:33:44.698028Z.results.json b/test/indexer/data/aaa96233-bf27-44c7-82df-b4dc15ad4d9d.2018-11-02T11:33:44.698028Z.results.json index c170e20882..febff19e83 100644 --- a/test/indexer/data/aaa96233-bf27-44c7-82df-b4dc15ad4d9d.2018-11-02T11:33:44.698028Z.results.json +++ b/test/indexer/data/aaa96233-bf27-44c7-82df-b4dc15ad4d9d.2018-11-02T11:33:44.698028Z.results.json @@ -3471,7 +3471,7 @@ } }, "entity_id": "0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb", - "replica_type": "file", + "replica_type": "sequence_file", "hub_ids": [ "0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb" ] @@ -3541,7 +3541,7 @@ } }, "entity_id": "70d1af4a-82c8-478a-8960-e9028b3616ca", - "replica_type": "file", + "replica_type": "sequence_file", "hub_ids": [ "70d1af4a-82c8-478a-8960-e9028b3616ca" ] @@ -3593,7 +3593,7 @@ } }, "entity_id": "a21dc760-a500-4236-bcff-da34a0e873d2", - "replica_type": "sample", + "replica_type": "specimen_from_organism", "hub_ids": [ "0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb", "70d1af4a-82c8-478a-8960-e9028b3616ca" diff --git a/test/service/data/verbatim/hca/pfb_entities.json b/test/service/data/verbatim/hca/pfb_entities.json index dd65d389fe..cdb1266adb 100644 --- a/test/service/data/verbatim/hca/pfb_entities.json +++ b/test/service/data/verbatim/hca/pfb_entities.json @@ -14,28 +14,28 @@ }, { "links": [], - "name": "file", + "name": "links", "ontology_reference": "", "properties": [], "values": {} }, { "links": [], - "name": "links", + "name": "project", "ontology_reference": "", "properties": [], "values": {} }, { "links": [], - "name": "project", + "name": "sequence_file", "ontology_reference": "", "properties": [], "values": {} }, { "links": [], - "name": "sample", + "name": "specimen_from_organism", "ontology_reference": "", "properties": [], "values": {} @@ -46,7 +46,7 @@ }, { "id": "70d1af4a-82c8-478a-8960-e9028b3616ca", - "name": "file", + "name": "sequence_file", "object": { "describedBy": "https://schema.humancellatlas.org/type/file/6.5.2/sequence_file", "file_core": { @@ -69,7 +69,7 @@ }, { "id": "0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb", - "name": "file", + "name": "sequence_file", "object": { "describedBy": "https://schema.humancellatlas.org/type/file/6.5.2/sequence_file", "file_core": { @@ -122,7 +122,7 @@ }, { "id": "a21dc760-a500-4236-bcff-da34a0e873d2", - "name": "sample", + "name": "specimen_from_organism", "object": { "biomaterial_core": { "biomaterial_id": "DID_scRSq06_pancreas", diff --git a/test/service/data/verbatim/hca/pfb_schema.json b/test/service/data/verbatim/hca/pfb_schema.json index 07228a90ff..af50aa32f7 100644 --- a/test/service/data/verbatim/hca/pfb_schema.json +++ b/test/service/data/verbatim/hca/pfb_schema.json @@ -222,85 +222,6 @@ "name": "cell_suspension", "type": "record" }, - { - "fields": [ - { - "name": "describedBy", - "namespace": "file", - "type": "string" - }, - { - "name": "file_core", - "namespace": "file", - "type": { - "fields": [ - { - "name": "file_format", - "namespace": "file.file_core", - "type": "string" - }, - { - "name": "file_name", - "namespace": "file.file_core", - "type": "string" - } - ], - "name": "file.file_core", - "type": "record" - } - }, - { - "name": "insdc_run", - "namespace": "file", - "type": { - "items": "string", - "type": "array" - } - }, - { - "name": "provenance", - "namespace": "file", - "type": { - "fields": [ - { - "name": "document_id", - "namespace": "file.provenance", - "type": "string" - }, - { - "name": "submission_date", - "namespace": "file.provenance", - "type": "string" - }, - { - "name": "update_date", - "namespace": "file.provenance", - "type": "string" - } - ], - "name": "file.provenance", - "type": "record" - } - }, - { - "name": "read_index", - "namespace": "file", - "type": "string" - }, - { - "name": "read_length", - "namespace": "file", - "type": "long" - }, - { - "name": "schema_type", - "namespace": "file", - "type": "string" - } - ], - "name": "file", - "type": "record" - }, { "fields": [ { @@ -592,59 +513,138 @@ "name": "project", "type": "record" }, + { + "fields": [ + { + "name": "describedBy", + "namespace": "sequence_file", + "type": "string" + }, + { + "name": "file_core", + "namespace": "sequence_file", + "type": { + "fields": [ + { + "name": "file_format", + "namespace": "sequence_file.file_core", + "type": "string" + }, + { + "name": "file_name", + "namespace": "sequence_file.file_core", + "type": "string" + } + ], + "name": "sequence_file.file_core", + "type": "record" + } + }, + { + "name": "insdc_run", + "namespace": "sequence_file", + "type": { + "items": "string", + "type": "array" + } + }, + { + "name": "provenance", + "namespace": "sequence_file", + "type": { + "fields": [ + { + "name": "document_id", + "namespace": "sequence_file.provenance", + "type": "string" + }, + { + "name": "submission_date", + "namespace": "sequence_file.provenance", + "type": "string" + }, + { + "name": "update_date", + "namespace": "sequence_file.provenance", + "type": "string" + } + ], + "name": "sequence_file.provenance", + "type": "record" + } + }, + { + "name": "read_index", + "namespace": "sequence_file", + "type": "string" + }, + { + "name": "read_length", + "namespace": "sequence_file", + "type": "long" + }, + { + "name": "schema_type", + "namespace": "sequence_file", + "type": "string" + } + ], + "name": "sequence_file", + "type": "record" + }, { "fields": [ { "name": "biomaterial_core", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": { "fields": [ { "name": "biomaterial_id", - "namespace": "sample.biomaterial_core", + "namespace": "specimen_from_organism.biomaterial_core", "type": "string" }, { "name": "ncbi_taxon_id", - "namespace": "sample.biomaterial_core", + "namespace": "specimen_from_organism.biomaterial_core", "type": { "items": "long", "type": "array" } } ], - "name": "sample.biomaterial_core", + "name": "specimen_from_organism.biomaterial_core", "type": "record" } }, { "name": "describedBy", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": "string" }, { "name": "diseases", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": { "items": { "fields": [ { "name": "ontology", - "namespace": "sample.diseases", + "namespace": "specimen_from_organism.diseases", "type": "string" }, { "name": "ontology_label", - "namespace": "sample.diseases", + "namespace": "specimen_from_organism.diseases", "type": "string" }, { "name": "text", - "namespace": "sample.diseases", + "namespace": "specimen_from_organism.diseases", "type": "string" } ], - "name": "sample.diseases", + "name": "specimen_from_organism.diseases", "type": "record" }, "type": "array" @@ -652,27 +652,27 @@ }, { "name": "genus_species", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": { "items": { "fields": [ { "name": "ontology", - "namespace": "sample.genus_species", + "namespace": "specimen_from_organism.genus_species", "type": "string" }, { "name": "ontology_label", - "namespace": "sample.genus_species", + "namespace": "specimen_from_organism.genus_species", "type": "string" }, { "name": "text", - "namespace": "sample.genus_species", + "namespace": "specimen_from_organism.genus_species", "type": "string" } ], - "name": "sample.genus_species", + "name": "specimen_from_organism.genus_species", "type": "record" }, "type": "array" @@ -680,86 +680,86 @@ }, { "name": "organ", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": { "fields": [ { "name": "ontology", - "namespace": "sample.organ", + "namespace": "specimen_from_organism.organ", "type": "string" }, { "name": "ontology_label", - "namespace": "sample.organ", + "namespace": "specimen_from_organism.organ", "type": "string" }, { "name": "text", - "namespace": "sample.organ", + "namespace": "specimen_from_organism.organ", "type": "string" } ], - "name": "sample.organ", + "name": "specimen_from_organism.organ", "type": "record" } }, { "name": "organ_part", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": { "fields": [ { "name": "ontology", - "namespace": "sample.organ_part", + "namespace": "specimen_from_organism.organ_part", "type": "string" }, { "name": "ontology_label", - "namespace": "sample.organ_part", + "namespace": "specimen_from_organism.organ_part", "type": "string" }, { "name": "text", - "namespace": "sample.organ_part", + "namespace": "specimen_from_organism.organ_part", "type": "string" } ], - "name": "sample.organ_part", + "name": "specimen_from_organism.organ_part", "type": "record" } }, { "name": "provenance", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": { "fields": [ { "name": "document_id", - "namespace": "sample.provenance", + "namespace": "specimen_from_organism.provenance", "type": "string" }, { "name": "submission_date", - "namespace": "sample.provenance", + "namespace": "specimen_from_organism.provenance", "type": "string" }, { "name": "update_date", - "namespace": "sample.provenance", + "namespace": "specimen_from_organism.provenance", "type": "string" } ], - "name": "sample.provenance", + "name": "specimen_from_organism.provenance", "type": "record" } }, { "name": "schema_type", - "namespace": "sample", + "namespace": "specimen_from_organism", "type": "string" } ], - "name": "sample", + "name": "specimen_from_organism", "type": "record" } ] diff --git a/test/service/test_manifest.py b/test/service/test_manifest.py index f1f955f18a..0bd177bfbb 100644 --- a/test/service/test_manifest.py +++ b/test/service/test_manifest.py @@ -1340,9 +1340,9 @@ def test_verbatim_jsonl_manifest(self): ('links', 'links.json'), ('cell_suspension', 'cell_suspension_0.json'), ('project', 'project_0.json'), - ('file', 'sequence_file_0.json'), - ('file', 'sequence_file_1.json'), - ('sample', 'specimen_from_organism_0.json') + ('sequence_file', 'sequence_file_0.json'), + ('sequence_file', 'sequence_file_1.json'), + ('specimen_from_organism', 'specimen_from_organism_0.json') ] ] response = self._get_manifest(ManifestFormat.verbatim_jsonl, {}) From 606e7171dcc2cd1091a14d6dc44b15b1c10aa59b Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Mon, 15 Jul 2024 17:29:21 -0700 Subject: [PATCH 03/11] Fix: Service responses are not compressed (#6360) --- lambdas/indexer/.chalice/config.json.template.py | 1 - lambdas/service/.chalice/config.json.template.py | 1 - src/azul/terraform.py | 14 ++++++++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lambdas/indexer/.chalice/config.json.template.py b/lambdas/indexer/.chalice/config.json.template.py index 6751ef223e..dd77ae946b 100644 --- a/lambdas/indexer/.chalice/config.json.template.py +++ b/lambdas/indexer/.chalice/config.json.template.py @@ -25,7 +25,6 @@ "manage_iam_role": False, "iam_role_arn": "${aws_iam_role.%s.arn}" % app_name, "environment_variables": config.lambda_env, - "minimum_compression_size": config.minimum_compression_size, "lambda_timeout": config.api_gateway_lambda_timeout, "lambda_memory_size": 128, **chalice.vpc_lambda_config(app_name), diff --git a/lambdas/service/.chalice/config.json.template.py b/lambdas/service/.chalice/config.json.template.py index 46747fb3d4..c40dff9442 100644 --- a/lambdas/service/.chalice/config.json.template.py +++ b/lambdas/service/.chalice/config.json.template.py @@ -25,7 +25,6 @@ "manage_iam_role": False, "iam_role_arn": "${aws_iam_role.%s.arn}" % app_name, "environment_variables": config.lambda_env, - "minimum_compression_size": config.minimum_compression_size, "lambda_timeout": config.api_gateway_lambda_timeout, "lambda_memory_size": 2048, **chalice.vpc_lambda_config(app_name), diff --git a/src/azul/terraform.py b/src/azul/terraform.py index 84b5fffad9..4c553eff21 100644 --- a/src/azul/terraform.py +++ b/src/azul/terraform.py @@ -793,6 +793,20 @@ def tf_config(self, app_name): del deployment['lifecycle'] deployment['triggers'] = {'redeployment': deployment.pop('stage_description')} + # Using Terraform to specify the REST API minimum compression size + # proved to be problematic as it would first make an UpdateRestApi call + # to set the property, followed by a PutRestApi call with mode=overwrite + # which would reset the property back to its default value (disabled). + # Setting this property using AWS API Gateway extensions to the OpenAPI + # specification works around this issue. + # + rest_api = resources['aws_api_gateway_rest_api'][app_name] + assert 'minimum_compression_size' not in rest_api, rest_api + openapi_spec = json.loads(locals[app_name]) + key = 'x-amazon-apigateway-minimum-compression-size' + openapi_spec[key] = config.minimum_compression_size + locals[app_name] = json.dumps(openapi_spec) + return { 'resource': resources, 'data': data, From 2ed4a01295a7fa074777eb4c0998d40b81dd3d7f Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Wed, 14 Aug 2024 09:43:49 -0700 Subject: [PATCH 04/11] Fix: Alarm api_unauthorized for HeadBucket from AWS Config (#6134) Originally committed as 00866ee1 with wrong title: [p] Fix: FedRAMP inventories review issues are created without labels (#6461) --- terraform/shared/shared.tf.json.template.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/shared/shared.tf.json.template.py b/terraform/shared/shared.tf.json.template.py index 6cadbf726f..70fe2b5e86 100644 --- a/terraform/shared/shared.tf.json.template.py +++ b/terraform/shared/shared.tf.json.template.py @@ -857,6 +857,9 @@ def conformance_pack(name: str) -> str: 'shared': { 'name': config.qualified_resource_name(config.aws_config_term), 's3_bucket_name': '${aws_s3_bucket.aws_config.bucket}', + 'snapshot_delivery_properties': { + 'delivery_frequency': 'TwentyFour_Hours' + }, 'depends_on': [ 'aws_config_configuration_recorder.shared' ] From 8dd10074fc52ae2169867a471ef84e73edfdf6b9 Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Fri, 12 Jul 2024 14:56:15 -0700 Subject: [PATCH 05/11] Replace Python image with the slim version (#6352) --- Dockerfile | 4 +++- environment.boot | 2 +- environment.py | 2 +- image_manifests.json | 12 ++++++------ 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 61de4fecb8..06aed2ecca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,9 @@ SHELL ["/bin/bash", "-c"] # are updated. # ARG azul_image_version=1 -RUN apt-get update && apt-get upgrade -y +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get -y install build-essential curl unzip # Install helper for access to ECR with credendtials from EC2 metadata service # diff --git a/environment.boot b/environment.boot index bb27731d73..2f91e803e1 100644 --- a/environment.boot +++ b/environment.boot @@ -1,4 +1,4 @@ azul_python_version=3.11.9 -azul_python_image=docker.io/library/python:3.11.9-bullseye +azul_python_image=docker.io/library/python:3.11.9-slim-bullseye azul_docker_version=27.1.1 azul_terraform_version=1.9.4 diff --git a/environment.py b/environment.py index 1c7dc1a0bb..e927f049f8 100644 --- a/environment.py +++ b/environment.py @@ -279,7 +279,7 @@ def env() -> Mapping[str, Optional[str]]: # Run `_refresh && make environment.boot` after modifying the Python # image reference. 'python': { - 'ref': 'docker.io/library/python:{azul_python_version}-bullseye', + 'ref': 'docker.io/library/python:{azul_python_version}-slim-bullseye', 'url': 'https://hub.docker.com/_/python', }, 'pycharm': { diff --git a/image_manifests.json b/image_manifests.json index 25c0751929..88340732bc 100644 --- a/image_manifests.json +++ b/image_manifests.json @@ -14,17 +14,17 @@ } } }, - "docker.io/library/python:3.11.9-bullseye": { - "digest": "sha256:c927121ba6f075a139438db25b22e0c21df8ecae790958f5f3764c32e3c25834", + "docker.io/library/python:3.11.9-slim-bullseye": { + "digest": "sha256:98d223d4801f1237560234274e23f7ed8571bdd5988879d37e5a1ff5f5764ecb", "manifests": { "linux/amd64": { - "digest": "sha256:a4639c0d6f9d27bd65c73159af557b66d22c512ad89c55198c9d812cb0d1fee0", - "id": "sha256:d893d69a12eec199d457c36f0944e933dd7f92910b462f651bdb1ec6014c394d", + "digest": "sha256:2fe85c25b98948eb8e9aa501c16e7a61324a03ad4ced3e54b1ae9e71c332ec8a", + "id": "sha256:41a460e44cd0023578a0fbe7155499a9f580a53c5854ce7871cf82d7eff57460", "platform": "linux/amd64" }, "linux/arm64": { - "digest": "sha256:8d89f62d1080cc609cd764b2bd06d3ce83e8da08fcc3ad0be0117c2baa593564", - "id": "sha256:1323e85d75bb4e77c548c25ed5c1aacba3d356c7330fc3ce427a957956dcd6f4", + "digest": "sha256:9e9102b040bf6e057e1080b0bb37081d42948aa140d8125aef8301dc6dbbb130", + "id": "sha256:3d6dd5e231cced1c7043e56c4dfd57102e056f0fd3cad2d71f4e0a695114c82c", "platform": "linux/arm64" } } From 966aa946e07b9779e1988c832ee0a5fffdb562bc Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Thu, 15 Aug 2024 09:08:57 -0700 Subject: [PATCH 06/11] [R] Ran `make requirements_update` --- requirements.all.txt | 2 +- requirements.dev.trans.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.all.txt b/requirements.all.txt index 7c110cabec..7e01089db0 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -10,7 +10,7 @@ blinker==1.8.2 boto3==1.28.63 boto3-stubs==1.28.63 botocore==1.31.63 -botocore-stubs==1.34.160 +botocore-stubs==1.34.161 brotli==1.1.0 cachetools==5.4.0 certifi==2024.7.4 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index a1f9516ee1..5056064b49 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -1,6 +1,6 @@ blessed==1.20.0 blinker==1.8.2 -botocore-stubs==1.34.160 +botocore-stubs==1.34.161 brotli==1.1.0 click==8.1.7 colorama==0.4.4 From f712668594696026b112a775e95ed604c6800530 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Thu, 15 Aug 2024 14:22:19 -0700 Subject: [PATCH 07/11] [R] Ran `make requirements_update` again --- requirements.all.txt | 2 +- requirements.dev.trans.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.all.txt b/requirements.all.txt index 7e01089db0..16c177e71a 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -10,7 +10,7 @@ blinker==1.8.2 boto3==1.28.63 boto3-stubs==1.28.63 botocore==1.31.63 -botocore-stubs==1.34.161 +botocore-stubs==1.34.162 brotli==1.1.0 cachetools==5.4.0 certifi==2024.7.4 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index 5056064b49..a7842ddfa5 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -1,6 +1,6 @@ blessed==1.20.0 blinker==1.8.2 -botocore-stubs==1.34.161 +botocore-stubs==1.34.162 brotli==1.1.0 click==8.1.7 colorama==0.4.4 From 3b6c69a7656f86bf449fc6e84dfa43b9463cf6a9 Mon Sep 17 00:00:00 2001 From: Noa Aviel Dove Date: Tue, 6 Aug 2024 22:06:12 -0700 Subject: [PATCH 08/11] Fix incorrect expected test value --- test/service/test_response.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/service/test_response.py b/test/service/test_response.py index a464cc54af..7606ce6c5b 100644 --- a/test/service/test_response.py +++ b/test/service/test_response.py @@ -3498,11 +3498,13 @@ def test_projects_response(self): self.assertEqual(expected_bionetwork_name, project['bionetworkName']) self.assertTrue(project['isTissueAtlasProject']) - tissue_atlas = { + tissue_atlas = response_json['termFacets']['tissueAtlas'] + self.assertEqual(5, tissue_atlas['total']) + terms = { entry['term']: entry['count'] - for entry in response_json['termFacets']['tissueAtlas']['terms'] + for entry in tissue_atlas['terms'] } - self.assertEqual({None: 1, 'Lung': 1, 'Retina': 1, 'Blood': 1}, tissue_atlas) + self.assertEqual({None: 2, 'Lung': 1, 'Retina': 1, 'Blood': 1}, terms) class TestUnpopulatedIndexResponse(IndexResponseTestCase): From d8918d6cd8a86f501261f2482cf135031ccd467c Mon Sep 17 00:00:00 2001 From: Noa Aviel Dove Date: Tue, 6 Aug 2024 22:06:42 -0700 Subject: [PATCH 09/11] Fix: Inconsistent null count in tissue atlas term facet (#6458) --- src/azul/service/elasticsearch_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/azul/service/elasticsearch_service.py b/src/azul/service/elasticsearch_service.py index 3fc1594f51..2b6b81631c 100644 --- a/src/azul/service/elasticsearch_service.py +++ b/src/azul/service/elasticsearch_service.py @@ -336,7 +336,7 @@ def _prepare_aggregation(self, *, facet: str, facet_path: FieldPath) -> Agg: agg_type='terms', field=path, size=config.terms_aggregation_size) - agg.bucket('untagged', 'missing', field=path) + nested_agg.bucket('untagged', 'missing', field=path) return agg def _annotate_aggs_for_translation(self, request: Search): @@ -369,7 +369,7 @@ def _flatten_nested_aggs(self, aggs: MutableJSON): except KeyError: pass else: - agg['myTerms'] = nested_agg['myTerms'] + agg.update(nested_agg) def _translate_response_aggs(self, aggs: MutableJSON): """ From caec84118c239801cb0b3eac4efa8885d9afe311 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Thu, 15 Aug 2024 13:39:25 -0700 Subject: [PATCH 10/11] [r] Index dcp41 in prod (#6501) --- deployments/prod/environment.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/deployments/prod/environment.py b/deployments/prod/environment.py index c082e0d92b..13db6d94cf 100644 --- a/deployments/prod/environment.py +++ b/deployments/prod/environment.py @@ -1093,6 +1093,18 @@ def mkdict(previous_catalog: dict[str, str], mksrc('datarepo-bde87024', 'hca_prod_dc0b65b0771346f0a3390b03ea786046__20230427_dcp2_20240711_dcp40', 5) ])) +dcp41_sources = mkdict(dcp40_sources, 463, mkdelta([ + mksrc('datarepo-ed01025c', 'hca_prod_0cc58d0b17344e1d9113b32e52f75e36__20240531_dcp2_20240604_dcp39', 990), + mksrc('datarepo-50b00aaf', 'hca_prod_2079bb2e676e4bbf8c68f9c6459edcbb__20240327_dcp2_20240807_dcp41', 232), + mksrc('datarepo-92ae9f34', 'hca_prod_248c5dc36b754fb4ad8acc771968483f__20240806_dcp2_20240807_dcp41', 140), + mksrc('datarepo-32cb91ae', 'hca_prod_4bcc16b57a4745bbb9c0be9d5336df2d__20240327_dcp2_20240807_dcp41', 613), + mksrc('datarepo-17cfd151', 'hca_prod_660fc8b58fb840508c57e6313195bc81__20240806_dcp2_20240807_dcp41', 240), + mksrc('datarepo-cc9e8ac9', 'hca_prod_815c5ef50fb14eb798821d160362468e__20240806_dcp2_20240807_dcp41', 50), + mksrc('datarepo-c2886bdd', 'hca_prod_838d46603d624b08b32ddc5cbd93919d__20240531_dcp2_20240807_dcp41', 15), + mksrc('datarepo-832dbfa1', 'hca_prod_c16a754f5da346ed8c1e6426af2ef625__20220519_dcp2_20240807_dcp41', 220), + mksrc('datarepo-65bb12f3', 'hca_prod_e870ab5635374b6da66f534fbf8cc57f__20240806_dcp2_20240807_dcp41', 2) +])) + pilot1_sources = mkdict({}, 4, mkdelta([ mksrc('datarepo-11e4dc06', 'hca_prod_59b3bfd9cf454d538c8ee240273cba71__20240410_dcp2_20240410_dcpPilot', 3), mksrc('datarepo-9ebf5be4', 'hca_prod_5bbd9f925bf447cb91999a9750d3fbcd__20240410_dcp2_20240410_dcpPilot', 3), @@ -1176,6 +1188,7 @@ def env() -> Mapping[str, Optional[str]]: sources=mklist(sources)) for atlas, catalog, sources in [ ('hca', 'dcp40', dcp40_sources), + ('hca', 'dcp41', dcp41_sources), ('hca', 'pilot1', pilot1_sources), ('lungmap', 'lm7', lm7_sources) ] for suffix, internal in [ From 347b2fb7a305879a90f0fcdd9ded1109ba60184d Mon Sep 17 00:00:00 2001 From: Daniel Sotirhos Date: Wed, 28 Aug 2024 17:21:43 -0700 Subject: [PATCH 11/11] [H r] Remove project 248c5dc3 from dcp41 (#6521) --- deployments/prod/environment.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/deployments/prod/environment.py b/deployments/prod/environment.py index 13db6d94cf..c10b23fa51 100644 --- a/deployments/prod/environment.py +++ b/deployments/prod/environment.py @@ -1093,10 +1093,9 @@ def mkdict(previous_catalog: dict[str, str], mksrc('datarepo-bde87024', 'hca_prod_dc0b65b0771346f0a3390b03ea786046__20230427_dcp2_20240711_dcp40', 5) ])) -dcp41_sources = mkdict(dcp40_sources, 463, mkdelta([ +dcp41_sources = mkdict(dcp40_sources, 462, mkdelta([ mksrc('datarepo-ed01025c', 'hca_prod_0cc58d0b17344e1d9113b32e52f75e36__20240531_dcp2_20240604_dcp39', 990), mksrc('datarepo-50b00aaf', 'hca_prod_2079bb2e676e4bbf8c68f9c6459edcbb__20240327_dcp2_20240807_dcp41', 232), - mksrc('datarepo-92ae9f34', 'hca_prod_248c5dc36b754fb4ad8acc771968483f__20240806_dcp2_20240807_dcp41', 140), mksrc('datarepo-32cb91ae', 'hca_prod_4bcc16b57a4745bbb9c0be9d5336df2d__20240327_dcp2_20240807_dcp41', 613), mksrc('datarepo-17cfd151', 'hca_prod_660fc8b58fb840508c57e6313195bc81__20240806_dcp2_20240807_dcp41', 240), mksrc('datarepo-cc9e8ac9', 'hca_prod_815c5ef50fb14eb798821d160362468e__20240806_dcp2_20240807_dcp41', 50),