Skip to content

Commit

Permalink
[r R] Promotion 2024-08-27 prod (#6519, PR #6525)
Browse files Browse the repository at this point in the history
  • Loading branch information
dsotirho-ucsc committed Aug 30, 2024
2 parents 27c4aef + 347b2fb commit 32ba09e
Show file tree
Hide file tree
Showing 19 changed files with 178 additions and 147 deletions.
4 changes: 3 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@ SHELL ["/bin/bash", "-c"]
# are updated.
#
ARG azul_image_version=1
RUN apt-get update && apt-get upgrade -y
RUN apt-get update \
&& apt-get upgrade -y \
&& apt-get -y install build-essential curl unzip

# Install helper for access to ECR with credendtials from EC2 metadata service
#
Expand Down
12 changes: 12 additions & 0 deletions deployments/prod/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -1093,6 +1093,17 @@ def mkdict(previous_catalog: dict[str, str],
mksrc('datarepo-bde87024', 'hca_prod_dc0b65b0771346f0a3390b03ea786046__20230427_dcp2_20240711_dcp40', 5)
]))

dcp41_sources = mkdict(dcp40_sources, 462, mkdelta([
mksrc('datarepo-ed01025c', 'hca_prod_0cc58d0b17344e1d9113b32e52f75e36__20240531_dcp2_20240604_dcp39', 990),
mksrc('datarepo-50b00aaf', 'hca_prod_2079bb2e676e4bbf8c68f9c6459edcbb__20240327_dcp2_20240807_dcp41', 232),
mksrc('datarepo-32cb91ae', 'hca_prod_4bcc16b57a4745bbb9c0be9d5336df2d__20240327_dcp2_20240807_dcp41', 613),
mksrc('datarepo-17cfd151', 'hca_prod_660fc8b58fb840508c57e6313195bc81__20240806_dcp2_20240807_dcp41', 240),
mksrc('datarepo-cc9e8ac9', 'hca_prod_815c5ef50fb14eb798821d160362468e__20240806_dcp2_20240807_dcp41', 50),
mksrc('datarepo-c2886bdd', 'hca_prod_838d46603d624b08b32ddc5cbd93919d__20240531_dcp2_20240807_dcp41', 15),
mksrc('datarepo-832dbfa1', 'hca_prod_c16a754f5da346ed8c1e6426af2ef625__20220519_dcp2_20240807_dcp41', 220),
mksrc('datarepo-65bb12f3', 'hca_prod_e870ab5635374b6da66f534fbf8cc57f__20240806_dcp2_20240807_dcp41', 2)
]))

pilot1_sources = mkdict({}, 4, mkdelta([
mksrc('datarepo-11e4dc06', 'hca_prod_59b3bfd9cf454d538c8ee240273cba71__20240410_dcp2_20240410_dcpPilot', 3),
mksrc('datarepo-9ebf5be4', 'hca_prod_5bbd9f925bf447cb91999a9750d3fbcd__20240410_dcp2_20240410_dcpPilot', 3),
Expand Down Expand Up @@ -1176,6 +1187,7 @@ def env() -> Mapping[str, Optional[str]]:
sources=mklist(sources))
for atlas, catalog, sources in [
('hca', 'dcp40', dcp40_sources),
('hca', 'dcp41', dcp41_sources),
('hca', 'pilot1', pilot1_sources),
('lungmap', 'lm7', lm7_sources)
] for suffix, internal in [
Expand Down
2 changes: 1 addition & 1 deletion environment.boot
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
azul_python_version=3.11.9
azul_python_image=docker.io/library/python:3.11.9-bullseye
azul_python_image=docker.io/library/python:3.11.9-slim-bullseye
azul_docker_version=27.1.1
azul_terraform_version=1.9.4
2 changes: 1 addition & 1 deletion environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ def env() -> Mapping[str, Optional[str]]:
# Run `_refresh && make environment.boot` after modifying the Python
# image reference.
'python': {
'ref': 'docker.io/library/python:{azul_python_version}-bullseye',
'ref': 'docker.io/library/python:{azul_python_version}-slim-bullseye',
'url': 'https://hub.docker.com/_/python',
},
'pycharm': {
Expand Down
12 changes: 6 additions & 6 deletions image_manifests.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,17 @@
}
}
},
"docker.io/library/python:3.11.9-bullseye": {
"digest": "sha256:c927121ba6f075a139438db25b22e0c21df8ecae790958f5f3764c32e3c25834",
"docker.io/library/python:3.11.9-slim-bullseye": {
"digest": "sha256:98d223d4801f1237560234274e23f7ed8571bdd5988879d37e5a1ff5f5764ecb",
"manifests": {
"linux/amd64": {
"digest": "sha256:a4639c0d6f9d27bd65c73159af557b66d22c512ad89c55198c9d812cb0d1fee0",
"id": "sha256:d893d69a12eec199d457c36f0944e933dd7f92910b462f651bdb1ec6014c394d",
"digest": "sha256:2fe85c25b98948eb8e9aa501c16e7a61324a03ad4ced3e54b1ae9e71c332ec8a",
"id": "sha256:41a460e44cd0023578a0fbe7155499a9f580a53c5854ce7871cf82d7eff57460",
"platform": "linux/amd64"
},
"linux/arm64": {
"digest": "sha256:8d89f62d1080cc609cd764b2bd06d3ce83e8da08fcc3ad0be0117c2baa593564",
"id": "sha256:1323e85d75bb4e77c548c25ed5c1aacba3d356c7330fc3ce427a957956dcd6f4",
"digest": "sha256:9e9102b040bf6e057e1080b0bb37081d42948aa140d8125aef8301dc6dbbb130",
"id": "sha256:3d6dd5e231cced1c7043e56c4dfd57102e056f0fd3cad2d71f4e0a695114c82c",
"platform": "linux/arm64"
}
}
Expand Down
1 change: 0 additions & 1 deletion lambdas/indexer/.chalice/config.json.template.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
"manage_iam_role": False,
"iam_role_arn": "${aws_iam_role.%s.arn}" % app_name,
"environment_variables": config.lambda_env,
"minimum_compression_size": config.minimum_compression_size,
"lambda_timeout": config.api_gateway_lambda_timeout,
"lambda_memory_size": 128,
**chalice.vpc_lambda_config(app_name),
Expand Down
1 change: 0 additions & 1 deletion lambdas/service/.chalice/config.json.template.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
"manage_iam_role": False,
"iam_role_arn": "${aws_iam_role.%s.arn}" % app_name,
"environment_variables": config.lambda_env,
"minimum_compression_size": config.minimum_compression_size,
"lambda_timeout": config.api_gateway_lambda_timeout,
"lambda_memory_size": 2048,
**chalice.vpc_lambda_config(app_name),
Expand Down
2 changes: 1 addition & 1 deletion requirements.all.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ blinker==1.8.2
boto3==1.28.63
boto3-stubs==1.28.63
botocore==1.31.63
botocore-stubs==1.34.160
botocore-stubs==1.34.162
brotli==1.1.0
cachetools==5.4.0
certifi==2024.7.4
Expand Down
2 changes: 1 addition & 1 deletion requirements.dev.trans.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
blessed==1.20.0
blinker==1.8.2
botocore-stubs==1.34.160
botocore-stubs==1.34.162
brotli==1.1.0
click==8.1.7
colorama==0.4.4
Expand Down
4 changes: 2 additions & 2 deletions src/azul/plugins/metadata/hca/indexer/transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -462,8 +462,8 @@ class BaseTransformer(Transformer, metaclass=ABCMeta):
api_bundle: api.Bundle

def replica_type(self, entity: EntityReference) -> str:
assert entity.entity_type == self.entity_type(), entity
return entity.entity_type.removesuffix('s')
api_entity = self.api_bundle.entities[UUID(entity.entity_id)]
return api_entity.schema_name

@classmethod
def aggregator(cls, entity_type: EntityType) -> Optional[EntityAggregator]:
Expand Down
4 changes: 2 additions & 2 deletions src/azul/service/elasticsearch_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,7 @@ def _prepare_aggregation(self, *, facet: str, facet_path: FieldPath) -> Agg:
agg_type='terms',
field=path,
size=config.terms_aggregation_size)
agg.bucket('untagged', 'missing', field=path)
nested_agg.bucket('untagged', 'missing', field=path)
return agg

def _annotate_aggs_for_translation(self, request: Search):
Expand Down Expand Up @@ -369,7 +369,7 @@ def _flatten_nested_aggs(self, aggs: MutableJSON):
except KeyError:
pass
else:
agg['myTerms'] = nested_agg['myTerms']
agg.update(nested_agg)

def _translate_response_aggs(self, aggs: MutableJSON):
"""
Expand Down
2 changes: 1 addition & 1 deletion src/azul/service/manifest_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -2081,7 +2081,7 @@ def create_file(self) -> tuple[str, Optional[str]]:
with open(path, 'w') as f:
for replica in self._all_replicas():
entry = {
'contents': replica['contents'],
'value': replica['contents'],
'type': replica['replica_type']
}
json.dump(entry, f)
Expand Down
14 changes: 14 additions & 0 deletions src/azul/terraform.py
Original file line number Diff line number Diff line change
Expand Up @@ -793,6 +793,20 @@ def tf_config(self, app_name):
del deployment['lifecycle']
deployment['triggers'] = {'redeployment': deployment.pop('stage_description')}

# Using Terraform to specify the REST API minimum compression size
# proved to be problematic as it would first make an UpdateRestApi call
# to set the property, followed by a PutRestApi call with mode=overwrite
# which would reset the property back to its default value (disabled).
# Setting this property using AWS API Gateway extensions to the OpenAPI
# specification works around this issue.
#
rest_api = resources['aws_api_gateway_rest_api'][app_name]
assert 'minimum_compression_size' not in rest_api, rest_api
openapi_spec = json.loads(locals[app_name])
key = 'x-amazon-apigateway-minimum-compression-size'
openapi_spec[key] = config.minimum_compression_size
locals[app_name] = json.dumps(openapi_spec)

return {
'resource': resources,
'data': data,
Expand Down
3 changes: 3 additions & 0 deletions terraform/shared/shared.tf.json.template.py
Original file line number Diff line number Diff line change
Expand Up @@ -857,6 +857,9 @@ def conformance_pack(name: str) -> str:
'shared': {
'name': config.qualified_resource_name(config.aws_config_term),
's3_bucket_name': '${aws_s3_bucket.aws_config.bucket}',
'snapshot_delivery_properties': {
'delivery_frequency': 'TwentyFour_Hours'
},
'depends_on': [
'aws_config_configuration_recorder.shared'
]
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions test/service/data/verbatim/hca/pfb_entities.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,28 +14,28 @@
},
{
"links": [],
"name": "file",
"name": "links",
"ontology_reference": "",
"properties": [],
"values": {}
},
{
"links": [],
"name": "links",
"name": "project",
"ontology_reference": "",
"properties": [],
"values": {}
},
{
"links": [],
"name": "project",
"name": "sequence_file",
"ontology_reference": "",
"properties": [],
"values": {}
},
{
"links": [],
"name": "sample",
"name": "specimen_from_organism",
"ontology_reference": "",
"properties": [],
"values": {}
Expand All @@ -46,7 +46,7 @@
},
{
"id": "70d1af4a-82c8-478a-8960-e9028b3616ca",
"name": "file",
"name": "sequence_file",
"object": {
"describedBy": "https://schema.humancellatlas.org/type/file/6.5.2/sequence_file",
"file_core": {
Expand All @@ -69,7 +69,7 @@
},
{
"id": "0c5ac7c0-817e-40d4-b1b1-34c3d5cfecdb",
"name": "file",
"name": "sequence_file",
"object": {
"describedBy": "https://schema.humancellatlas.org/type/file/6.5.2/sequence_file",
"file_core": {
Expand Down Expand Up @@ -122,7 +122,7 @@
},
{
"id": "a21dc760-a500-4236-bcff-da34a0e873d2",
"name": "sample",
"name": "specimen_from_organism",
"object": {
"biomaterial_core": {
"biomaterial_id": "DID_scRSq06_pancreas",
Expand Down
Loading

0 comments on commit 32ba09e

Please sign in to comment.