From 049efb0fc706b76fe66a6ae866dab37fec46355b Mon Sep 17 00:00:00 2001 From: Noa Aviel Dove Date: Tue, 7 Nov 2023 19:13:04 -0800 Subject: [PATCH] fixup! [a r] Index dataset description from Terra API (#5547) --- .../plugins/metadata/anvil/indexer/transform.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/azul/plugins/metadata/anvil/indexer/transform.py b/src/azul/plugins/metadata/anvil/indexer/transform.py index a57b85a65b..d09775887b 100644 --- a/src/azul/plugins/metadata/anvil/indexer/transform.py +++ b/src/azul/plugins/metadata/anvil/indexer/transform.py @@ -35,6 +35,9 @@ from azul import ( JSON, ) +from azul.collections import ( + deep_dict_merge, +) from azul.indexer import ( BundleFQID, BundlePartition, @@ -428,7 +431,18 @@ def reconcile_inner_entities(cls, ) -> tuple[JSON, BundleFQID]: this_entity, this_bundle = this that_entity, that_bundle = that - return that if that_bundle.version > this_bundle.version else this + if this_entity.keys() == that_entity.keys(): + return that if that_bundle.version > this_bundle.version else this + else: + assert entity_type == 'datasets', (entity_type, this, that) + assert this_bundle.version == that_bundle.version, (this, that) + merged = deep_dict_merge((this_entity, that_entity)) + # Confirm that we combined a regular dataset with a DUOS stub to + # produce the complete set of expected fields + assert merged.keys() == cls.field_types()[entity_type].keys(), (this, that) + # We can safely discard that_bundle because only the version is + # used by the caller, and we know the versions are equal. + return merged, this_bundle class ActivityTransformer(BaseTransformer):