From 8de3bfb8ae7f0704e45798c23f2271eda557c79b Mon Sep 17 00:00:00 2001 From: Noa Aviel Dove Date: Tue, 7 Nov 2023 01:12:25 -0800 Subject: [PATCH] fixup! [a r] Index dataset description from Terra API (#5547) --- .../metadata/anvil/indexer/transform.py | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/azul/plugins/metadata/anvil/indexer/transform.py b/src/azul/plugins/metadata/anvil/indexer/transform.py index a57b85a65..149e389bb 100644 --- a/src/azul/plugins/metadata/anvil/indexer/transform.py +++ b/src/azul/plugins/metadata/anvil/indexer/transform.py @@ -35,8 +35,10 @@ from azul import ( JSON, ) +from azul.collections import ( + deep_dict_merge, +) from azul.indexer import ( - BundleFQID, BundlePartition, ) from azul.indexer.aggregate import ( @@ -70,6 +72,10 @@ DonorAggregator, FileAggregator, ) +from azul.plugins.repository.tdr_anvil import ( + AnvilBundleFQID, + BundleEntityType, +) from azul.strings import ( pluralize, ) @@ -423,12 +429,25 @@ def inner_entity_id(cls, entity_type: EntityType, entity: JSON) -> EntityID: def reconcile_inner_entities(cls, entity_type: EntityType, *, - this: tuple[JSON, BundleFQID], - that: tuple[JSON, BundleFQID] - ) -> tuple[JSON, BundleFQID]: + this: tuple[JSON, AnvilBundleFQID], + that: tuple[JSON, AnvilBundleFQID] + ) -> tuple[JSON, AnvilBundleFQID]: this_entity, this_bundle = this that_entity, that_bundle = that - return that if that_bundle.version > this_bundle.version else this + if this_bundle.version < that_bundle.version: + return that + else: + if this_entity.keys() == that_entity.keys(): + return this + else: + assert entity_type == 'datasets', entity_type + this_is_duos = this_bundle.entity_type is BundleEntityType.duos + that_is_duos = that_bundle.entity_type is BundleEntityType.duos + assert this_is_duos != that_is_duos, (this_bundle, that_bundle) + return ( + deep_dict_merge((this_entity, that_entity)), + that_bundle if this_is_duos else this_bundle + ) class ActivityTransformer(BaseTransformer):