Fix: Invalid columns in compact manifest for AnVIL (#6110)

src/azul/plugins/__init__.py

@@ -18,6 +18,7 @@
 from typing import (
     ClassVar,
     Generic,
+    NamedTuple,
     Optional,
     TYPE_CHECKING,
     Type,
@@ -228,6 +229,13 @@ def _load(cls, plugin_type_name: str, plugin_package_name: str) -> Type[T]:
         return plugin_cls
 
 
+class FieldLabel(NamedTuple):
+    # The field's name in an API response
+    name: str
+    # The column's name in a manifest (None to use `name`)
+    column: str | None = None
+
+
 class MetadataPlugin(Plugin[BUNDLE]):
 
     @classmethod
@@ -386,6 +394,8 @@ def invert(v: MetadataPlugin._FieldMapping,
                 for k, v in v.items():
                     assert isinstance(k, FieldPathElement)
                     yield from invert(v, *path, k)
+            elif isinstance(v, FieldLabel):
+                yield v.name, path
             elif isinstance(v, FieldName):
                 yield v, path
             else:

src/azul/plugins/metadata/anvil/__init__.py

@@ -24,6 +24,7 @@
 )
 from azul.plugins import (
     DocumentSlice,
+    FieldLabel,
     ManifestConfig,
     MetadataPlugin,
     Sorting,
@@ -140,12 +141,16 @@ def _field_mapping(self) -> MetadataPlugin._FieldMapping:
         return {
             'entity_id': 'entryId',
             'bundles': {
-                'uuid': self.special_fields.bundle_uuid,
-                'version': self.special_fields.bundle_version
+                'uuid': FieldLabel(name=self.special_fields.bundle_uuid,
+                                   column='files.bundle_uuid'),
+                'version': FieldLabel(name=self.special_fields.bundle_version,
+                                      column='files.bundle_version')
             },
             'sources': {
-                'id': self.special_fields.source_id,
-                'spec': self.special_fields.source_spec
+                'id': FieldLabel(name=self.special_fields.source_id,
+                                 column='sources.source_id'),
+                'spec': FieldLabel(name=self.special_fields.source_spec,
+                                   column='sources.source_spec')
             },
             'contents': {
                 'datasets': {
@@ -282,17 +287,22 @@ def recurse(mapping: MetadataPlugin._FieldMapping, path: FieldPath):
                 new_path = (*path, path_element)
                 if isinstance(name_or_type, dict):
                     recurse(name_or_type, new_path)
-                elif isinstance(name_or_type, str):
+                elif isinstance(name_or_type, (str, FieldLabel)):
                     if new_path == ('entity_id',):
                         pass
+                    elif new_path == ('contents', 'activities', 'activity_table'):
+                        # Remove column from manifest
+                        result[path][path_element] = None
                     elif new_path == ('contents', 'files', 'uuid'):
                         # Request the injection of a file URL …
-                        result[path]['file_url'] = 'files.file_url'
+                        result[path]['file_url'] = 'files.azul_file_url'
                         # … but suppress the columns for the fields …
                         result[path][path_element] = None
                     elif new_path == ('contents', 'files', 'version'):
                         # … only used by that injection.
                         result[path][path_element] = None
+                    elif isinstance(name_or_type, FieldLabel):
+                        result[path][path_element] = name_or_type.column
                     else:
                         result[path][path_element] = name_or_type
                 else:

test/service/test_manifest.py

@@ -1709,25 +1709,25 @@ def test_compact_manifest(self):
         self.assertEqual(200, response.status_code)
         expected = [
             (
-                'bundle_uuid',
+                'files.bundle_uuid',
                 '6b0f6c0f-5d80-a242-accb-840921351cd5',
                 '826dea02-e274-affe-aabc-eb3db63ad068',
                 '826dea02-e274-affe-aabc-eb3db63ad068'
             ),
             (
-                'bundle_version',
+                'files.bundle_version',
                 '2022-06-01T00:00:00.000000Z',
                 '2022-06-01T00:00:00.000000Z',
                 '2022-06-01T00:00:00.000000Z'
             ),
             (
-                'source_id',
+                'sources.source_id',
                 '6c87f0e1-509d-46a4-b845-7584df39263b',
                 '6c87f0e1-509d-46a4-b845-7584df39263b',
                 '6c87f0e1-509d-46a4-b845-7584df39263b'
             ),
             (
-                'source_spec',
+                'sources.source_spec',
                 'tdr:bigquery:gcp:test_anvil_project:anvil_snapshot:/2',
                 'tdr:bigquery:gcp:test_anvil_project:anvil_snapshot:/2',
                 'tdr:bigquery:gcp:test_anvil_project:anvil_snapshot:/2'
@@ -1966,12 +1966,6 @@ def test_compact_manifest(self):
                 '18b3be87-e26b-4376-0d8d-c1e370e90e07',
                 'a60c5138-3749-f7cb-8714-52d389ad5231'
             ),
-            (
-                'activities.activity_table',
-                '',
-                'sequencingactivity',
-                'sequencingactivity'
-            ),
             (
                 'activities.activity_type',
                 '',
@@ -2075,7 +2069,7 @@ def test_compact_manifest(self):
                 self._drs_uri('v1_6c87f0e1-509d-46a4-b845-7584df39263b_8b722e88-8103-49c1-b351-e64fa7c6ab37')
             ),
             (
-                'files.file_url',
+                'files.azul_file_url',
                 self._file_url('6b0f6c0f-5d80-4242-accb-840921351cd5', self.version),
                 self._file_url('15b76f9c-6b46-433f-851d-34e89f1b9ba6', self.version),
                 self._file_url('3b17377b-16b1-431c-9967-e5d01fc5923f', self.version)