Support dynamic endpoints retrieve from ARM Cloud Metadata API

src/aaz_dev/cli/controller/az_client_generator.py

@@ -79,6 +79,10 @@ def __init__(self, endpoints):
     def type(self):
         return self._endpoints.type
 
+    @property
+    def cloud_metadata(self):
+        return self._endpoints.cloud_metadata
+
     def iter_hosts(self):
         for template in self._endpoints.templates:
             yield template.cloud, template.template
@@ -108,4 +112,3 @@ def __init__(self, endpoints, cmd_ctx):
     @property
     def type(self):
         return self._endpoints.type
-

src/aaz_dev/cli/templates/aaz/profile/_clients.py.j2

@@ -21,8 +21,13 @@ class {{ client.cls_name }}(AAZBaseClient):
         CloudNameEnum.{{cloud}}: {{ template|constant_convert }},
         {%- endfor %}
     }
+    {%- if client.endpoints.cloud_metadata %}
+    _CLOUD_HOST_METADATA_INDEX = {{ client.endpoints.cloud_metadata.selector_index|constant_convert }}
+    {%- if client.endpoints.cloud_metadata.prefix_template %}
+    _CLOUD_HOST_METADATA_PREFIX_TEMPLATE = {{ client.endpoints.cloud_metadata.prefix_template|constant_convert }}
+    {%- endif %}
+    {%- endif %}
     {%- elif client.endpoints.type == "http-operation" %}
-
     {%- endif %}
 
     _AAD_CREDENTIAL_SCOPES = [
@@ -34,13 +39,25 @@ class {{ client.cls_name }}(AAZBaseClient):
     @classmethod
     def _build_base_url(cls, ctx, **kwargs):
         {%- if client.endpoints.type == "template" %}
-        return cls._CLOUD_HOST_TEMPLATES.get(ctx.cli_ctx.cloud.name, None)
+        {%- if client.endpoints.cloud_metadata %}
+        {%- if client.endpoints.cloud_metadata.prefix_template %}
+        endpoint = None
+        suffix = cls.get_cloud_suffix(ctx, cls._CLOUD_HOST_METADATA_INDEX)
+        if suffix:
+            endpoint = cls._CLOUD_HOST_METADATA_PREFIX_TEMPLATE + suffix
+        {%- else %}
+        endpoint = cls.get_cloud_endpoint(ctx, cls._CLOUD_HOST_METADATA_INDEX)
+        {%- endif %}
+        {%- endif %}
+        if not endpoint:
+            endpoint = cls._CLOUD_HOST_TEMPLATES.get(ctx.cli_ctx.cloud.name, None)
+        return endpoint
         {%- elif client.endpoints.type == "http-operation" %}
         cls._fetch_endpoint(ctx, **kwargs)
-        host = ctx.selectors.{{client.endpoints.selector.name}}.required().to_serialized_data()
-        if not isinstance(host, str):
-            raise ValueError(f"Invalid host value: '{host}'")
-        return host
+        endpoint = ctx.selectors.{{client.endpoints.selector.name}}.required().to_serialized_data()
+        if not isinstance(endpoint, str):
+            raise ValueError(f"Invalid host value type: '{type(endpoint)}' expected 'str'")
+        return endpoint
         {%- endif %}
 
     @classmethod

src/aaz_dev/cli/tests/common.py

@@ -2810,6 +2810,10 @@ def prepare_monitor_metrics_aaz_2023_05_01_preview(self, ws_name):
                         "template": "https://{region}.metrics.monitor.cloudapi.de"
                     },
                 ],
+                "cloudMetadata": {
+                    "selectorIndex": "suffixes.monitorMetrics",
+                    "prefixTemplate": "https://{region}",
+                }
             })
             self.assertTrue(rv.status_code == 200)
 

src/aaz_dev/command/api/editor.py

@@ -128,6 +128,7 @@ def editor_workspace_client_config(name):
         cfg_editor = manager.create_cfg_editor(
             auth=data['auth'],
             templates=data.get('templates', None),
+            cloud_medadata=data.get('cloudMetadata', None),
             arm_resource=data.get('resource', None),
         )
         manager.save()
@@ -136,6 +137,7 @@ def editor_workspace_client_config(name):
     result = cfg_editor.cfg.to_primitive()
     return jsonify(result)
 
+
 @bp.route("/Workspaces/<name>/ClientConfig/AAZ/Compare", methods=("POST",))
 def compare_workspace_client_config_version_with_aaz(name):
     manager = WorkspaceManager(name)

src/aaz_dev/command/controller/workspace_client_cfg_editor.py

@@ -5,7 +5,9 @@
 
 from .client_cfg_reader import ClientCfgReader
 from .workspace_helper import ArgumentUpdateMixin
-from command.model.configuration import CMDClientConfig, CMDDiffLevelEnum, CMDClientEndpointsByTemplate, CMDClientEndpointsByHttpOperation, CMDClientEndpoints
+from command.model.configuration import (CMDClientConfig, CMDDiffLevelEnum, CMDClientEndpointsByTemplate,
+                                         CMDClientEndpointsByHttpOperation, CMDClientEndpoints,
+                                         CMDClientEndpointCloudMetadataTemplate)
 
 logger = logging.getLogger('backend')
 
@@ -50,10 +52,12 @@ def new_client_cfg(cls, plane, auth, endpoints, ref_cfg: ClientCfgReader = None)
         return cfg_editor
 
     @classmethod
-    def new_client_endpoints_by_template(cls, templates):
+    def new_client_endpoints_by_template(cls, templates, cloud_medadata):
         endpoints = CMDClientEndpointsByTemplate(raw_data={
-            "templates": templates
+            "templates": templates,
         })
+        if cloud_medadata:
+            endpoints.cloud_metadata = CMDClientEndpointCloudMetadataTemplate(raw_data=cloud_medadata)
         endpoints.prepare()
         return endpoints
 

src/aaz_dev/command/controller/workspace_manager.py

@@ -1048,10 +1048,10 @@ def find_similar_args(self, *cmd_names, arg):
 
     # client config
 
-    def create_cfg_editor(self, auth, templates=None, arm_resource=None):
+    def create_cfg_editor(self, auth, templates=None, cloud_medadata=None, arm_resource=None):
         ref_cfg = self.load_client_cfg_editor()
         if templates:
-            endpoints = WorkspaceClientCfgEditor.new_client_endpoints_by_template(templates)
+            endpoints = WorkspaceClientCfgEditor.new_client_endpoints_by_template(templates, cloud_medadata)
         elif arm_resource:
             # arm_resrouce should have these keys: "module", "version", "id", "subresource"
             mod_names = arm_resource['module']

src/aaz_dev/command/model/configuration/__init__.py

@@ -25,7 +25,9 @@
     CMDArgPromptInput, CMDPasswordArgPromptInput
 from ._arg_builder import CMDArgBuilder
 from ._arg_group import CMDArgGroup
-from ._client import CMDClientConfig, CMDClientEndpoints, CMDClientEndpointsByHttpOperation, CMDClientEndpointsByTemplate, CMDClientEndpointTemplate, CMDClientAuth, CMDClientAADAuthConfig
+from ._client import (CMDClientConfig, CMDClientEndpoints, CMDClientEndpointsByHttpOperation,
+                      CMDClientEndpointsByTemplate, CMDClientEndpointTemplate, CMDClientAuth, CMDClientAADAuthConfig,
+                      CMDClientEndpointCloudMetadataTemplate)
 from ._command import CMDCommand
 from ._command_group import CMDCommandGroup
 from ._condition import CMDConditionOperator, \

src/aaz_dev/command/model/configuration/_client.py

@@ -77,36 +77,16 @@ def diff(self, old, level):
         return diff
 
 
-class CMDClientEndpointTemplate(Model):
-    cloud = CloudField(required=True)
-
-    # https://{accountName}.{zone}.blob.storage.azure.net
-    template = StringType(required=True)
-
-    class Options:
-        serialize_when_none = False
-
-    def reformat(self, **kwargs):
-        parsed = urlparse(self.template)
-        if parsed.path:
-            if parsed.path == '/' and not parsed.params and not parsed.query and not parsed.fragment:
-                self.template = self.template.rstrip('/')
-            else:
-                raise exceptions.VerificationError('Invalid endpoints', details='"{}" contains path'.format(self.template))
-        if not parsed.scheme or not parsed.netloc:
-            raise exceptions.VerificationError('Invalid endpoints', details='"{}" has no schema or hostname'.format(self.template))
-
-    def iter_placeholders(self):
-        parsed = urlparse(self.template)
-        return self._iter_placeholders(parsed.netloc)
-
-    def _iter_placeholders(self, endpoint):
+class _EndpointTemplateMixin:
+    @staticmethod
+    def _iter_placeholders(template):
+        endpoint = urlparse(template).netloc
         while True:
             idx = 0
             required = True
             while idx < len(endpoint) and endpoint[idx] != '{':
                 idx += 1
-            endpoint = endpoint[idx+1:]
+            endpoint = endpoint[idx + 1:]
             if not endpoint:
                 # not found '{'
                 return
@@ -122,6 +102,36 @@ def _iter_placeholders(self, endpoint):
 
             yield placeholder, required
 
+    @staticmethod
+    def _reformat(template):
+        parsed = urlparse(template)
+        if parsed.path:
+            if parsed.path == '/' and not parsed.params and not parsed.query and not parsed.fragment:
+                return template.rstrip('/')
+            else:
+                raise exceptions.VerificationError('Invalid endpoints', details='"{}" contains path'.format(template))
+        if not parsed.scheme or not parsed.netloc:
+            raise exceptions.VerificationError('Invalid endpoints', details='"{}" has no schema or hostname'.format(template))
+
+        return template
+
+
+class CMDClientEndpointTemplate(Model, _EndpointTemplateMixin):
+
+    cloud = CloudField(required=True)
+
+    # https://{accountName}.{zone}.blob.storage.azure.net
+    template = StringType(required=True)
+
+    class Options:
+        serialize_when_none = False
+
+    def iter_placeholders(self):
+        return self._iter_placeholders(self.template)
+
+    def reformat(self, **kwargs):
+        self.template = self._reformat(self.template)
+
     def diff(self, old, level):
         if type(self) is not type(old):
             return f"Type: {type(old)} != {type(self)}"
@@ -134,6 +144,44 @@ def diff(self, old, level):
         return diff
 
 
+class CMDClientEndpointCloudMetadataTemplate(Model, _EndpointTemplateMixin):
+
+    selector_index = StringType(
+        required=True,
+        serialized_name="selectorIndex",
+        deserialize_from="selectorIndex",
+    )  # index, used to retrieve property in arm cloud metadata endpoints.
+    prefix_template = StringType(
+        serialized_name="prefixTemplate",
+        deserialize_from="prefixTemplate",
+    )  # prefix template, required for suffixes.
+
+    class Options:
+        serialize_when_none = False
+
+    def iter_placeholders(self):
+        if self.prefix_template is None:
+            return
+        for placeholder in self._iter_placeholders(self.prefix_template):
+            yield placeholder
+
+    def reformat(self, **kwargs):
+        if self.prefix_template is None:
+            return
+        self.prefix_template = self._reformat(self.prefix_template)
+
+    def diff(self, old, level):
+        if type(self) is not type(old):
+            return f"Type: {type(old)} != {type(self)}"
+        diff = {}
+        if level >= CMDDiffLevelEnum.BreakingChange:
+            if self.selector_index != old.selector_index:
+                diff['selector_index'] = f"{old.selector_index} != {self.selector_index}"
+            if self.prefix_template != old.prefix_template:
+                diff['prefix_template'] = f"{old.prefix_template} != {self.prefix_template}"
+        return diff
+
+
 class CMDClientEndpoints(Model):
     # properties as tags
     TYPE_VALUE = None  # types: "template", "http-operation"
@@ -182,12 +230,17 @@ class CMDClientEndpointsByTemplate(CMDClientEndpoints):
     TYPE_VALUE = 'template'
 
     templates = ListType(ModelType(CMDClientEndpointTemplate), required=True, min_size=1)
+    cloud_metadata = ModelType(CMDClientEndpointCloudMetadataTemplate,
+                               serialized_name="cloudMetadata",
+                               deserialize_from="cloudMetadata")
     params = ListType(CMDSchemaField())
 
     def reformat(self, **kwargs):
         for template in self.templates:
             template.reformat(**kwargs)
         self.templates = sorted(self.templates, key=lambda e: e.cloud)
+        if self.cloud_metadata:
+            self.cloud_metadata.reformat(**kwargs)
 
         # make sure the placeholders across all the endpoints are consistent
         placeholders = {}
@@ -202,9 +255,22 @@ def reformat(self, **kwargs):
                         "required": required,
                         "count": 1
                     }
+        expected_count = len(self.templates)
+        if self.cloud_metadata:
+            for placeholder, required in self.cloud_metadata.iter_placeholders():
+                if placeholder in placeholders:
+                    placeholders[placeholder]['count'] += 1
+                    if placeholders[placeholder]['required'] != required:
+                        raise exceptions.VerificationError('Invalid endpoints', details='Inconsistent required for placeholder "{}" in endpoints or cloud metadata'.format(placeholder))
+                else:
+                    placeholders[placeholder] = {
+                        "required": required,
+                        "count": 1
+                    }
+            expected_count += 1
         for item in placeholders.values():
-            if item['count'] != len(self.templates):
-                raise exceptions.VerificationError('Invalid endpoints', details='placeholder "{}" is missed in some endpoints'.format(placeholder))
+            if item['count'] != expected_count:
+                raise exceptions.VerificationError('Invalid endpoints', details='placeholder "{}" is missed in some endpoints or cloud metadata'.format(placeholder))
 
         # make sure the parameters are consistent with the placeholders
         if self.params:
@@ -229,6 +295,14 @@ def prepare(self):
                         "required": required,
                         "skip_url_encoding": True,
                     })
+        if self.cloud_metadata:
+            for placeholder, required in self.cloud_metadata.iter_placeholders():
+                if placeholder not in params:
+                    params[placeholder] = CMDStringSchema({
+                        "name": placeholder,
+                        "required": required,
+                        "skip_url_encoding": True,
+                    })
         self.params = sorted(params.values(), key=lambda p: p.name) or None
 
     def generate_args(self, ref_args):
@@ -247,6 +321,12 @@ def diff(self, old, level):
             return f"Type: {type(old)} != {type(self)}"
         diff = {}
         if level >= CMDDiffLevelEnum.BreakingChange:
+            if (not self.cloud_metadata) != (not old.cloud_metadata):
+                diff['cloud_metadata'] = "Add cloud metadata" if self.cloud_metadata else "Remove cloud metadata"
+            elif self.cloud_metadata:
+                if cloud_metadata_diff := self.cloud_metadata.diff(old.cloud_metadata, level):
+                    diff['cloud_metadata'] = cloud_metadata_diff
+
             if len(self.templates) != len(old.templates):
                 diff['templates'] = "template not match"
             else:

src/aaz_dev/command/tests/api_tests/test_editor.py

@@ -2226,9 +2226,19 @@ def test_dataplane_monitor_metrics(self, ws_name):
                         "template": "https://{region}.metrics.monitor.cloudapi.de"
                     },
                 ],
+                "cloudMetadata": {
+                    "selectorIndex": "suffixes.monitorMetrics",
+                    "prefixTemplate": "https://{region}",
+                }
             })
             self.assertTrue(rv.status_code == 200)
 
+            client_config = rv.get_json()
+            self.assertEqual(client_config['endpoints']['cloudMetadata'], {
+                "selectorIndex": "suffixes.monitorMetrics",
+                "prefixTemplate": "https://{region}",
+            })
+
             # update client arguments
             rv = c.get(f"{ws_url}/ClientConfig/Arguments/$Client.Endpoint.region")
             self.assertTrue(rv.status_code == 200)