implement to load the powershell modules

Author: kairu-ms

src/aaz_dev/ps/api/powershell.py

@@ -54,8 +54,8 @@ def powershell_modules():
 def powershell_module(module_names):
     manager = PSModuleManager()
     if request.method == "GET":
-        result = manager.load_module(module_names)
-        # result = module.to_primitive()
+        module = manager.load_module(module_names)
+        result = module.to_primitive()
         result['url'] = url_for('powershell.powershell_module', module_names=result['name'])
     elif request.method == "PUT":
         raise NotImplementedError()

src/aaz_dev/ps/controller/autorest_configuration_generator.py

@@ -106,7 +106,8 @@ def generate_config(self):
         if not swagger_resources:
             raise ResourceNotFind("Resources not find in Swagger")
 
-        readme_parts= rp._readme_path.split(os.sep)
+        # TODO: use the correct readme file
+        readme_parts= rp._readme_paths[0].split(os.sep)
         ps_cfg.readme_file = '/'.join(readme_parts[readme_parts.index("specification"):])
         ps_cfg.version = "0.1.0"
         ps_cfg.module_name = mod_names.split("/")[0]

src/aaz_dev/ps/controller/ps_module_manager.py

@@ -1,9 +1,15 @@
 import logging
 import os
-import yaml
 
 from utils.config import Config
-
+from utils.plane import PlaneEnum
+from utils.readme_helper import parse_readme_file
+from ps.model import PSModuleConfig
+from swagger.controller.specs_manager import SwaggerSpecsManager
+from swagger.model.specs import SwaggerModule
+from command.controller.specs_manager import AAZSpecsManager
+from swagger.model.specs import OpenAPIResourceProvider
+from swagger.utils.tools import resolve_path_to_uri
 logger = logging.getLogger('backend')
 
 
@@ -12,6 +18,20 @@ class PSModuleManager:
     def __init__(self):
         module_folder = self._find_module_folder()
         self.folder = module_folder
+        self._aaz_specs = None
+        self._swagger_specs = None
+
+    @property
+    def aaz_specs(self):
+        if not self._aaz_specs:
+            self._aaz_specs = AAZSpecsManager()
+        return self._aaz_specs
+
+    @property
+    def swagger_specs(self):
+        if not self._swagger_specs:
+            self._swagger_specs = SwaggerSpecsManager()
+        return self._swagger_specs
 
     def _find_module_folder(self):
         powershell_folder = Config.POWERSHELL_PATH
@@ -48,12 +68,8 @@ def load_module(self, module_names):
         folder = os.path.join(self.folder, *module_names)
         if not os.path.exists(folder):
             raise ValueError(f"Module folder not found: '{folder}'")
-        autorest_config = self.load_autorest_config(module_names)
-        return {
-            **autorest_config,
-            "name": "/".join(module_names),
-            "folder": folder
-        }
+        config = self.load_module_config(module_names)
+        return config
 
     def load_autorest_config(self, module_names):
         if isinstance(module_names, str):
@@ -62,33 +78,96 @@ def load_autorest_config(self, module_names):
         readme_file = os.path.join(folder, "README.md")
         if not os.path.exists(readme_file):
             raise ValueError(f"README.md not found in: '{readme_file}'")
-        with open(readme_file, "r") as f:
-            content = f.readlines()
-        autorest_config = []
-        in_autorest_config_section = False
-        in_yaml_section = False
-        for line in content:
-            if line.strip().startswith("### AutoRest Configuration"):
-                in_autorest_config_section = True
-            elif in_autorest_config_section:
-                if line.strip().startswith("###"):
-                    break
-                if line.strip().startswith("```") and 'yaml' in line:
-                    in_yaml_section = True
-                elif in_yaml_section:
-                    if line.strip().startswith("```"):
-                        in_yaml_section = False
-                    else:
-                        if line.strip():
-                            autorest_config.append(line)
-                        else:
-                            autorest_config.append("")
-        autorest_config_raw = "\n".join(autorest_config)
+        content = parse_readme_file(readme_file)
+        return content['config'], content['title']
+
+    def load_module_config(self, module_names):
         try:
-            yaml_config = yaml.load(autorest_config_raw, Loader=yaml.FullLoader)
-        except Exception as e:
-            raise ValueError(f"Failed to parse autorest config: {e} for readme_file: {readme_file}")
-        return {
-            "autorest_config": yaml_config,
-            # "raw": autorest_config_raw  # can be used for directive merging
-        }
+            autorest_config, readme_title = self.load_autorest_config(module_names)
+        except:
+            logger.error(f"Failed to load autorest config for module: {module_names}, error: {e}")
+            raise
+        
+        config = PSModuleConfig()
+        config.name = "/".join(module_names)
+        config.folder = self.folder
+        if not autorest_config:
+            raise ValueError(f"autorest config not found in README.md for module: {config.name}")
+
+        # config.swagger = autorest_config
+        repo = autorest_config.get('repo', "https://github.com/Azure/azure-rest-api-specs/blob/$(commit)")
+        if commit := autorest_config.get('commit'):
+            repo = repo.replace("$(commit)", commit)
+        if "$(commit)" in repo:
+            # make sure the repo is valid https link or valid folder path
+            raise ValueError(f"commit is not defined in autorest config for module: {config.name}")
+        config.repo = repo
+
+        readme_file = None
+        for required_file in autorest_config['require']:
+            if required_file.startswith('$(repo)/') and required_file.endswith('/readme.md'):
+                readme_file = required_file.replace('$(repo)/', '')
+                break
+
+        if not readme_file:
+            # search the readme.md in the swagger specs folder
+            for input_file in autorest_config.get('input-file', []):
+                if "/specification/" in input_file:
+                    folder_names = input_file.split("/specification/")[1].split("/")[:-1]
+                    path = os.path.join(self.swagger_specs.specs.spec_folder_path, *folder_names)
+                    while path != self.swagger_specs.specs.spec_folder_path:
+                        if os.path.exists(os.path.join(path, "readme.md")):
+                            readme_file = os.path.join(path, "readme.md")
+                            break
+                        path = os.path.dirname(path)
+                    if readme_file:
+                        readme_file = resolve_path_to_uri(readme_file)
+                        break
+        if not readme_file:
+            raise ValueError(f"swagger readme.md not defined in autorest config for module: {config.name}")
+        
+        # use the local swagger specs to find the resource provider even the repo is in remote
+        # we can always suppose the local swagger specs will always be newer than the used commit in submitted azure.powershell code
+        rp = None
+        readme_config = None
+        plane = PlaneEnum.Mgmt if "resource-manager" in readme_file else PlaneEnum._Data
+        for module in self.swagger_specs.get_modules(plane):
+            module_relative_path = resolve_path_to_uri(module.folder_path) + "/"
+            if readme_file.startswith(module_relative_path):
+                for resource_provider in module.get_resource_providers():
+                    if not isinstance(resource_provider, OpenAPIResourceProvider):
+                        continue
+                    readme_config = resource_provider.load_readme_config(readme_file)
+                    if readme_config:
+                        rp = resource_provider
+                        break
+                if rp:
+                    break
+        if not rp:
+            raise ValueError(f"Resource provider not found in autorest config for module: {config.name}")
+        config.rp = rp
+        config.swagger = str(rp)
+
+        if tag := autorest_config.get('tag'):
+            config.tag = tag
+        if input_files := autorest_config.get('input-file'):
+            config.input_files = []
+            for input_file in input_files:
+                if input_file.startswith('$(repo)/'):
+                    input_file = input_file.replace('$(repo)/', '')
+                config.input_files.append(input_file)
+        if not config.input_files and not config.tag:
+            config.tag = readme_config.get('tag', None)
+
+        if readme_title.startswith("Az."):
+            config.service_name = readme_title.split(".")[1]
+        if title := autorest_config.get('title'):
+            config.title = title
+        else:
+            # get title from swagger readme
+            config.title = readme_config.get('title', None)
+        
+        if not config.title:
+            raise ValueError(f"Title not found in autorest config or swagger readme for module: {config.name}")
+
+        return config

src/aaz_dev/ps/model/__init__.py

@@ -0,0 +1 @@
+from ._module_config import PSModuleConfig

src/aaz_dev/ps/model/_module_config.py

@@ -0,0 +1,81 @@
+
+
+from schematics.models import Model
+from schematics.types import ModelType, DictType, StringType, ListType
+
+
+class PSModuleConfig(Model):
+    name = StringType(required=True)
+    folder = StringType(required=True)
+    repo = StringType(required=True)    # swagger repo path, https://github.com/Azure/<repo_name>/tree/<commit> or $(this-folder)/../../../<repo_name>
+    swagger = StringType(required=True) # swagger resource provider, <plane>/<path:mod_names>/ResourceProviders/<rp_name>
+
+    # use tag or input files to select the swagger apis
+    tag = StringType() # if the tag selected, the input_files will be ignored
+    input_files = ListType(
+        StringType(),
+        serialized_name='inputFiles',
+        deserialize_from='inputFiles',
+    )  # The input file should not contain $(repo) and can be directly appended to the repo
+
+    title = StringType(required=True) # the required value for the autorest configuration
+    service_name = StringType(
+        required=True,
+        serialized_name='serviceName',
+        deserialize_from='serviceName',
+    )  # by default calculated from the title with this implementation https://github.com/Azure/autorest.powershell/blob/main/powershell/plugins/plugin-tweak-model.ts#L25-L33
+
+    # those default value defined in the noprofile.md configuration https://github.com/Azure/azure-powershell/blob/generation/src/readme.azure.noprofile.md
+    module_name = StringType(
+        required=True,
+        serialized_name='moduleName',
+        deserialize_from='moduleName',
+        default='$(prefix).$(service-name)'
+    )  # by default $(prefix).$(service-name)
+    namespace = StringType(
+        required=True,
+        default='Microsoft.Azure.PowerShell.Cmdlets.$(service-name)'
+    )  # used for sub module to define the powershell class namespace, by default Microsoft.Azure.PowerShell.Cmdlets.$(service-name)
+    subject_prefix = StringType(
+        required=True,
+        serialized_name='subjectPrefix',
+        deserialize_from='subjectPrefix',
+        default='$(service-name)'
+    )  # the default value $(service-name)
+    # root_module_name = StringType()   # used for sub module to generate the code in root module if there are multiple sub modules
+
+    prefix = StringType(
+        default='Az',
+    )  # Not allowed to change
+
+    class Options:
+        serialize_when_none = False
+
+    # swagger related properties
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.rp = None
+
+    @property
+    def repo_name(self):
+        return self.repo.split('/tree/', 1)[0].split('/')[-1]
+    
+    @property
+    def commit(self):
+        parts = self.repo.split('/tree/', 1)
+        if len(parts) == 2:
+            return parts[1].split('/')[0]
+        return None
+    
+    @property
+    def plane(self):
+        return self.swagger.split('/')[0]
+
+    @property
+    def mod_names(self):
+        return self.swagger.split("/ResourceProviders/")[0].split('/')[1:]
+
+    @property
+    def rp_name(self):
+        return self.swagger.split("/ResourceProviders/")[1].split('/')[0]

src/aaz_dev/ps/tests/api_tests/test_powershell.py

@@ -1,11 +1,5 @@
 from ps.tests.common import CommandTestCase
 from utils.config import Config
-from utils.base64 import b64encode_str
-from utils.stage import AAZStageEnum
-# from cli.controller.az_module_manager import AzMainManager, AzExtensionManager
-import os
-import shutil
-import yaml
 
 
 class APIPowerShellTest(CommandTestCase):
@@ -19,45 +13,20 @@ def test_get_powershell_path(self):
             self.assertTrue(data["path"] == Config.POWERSHELL_PATH)
 
     def test_list_powershell_modules(self):
-        config_dict = {}
         with self.app.test_client() as c:
             rv = c.get("/PS/Powershell/Modules")
             self.assertTrue(rv.status_code == 200)
             data = rv.get_json()
             self.assertTrue(len(data) > 100)
             self.assertTrue(all(module["name"].endswith(".Autorest") for module in data))
             for module in data:
+                if module["name"] in [
+                    "Communication/EmailServicedata.Autorest",
+                    "ManagedServiceIdentity/ManagedServiceIdentity.Autorest", "VoiceServices/VoiceServices.Autorest",
+                    "Resources/MSGraph.Autorest", "Migrate/Migrate.Autorest"
+                ]:
+                    continue
                 request_url = module["url"]
                 rv = c.get(request_url)
                 self.assertTrue(rv.status_code == 200)
                 data = rv.get_json()
-                if data["autorest_config"] is None:
-                    continue
-                for key, value in data["autorest_config"].items():
-                    if key in ["directive", "commit", "input-file", "title", "module-version"]:
-                        continue
-                    if key not in config_dict:
-                        config_dict[key] = {
-                            "list": set(),
-                            "dict": {},
-                            "basic": set(),
-                        }
-                    if isinstance(value, list):
-                        config_dict[key]["list"].update(value)
-                    elif isinstance(value, dict):
-                        config_dict[key]["dict"].update(value)
-                    else:
-                        config_dict[key]["basic"].add(value)
-        for key, value in config_dict.items():
-            if not len(value["list"]):
-                del value["list"]
-            else:
-                value["list"] = sorted(list(value["list"]))
-            if not len(value["dict"]):
-                del value["dict"]
-            if not len(value["basic"]):
-                del value["basic"]
-            else:
-                value["basic"] = sorted(list(value["basic"]))
-        # with open("ps/templates/autorest/config_common_used_props.yaml", "w") as f:
-        #     yaml.dump(config_dict, f)

src/aaz_dev/swagger/model/specs/_resource_provider.py

@@ -80,29 +80,6 @@ def load_readme_config(self, readme_file):
                 return parse_readme_file(readme_path)['config']
         return None
 
-    @property
-    def default_tag(self):
-        if self._readme_path is None:
-            return None
-
-        with open(self._readme_path, 'r', encoding='utf-8') as f:
-            readme = f.read()
-        lines = readme.split('\n')
-        for i in range(len(lines)):
-            line = lines[i]
-            if line.startswith('### Basic Information'):
-                lines = lines[i+1:]
-                break
-        latest_tag = None
-        for i in range(len(lines)):
-            line = lines[i]
-            if line.startswith('##'):
-                break
-            if line.startswith('tag:'):
-                latest_tag = line.split(':')[-1].strip()
-                break
-        return latest_tag
-
     @property
     def tags(self):
         if self._tags is None: