split emitter into autorest.python and pygen (#2588)

* initial moving over into pygen core * split package into pygen/core and autorest.python * black * pylint and mypy * add editable install of pygen into autorest.python * move files over to pygen * basic generaiton possible from autorest with script files moved to pygen * autorest generation working from scripts in pygen file * require changelog check * add changeset * add pcgc to changelog * comment out pylinting * add back linting and check * list installed packages * list installed packages * activate env in pipeline * add pygen to common reqs * change path to be relative from autorest.python * pylint * pyright * add pygen to setup.py, remove from requirements.txt * Revert "add pygen to setup.py, remove from requirements.txt" This reverts commit ea73b14. * add pygen to unittests requirements.txt * update unittests * hopefully fix multiapi * keep it as just one venv in pygen * typespec compiling with pygen inside dist * can copy pygen into dist when building and generate all typespec * trying to remove packaging files from pygen * typespec and autorest.python working with vendored pygen * update chronus * fix version mismatch * lockfile * add venv folder path * fix template * add venv folder path with $ sign * don't put venv in the venvFolderPath * add fi to venv script * add tilde to semver * rmeove venv from pipelines, too complicated * udpate lockfile * revert dev_requirements.txt back to current in main * remove pygen from reqs list * add pygen to dev reqs * add pygen back to unittests reqs * temp * continue moving pygen to into typespec-python * add dep on typespec-python in autorest.python * can generate autorest * can generate typespec locally * update reqs * duplicate runpython3 script in autorest, fix dev reqs path * move scripts for typespec out of pygen folder * add scripts to incldued files * copyfiles venv into autorest.python * duplicate venv * move venv into pygen folder * can generate typespec * fix installation for tsp and try to copy pygen into autorest.python * use own script for copying pygen * fix readme and setup.py of autorest.pyton * copy pygen into autorest.python * try moving pygen up a folder * make pygen a package again * keep trying with setup.py file * remove generator from autorest.python committed code, can generate typespec * move venv into top level of package * get autorest.python running * add generator to dev reqs * fix venvtools path * update dev reqs * add generator to published files * update script * update * update --------- Co-authored-by: iscai-msft <[email protected]> Co-authored-by: Chenjie Shi <[email protected]>
Azure · Jul 3, 2024 · 48d1426 · 48d1426
1 parent ad59817
commit 48d1426
Show file tree

Hide file tree

Showing 147 changed files with 949 additions and 347 deletions.
diff --git a/.chronus/changes/link_emitters-2024-4-13-14-39-2.md b/.chronus/changes/link_emitters-2024-4-13-14-39-2.md
@@ -0,0 +1,8 @@
+---
+changeKind: feature
+packages:
+  - "@autorest/python"
+  - "@azure-tools/typespec-python"
+---
+
+add package pygen that both autorest.python and typespec-python will rely on
diff --git a/.gitignore b/.gitignore
@@ -125,3 +125,4 @@ node_modules/
 
 # Generated test folders
 test/services/*/_generated
+**/autorest.python/generator
diff --git a/eng/pipelines/ci-template.yml b/eng/pipelines/ci-template.yml
@@ -69,10 +69,18 @@ steps:
       displayName: Build project
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/
 
+    - script: pip list
+      displayName: List installed packages
+      workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
+
     - script: pip install -r dev_requirements.txt
       displayName: Pip install dev requirements
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
 
+    - script: pip list
+      displayName: List installed packages
+      workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}
+
     - script: pylint ${{parameters.pythonFolderName}}
       displayName: Pylint
       workingDirectory: $(Build.SourcesDirectory)/autorest.python/packages/${{parameters.folderName}}

diff --git a/packages/autorest.python/autorest/__init__.py b/packages/autorest.python/autorest/__init__.py
@@ -5,59 +5,18 @@
 # --------------------------------------------------------------------------
 import logging
 from pathlib import Path
-import json
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from typing import Any, Dict, Union, List
 
 import yaml
 
+from pygen import ReaderAndWriter, Plugin, YamlUpdatePlugin
 from .jsonrpc import AutorestAPI
-from ._version import VERSION
 
 
-__version__ = VERSION
 _LOGGER = logging.getLogger(__name__)
 
 
-class ReaderAndWriter:
-    def __init__(self, *, output_folder: Union[str, Path], **kwargs: Any) -> None:
-        self.output_folder = Path(output_folder)
-        self._list_file: List[str] = []
-        try:
-            with open(
-                Path(self.output_folder) / Path("..") / Path("python.json"),
-                "r",
-                encoding="utf-8-sig",
-            ) as fd:
-                python_json = json.load(fd)
-        except Exception:  # pylint: disable=broad-except
-            python_json = {}
-        self.options = kwargs
-        if python_json:
-            _LOGGER.warning("Loading python.json file. This behavior will be depreacted")
-        self.options.update(python_json)
-
-    def read_file(self, path: Union[str, Path]) -> str:
-        """Directly reading from disk"""
-        # make path relative to output folder
-        try:
-            with open(self.output_folder / Path(path), "r", encoding="utf-8-sig") as fd:
-                return fd.read()
-        except FileNotFoundError:
-            return ""
-
-    def write_file(self, filename: Union[str, Path], file_content: str) -> None:
-        """Directly writing to disk"""
-        file_folder = Path(filename).parent
-        if not Path.is_dir(self.output_folder / file_folder):
-            Path.mkdir(self.output_folder / file_folder, parents=True)
-        with open(self.output_folder / Path(filename), "w", encoding="utf-8") as fd:
-            fd.write(file_content)
-
-    def list_file(self) -> List[str]:
-        return [str(f) for f in self.output_folder.glob("**/*") if f.is_file()]
-
-
 class ReaderAndWriterAutorest(ReaderAndWriter):
     def __init__(self, *, output_folder: Union[str, Path], autorestapi: AutorestAPI) -> None:
         super().__init__(output_folder=output_folder)
@@ -73,23 +32,6 @@ def list_file(self) -> List[str]:
         return self._autorestapi.list_inputs()
 
 
-class Plugin(ReaderAndWriter, ABC):
-    """A base class for autorest plugin.
-
-    :param autorestapi: An autorest API instance
-    """
-
-    @abstractmethod
-    def process(self) -> bool:
-        """The plugin process.
-
-        :rtype: bool
-        :returns: True if everything's ok, False optherwise
-        :raises Exception: Could raise any exception, stacktrace will be sent to autorest API
-        """
-        raise NotImplementedError()
-
-
 class PluginAutorest(Plugin, ReaderAndWriterAutorest):
     """For our Autorest plugins, we want to take autorest api as input as options, then pass it to the Plugin"""
 
@@ -102,39 +44,6 @@ def get_options(self) -> Dict[str, Any]:
         """Get the options bag using the AutorestAPI that we send to the parent plugin"""
 
 
-class YamlUpdatePlugin(Plugin):
-    """A plugin that update the YAML as input."""
-
-    def get_yaml(self) -> Dict[str, Any]:
-        # cadl file doesn't have to be relative to output folder
-        with open(self.options["cadl_file"], "r", encoding="utf-8-sig") as fd:
-            return yaml.safe_load(fd.read())
-
-    def write_yaml(self, yaml_string: str) -> None:
-        with open(self.options["cadl_file"], "w", encoding="utf-8-sig") as fd:
-            fd.write(yaml_string)
-
-    def process(self) -> bool:
-        # List the input file, should be only one
-        yaml_data = self.get_yaml()
-
-        self.update_yaml(yaml_data)
-
-        yaml_string = yaml.safe_dump(yaml_data)
-
-        self.write_yaml(yaml_string)
-        return True
-
-    @abstractmethod
-    def update_yaml(self, yaml_data: Dict[str, Any]) -> None:
-        """The code-model-v4-no-tags yaml model tree.
-
-        :rtype: updated yaml
-        :raises Exception: Could raise any exception, stacktrace will be sent to autorest API
-        """
-        raise NotImplementedError()
-
-
 class YamlUpdatePluginAutorest(YamlUpdatePlugin, PluginAutorest):  # pylint: disable=abstract-method
     def get_yaml(self) -> Dict[str, Any]:
         return yaml.safe_load(self.read_file("code-model-v4-no-tags.yaml"))

diff --git a/packages/autorest.python/autorest/black.py b/packages/autorest.python/autorest/black.py
@@ -0,0 +1,13 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Dict, Any
+from pygen.black import BlackScriptPlugin
+from . import PluginAutorest
+
+
+class BlackScriptPluginAutorest(BlackScriptPlugin, PluginAutorest):
+    def get_options(self) -> Dict[str, Any]:
+        return {"output_folder": self._autorestapi.get_value("outputFolderUri")}
diff --git a/packages/autorest.python/autorest/codegen.py b/packages/autorest.python/autorest/codegen.py
@@ -0,0 +1,117 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+import logging
+from typing import Dict, Any, Union
+from pathlib import Path
+import yaml
+
+from pygen.codegen import CodeGenerator
+from pygen.codegen.models import CodeModel
+from pygen.codegen.serializers import JinjaSerializer
+
+from . import ReaderAndWriterAutorest
+from .jsonrpc import AutorestAPI
+from . import PluginAutorest
+
+
+_LOGGER = logging.getLogger(__name__)
+
+
+class JinjaSerializerAutorest(JinjaSerializer, ReaderAndWriterAutorest):
+    def __init__(
+        self,
+        autorestapi: AutorestAPI,
+        code_model: CodeModel,
+        *,
+        output_folder: Union[str, Path],
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(  # type: ignore
+            autorestapi=autorestapi,
+            code_model=code_model,
+            output_folder=output_folder,
+            **kwargs,
+        )
+
+
+class CodeGeneratorAutorest(CodeGenerator, PluginAutorest):
+    def get_options(self) -> Dict[str, Any]:
+        if self._autorestapi.get_boolean_value("python3-only") is False:
+            _LOGGER.warning("You have passed in --python3-only=False. We have force overriden this to True.")
+        if self._autorestapi.get_boolean_value("add-python3-operation-files"):
+            _LOGGER.warning(
+                "You have passed in --add-python3-operation-files. "
+                "This flag no longer has an effect bc all SDKs are now Python3 only."
+            )
+        if self._autorestapi.get_boolean_value("reformat-next-link"):
+            _LOGGER.warning(
+                "You have passed in --reformat-next-link. We have force overriden "
+                "this to False because we no longer reformat initial query parameters into next "
+                "calls unless explicitly defined in the service definition."
+            )
+        options = {
+            "azure-arm": self._autorestapi.get_boolean_value("azure-arm"),
+            "header-text": self._autorestapi.get_value("header-text"),
+            "low-level-client": self._autorestapi.get_boolean_value("low-level-client", False),
+            "version-tolerant": self._autorestapi.get_boolean_value("version-tolerant", True),
+            "show-operations": self._autorestapi.get_boolean_value("show-operations"),
+            "python3-only": self._autorestapi.get_boolean_value("python3-only"),
+            "head-as-boolean": self._autorestapi.get_boolean_value("head-as-boolean", False),
+            "keep-version-file": self._autorestapi.get_boolean_value("keep-version-file"),
+            "no-async": self._autorestapi.get_boolean_value("no-async"),
+            "no-namespace-folders": self._autorestapi.get_boolean_value("no-namespace-folders"),
+            "basic-setup-py": self._autorestapi.get_boolean_value("basic-setup-py"),
+            "package-name": self._autorestapi.get_value("package-name"),
+            "package-version": self._autorestapi.get_value("package-version"),
+            "client-side-validation": self._autorestapi.get_boolean_value("client-side-validation"),
+            "tracing": self._autorestapi.get_boolean_value("trace"),
+            "multiapi": self._autorestapi.get_boolean_value("multiapi", False),
+            "polymorphic-examples": self._autorestapi.get_value("polymorphic-examples"),
+            "models-mode": self._autorestapi.get_value("models-mode"),
+            "builders-visibility": self._autorestapi.get_value("builders-visibility"),
+            "show-send-request": self._autorestapi.get_boolean_value("show-send-request"),
+            "only-path-and-body-params-positional": self._autorestapi.get_boolean_value(
+                "only-path-and-body-params-positional"
+            ),
+            "combine-operation-files": self._autorestapi.get_boolean_value("combine-operation-files"),
+            "package-mode": self._autorestapi.get_value("package-mode"),
+            "package-pprint-name": self._autorestapi.get_value("package-pprint-name"),
+            "packaging-files-config": self._autorestapi.get_value("package-configuration"),
+            "default-optional-constants-to-none": self._autorestapi.get_boolean_value(
+                "default-optional-constants-to-none"
+            ),
+            "generate-sample": self._autorestapi.get_boolean_value("generate-sample"),
+            "generate-test": self._autorestapi.get_boolean_value("generate-test"),
+            "default-api-version": self._autorestapi.get_value("default-api-version"),
+        }
+        return {k: v for k, v in options.items() if v is not None}
+
+    def get_yaml(self) -> Dict[str, Any]:
+        inputs = self._autorestapi.list_inputs()
+        _LOGGER.debug("Possible Inputs: %s", inputs)
+        if "code-model-v4-no-tags.yaml" not in inputs:
+            raise ValueError("code-model-v4-no-tags.yaml must be a possible input")
+
+        if self._autorestapi.get_value("input-yaml"):
+            input_yaml = self._autorestapi.get_value("input-yaml")
+            file_content = self._autorestapi.read_file(input_yaml)
+        else:
+            inputs = self._autorestapi.list_inputs()
+            _LOGGER.debug("Possible Inputs: %s", inputs)
+            if "code-model-v4-no-tags.yaml" not in inputs:
+                raise ValueError("code-model-v4-no-tags.yaml must be a possible input")
+
+            file_content = self._autorestapi.read_file("code-model-v4-no-tags.yaml")
+
+        # Parse the received YAML
+        return yaml.safe_load(file_content)
+
+    def get_serializer(self, code_model: CodeModel):
+        return JinjaSerializerAutorest(
+            self._autorestapi,
+            code_model,
+            output_folder=self.output_folder,
+        )
diff --git a/packages/autorest.python/autorest/m2r.py b/packages/autorest.python/autorest/m2r.py
@@ -0,0 +1,16 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""An autorest MD to RST plugin.
+"""
+from typing import Any, Dict
+
+from pygen.m2r import M2R
+from . import YamlUpdatePluginAutorest
+
+
+class M2RAutorest(YamlUpdatePluginAutorest, M2R):
+    def get_options(self) -> Dict[str, Any]:
+        return {}
diff --git a/packages/autorest.python/autorest/m4reformatter/__init__.py b/packages/autorest.python/autorest/m4reformatter/__init__.py
@@ -11,7 +11,7 @@
 import logging
 from typing import Callable, Dict, Any, Iterable, List, Optional, Set
 
-from .._utils import (
+from pygen.utils import (
     to_snake_case,
     KNOWN_TYPES,
     get_body_type_for_description,

diff --git a/packages/autorest.python/autorest/multiapi/__init__.py b/packages/autorest.python/autorest/multiapi/__init__.py
@@ -10,12 +10,13 @@
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Optional, cast, Any
+from pygen import Plugin, ReaderAndWriter
 
 from .serializers import MultiAPISerializer, MultiAPISerializerAutorest
 from .models import CodeModel
 from .utils import _get_default_api_version_from_list
 
-from .. import Plugin, PluginAutorest, ReaderAndWriter, ReaderAndWriterAutorest
+from .. import PluginAutorest, ReaderAndWriterAutorest
 
 _LOGGER = logging.getLogger(__name__)
 

diff --git a/packages/autorest.python/autorest/multiapi/serializers/__init__.py b/packages/autorest.python/autorest/multiapi/serializers/__init__.py
@@ -6,13 +6,15 @@
 from pathlib import Path
 from typing import Any, Optional, Union, List
 from jinja2 import PackageLoader, Environment
+from pygen import ReaderAndWriter
+from pygen.utils import build_policies
 
 from .import_serializer import FileImportSerializer
 
 from ...jsonrpc import AutorestAPI
 from ..models import CodeModel, GlobalParameter
-from ... import ReaderAndWriter, ReaderAndWriterAutorest
-from ..._utils import build_policies
+from ... import ReaderAndWriterAutorest
+
 
 __all__ = [
     "MultiAPISerializer",
@@ -123,7 +125,7 @@ def serialize(self, code_model: CodeModel, no_async: Optional[bool]) -> None:
 
         if not code_model.client.client_side_validation:
             codegen_env = Environment(
-                loader=PackageLoader("autorest.codegen", "templates"),
+                loader=PackageLoader("pygen.codegen", "templates"),
                 keep_trailing_newline=True,
                 line_statement_prefix="##",
                 line_comment_prefix="###",

diff --git a/packages/autorest.python/autorest/multiclient/__init__.py b/packages/autorest.python/autorest/multiclient/__init__.py
@@ -7,7 +7,8 @@
 from typing import Any, Dict
 from pathlib import Path
 from jinja2 import Environment, PackageLoader
-from .. import Plugin, PluginAutorest
+from pygen import Plugin
+from .. import PluginAutorest
 
 _LOGGER = logging.getLogger(__name__)
 

diff --git a/packages/autorest.python/autorest/postprocess.py b/packages/autorest.python/autorest/postprocess.py
@@ -0,0 +1,14 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+from typing import Any, Dict
+
+from pygen.postprocess import PostProcessPlugin
+from . import PluginAutorest
+
+
+class PostProcessPluginAutorest(PostProcessPlugin, PluginAutorest):
+    def get_options(self) -> Dict[str, Any]:
+        return {"outputFolderUri": self._autorestapi.get_value("outputFolderUri")}