[tools/generator] add automation and release command for track2

Gopkg.toml

@@ -19,7 +19,7 @@
 #  name = "github.com/x/y"
 #  version = "2.4.0"
 
-ignored = ["github.com/ahmetb/go-linq/*", "github.com/google/go-github/*"]
+ignored = ["github.com/ahmetb/go-linq/*", "github.com/google/go-github/*", "github.com/go-git/go-git/*"]
 
 [prune]
   unused-packages = true

eng/scripts/automation_init.sh

@@ -0,0 +1,19 @@
+set -x
+set -e
+export GOPATH="$(realpath ../../../..)"
+if [ ! -d "$GOPATH/bin" ]
+then
+  mkdir $GOPATH/bin
+fi
+PATH=$PATH:$GOPATH/bin
+export GO111MODULE=on
+cd tools/generator && go build && cp generator $GOPATH/bin && cd ../..
+ln -s /usr/bin/pwsh $GOPATH/bin/pwsh.exe
+cat > $2 << EOF
+{
+  "envs": {
+    "PATH": "$PATH:$GOPATH",
+    "GOPATH": "$GOPATH"
+  }
+}
+EOF

eng/scripts/build.ps1

@@ -63,7 +63,7 @@ $root = Resolve-Path ($PSScriptRoot + "/../..")
 Set-Location $root
 $sdks = @{};
 
-foreach ($sdk in (Get-ModuleDirs 'sdk/...')) {
+foreach ($sdk in (Get-ModuleDirs 'sdk/')) {
     $name = $sdk | split-path -leaf
     $sdks[$name] = @{
         'path' = $sdk;

eng/swagger_to_sdk_config.json

@@ -0,0 +1,28 @@
+{
+  "$schema": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/documentation/sdkautomation/SwaggerToSdkConfigSchema.json",
+  "generateOptions": {
+    "generateScript": {
+      "path": "generator automation-v2",
+      "stderr": {
+        "showInComment": "^\\[AUTOREST\\]",
+        "scriptError": "^\\[ERROR\\]",
+        "scriptWarning": "^\\[WARNING\\]"
+      }
+    }
+  },
+  "advancedOptions": {
+    "cloneDir": "src/github.com/Azure/azure-sdk-for-go",
+    "breakingChangeTracking": true
+  },
+  "initOptions": {
+    "initScript": {
+      "path": "sh ./eng/scripts/automation_init.sh",
+      "stderr": {
+        "scriptWarning": false
+      }
+    }
+  },
+  "packageOptions": {
+    "breakingChangeLabel": "CI-BreakingChange-Go-V2"
+  }
+}

tools/generator/cmd/root.go

@@ -10,6 +10,8 @@ import (
 	"github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation"
 	"github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue"
 	"github.com/Azure/azure-sdk-for-go/tools/generator/cmd/template"
+	automation_v2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/automation"
+	release_v2 "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/release"
 	"github.com/spf13/cobra"
 )
 
@@ -29,6 +31,8 @@ func Command() *cobra.Command {
 
 	rootCmd.AddCommand(
 		automation.Command(),
+		automation_v2.Command(),
+		release_v2.Command(),
 		issue.Command(),
 		template.Command(),
 	)

tools/generator/cmd/v2/automation/automationCmd.go

@@ -0,0 +1,150 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+package automation
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/pipeline"
+	"github.com/Azure/azure-sdk-for-go/tools/generator/cmd/v2/common"
+	"github.com/Azure/azure-sdk-for-go/tools/internal/utils"
+	"github.com/spf13/cobra"
+)
+
+// Command returns the automation v2 command. Note that this command is designed to run in the root directory of
+// azure-sdk-for-go. It does not work if you are running this tool in somewhere else
+func Command() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:  "automation-v2 <generate input filepath> <generate output filepath>",
+		Args: cobra.ExactArgs(2),
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			log.SetFlags(0) // remove the time stamp prefix
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := execute(args[0], args[1]); err != nil {
+				logError(err)
+				return err
+			}
+			return nil
+		},
+		SilenceUsage: true, // this command is used for a pipeline, the usage should never show
+	}
+
+	return cmd
+}
+
+func execute(inputPath, outputPath string) error {
+	log.Printf("Reading generate input file from '%s'...", inputPath)
+	input, err := pipeline.ReadInput(inputPath)
+	if err != nil {
+		return fmt.Errorf("cannot read generate input: %+v", err)
+	}
+	log.Printf("Generating using the following GenerateInput...\n%s", input.String())
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	log.Printf("Using current directory as SDK root: %s", cwd)
+
+	ctx := automationContext{
+		sdkRoot:    utils.NormalizePath(cwd),
+		specRoot:   input.SpecFolder,
+		commitHash: input.HeadSha,
+	}
+	output, err := ctx.generate(input)
+	if err != nil {
+		return err
+	}
+	log.Printf("Output generated: \n%s", output.String())
+	log.Printf("Writing output to file '%s'...", outputPath)
+	if err := pipeline.WriteOutput(outputPath, output); err != nil {
+		return fmt.Errorf("cannot write generate output: %+v", err)
+	}
+	return nil
+}
+
+type automationContext struct {
+	sdkRoot    string
+	specRoot   string
+	commitHash string
+}
+
+// TODO -- support dry run
+func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) {
+	if input.DryRun {
+		return nil, fmt.Errorf("dry run not supported yet")
+	}
+
+	// iterate over all the readme
+	results := make([]pipeline.PackageResult, 0)
+	errorBuilder := generateErrorBuilder{}
+	for _, readme := range input.RelatedReadmeMdFiles {
+		log.Printf("Start to process readme file: %s", readme)
+		generateCtx := common.GenerateContext{
+			SdkPath:    ctx.sdkRoot,
+			SpecPath:   ctx.specRoot,
+			CommitHash: ctx.commitHash,
+		}
+
+		namespaceResults, errors := generateCtx.GenerateForAutomation(readme, input.RepoHTTPSURL)
+		if len(errors) != 0 {
+			errorBuilder.add(errors...)
+			continue
+		}
+
+		for _, namespaceResult := range namespaceResults {
+			content := namespaceResult.ChangelogMd
+			breaking := namespaceResult.Changelog.HasBreakingChanges()
+			breakingChangeItems := namespaceResult.Changelog.GetBreakingChangeItems()
+
+			results = append(results, pipeline.PackageResult{
+				Version:     namespaceResult.Version,
+				PackageName: namespaceResult.PackageName,
+				Path:        []string{fmt.Sprintf("sdk/%s/%s", namespaceResult.RpName, namespaceResult.PackageName)},
+				ReadmeMd:    []string{readme},
+				Changelog: &pipeline.Changelog{
+					Content:             &content,
+					HasBreakingChange:   &breaking,
+					BreakingChangeItems: &breakingChangeItems,
+				},
+			})
+		}
+		log.Printf("Finish to process readme file: %s", readme)
+	}
+
+	return &pipeline.GenerateOutput{
+		Packages: results,
+	}, errorBuilder.build()
+}
+
+type generateErrorBuilder struct {
+	errors []error
+}
+
+func (b *generateErrorBuilder) add(err ...error) {
+	b.errors = append(b.errors, err...)
+}
+
+func (b *generateErrorBuilder) build() error {
+	if len(b.errors) == 0 {
+		return nil
+	}
+	var messages []string
+	for _, err := range b.errors {
+		messages = append(messages, err.Error())
+	}
+	return fmt.Errorf("total %d error(s): \n%s", len(b.errors), strings.Join(messages, "\n"))
+}
+
+func logError(err error) {
+	for _, line := range strings.Split(err.Error(), "\n") {
+		if l := strings.TrimSpace(line); l != "" {
+			log.Printf("[ERROR] %s", l)
+		}
+	}
+}

tools/generator/cmd/v2/common/cmdProcessor.go

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+package common
+
+import (
+	"fmt"
+	"log"
+	"os/exec"
+)
+
+// execute `go generate` command and fetch result
+func ExecuteGoGenerate(path string) error {
+	cmd := exec.Command("go", "generate")
+	cmd.Dir = path
+	output, err := cmd.CombinedOutput()
+	log.Printf("Result of `go generate` execution: \n%s", string(output))
+	if err != nil {
+		return fmt.Errorf("failed to execute go generate '%s': %+v", string(output), err)
+	}
+	return nil
+}