From 1874a712b4f9ba29b39133508a16f07a7c83aee4 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 15:05:26 +0800 Subject: [PATCH 01/11] move the root command to automation command --- swagger_to_sdk_config.json | 2 +- tools/generator/README.md | 3 + .../generator/cmd/automation/automationCmd.go | 366 ++++++++++++++++++ .../cmd/{ => automation}/categorization.go | 2 +- tools/generator/cmd/{ => automation}/clean.go | 2 +- .../cmd/{ => automation}/generation.go | 4 +- .../cmd/{ => automation}/readVersion.go | 2 +- .../{ => automation}/validate/validation.go | 0 tools/generator/cmd/root.go | 354 +---------------- tools/generator/common/constants.go | 8 + 10 files changed, 390 insertions(+), 353 deletions(-) create mode 100644 tools/generator/README.md create mode 100644 tools/generator/cmd/automation/automationCmd.go rename tools/generator/cmd/{ => automation}/categorization.go (98%) rename tools/generator/cmd/{ => automation}/clean.go (98%) rename tools/generator/cmd/{ => automation}/generation.go (97%) rename tools/generator/cmd/{ => automation}/readVersion.go (97%) rename tools/generator/cmd/{ => automation}/validate/validation.go (100%) create mode 100644 tools/generator/common/constants.go diff --git a/swagger_to_sdk_config.json b/swagger_to_sdk_config.json index 396dcd5ff359..75ca61c2f749 100644 --- a/swagger_to_sdk_config.json +++ b/swagger_to_sdk_config.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/documentation/sdkautomation/SwaggerToSdkConfigSchema.json", "generateOptions": { "generateScript": { - "path": "go run ./tools/generator/main.go", + "path": "go run ./tools/generator/main.go automation", "stderr": { "showInComment": "^\\[AUTOREST\\]", "scriptError": "^\\[ERROR\\]", diff --git a/tools/generator/README.md b/tools/generator/README.md new file mode 100644 index 000000000000..c1ba83f18c0e --- /dev/null +++ b/tools/generator/README.md @@ -0,0 +1,3 @@ +# Generator + +This is a command line tool for generating new releases for `github.com/Azure/azure-sdk-for-go`. \ No newline at end of file diff --git a/tools/generator/cmd/automation/automationCmd.go b/tools/generator/cmd/automation/automationCmd.go new file mode 100644 index 000000000000..c80ff5582aff --- /dev/null +++ b/tools/generator/cmd/automation/automationCmd.go @@ -0,0 +1,366 @@ +package automation + +import ( + "bufio" + "fmt" + "log" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" + "github.com/Azure/azure-sdk-for-go/tools/generator/common" + "github.com/Azure/azure-sdk-for-go/tools/generator/pipeline" + "github.com/Azure/azure-sdk-for-go/tools/internal/exports" + "github.com/Azure/azure-sdk-for-go/tools/internal/packages/track1" + "github.com/Azure/azure-sdk-for-go/tools/internal/utils" + "github.com/spf13/cobra" +) + +func Command() *cobra.Command { + automationCmd := &cobra.Command{ + Use: "automation ", + Args: cobra.ExactArgs(2), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + log.SetFlags(0) // remove the time stamp prefix + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + optionPath, err := cmd.Flags().GetString("options") + if err != nil { + return err + } + return execute(args[0], args[1], Flags{ + OptionPath: optionPath, + }) + }, + SilenceUsage: true, // this command is used for a pipeline, the usage should never show + } + + flags := automationCmd.Flags() + flags.String("options", common.DefaultOptionPath, "Specify a file with the autorest options") + + return automationCmd +} + +// Flags ... +type Flags struct { + OptionPath string +} + +func execute(inputPath, outputPath string, flags Flags) error { + log.Printf("Reading generate input file from '%s'...", inputPath) + input, err := pipeline.ReadInput(inputPath) + if err != nil { + return fmt.Errorf("cannot read generate input: %+v", err) + } + log.Printf("Generating using the following GenerateInput...\n%s", input.String()) + cwd, err := os.Getwd() + if err != nil { + return err + } + log.Printf("Using current directory as SDK root: %s", cwd) + + ctx := automationContext{ + sdkRoot: utils.NormalizePath(cwd), + specRoot: input.SpecFolder, + commitHash: input.HeadSha, + optionPath: flags.OptionPath, + } + output, err := ctx.generate(input) + if err != nil { + return err + } + log.Printf("Output generated: \n%s", output.String()) + log.Printf("Writing output to file '%s'...", outputPath) + if err := pipeline.WriteOutput(outputPath, output); err != nil { + return fmt.Errorf("cannot write generate output: %+v", err) + } + return nil +} + +func tempDir() string { + if dir := os.Getenv("TMP_DIR"); dir != "" { + return dir + } + return os.TempDir() +} + +type automationContext struct { + sdkRoot string + specRoot string + commitHash string + optionPath string + + repoContent map[string]exports.Content + + sdkVersion string + + existingPackages existingPackageMap + + defaultOptions model.Options + additionalOptions []model.Option +} + +func (ctx *automationContext) categorizePackages() error { + ctx.existingPackages = existingPackageMap{} + + serviceRoot := filepath.Join(ctx.sdkRoot, "services") + m, err := autorest.CollectGenerationMetadata(serviceRoot) + if err != nil { + return err + } + + for path, metadata := range m { + // the path in the metadata map is the absolute path + relPath, err := filepath.Rel(ctx.sdkRoot, path) + if err != nil { + return err + } + ctx.existingPackages.add(utils.NormalizePath(relPath), metadata) + } + + return nil +} + +func (ctx *automationContext) readDefaultOptions() error { + log.Printf("Reading defaultOptions from file '%s'...", ctx.optionPath) + optionFile, err := os.Open(ctx.optionPath) + if err != nil { + return err + } + + generateOptions, err := model.NewGenerateOptionsFrom(optionFile) + if err != nil { + return err + } + + // parsing the default options + defaultOptions, err := model.ParseOptions(generateOptions.AutorestArguments) + if err != nil { + return fmt.Errorf("cannot parse default options from %v: %+v", generateOptions.AutorestArguments, err) + } + + // remove the `--multiapi` in default options + var options []model.Option + for _, o := range defaultOptions.Arguments() { + if v, ok := o.(model.FlagOption); ok && v.Flag() == "multiapi" { + continue + } + options = append(options, o) + } + + ctx.defaultOptions = model.NewOptions(options...) + log.Printf("Autorest defaultOptions: \n%+v", ctx.defaultOptions.Arguments()) + + // parsing the additional options + additionalOptions, err := model.ParseOptions(generateOptions.AdditionalOptions) + if err != nil { + return fmt.Errorf("cannot parse additional options from %v: %+v", generateOptions.AdditionalOptions, err) + } + ctx.additionalOptions = additionalOptions.Arguments() + + return nil +} + +// TODO -- support dry run +func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) { + if input.DryRun { + return nil, fmt.Errorf("dry run not supported yet") + } + + log.Printf("Reading packages in azure-sdk-for-go...") + if err := ctx.readRepoContent(); err != nil { + return nil, err + } + + log.Printf("Reading metadata information in azure-sdk-for-go...") + if err := ctx.categorizePackages(); err != nil { + return nil, err + } + + log.Printf("Reading default options...") + if err := ctx.readDefaultOptions(); err != nil { + return nil, err + } + + log.Printf("Reading version number...") + if err := ctx.readVersion(); err != nil { + return nil, err + } + + // iterate over all the readme + results := make([]pipeline.PackageResult, 0) + errorBuilder := generateErrorBuilder{} + for _, readme := range input.RelatedReadmeMdFiles { + generateCtx := generateContext{ + sdkRoot: ctx.sdkRoot, + specRoot: ctx.specRoot, + commitHash: ctx.commitHash, + repoContent: ctx.repoContent, + existingPackages: ctx.existingPackages[readme], + defaultOptions: ctx.defaultOptions, + } + + packageResults, errors := generateCtx.generate(readme) + if len(errors) != 0 { + errorBuilder.add(errors...) + continue + } + + // iterate over the changed packages + set := packageResultSet{} + for _, p := range packageResults { + log.Printf("Getting package result for package '%s'", p.Package.PackageName) + content := p.Package.Changelog.ToCompactMarkdown() + breaking := p.Package.Changelog.HasBreakingChanges() + breakingChangeItems := p.Package.Changelog.GetBreakingChangeItems() + set.add(pipeline.PackageResult{ + Version: ctx.sdkVersion, + PackageName: getPackageIdentifier(p.Package.PackageName), + Path: []string{p.Package.PackageName}, + ReadmeMd: []string{readme}, + Changelog: &pipeline.Changelog{ + Content: &content, + HasBreakingChange: &breaking, + BreakingChangeItems: &breakingChangeItems, + }, + }) + } + results = append(results, set.toSlice()...) + } + + // validate the sdk structure + log.Printf("Validating services directory structure...") + exceptions, err := loadExceptions(filepath.Join(ctx.sdkRoot, "tools/pkgchk/exceptions.txt")) + if err != nil { + return nil, err + } + if err := track1.VerifyWithDefaultVerifiers(filepath.Join(ctx.sdkRoot, "services"), exceptions); err != nil { + return nil, err + } + + return &pipeline.GenerateOutput{ + Packages: results, + }, errorBuilder.build() +} + +func (ctx *automationContext) readRepoContent() error { + ctx.repoContent = make(map[string]exports.Content) + pkgs, err := track1.List(filepath.Join(ctx.sdkRoot, "services")) + if err != nil { + return fmt.Errorf("failed to list track 1 packages: %+v", err) + } + + for _, pkg := range pkgs { + relativePath, err := filepath.Rel(ctx.sdkRoot, pkg.FullPath()) + if err != nil { + return err + } + relativePath = utils.NormalizePath(relativePath) + if _, ok := ctx.repoContent[relativePath]; ok { + return fmt.Errorf("duplicate package: %s", pkg.Path()) + } + exp, err := exports.Get(pkg.FullPath()) + if err != nil { + return err + } + ctx.repoContent[relativePath] = exp + } + + return nil +} + +func (ctx *automationContext) readVersion() error { + v, err := ReadVersion(filepath.Join(ctx.sdkRoot, "version")) + if err != nil { + return err + } + ctx.sdkVersion = v + return nil +} + +func contains(array []autorest.GenerateResult, item string) bool { + for _, r := range array { + if utils.NormalizePath(r.Package.PackageName) == utils.NormalizePath(item) { + return true + } + } + return false +} + +type generateErrorBuilder struct { + errors []error +} + +func (b *generateErrorBuilder) add(err ...error) { + b.errors = append(b.errors, err...) +} + +func (b *generateErrorBuilder) build() error { + if len(b.errors) == 0 { + return nil + } + var messages []string + for _, err := range b.errors { + messages = append(messages, err.Error()) + } + return fmt.Errorf("total %d error(s): \n%s", len(b.errors), strings.Join(messages, "\n")) +} + +type packageResultSet map[string]pipeline.PackageResult + +func (s packageResultSet) contains(r pipeline.PackageResult) bool { + _, ok := s[r.PackageName] + return ok +} + +func (s packageResultSet) add(r pipeline.PackageResult) { + if s.contains(r) { + log.Printf("[WARNING] The result set already contains key %s with value %+v, but we are still trying to insert a new value %+v on the same key", r.PackageName, s[r.PackageName], r) + } + s[r.PackageName] = r +} + +func (s packageResultSet) toSlice() []pipeline.PackageResult { + results := make([]pipeline.PackageResult, 0) + for _, r := range s { + results = append(results, r) + } + // sort the results + sort.SliceStable(results, func(i, j int) bool { + // we first clip the preview segment and then sort by string literal + pI := strings.Replace(results[i].PackageName, "preview/", "/", 1) + pJ := strings.Replace(results[j].PackageName, "preview/", "/", 1) + return pI > pJ + }) + return results +} + +func getPackageIdentifier(pkg string) string { + return strings.TrimPrefix(utils.NormalizePath(pkg), "services/") +} + +func loadExceptions(exceptFile string) (map[string]bool, error) { + if exceptFile == "" { + return nil, nil + } + f, err := os.Open(exceptFile) + if err != nil { + return nil, err + } + defer f.Close() + + exceptions := make(map[string]bool) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + exceptions[scanner.Text()] = true + } + if err = scanner.Err(); err != nil { + return nil, err + } + + return exceptions, nil +} diff --git a/tools/generator/cmd/categorization.go b/tools/generator/cmd/automation/categorization.go similarity index 98% rename from tools/generator/cmd/categorization.go rename to tools/generator/cmd/automation/categorization.go index caecd6e08569..4a636618824f 100644 --- a/tools/generator/cmd/categorization.go +++ b/tools/generator/cmd/automation/categorization.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package cmd +package automation import "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" diff --git a/tools/generator/cmd/clean.go b/tools/generator/cmd/automation/clean.go similarity index 98% rename from tools/generator/cmd/clean.go rename to tools/generator/cmd/automation/clean.go index a3652f6087af..e394a7ced235 100644 --- a/tools/generator/cmd/clean.go +++ b/tools/generator/cmd/automation/clean.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package cmd +package automation import ( "fmt" diff --git a/tools/generator/cmd/generation.go b/tools/generator/cmd/automation/generation.go similarity index 97% rename from tools/generator/cmd/generation.go rename to tools/generator/cmd/automation/generation.go index 88af4a65b212..089c121e3d80 100644 --- a/tools/generator/cmd/generation.go +++ b/tools/generator/cmd/automation/generation.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package cmd +package automation import ( "fmt" @@ -12,7 +12,7 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" - "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/validate" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/validate" "github.com/Azure/azure-sdk-for-go/tools/internal/exports" "github.com/Azure/azure-sdk-for-go/tools/internal/utils" ) diff --git a/tools/generator/cmd/readVersion.go b/tools/generator/cmd/automation/readVersion.go similarity index 97% rename from tools/generator/cmd/readVersion.go rename to tools/generator/cmd/automation/readVersion.go index 7a7952775ee0..fddbcb8e66b3 100644 --- a/tools/generator/cmd/readVersion.go +++ b/tools/generator/cmd/automation/readVersion.go @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -package cmd +package automation import ( "fmt" diff --git a/tools/generator/cmd/validate/validation.go b/tools/generator/cmd/automation/validate/validation.go similarity index 100% rename from tools/generator/cmd/validate/validation.go rename to tools/generator/cmd/automation/validate/validation.go diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index efc5fd98bb00..53b4f4255c7c 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -4,371 +4,31 @@ package cmd import ( - "bufio" "fmt" "log" - "os" - "path/filepath" - "sort" - "strings" - "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" - "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" - "github.com/Azure/azure-sdk-for-go/tools/generator/pipeline" - "github.com/Azure/azure-sdk-for-go/tools/internal/exports" - "github.com/Azure/azure-sdk-for-go/tools/internal/packages/track1" - "github.com/Azure/azure-sdk-for-go/tools/internal/utils" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation" "github.com/spf13/cobra" ) -const ( - defaultOptionPath = "generate_options.json" -) - // Command returns the command for the generator. Note that this command is designed to run in the root directory of // azure-sdk-for-go. It does not work if you are running this tool in somewhere else func Command() *cobra.Command { rootCmd := &cobra.Command{ - Use: "generator ", - Args: cobra.ExactArgs(2), + Use: "generator", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { log.SetFlags(0) // remove the time stamp prefix return nil }, RunE: func(cmd *cobra.Command, args []string) error { - optionPath, err := cmd.Flags().GetString("options") - if err != nil { - return err - } - return execute(args[0], args[1], Flags{ - OptionPath: optionPath, - }) + return fmt.Errorf("please specify a subcommand") }, - SilenceUsage: true, // this command is used for a pipeline, the usage should never show + Hidden: true, } - flags := rootCmd.Flags() - flags.String("options", defaultOptionPath, "Specify a file with the autorest options") + rootCmd.AddCommand( + automation.Command(), + ) return rootCmd } - -// Flags ... -type Flags struct { - OptionPath string -} - -func execute(inputPath, outputPath string, flags Flags) error { - log.Printf("Reading generate input file from '%s'...", inputPath) - input, err := pipeline.ReadInput(inputPath) - if err != nil { - return fmt.Errorf("cannot read generate input: %+v", err) - } - log.Printf("Generating using the following GenerateInput...\n%s", input.String()) - cwd, err := os.Getwd() - if err != nil { - return err - } - log.Printf("Using current directory as SDK root: %s", cwd) - - ctx := automationContext{ - sdkRoot: utils.NormalizePath(cwd), - specRoot: input.SpecFolder, - commitHash: input.HeadSha, - optionPath: flags.OptionPath, - } - output, err := ctx.generate(input) - if err != nil { - return err - } - log.Printf("Output generated: \n%s", output.String()) - log.Printf("Writing output to file '%s'...", outputPath) - if err := pipeline.WriteOutput(outputPath, output); err != nil { - return fmt.Errorf("cannot write generate output: %+v", err) - } - return nil -} - -func tempDir() string { - if dir := os.Getenv("TMP_DIR"); dir != "" { - return dir - } - return os.TempDir() -} - -type automationContext struct { - sdkRoot string - specRoot string - commitHash string - optionPath string - - repoContent map[string]exports.Content - - sdkVersion string - - existingPackages existingPackageMap - - defaultOptions model.Options - additionalOptions []model.Option -} - -func (ctx *automationContext) categorizePackages() error { - ctx.existingPackages = existingPackageMap{} - - serviceRoot := filepath.Join(ctx.sdkRoot, "services") - m, err := autorest.CollectGenerationMetadata(serviceRoot) - if err != nil { - return err - } - - for path, metadata := range m { - // the path in the metadata map is the absolute path - relPath, err := filepath.Rel(ctx.sdkRoot, path) - if err != nil { - return err - } - ctx.existingPackages.add(utils.NormalizePath(relPath), metadata) - } - - return nil -} - -func (ctx *automationContext) readDefaultOptions() error { - log.Printf("Reading defaultOptions from file '%s'...", ctx.optionPath) - optionFile, err := os.Open(ctx.optionPath) - if err != nil { - return err - } - - generateOptions, err := model.NewGenerateOptionsFrom(optionFile) - if err != nil { - return err - } - - // parsing the default options - defaultOptions, err := model.ParseOptions(generateOptions.AutorestArguments) - if err != nil { - return fmt.Errorf("cannot parse default options from %v: %+v", generateOptions.AutorestArguments, err) - } - - // remove the `--multiapi` in default options - var options []model.Option - for _, o := range defaultOptions.Arguments() { - if v, ok := o.(model.FlagOption); ok && v.Flag() == "multiapi" { - continue - } - options = append(options, o) - } - - ctx.defaultOptions = model.NewOptions(options...) - log.Printf("Autorest defaultOptions: \n%+v", ctx.defaultOptions.Arguments()) - - // parsing the additional options - additionalOptions, err := model.ParseOptions(generateOptions.AdditionalOptions) - if err != nil { - return fmt.Errorf("cannot parse additional options from %v: %+v", generateOptions.AdditionalOptions, err) - } - ctx.additionalOptions = additionalOptions.Arguments() - - return nil -} - -// TODO -- support dry run -func (ctx *automationContext) generate(input *pipeline.GenerateInput) (*pipeline.GenerateOutput, error) { - if input.DryRun { - return nil, fmt.Errorf("dry run not supported yet") - } - - log.Printf("Reading packages in azure-sdk-for-go...") - if err := ctx.readRepoContent(); err != nil { - return nil, err - } - - log.Printf("Reading metadata information in azure-sdk-for-go...") - if err := ctx.categorizePackages(); err != nil { - return nil, err - } - - log.Printf("Reading default options...") - if err := ctx.readDefaultOptions(); err != nil { - return nil, err - } - - log.Printf("Reading version number...") - if err := ctx.readVersion(); err != nil { - return nil, err - } - - // iterate over all the readme - results := make([]pipeline.PackageResult, 0) - errorBuilder := generateErrorBuilder{} - for _, readme := range input.RelatedReadmeMdFiles { - generateCtx := generateContext{ - sdkRoot: ctx.sdkRoot, - specRoot: ctx.specRoot, - commitHash: ctx.commitHash, - repoContent: ctx.repoContent, - existingPackages: ctx.existingPackages[readme], - defaultOptions: ctx.defaultOptions, - } - - packageResults, errors := generateCtx.generate(readme) - if len(errors) != 0 { - errorBuilder.add(errors...) - continue - } - - // iterate over the changed packages - set := packageResultSet{} - for _, p := range packageResults { - log.Printf("Getting package result for package '%s'", p.Package.PackageName) - content := p.Package.Changelog.ToCompactMarkdown() - breaking := p.Package.Changelog.HasBreakingChanges() - breakingChangeItems := p.Package.Changelog.GetBreakingChangeItems() - set.add(pipeline.PackageResult{ - Version: ctx.sdkVersion, - PackageName: getPackageIdentifier(p.Package.PackageName), - Path: []string{p.Package.PackageName}, - ReadmeMd: []string{readme}, - Changelog: &pipeline.Changelog{ - Content: &content, - HasBreakingChange: &breaking, - BreakingChangeItems: &breakingChangeItems, - }, - }) - } - results = append(results, set.toSlice()...) - } - - // validate the sdk structure - log.Printf("Validating services directory structure...") - exceptions, err := loadExceptions(filepath.Join(ctx.sdkRoot, "tools/pkgchk/exceptions.txt")) - if err != nil { - return nil, err - } - if err := track1.VerifyWithDefaultVerifiers(filepath.Join(ctx.sdkRoot, "services"), exceptions); err != nil { - return nil, err - } - - return &pipeline.GenerateOutput{ - Packages: results, - }, errorBuilder.build() -} - -func (ctx *automationContext) readRepoContent() error { - ctx.repoContent = make(map[string]exports.Content) - pkgs, err := track1.List(filepath.Join(ctx.sdkRoot, "services")) - if err != nil { - return fmt.Errorf("failed to list track 1 packages: %+v", err) - } - - for _, pkg := range pkgs { - relativePath, err := filepath.Rel(ctx.sdkRoot, pkg.FullPath()) - if err != nil { - return err - } - relativePath = utils.NormalizePath(relativePath) - if _, ok := ctx.repoContent[relativePath]; ok { - return fmt.Errorf("duplicate package: %s", pkg.Path()) - } - exp, err := exports.Get(pkg.FullPath()) - if err != nil { - return err - } - ctx.repoContent[relativePath] = exp - } - - return nil -} - -func (ctx *automationContext) readVersion() error { - v, err := ReadVersion(filepath.Join(ctx.sdkRoot, "version")) - if err != nil { - return err - } - ctx.sdkVersion = v - return nil -} - -func contains(array []autorest.GenerateResult, item string) bool { - for _, r := range array { - if utils.NormalizePath(r.Package.PackageName) == utils.NormalizePath(item) { - return true - } - } - return false -} - -type generateErrorBuilder struct { - errors []error -} - -func (b *generateErrorBuilder) add(err ...error) { - b.errors = append(b.errors, err...) -} - -func (b *generateErrorBuilder) build() error { - if len(b.errors) == 0 { - return nil - } - var messages []string - for _, err := range b.errors { - messages = append(messages, err.Error()) - } - return fmt.Errorf("total %d error(s): \n%s", len(b.errors), strings.Join(messages, "\n")) -} - -type packageResultSet map[string]pipeline.PackageResult - -func (s packageResultSet) contains(r pipeline.PackageResult) bool { - _, ok := s[r.PackageName] - return ok -} - -func (s packageResultSet) add(r pipeline.PackageResult) { - if s.contains(r) { - log.Printf("[WARNING] The result set already contains key %s with value %+v, but we are still trying to insert a new value %+v on the same key", r.PackageName, s[r.PackageName], r) - } - s[r.PackageName] = r -} - -func (s packageResultSet) toSlice() []pipeline.PackageResult { - results := make([]pipeline.PackageResult, 0) - for _, r := range s { - results = append(results, r) - } - // sort the results - sort.SliceStable(results, func(i, j int) bool { - // we first clip the preview segment and then sort by string literal - pI := strings.Replace(results[i].PackageName, "preview/", "/", 1) - pJ := strings.Replace(results[j].PackageName, "preview/", "/", 1) - return pI > pJ - }) - return results -} - -func getPackageIdentifier(pkg string) string { - return strings.TrimPrefix(utils.NormalizePath(pkg), "services/") -} - -func loadExceptions(exceptFile string) (map[string]bool, error) { - if exceptFile == "" { - return nil, nil - } - f, err := os.Open(exceptFile) - if err != nil { - return nil, err - } - defer f.Close() - - exceptions := make(map[string]bool) - scanner := bufio.NewScanner(f) - for scanner.Scan() { - exceptions[scanner.Text()] = true - } - if err = scanner.Err(); err != nil { - return nil, err - } - - return exceptions, nil -} diff --git a/tools/generator/common/constants.go b/tools/generator/common/constants.go new file mode 100644 index 000000000000..fe10e39e7d95 --- /dev/null +++ b/tools/generator/common/constants.go @@ -0,0 +1,8 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package common + +const ( + DefaultOptionPath = "generate_options.json" +) From 9ddcc16bded318658051910042c09d67ca06cc6d Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 15:07:57 +0800 Subject: [PATCH 02/11] refactor --- tools/generator/{ => cmd/automation}/pipeline/io.go | 0 tools/generator/{ => cmd/automation}/pipeline/model.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tools/generator/{ => cmd/automation}/pipeline/io.go (100%) rename tools/generator/{ => cmd/automation}/pipeline/model.go (100%) diff --git a/tools/generator/pipeline/io.go b/tools/generator/cmd/automation/pipeline/io.go similarity index 100% rename from tools/generator/pipeline/io.go rename to tools/generator/cmd/automation/pipeline/io.go diff --git a/tools/generator/pipeline/model.go b/tools/generator/cmd/automation/pipeline/model.go similarity index 100% rename from tools/generator/pipeline/model.go rename to tools/generator/cmd/automation/pipeline/model.go From fdb2144187dbc245eaf4502878b7b5be554db823 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 15:10:05 +0800 Subject: [PATCH 03/11] fix compile error --- tools/generator/cmd/automation/automationCmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generator/cmd/automation/automationCmd.go b/tools/generator/cmd/automation/automationCmd.go index c80ff5582aff..856127d901d2 100644 --- a/tools/generator/cmd/automation/automationCmd.go +++ b/tools/generator/cmd/automation/automationCmd.go @@ -11,8 +11,8 @@ import ( "github.com/Azure/azure-sdk-for-go/tools/generator/autorest" "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation/pipeline" "github.com/Azure/azure-sdk-for-go/tools/generator/common" - "github.com/Azure/azure-sdk-for-go/tools/generator/pipeline" "github.com/Azure/azure-sdk-for-go/tools/internal/exports" "github.com/Azure/azure-sdk-for-go/tools/internal/packages/track1" "github.com/Azure/azure-sdk-for-go/tools/internal/utils" From baf911f3f319658a9652fb5407c48ae7c2b264d9 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 15:57:10 +0800 Subject: [PATCH 04/11] fix ci --- tools/generator/cmd/root.go | 2 +- tools/generator/common/constants.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index 53b4f4255c7c..13cd8b2847c8 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -15,7 +15,7 @@ import ( // azure-sdk-for-go. It does not work if you are running this tool in somewhere else func Command() *cobra.Command { rootCmd := &cobra.Command{ - Use: "generator", + Use: "generator", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { log.SetFlags(0) // remove the time stamp prefix return nil diff --git a/tools/generator/common/constants.go b/tools/generator/common/constants.go index fe10e39e7d95..ca5de0c82afc 100644 --- a/tools/generator/common/constants.go +++ b/tools/generator/common/constants.go @@ -4,5 +4,6 @@ package common const ( + // DefaultOptionPath the default path of the option file DefaultOptionPath = "generate_options.json" ) From 88746a1dc9ecd7a0b8a3f0c07824bc8b10fee63a Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 16:46:29 +0800 Subject: [PATCH 05/11] update comment of Command group --- tools/generator/cmd/automation/automationCmd.go | 2 ++ tools/generator/cmd/root.go | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/generator/cmd/automation/automationCmd.go b/tools/generator/cmd/automation/automationCmd.go index 856127d901d2..5b2f9ad48591 100644 --- a/tools/generator/cmd/automation/automationCmd.go +++ b/tools/generator/cmd/automation/automationCmd.go @@ -19,6 +19,8 @@ import ( "github.com/spf13/cobra" ) +// Command returns the automation command. Note that this command is designed to run in the root directory of +// azure-sdk-for-go. It does not work if you are running this tool in somewhere else func Command() *cobra.Command { automationCmd := &cobra.Command{ Use: "automation ", diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index 13cd8b2847c8..f280dfb1b0bf 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -11,8 +11,7 @@ import ( "github.com/spf13/cobra" ) -// Command returns the command for the generator. Note that this command is designed to run in the root directory of -// azure-sdk-for-go. It does not work if you are running this tool in somewhere else +// Command returns the command for the generator func Command() *cobra.Command { rootCmd := &cobra.Command{ Use: "generator", From 7e1dd072e98963d6231c30b499e27c269bf9c8ee Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 17:07:33 +0800 Subject: [PATCH 06/11] add copy right headers --- tools/generator/autorest/model/options_test.go | 3 +++ tools/generator/autorest/validate.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tools/generator/autorest/model/options_test.go b/tools/generator/autorest/model/options_test.go index b291241b3be9..8af1565cceb8 100644 --- a/tools/generator/autorest/model/options_test.go +++ b/tools/generator/autorest/model/options_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package model_test import ( diff --git a/tools/generator/autorest/validate.go b/tools/generator/autorest/validate.go index 75afdaacc63b..eea213c61dac 100644 --- a/tools/generator/autorest/validate.go +++ b/tools/generator/autorest/validate.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package autorest import "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" From f012aa36ecdd2ddab6a2742d4752a0da8f85cc40 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 18:27:11 +0800 Subject: [PATCH 07/11] move the [ERROR] prefix to automation command - other command does not need this prefix --- tools/generator/cmd/automation/automationCmd.go | 17 +++++++++++++++-- tools/generator/main.go | 5 ----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tools/generator/cmd/automation/automationCmd.go b/tools/generator/cmd/automation/automationCmd.go index 5b2f9ad48591..fc1737803de6 100644 --- a/tools/generator/cmd/automation/automationCmd.go +++ b/tools/generator/cmd/automation/automationCmd.go @@ -32,11 +32,16 @@ func Command() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { optionPath, err := cmd.Flags().GetString("options") if err != nil { + logError(err) return err } - return execute(args[0], args[1], Flags{ + if err := execute(args[0], args[1], Flags{ OptionPath: optionPath, - }) + }); err != nil { + logError(err) + return err + } + return nil }, SilenceUsage: true, // this command is used for a pipeline, the usage should never show } @@ -366,3 +371,11 @@ func loadExceptions(exceptFile string) (map[string]bool, error) { return exceptions, nil } + +func logError(err error) { + for _, line := range strings.Split(err.Error(), "\n") { + if l := strings.TrimSpace(line); l != "" { + log.Printf("[ERROR] %s", l) + } + } +} diff --git a/tools/generator/main.go b/tools/generator/main.go index 120516e4cdb6..90ae165de94a 100644 --- a/tools/generator/main.go +++ b/tools/generator/main.go @@ -4,18 +4,13 @@ package main import ( - "log" "os" - "strings" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd" ) func main() { if err := cmd.Command().Execute(); err != nil { - for _, line := range strings.Split(err.Error(), "\n") { - log.Printf("[ERROR] %s", line) - } os.Exit(1) } } From 35f65b1789354c232b013dd654db61aac38c9882 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Tue, 1 Jun 2021 18:14:56 +0800 Subject: [PATCH 08/11] initial commit of the command migration --- tools/generator/README.md | 43 ++- tools/generator/cmd/issue/issueCmd.go | 265 ++++++++++++++++++ tools/generator/cmd/issue/link/commitLink.go | 70 +++++ .../cmd/issue/link/commitLink_test.go | 69 +++++ tools/generator/cmd/issue/link/common.go | 102 +++++++ tools/generator/cmd/issue/link/common_test.go | 44 +++ tools/generator/cmd/issue/link/consts.go | 18 ++ .../generator/cmd/issue/link/directoryLink.go | 53 ++++ tools/generator/cmd/issue/link/fileLink.go | 53 ++++ tools/generator/cmd/issue/link/link.go | 90 ++++++ .../cmd/issue/link/pullRequestLink.go | 98 +++++++ .../cmd/issue/link/pullRequestLink_test.go | 46 +++ tools/generator/cmd/issue/query/login.go | 59 ++++ tools/generator/cmd/issue/query/query.go | 33 +++ tools/generator/cmd/issue/request/handlers.go | 46 +++ tools/generator/cmd/issue/request/request.go | 194 +++++++++++++ .../cmd/issue/request/request_test.go | 52 ++++ tools/generator/cmd/root.go | 2 + tools/generator/common/constants.go | 7 + tools/generator/config/config.go | 23 ++ tools/generator/config/parse.go | 31 ++ tools/generator/config/parse_test.go | 137 +++++++++ tools/generator/config/reader.go | 26 ++ tools/generator/config/refresh.go | 51 ++++ tools/generator/config/releaseRequests.go | 81 ++++++ .../config/validate/localValidator.go | 91 ++++++ .../config/validate/remoteValidator.go | 104 +++++++ tools/generator/config/validate/validator.go | 25 ++ tools/generator/flags/flags.go | 47 ++++ tools/generator/go.mod | 5 + tools/generator/go.sum | 13 + 31 files changed, 1977 insertions(+), 1 deletion(-) create mode 100644 tools/generator/cmd/issue/issueCmd.go create mode 100644 tools/generator/cmd/issue/link/commitLink.go create mode 100644 tools/generator/cmd/issue/link/commitLink_test.go create mode 100644 tools/generator/cmd/issue/link/common.go create mode 100644 tools/generator/cmd/issue/link/common_test.go create mode 100644 tools/generator/cmd/issue/link/consts.go create mode 100644 tools/generator/cmd/issue/link/directoryLink.go create mode 100644 tools/generator/cmd/issue/link/fileLink.go create mode 100644 tools/generator/cmd/issue/link/link.go create mode 100644 tools/generator/cmd/issue/link/pullRequestLink.go create mode 100644 tools/generator/cmd/issue/link/pullRequestLink_test.go create mode 100644 tools/generator/cmd/issue/query/login.go create mode 100644 tools/generator/cmd/issue/query/query.go create mode 100644 tools/generator/cmd/issue/request/handlers.go create mode 100644 tools/generator/cmd/issue/request/request.go create mode 100644 tools/generator/cmd/issue/request/request_test.go create mode 100644 tools/generator/config/config.go create mode 100644 tools/generator/config/parse.go create mode 100644 tools/generator/config/parse_test.go create mode 100644 tools/generator/config/reader.go create mode 100644 tools/generator/config/refresh.go create mode 100644 tools/generator/config/releaseRequests.go create mode 100644 tools/generator/config/validate/localValidator.go create mode 100644 tools/generator/config/validate/remoteValidator.go create mode 100644 tools/generator/config/validate/validator.go create mode 100644 tools/generator/flags/flags.go diff --git a/tools/generator/README.md b/tools/generator/README.md index c1ba83f18c0e..e168352d69f2 100644 --- a/tools/generator/README.md +++ b/tools/generator/README.md @@ -1,3 +1,44 @@ # Generator -This is a command line tool for generating new releases for `github.com/Azure/azure-sdk-for-go`. \ No newline at end of file +This is a command line tool for generating new releases for `github.com/Azure/azure-sdk-for-go`. + +## Commands + +This CLI tool provides 2 commands now: `automation`, `issue`. + +### The `issue` command and the configuration file + +The `issue` command fetches the release request issues from [GitHub](https://github.com/Azure/sdk-release-request/issues?q=is%3Aissue+is%3Aopen+label%3AGo) and parses them into the configuration that other commands consume. The configuration will output to stdout. + +The configuration is a JSON string, which has the following pattern: +```json +{ + "track1Requests": { + "specification/network/resource-manager/readme.md": { + "package-2020-12-01": [ + { + "targetDate": "2021-02-11T00:00:00Z", + "requestLink": "https://github.com/Azure/sdk-release-request/issues/1212" + } + ] + } + }, + "track2Requests": {}, + "refresh": {} +} +``` +The keys of this JSON is the relative path of the `readme.md` file in `azure-rest-api-specs`. + +To authenticate this command, you need to either +1. Populate a personal access token by assigning the `-t` flag. +1. Populate the username, password (and OTP if necessary) by assigning the `-u`, `-p` and `--otp` flags. + +**Important notice:** +1. A release request by design can only have one RP in them, therefore if a release request is referencing a PR that contains changes of multiple RPs, the tool will just give an error. +1. A release request by design can only have one tag in them, therefore if a release request is requesting release on multiple tags, the tool will not give an error but output the plain value of the multiple tags without splitting them. +1. This command will try to output everything that it is able to parse, even some errors occur. + +Example usage: +```shell +generator issue -t $YOUR_PERSONAL_ACCESS_TOKEN > sdk-release.json +``` \ No newline at end of file diff --git a/tools/generator/cmd/issue/issueCmd.go b/tools/generator/cmd/issue/issueCmd.go new file mode 100644 index 000000000000..71bf0ad8b09f --- /dev/null +++ b/tools/generator/cmd/issue/issueCmd.go @@ -0,0 +1,265 @@ +package issue + +import ( + "context" + "encoding/json" + "fmt" + "log" + "time" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/request" + "github.com/Azure/azure-sdk-for-go/tools/generator/config" + "github.com/Azure/azure-sdk-for-go/tools/generator/config/validate" + "github.com/Azure/azure-sdk-for-go/tools/generator/flags" + "github.com/google/go-github/v32/github" + "github.com/hashicorp/go-multierror" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const ( + Repo = "sdk-release-request" + Owner = "Azure" +) + +// Command returns the issue command +func Command() *cobra.Command { + issueCmd := &cobra.Command{ + Use: "issue", + Short: "Fetch and parse the release request issues to get the configuration of the release", + Long: `This command fetches the release request from https://github.com/Azure/sdk-release-request/issues?q=is%3Aissue+is%3Aopen+label%3AGo +and produces the configuration to stdout which other command in this tool can consume. + +In order to query the issues from GitHub, you need to provide some authentication information to this command. +You can either use populate the personal access token by assigning the flag '-t', or you can use your account and +password (also otp if needed). + +WARNING: This command is still working in progress. The current version of this command cannot handle the request of +a data plane RP at all (an error will be thrown out). Use with caution. +`, + RunE: func(cmd *cobra.Command, args []string) error { + info := query.Info{ + UserInfo: query.UserInfo{ + Username: flags.GetString(cmd.Flags(), "username"), + Password: flags.GetString(cmd.Flags(), "password"), + Otp: flags.GetString(cmd.Flags(), "otp"), + }, + Token: flags.GetString(cmd.Flags(), "token"), + } + ctx := context.Background() + cmdCtx := &commandContext{ + ctx: ctx, + client: query.Login(ctx, info), + flags: ParseFlags(cmd.Flags()), + } + return cmdCtx.execute() + }, + } + + BindFlags(issueCmd.Flags()) + + return issueCmd +} + +// BindFlags binds the flags to this command +func BindFlags(flagSet *pflag.FlagSet) { + flagSet.StringP("username", "u", "", "Specify username of github account") + flagSet.StringP("password", "p", "", "Specify the password of github account") + flagSet.String("otp", "", "Specify the two-factor authentication code") + flagSet.StringP("token", "t", "", "Specify the personal access token") + flagSet.Bool("include-data-plane", false, "Specify whether we include the requests from data plane RPs") + flagSet.BoolP("skip-validate", "l", false, "Skip the validate for readme files and tags.") + flagSet.IntSlice("request-issues", []int{}, "Specify the release request IDs to parse.") + flagSet.StringSlice("additional-options", []string{"--enum-prefix"}, "Specify the default additional options for the upcoming new version of SDK.") +} + +// ParseFlags parses the flags to a Flags struct +func ParseFlags(flagSet *pflag.FlagSet) Flags { + return Flags{ + IncludeDataPlaneRequests: flags.GetBool(flagSet, "include-data-plane"), + SkipValidate: flags.GetBool(flagSet, "skip-validate"), + ReleaseRequestIDs: flags.GetIntSlice(flagSet, "request-issues"), + AdditionalOptions: flags.GetStringSlice(flagSet, "additional-options"), + } +} + +// Flags ... +type Flags struct { + IncludeDataPlaneRequests bool + SkipValidate bool + ReleaseRequestIDs []int + AdditionalOptions []string +} + +type commandContext struct { + ctx context.Context + client *query.Client + + flags Flags +} + +func (c *commandContext) execute() error { + issues, err := c.listIssues() + if err != nil { + return err + } + requests, reqErr := c.parseIssues(issues) + if reqErr != nil { + log.Printf("[ERROR] We are getting errors during parsing the release requests: %+v", reqErr) + } + log.Printf("Successfully parsed %d request(s)", len(requests)) + cfg, err := c.buildConfig(requests) + if err != nil { + return err + } + // write config to stdout + b, err := json.MarshalIndent(*cfg, "", " ") + if err != nil { + return err + } + fmt.Println(string(b)) + // we first output the config to stdout, then validate it so that the user could always get a usable config + // validate the config + if err := c.validateConfig(*cfg); err != nil { + return err + } + return reqErr +} + +func (c *commandContext) listIssues() ([]*github.Issue, error) { + if len(c.flags.ReleaseRequestIDs) == 0 { + return c.listOpenIssues() + } + + return c.listSpecifiedIssues(c.flags.ReleaseRequestIDs) +} + +func (c *commandContext) listOpenIssues() ([]*github.Issue, error) { + opt := &github.IssueListByRepoOptions{ + Labels: []string{"Go"}, + ListOptions: github.ListOptions{ + PerPage: 10, + }, + } + var issues []*github.Issue + for { + r, resp, err := c.client.Issues.ListByRepo(c.ctx, Owner, Repo, opt) + if err != nil { + return nil, err + } + issues = append(issues, r...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + return issues, nil +} + +func (c *commandContext) listSpecifiedIssues(ids []int) ([]*github.Issue, error) { + var issues []*github.Issue + for _, id := range ids { + issue, _, err := c.client.Issues.Get(c.ctx, Owner, Repo, id) + if err != nil { + return nil, err + } + + if !isGoReleaseRequest(issue) { + return nil, fmt.Errorf("release request '%s' is not a Go SDK release request", issue.GetHTMLURL()) + } + + issues = append(issues, issue) + } + + return issues, nil +} + +func issueHasLabel(issue *github.Issue, label string) bool { + if issue == nil { + return false + } + + for _, l := range issue.Labels { + if l.GetName() == label { + return true + } + } + + return false +} + +func isGoReleaseRequest(issue *github.Issue) bool { + return issueHasLabel(issue, "Go") +} + +func isTrack2ReleaseRequest(issue *github.Issue) bool { + return issueHasLabel(issue, "Track2") +} + +func (c *commandContext) parseIssues(issues []*github.Issue) ([]request.Request, error) { + var requests []request.Request + var errResult error + for _, issue := range issues { + if issue == nil { + continue + } + log.Printf("Parsing issue %s (%s)", issue.GetHTMLURL(), issue.GetTitle()) + req, err := request.ParseIssue(c.ctx, c.client, *issue, request.ParsingOptions{ + IncludeDataPlaneRequests: c.flags.IncludeDataPlaneRequests, + }) + if err != nil { + log.Printf("[ERROR] Cannot parse release request %s: %+v", issue.GetHTMLURL(), err) + errResult = multierror.Append(errResult, err) + continue + } + if req == nil { + continue + } + requests = append(requests, *req) + } + return requests, errResult +} + +func (c *commandContext) buildConfig(requests []request.Request) (*config.Config, error) { + track1Requests := config.Track1ReleaseRequests{} + track2Requests := config.Track2ReleaseRequests{} + for _, req := range requests { + switch req.Track { + case request.Track1: + track1Requests.Add(req.ReadmePath, req.Tag, config.ReleaseRequestInfo{ + TargetDate: timePtr(req.TargetDate), + RequestLink: req.RequestLink, + }) + case request.Track2: + track2Requests.Add(req.ReadmePath, config.Track2Request{ + ReleaseRequestInfo: config.ReleaseRequestInfo{ + TargetDate: timePtr(req.TargetDate), + RequestLink: req.RequestLink, + }, + PackageFlag: req.Tag, // TODO -- we need a better place to put this in the request + }) + default: + panic("unhandled track " + req.Track) + } + } + return &config.Config{ + Track1Requests: track1Requests, + Track2Requests: track2Requests, + AdditionalFlags: c.flags.AdditionalOptions, + }, nil +} + +func (c *commandContext) validateConfig(cfg config.Config) error { + if c.flags.SkipValidate { + return nil + } + log.Printf("Validating the generated config...") + validator := validate.NewRemoteValidator(c.ctx, c.client) + return validator.Validate(cfg) +} + +func timePtr(t time.Time) *time.Time { + return &t +} diff --git a/tools/generator/cmd/issue/link/commitLink.go b/tools/generator/cmd/issue/link/commitLink.go new file mode 100644 index 000000000000..c7f158e737df --- /dev/null +++ b/tools/generator/cmd/issue/link/commitLink.go @@ -0,0 +1,70 @@ +package link + +import ( + "context" + "fmt" + "regexp" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/ahmetb/go-linq/v3" + "github.com/google/go-github/v32/github" +) + +var ( + splitRegex = regexp.MustCompile(`[^A-Fa-f0-9]`) +) + +type commitLink struct { + linkBase + + rawLink string +} + +var _ CommitHashLink = (*commitLink)(nil) + +// NewCommitLink parses a commit link to its corresponding readme.md file link +func NewCommitLink(ctx context.Context, client *query.Client, requestLink, releaseLink string) Resolver { + segments := splitRegex.Split(strings.TrimPrefix(releaseLink, CommitPrefix), -1) + realLink := fmt.Sprintf("%s%s", CommitPrefix, segments[0]) + return &commitLink{ + linkBase: linkBase{ + ctx: ctx, + client: client, + releaseLink: realLink, + requestLink: requestLink, + }, + rawLink: releaseLink, + } +} + +// Resolve ... +func (l commitLink) Resolve() (ResolveResult, error) { + hash, err := l.GetCommitHash() + if err != nil { + return nil, err + } + commit, _, err := l.client.Repositories.GetCommit(l.ctx, SpecOwner, SpecRepo, hash) + if err != nil { + return nil, err + } + var filePaths []string + linq.From(commit.Files).Select(func(item interface{}) interface{} { + return item.(*github.CommitFile).GetFilename() + }).ToSlice(&filePaths) + readme, err := GetReadmePathFromChangedFiles(l.ctx, l.client, filePaths) + if err != nil { + return nil, fmt.Errorf("cannot resolve commit link '%s': %+v", l.GetReleaseLink(), err) + } + return getResult(readme), nil +} + +// GetCommitHash ... +func (l commitLink) GetCommitHash() (string, error) { + return getCommitRefFromLink(l.GetReleaseLink(), CommitPrefix) +} + +// String ... +func (l commitLink) String() string { + return l.GetReleaseLink() +} diff --git a/tools/generator/cmd/issue/link/commitLink_test.go b/tools/generator/cmd/issue/link/commitLink_test.go new file mode 100644 index 000000000000..dfe126dbc3a1 --- /dev/null +++ b/tools/generator/cmd/issue/link/commitLink_test.go @@ -0,0 +1,69 @@ +package link_test + +import ( + "context" + "testing" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/link" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +func TestCommitLink_GetCommitHash(t *testing.T) { + testdata := []struct { + input string + expected string + }{ + { + input: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + expected: "c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + }, + { + input: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c#diff-708c2fb843b022cac4af8c6f996a527440c1e0d328abb81f54670747bf14ab1a", + expected: "c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + }, + { + input: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c/somethingElse", + expected: "c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + }, + } + ctx := context.Background() + client := query.NewClient() + for _, c := range testdata { + l := link.NewCommitLink(ctx, client, "", c.input) + hash, err := l.(link.CommitHashLink).GetCommitHash() + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if hash != c.expected { + t.Fatalf("expected %s but got %s", c.expected, hash) + } + } +} + +func TestNewCommitLink(t *testing.T) { + testdata := []struct { + input string + expected string + }{ + { + input: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + expected: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + }, + { + input: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c#diff-708c2fb843b022cac4af8c6f996a527440c1e0d328abb81f54670747bf14ab1a", + expected: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + }, + { + input: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c/somethingElse", + expected: "commit/c40f8fafa2c9b1bc3bb81e1feea4ff715d48e00c", + }, + } + ctx := context.Background() + client := query.NewClient() + for _, c := range testdata { + l := link.NewCommitLink(ctx, client, "", c.input) + if l.GetReleaseLink() != c.expected { + t.Fatalf("expected %s but got %s", c.expected, l.GetReleaseLink()) + } + } +} diff --git a/tools/generator/cmd/issue/link/common.go b/tools/generator/cmd/issue/link/common.go new file mode 100644 index 000000000000..f91b972ff520 --- /dev/null +++ b/tools/generator/cmd/issue/link/common.go @@ -0,0 +1,102 @@ +package link + +import ( + "context" + "fmt" + "log" + "path/filepath" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/ahmetb/go-linq/v3" +) + +func init() { + cache = make(map[Readme]bool) +} + +const ( + mgmtSegment = "resource-manager" +) + +// Readme represents a readme filepath +type Readme string + +// IsMgmt returns true when the readme belongs to a mgmt plane package +func (r Readme) IsMgmt() bool { + return strings.Index(string(r), mgmtSegment) >= 0 +} + +// GetReadmePathFromChangedFiles ... +func GetReadmePathFromChangedFiles(ctx context.Context, client *query.Client, files []string) (Readme, error) { + // find readme files one by one + readmeFiles := make(map[Readme]bool) + for _, file := range files { + readme, err := GetReadmeFromPath(ctx, client, file) + if err != nil { + log.Printf("Changed file '%s' does not belong to any RP, ignoring", file) + continue + } + readmeFiles[readme] = true + } + if len(readmeFiles) > 1 { + return "", fmt.Errorf("cannot determine which RP to release because we have the following readme files involved: %+v", getMapKeys(readmeFiles)) + } + if len(readmeFiles) == 0 { + return "", fmt.Errorf("cannot get any readme files from these changed files: [%s]", strings.Join(files, ", ")) + } + // we only have one readme file + return getMapKeys(readmeFiles)[0], nil +} + +func getMapKeys(m map[Readme]bool) []Readme { + var result []Readme + linq.From(m).Select(func(kv interface{}) interface{} { + return kv.(linq.KeyValue).Key + }).ToSlice(&result) + return result +} + +var ( + cache map[Readme]bool +) + +// GetReadmeFromPath ... +func GetReadmeFromPath(ctx context.Context, client *query.Client, path string) (Readme, error) { + // we do not need to determine this is a path of file or directory + // we could always assume this is a directory path even if it is a file path - just waste a try in the first attempt if it is a filepath + return getReadmeFromDirectoryPath(ctx, client, path) +} + +func getReadmeFromDirectoryPath(ctx context.Context, client *query.Client, dir string) (Readme, error) { + if len(dir) == 0 || dir == "." { + return "", fmt.Errorf("cannot determine the readme.md path") + } + file := tryReadmePath(dir) + // use cache to short cut + if y, ok := cache[file]; ok { + if y { + return file, nil + } else { + return getReadmeFromDirectoryPath(ctx, client, filepathDir(dir)) + } + } + _, _, resp, err := client.Repositories.GetContents(ctx, SpecOwner, SpecRepo, string(file), nil) + if err == nil { + cache[file] = true + return file, nil + } + if resp != nil && resp.StatusCode == 404 { + cache[file] = false + return getReadmeFromDirectoryPath(ctx, client, filepathDir(dir)) + } + return "", err +} + +func tryReadmePath(base string) Readme { + return Readme(fmt.Sprintf("%s/readme.md", base)) +} + +func filepathDir(path string) string { + return strings.ReplaceAll(filepath.Dir(path), "\\", "/") +} diff --git a/tools/generator/cmd/issue/link/common_test.go b/tools/generator/cmd/issue/link/common_test.go new file mode 100644 index 000000000000..2a979dfe0737 --- /dev/null +++ b/tools/generator/cmd/issue/link/common_test.go @@ -0,0 +1,44 @@ +package link_test + +import ( + "context" + "os" + "testing" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/link" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +func TestGetReadmeFromPath(t *testing.T) { + token := os.Getenv("ACCESS_TOKEN") + if len(token) == 0 { + // skip this test when the token is not specified + t.Skip() + } + ctx := context.Background() + client := query.NewClientWithAccessToken(ctx, token) + + testdata := []struct { + input string + expected link.Readme + }{ + { + input: "specification/compute/resource-manager/Microsoft.Compute/stable/2020-12-01/compute.json", + expected: "specification/compute/resource-manager/readme.md", + }, + { + input: "specification/compute/resource-manager/Microsoft.Compute/stable/2020-12-01", + expected: "specification/compute/resource-manager/readme.md", + }, + } + + for _, c := range testdata { + readme, err := link.GetReadmeFromPath(ctx, client, c.input) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if readme != c.expected { + t.Fatalf("expected %s but got %s", c.expected, readme) + } + } +} diff --git a/tools/generator/cmd/issue/link/consts.go b/tools/generator/cmd/issue/link/consts.go new file mode 100644 index 000000000000..f3c96f22d3d3 --- /dev/null +++ b/tools/generator/cmd/issue/link/consts.go @@ -0,0 +1,18 @@ +package link + +const ( + // SpecOwner ... + SpecOwner = "Azure" + // SpecRepo ... + SpecRepo = "azure-rest-api-specs" + // SpecRepoPrefix ... + SpecRepoPrefix = "https://github.com/Azure/azure-rest-api-specs/" + // PullRequestPrefix ... + PullRequestPrefix = "pull/" + // DirectoryPrefix ... + DirectoryPrefix = "tree/" + // FilePrefix ... + FilePrefix = "blob/" + // CommitPrefix ... + CommitPrefix = "commit/" +) diff --git a/tools/generator/cmd/issue/link/directoryLink.go b/tools/generator/cmd/issue/link/directoryLink.go new file mode 100644 index 000000000000..74f84b2f5f10 --- /dev/null +++ b/tools/generator/cmd/issue/link/directoryLink.go @@ -0,0 +1,53 @@ +package link + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +type directoryLink struct { + linkBase + + path string +} + +var _ CommitHashLink = (*directoryLink)(nil) + +// NewDirectoryLink parses a directory link to its corresponding readme.md file link +func NewDirectoryLink(ctx context.Context, client *query.Client, requestLink, releaseLink string) Resolver { + return &directoryLink{ + linkBase: linkBase{ + ctx: ctx, + client: client, + releaseLink: releaseLink, + requestLink: requestLink, + }, + } +} + +// Resolve ... +func (l directoryLink) Resolve() (ResolveResult, error) { + commitRef, err := l.GetCommitHash() + if err != nil { + return nil, err + } + l.path = strings.TrimPrefix(l.GetReleaseLink(), DirectoryPrefix+commitRef+"/") + readme, err := GetReadmeFromPath(l.ctx, l.client, l.path) + if err != nil { + return nil, fmt.Errorf("cannot resolve directory link '%s': %+v", l.GetReleaseLink(), err) + } + return getResult(readme), nil +} + +// String ... +func (l directoryLink) String() string { + return l.GetReleaseLink() +} + +// GetCommitHash ... +func (l directoryLink) GetCommitHash() (string, error) { + return getCommitRefFromLink(l.GetReleaseLink(), DirectoryPrefix) +} diff --git a/tools/generator/cmd/issue/link/fileLink.go b/tools/generator/cmd/issue/link/fileLink.go new file mode 100644 index 000000000000..4bf6e3078af6 --- /dev/null +++ b/tools/generator/cmd/issue/link/fileLink.go @@ -0,0 +1,53 @@ +package link + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +type fileLink struct { + linkBase + + path string +} + +var _ CommitHashLink = (*fileLink)(nil) + +// NewFileLink parses a file link to its corresponding readme.md file link +func NewFileLink(ctx context.Context, client *query.Client, requestLink, releaseLink string) Resolver { + return &fileLink{ + linkBase: linkBase{ + ctx: ctx, + client: client, + releaseLink: releaseLink, + requestLink: requestLink, + }, + } +} + +// Resolve ... +func (l fileLink) Resolve() (ResolveResult, error) { + commitRef, err := l.GetCommitHash() + if err != nil { + return nil, err + } + l.path = strings.TrimPrefix(l.GetReleaseLink(), FilePrefix+commitRef+"/") + readme, err := GetReadmeFromPath(l.ctx, l.client, l.path) + if err != nil { + return nil, fmt.Errorf("cannot resolve file link '%s': %+v", l.GetReleaseLink(), err) + } + return getResult(readme), nil +} + +// String ... +func (l fileLink) String() string { + return l.GetReleaseLink() +} + +// GetCommitHash ... +func (l fileLink) GetCommitHash() (string, error) { + return getCommitRefFromLink(l.GetReleaseLink(), FilePrefix) +} diff --git a/tools/generator/cmd/issue/link/link.go b/tools/generator/cmd/issue/link/link.go new file mode 100644 index 000000000000..2922a243cd26 --- /dev/null +++ b/tools/generator/cmd/issue/link/link.go @@ -0,0 +1,90 @@ +package link + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +// Resolver represent a readme path resolver which resolves a link and produces a readme path +type Resolver interface { + GetReleaseLink() string + GetRequestLink() string + Resolve() (ResolveResult, error) +} + +// CommitHashLink ... +type CommitHashLink interface { + GetCommitHash() (string, error) +} + +func getCommitRefFromLink(l, prefix string) (string, error) { + if !strings.HasPrefix(l, prefix) { + return "", fmt.Errorf("link '%s' does not have prefix '%s'", l, prefix) + } + l = strings.TrimPrefix(l, prefix) + segments := strings.Split(l, "/") + return segments[0], nil +} + +// ResolveResult ... +type ResolveResult interface { + GetReadme() Readme + GetCode() Code +} + +type Code string + +const ( + // CodeSuccess marks the resolve is successful + CodeSuccess Code = "Success" + // CodeDataPlane marks the resolved readme belongs to a data plane package + CodeDataPlane Code = "DataPlaneRequest" + // CodePRNotMerged marks the resolve succeeds but the requested PR is not merged yet + CodePRNotMerged Code = "PRNotMerged" +) + +type result struct { + readme Readme + code Code +} + +// GetReadme ... +func (r result) GetReadme() Readme { + return r.readme +} + +// GetCode ... +func (r result) GetCode() Code { + return r.code +} + +type linkBase struct { + ctx context.Context + client *query.Client + releaseLink string + requestLink string +} + +// GetReleaseLink ... +func (l linkBase) GetReleaseLink() string { + return l.releaseLink +} + +// GetRequestLink ... +func (l linkBase) GetRequestLink() string { + return l.requestLink +} + +func getResult(readme Readme) ResolveResult { + code := CodeDataPlane + if readme.IsMgmt() { + code = CodeSuccess + } + return result{ + readme: readme, + code: code, + } +} diff --git a/tools/generator/cmd/issue/link/pullRequestLink.go b/tools/generator/cmd/issue/link/pullRequestLink.go new file mode 100644 index 000000000000..77d7565f9704 --- /dev/null +++ b/tools/generator/cmd/issue/link/pullRequestLink.go @@ -0,0 +1,98 @@ +package link + +import ( + "context" + "fmt" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/ahmetb/go-linq/v3" + "github.com/google/go-github/v32/github" +) + +type pullRequestLink struct { + linkBase +} + +// NewPullRequestLink parses a pull request link to its corresponding readme.md file link +func NewPullRequestLink(ctx context.Context, client *query.Client, requestLink, releaseLink string) Resolver { + return &pullRequestLink{ + linkBase: linkBase{ + ctx: ctx, + client: client, + releaseLink: releaseLink, + requestLink: requestLink, + }, + } +} + +// Resolve ... +func (l pullRequestLink) Resolve() (ResolveResult, error) { + n, err := l.getPullRequestNumber() + if err != nil { + return nil, fmt.Errorf("cannot resolve pull request number from '%s'", l) + } + files, err := l.listChangedFiles(SpecOwner, SpecRepo, n) + if err != nil { + return nil, err + } + var filePaths []string + linq.From(files).Select(func(item interface{}) interface{} { + return item.(*github.CommitFile).GetFilename() + }).ToSlice(&filePaths) + readme, err := GetReadmePathFromChangedFiles(l.ctx, l.client, filePaths) + if err != nil { + return nil, fmt.Errorf("cannot resolve pull request link '%s': %+v", l.GetReleaseLink(), err) + } + // we need to check if the associated PR has been merged + merged, err := l.checkStatus(SpecOwner, SpecRepo, n) + if err != nil { + return nil, err + } + if !merged { + return result{ + readme: readme, + code: CodePRNotMerged, + }, nil + } + return getResult(readme), nil +} + +// String ... +func (l pullRequestLink) String() string { + return l.GetReleaseLink() +} + +// getPullRequestNumber returns the PR number from a PR link which should be in this form: {number}(/something)? +func (l pullRequestLink) getPullRequestNumber() (int, error) { + segments := strings.Split(strings.TrimPrefix(l.GetReleaseLink(), PullRequestPrefix), "/") + return strconv.Atoi(segments[0]) +} + +func (l pullRequestLink) checkStatus(owner, repo string, number int) (bool, error) { + merged, _, err := l.client.PullRequests.IsMerged(l.ctx, owner, repo, number) + if err != nil { + return false, err + } + return merged, nil +} + +func (l pullRequestLink) listChangedFiles(owner, repo string, number int) ([]*github.CommitFile, error) { + opt := &github.ListOptions{ + PerPage: 10, + } + var files []*github.CommitFile + for { + f, resp, err := l.client.PullRequests.ListFiles(l.ctx, owner, repo, number, opt) + if err != nil { + return nil, err + } + files = append(files, f...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return files, nil +} diff --git a/tools/generator/cmd/issue/link/pullRequestLink_test.go b/tools/generator/cmd/issue/link/pullRequestLink_test.go new file mode 100644 index 000000000000..608211c3677d --- /dev/null +++ b/tools/generator/cmd/issue/link/pullRequestLink_test.go @@ -0,0 +1,46 @@ +package link_test + +import ( + "context" + "os" + "testing" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/link" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +func TestPullRequestLink_Resolve(t *testing.T) { + token := os.Getenv("ACCESS_TOKEN") + if len(token) == 0 { + // skip this test when the token is not specified + t.Skip() + } + ctx := context.Background() + client := query.NewClientWithAccessToken(ctx, token) + + testdata := []struct { + input string + expected link.Readme + code link.Code + }{ + { + input: "pull/13024", + expected: "specification/communication/resource-manager/readme.md", + code: link.CodeSuccess, + }, + } + + for _, c := range testdata { + l := link.NewPullRequestLink(ctx, client, "", c.input) + result, err := l.Resolve() + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if result.GetCode() != c.code { + t.Fatalf("expect code %v but got %v", c.code, result.GetCode()) + } + if result.GetReadme() != c.expected { + t.Fatalf("expect %s but got %s", c.expected, result.GetReadme()) + } + } +} diff --git a/tools/generator/cmd/issue/query/login.go b/tools/generator/cmd/issue/query/login.go new file mode 100644 index 000000000000..55f88a41ad26 --- /dev/null +++ b/tools/generator/cmd/issue/query/login.go @@ -0,0 +1,59 @@ +package query + +import ( + "context" + "log" + + "github.com/google/go-github/v32/github" + "golang.org/x/oauth2" +) + +// Login to github using the given credentials +func Login(ctx context.Context, info Info) *Client { + var client *Client + if info.UserInfo.IsValid() { + client = NewClientWithUserInfo(info.UserInfo) + } else if info.Token != "" { + client = NewClientWithAccessToken(ctx, info.Token) + } else { + client = NewClient() + } + return client +} + +func getGithubClientWithUserInfo(info UserInfo) *github.Client { + log.Printf("Loging in with username and password") + auth := &github.BasicAuthTransport{ + Username: info.Username, + Password: info.Password, + OTP: info.Otp, + Transport: nil, + } + return github.NewClient(auth.Client()) +} + +func getGithubClientWithAccessToken(ctx context.Context, token string) *github.Client { + log.Printf("Loging in with personal access token") + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + return github.NewClient(tc) +} + +// UserInfo ... +type UserInfo struct { + Username string + Password string + Otp string +} + +func (u UserInfo) IsValid() bool { + return u.Username != "" && u.Password != "" +} + +// Info represents the login info +type Info struct { + UserInfo UserInfo + Token string +} diff --git a/tools/generator/cmd/issue/query/query.go b/tools/generator/cmd/issue/query/query.go new file mode 100644 index 000000000000..a478f20dcce1 --- /dev/null +++ b/tools/generator/cmd/issue/query/query.go @@ -0,0 +1,33 @@ +package query + +import ( + "context" + + "github.com/google/go-github/v32/github" +) + +// Client ... +type Client struct { + *github.Client +} + +// NewClient returns a new Client without credential +func NewClient() *Client { + return &Client{ + Client: github.NewClient(nil), + } +} + +// NewClientWithUserInfo ... +func NewClientWithUserInfo(info UserInfo) *Client { + return &Client{ + Client: getGithubClientWithUserInfo(info), + } +} + +// NewClientWithAccessToken ... +func NewClientWithAccessToken(ctx context.Context, token string) *Client { + return &Client{ + Client: getGithubClientWithAccessToken(ctx, token), + } +} diff --git a/tools/generator/cmd/issue/request/handlers.go b/tools/generator/cmd/issue/request/handlers.go new file mode 100644 index 000000000000..74649d819721 --- /dev/null +++ b/tools/generator/cmd/issue/request/handlers.go @@ -0,0 +1,46 @@ +package request + +import ( + "context" + "log" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/link" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" +) + +func handleSuccess(_ context.Context, _ *query.Client, reqIssue ReleaseRequestIssue, result link.ResolveResult) (*Request, error) { + return &Request{ + RequestLink: reqIssue.IssueLink, + TargetDate: reqIssue.ReleaseDate, + ReadmePath: string(result.GetReadme()), + Tag: reqIssue.Tag, + Track: getTrack(reqIssue), + }, nil +} + +func handleDataPlane(_ context.Context, _ *query.Client, reqIssue ReleaseRequestIssue, result link.ResolveResult) (*Request, error) { + log.Printf("[WARNING] Release request %s is requesting a release from a data-plane readme file `%s`, treat this as a track 2 request by default", reqIssue.IssueLink, result.GetReadme()) + return &Request{ + RequestLink: reqIssue.IssueLink, + TargetDate: reqIssue.ReleaseDate, + ReadmePath: string(result.GetReadme()), + Tag: reqIssue.Tag, + Track: Track2, + }, nil +} + +func handlePRNotMerged(_ context.Context, _ *query.Client, reqIssue ReleaseRequestIssue, result link.ResolveResult) (*Request, error) { + log.Printf("[WARNING] Release request %s is requesting a release from a non-merged PR `%s`, discard this request", reqIssue.IssueLink, reqIssue.TargetLink) + // TODO -- add comment and close this issue + return nil, nil +} + +func getTrack(issue ReleaseRequestIssue) Track { + for _, l := range issue.Labels { + if l != nil && l.GetName() == "Track2" { + return Track2 + } + } + + return Track1 +} diff --git a/tools/generator/cmd/issue/request/request.go b/tools/generator/cmd/issue/request/request.go new file mode 100644 index 000000000000..56fa53c8ae15 --- /dev/null +++ b/tools/generator/cmd/issue/request/request.go @@ -0,0 +1,194 @@ +package request + +import ( + "context" + "fmt" + "regexp" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/link" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/google/go-github/v32/github" +) + +var ( + resultHandlerMap = map[link.Code]resultHandlerFunc{ + link.CodeSuccess: handleSuccess, + link.CodeDataPlane: handleDataPlane, + link.CodePRNotMerged: handlePRNotMerged, + } +) + +// ParsingOptions ... +type ParsingOptions struct { + IncludeDataPlaneRequests bool +} + +type resultHandlerFunc func(ctx context.Context, client *query.Client, reqIssue ReleaseRequestIssue, result link.ResolveResult) (*Request, error) + +// Request represents a parsed SDK release request +type Request struct { + RequestLink string + TargetDate time.Time + ReadmePath string + Tag string + Track Track +} + +// Track ... +type Track string + +const ( + // Track1 ... + Track1 Track = "Track1" + // Track2 ... + Track2 Track = "Track2" +) + +const ( + linkKeyword = "**Link**: " + tagKeyword = "**Readme Tag**: " + releaseDateKeyword = "**Target release date**: " +) + +type issueError struct { + issue github.Issue + err error +} + +// Error ... +func (e *issueError) Error() string { + return fmt.Sprintf("cannot parse release request from issue %s: %+v", e.issue.GetHTMLURL(), e.err) +} + +func initializeHandlers(options ParsingOptions) { + if options.IncludeDataPlaneRequests { + resultHandlerMap[link.CodeDataPlane] = handleSuccess + } +} + +// ParseIssue parses the release request issues to release requests +func ParseIssue(ctx context.Context, client *query.Client, issue github.Issue, options ParsingOptions) (*Request, error) { + initializeHandlers(options) + + reqIssue, err := NewReleaseRequestIssue(issue) + if err != nil { + return nil, err + } + result, err := ParseReadmeFromLink(ctx, client, *reqIssue) + if err != nil { + return nil, err + } + handler := resultHandlerMap[result.GetCode()] + if handler == nil { + panic(fmt.Sprintf("unhandled code '%s'", result.GetCode())) + } + return handler(ctx, client, *reqIssue, result) +} + +// ReleaseRequestIssue represents a release request issue +type ReleaseRequestIssue struct { + IssueLink string + TargetLink string + Tag string + ReleaseDate time.Time + Labels []*github.Label +} + +// NewReleaseRequestIssue ... +func NewReleaseRequestIssue(issue github.Issue) (*ReleaseRequestIssue, error) { + body := issue.GetBody() + contents := getRawContent(strings.Split(body, "\n"), []string{ + linkKeyword, tagKeyword, releaseDateKeyword, + }) + // get release date + releaseDate, err := time.Parse("2006-01-02", contents[releaseDateKeyword]) + if err != nil { + return nil, &issueError{ + issue: issue, + err: err, + } + } + return &ReleaseRequestIssue{ + IssueLink: issue.GetHTMLURL(), + TargetLink: parseLink(contents[linkKeyword]), + Tag: contents[tagKeyword], + ReleaseDate: releaseDate, + Labels: issue.Labels, + }, nil +} + +func getRawContent(lines []string, keywords []string) map[string]string { + result := make(map[string]string) + for _, line := range lines { + for _, keyword := range keywords { + raw := getContentByPrefix(line, keyword) + if raw != "" { + result[keyword] = raw + } + } + } + return result +} + +func getContentByPrefix(line, prefix string) string { + if strings.HasPrefix(line, prefix) { + return strings.TrimSpace(strings.TrimPrefix(line, prefix)) + } + return "" +} + +func parseLink(rawLink string) string { + regex := regexp.MustCompile(`^\[.+\]\((.+)\)$`) + r := regex.FindStringSubmatch(rawLink) + if len(r) < 1 { + return "" + } + return r[1] +} + +// ParseReadmeFromLink ... +func ParseReadmeFromLink(ctx context.Context, client *query.Client, reqIssue ReleaseRequestIssue) (link.ResolveResult, error) { + // check if invalid characters in a url + regex := regexp.MustCompile(`^[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\-._~:/\?#\[\]@!\$&'\(\)\*\+,;=]+$`) + if !regex.MatchString(reqIssue.TargetLink) { + return nil, fmt.Errorf("link '%s' contains invalid characters", reqIssue.TargetLink) + } + r, err := parseResolver(ctx, client, reqIssue.IssueLink, reqIssue.TargetLink) + if err != nil { + return nil, err + } + return r.Resolve() +} + +func parseResolver(ctx context.Context, client *query.Client, requestLink, releaseLink string) (link.Resolver, error) { + if !strings.HasPrefix(releaseLink, link.SpecRepoPrefix) { + return nil, fmt.Errorf("link '%s' is not from '%s'", releaseLink, link.SpecRepoPrefix) + } + releaseLink = strings.TrimPrefix(releaseLink, link.SpecRepoPrefix) + prefix, err := getLinkPrefix(releaseLink) + if err != nil { + return nil, fmt.Errorf("cannot resolve link '%s': %+v", releaseLink, err) + } + switch prefix { + case link.PullRequestPrefix: + return link.NewPullRequestLink(ctx, client, requestLink, releaseLink), nil + case link.DirectoryPrefix: + return link.NewDirectoryLink(ctx, client, requestLink, releaseLink), nil + case link.FilePrefix: + return link.NewFileLink(ctx, client, requestLink, releaseLink), nil + case link.CommitPrefix: + return link.NewCommitLink(ctx, client, requestLink, releaseLink), nil + default: + return nil, fmt.Errorf("prefix '%s' of link '%s' not supported yet", prefix, releaseLink) + } +} + +func getLinkPrefix(link string) (string, error) { + segments := strings.Split(link, "/") + if len(segments) < 2 { + return "", fmt.Errorf("cannot determine the prefix of link") + } + return segments[0] + "/", nil +} diff --git a/tools/generator/cmd/issue/request/request_test.go b/tools/generator/cmd/issue/request/request_test.go new file mode 100644 index 000000000000..b4b7b9cb44cc --- /dev/null +++ b/tools/generator/cmd/issue/request/request_test.go @@ -0,0 +1,52 @@ +package request_test + +import ( + "context" + "testing" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/link" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/request" + "github.com/google/go-github/v32/github" +) + +func TestParseReadmeFromLink(t *testing.T) { + testData := []struct { + name string + input string + expected link.Readme + }{ + { + name: "parse PR link", + input: "https://github.com/Azure/azure-rest-api-specs/pull/11377", + expected: "specification/security/resource-manager/readme.md", + }, + { + name: "parse github directory", + input: "https://github.com/Azure/azure-rest-api-specs/tree/636ade9c4b88ad7e408d85f564ddedbc638ae524/specification/monitor/resource-manager", + expected: "specification/monitor/resource-manager/readme.md", + }, + { + name: "parse github file", + input: "https://github.com/Azure/azure-rest-api-specs/blob/636ade9c4b88ad7e408d85f564ddedbc638ae524/specification/storage/resource-manager/readme.md", + expected: "specification/storage/resource-manager/readme.md", + }, + } + + for _, c := range testData { + t.Logf("Testing %s...", c.name) + client := &query.Client{ + Client: github.NewClient(nil), + } + result, err := request.ParseReadmeFromLink(context.Background(), client, request.ReleaseRequestIssue{ + IssueLink: "", + TargetLink: c.input, + }) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if result.GetReadme() != c.expected { + t.Fatalf("expected %q but got %q", c.expected, result.GetReadme()) + } + } +} diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index f280dfb1b0bf..718dd226c160 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -8,6 +8,7 @@ import ( "log" "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/automation" + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue" "github.com/spf13/cobra" ) @@ -27,6 +28,7 @@ func Command() *cobra.Command { rootCmd.AddCommand( automation.Command(), + issue.Command(), ) return rootCmd diff --git a/tools/generator/common/constants.go b/tools/generator/common/constants.go index ca5de0c82afc..913bad95b514 100644 --- a/tools/generator/common/constants.go +++ b/tools/generator/common/constants.go @@ -6,4 +6,11 @@ package common const ( // DefaultOptionPath the default path of the option file DefaultOptionPath = "generate_options.json" + + Services = "services" + ChangelogFilename = "CHANGELOG.md" + MetadataFilename = "_meta.json" + RelativeVersionGo = "version/version.go" + + Root = "github.com/Azure/azure-sdk-for-go" ) diff --git a/tools/generator/config/config.go b/tools/generator/config/config.go new file mode 100644 index 000000000000..247d00378f9f --- /dev/null +++ b/tools/generator/config/config.go @@ -0,0 +1,23 @@ +package config + +import ( + "encoding/json" + + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" +) + +type Config struct { + Track1Requests Track1ReleaseRequests `json:"track1Requests,omitempty"` + Track2Requests Track2ReleaseRequests `json:"track2Requests,omitempty"` + RefreshInfo RefreshInfo `json:"refresh,omitempty"` + AdditionalFlags []string `json:"additionalOptions,omitempty"` +} + +func (c Config) String() string { + b, _ := json.Marshal(c) + return string(b) +} + +func (c Config) AdditionalOptions() ([]model.Option, error) { + return parseAdditionalOptions(c.AdditionalFlags) +} diff --git a/tools/generator/config/parse.go b/tools/generator/config/parse.go new file mode 100644 index 000000000000..230111fabadd --- /dev/null +++ b/tools/generator/config/parse.go @@ -0,0 +1,31 @@ +package config + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" +) + +type parser struct { + reader io.Reader +} + +func FromReader(reader io.Reader) *parser { + return &parser{ + reader: reader, + } +} + +func (p *parser) Parse() (*Config, error) { + b, err := ioutil.ReadAll(p.reader) + if err != nil { + return nil, err + } + var config Config + if err := json.Unmarshal(b, &config); err != nil { + return nil, fmt.Errorf("content of configs must be a valid JSON: %+v", err) + } + + return &config, nil +} diff --git a/tools/generator/config/parse_test.go b/tools/generator/config/parse_test.go new file mode 100644 index 000000000000..f8f0af809e14 --- /dev/null +++ b/tools/generator/config/parse_test.go @@ -0,0 +1,137 @@ +package config_test + +import ( + "reflect" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/tools/generator/config" +) + +func TestParser_Parse(t *testing.T) { + testCase := []struct { + input string + expected *config.Config + }{ + { + input: `{ + "track1Requests": { + "readme1": { + "tag1": [{"requestLink": "link1" }] + } + } +}`, + expected: &config.Config{ + Track1Requests: config.Track1ReleaseRequests{ + "readme1": { + "tag1": []config.ReleaseRequestInfo{ + { + RequestLink: "link1", + }, + }, + }, + }, + }, + }, + { + input: `{ + "track1Requests": { + "readme1": { + "tag1": [{"requestLink": "link1"}, {"requestLink": "link2"} ] + } + } +}`, + expected: &config.Config{ + Track1Requests: config.Track1ReleaseRequests{ + "readme1": { + "tag1": []config.ReleaseRequestInfo{ + { + RequestLink: "link1", + }, + { + RequestLink: "link2", + }, + }, + }, + }, + }, + }, + { + input: `{ + "track1Requests": { + "readme1": { + "tag1": [{"requestLink": "link1"}, {"requestLink": "link2" }], + "tag2": [{"requestLink": "link3"}] + } + } +}`, + expected: &config.Config{ + Track1Requests: config.Track1ReleaseRequests{ + "readme1": { + "tag1": []config.ReleaseRequestInfo{ + { + RequestLink: "link1", + }, + { + RequestLink: "link2", + }, + }, + "tag2": []config.ReleaseRequestInfo{ + { + RequestLink: "link3", + }, + }, + }, + }, + }, + }, + { + input: `{ + "track1Requests": { + "readme1": { + "tag1": [{"requestLink": "link1"}, {"requestLink": "link2"}], + "tag2": [{"requestLink": "link3"}] + } + }, + "refresh": { + "additionalOptions": ["--use=somethingNew"] + } +}`, + expected: &config.Config{ + Track1Requests: config.Track1ReleaseRequests{ + "readme1": { + "tag1": []config.ReleaseRequestInfo{ + { + RequestLink: "link1", + }, + { + RequestLink: "link2", + }, + }, + "tag2": []config.ReleaseRequestInfo{ + { + RequestLink: "link3", + }, + }, + }, + }, + RefreshInfo: config.RefreshInfo{ + AdditionalFlags: []string{ + "--use=somethingNew", + }, + }, + }, + }, + } + + for i, c := range testCase { + t.Logf("testing %d", i) + cfg, err := config.FromReader(strings.NewReader(c.input)).Parse() + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if !reflect.DeepEqual(cfg, c.expected) { + t.Fatalf("expected '%s' but got '%s'", cfg.String(), c.expected.String()) + } + } +} diff --git a/tools/generator/config/reader.go b/tools/generator/config/reader.go new file mode 100644 index 000000000000..74dc7d894cd1 --- /dev/null +++ b/tools/generator/config/reader.go @@ -0,0 +1,26 @@ +package config + +import ( + "fmt" + "io" + "os" +) + +func ParseConfig(path string) (*Config, error) { + reader, err := getConfigReader(path) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %+v", err) + } + cfg, err := FromReader(reader).Parse() + if err != nil { + return nil, fmt.Errorf("failed to parse configs: %+v", err) + } + return cfg, nil +} + +func getConfigReader(config string) (io.Reader, error) { + if config == "" { + return os.Stdin, nil + } + return os.Open(config) +} diff --git a/tools/generator/config/refresh.go b/tools/generator/config/refresh.go new file mode 100644 index 000000000000..fb8d1ebd14fd --- /dev/null +++ b/tools/generator/config/refresh.go @@ -0,0 +1,51 @@ +package config + +import ( + "encoding/json" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model" + "github.com/Azure/azure-sdk-for-go/tools/generator/common" + "github.com/hashicorp/go-multierror" +) + +type RefreshInfo struct { + // AdditionalFlags are the additional options that will be used in the general refresh + AdditionalFlags []string `json:"additionalOptions,omitempty"` + // Packages are the full package identifier of the packages to refresh, eg 'github.com/Azure/azure-sdk-for-go/services/preview/securityinsight/mgmt/2019-01-01-preview/securityinsight' + Packages []string `json:"packages,omitempty"` +} + +func (r RefreshInfo) AdditionalOptions() ([]model.Option, error) { + return parseAdditionalOptions(r.AdditionalFlags) +} + +func (r RefreshInfo) RelativePackages() []string { + var packages []string + for _, p := range r.Packages { + l := strings.TrimPrefix(strings.TrimPrefix(p, common.Root), "/") + packages = append(packages, l) + } + + return packages +} + +func (r RefreshInfo) String() string { + b, _ := json.Marshal(r) + return string(b) +} + +func parseAdditionalOptions(input []string) ([]model.Option, error) { + var errResult error + var options []model.Option + for _, f := range input { + o, err := model.NewOption(f) + if err != nil { + errResult = multierror.Append(errResult, err) + continue + } + options = append(options, o) + } + + return options, errResult +} diff --git a/tools/generator/config/releaseRequests.go b/tools/generator/config/releaseRequests.go new file mode 100644 index 000000000000..8dcbd824f5e2 --- /dev/null +++ b/tools/generator/config/releaseRequests.go @@ -0,0 +1,81 @@ +package config + +import ( + "encoding/json" + "fmt" + "time" +) + +type Track2ReleaseRequests map[string][]Track2Request + +type Track2Request struct { + ReleaseRequestInfo + PackageFlag string `json:"packageFlag,omitempty"` +} + +func (c Track2ReleaseRequests) String() string { + b, _ := json.Marshal(c) + return string(b) +} + +func (c Track2ReleaseRequests) Add(readme string, info Track2Request) { + if !c.Contains(readme) { + c[readme] = make([]Track2Request, 0) + } + c[readme] = append(c[readme], info) +} + +func (c Track2ReleaseRequests) Contains(readme string) bool { + _, ok := c[readme] + return ok +} + +type Track1ReleaseRequests map[string]TagForRelease + +type TagForRelease map[string][]ReleaseRequestInfo + +type ReleaseRequestInfo struct { + TargetDate *time.Time `json:"targetDate,omitempty"` + RequestLink string `json:"requestLink,omitempty"` +} + +func (info ReleaseRequestInfo) HasTargetDate() bool { + return info.TargetDate != nil +} + +func (info ReleaseRequestInfo) String() string { + m := fmt.Sprintf("Release request '%s'", info.RequestLink) + if info.HasTargetDate() { + m = fmt.Sprintf("%s (Target date: %s)", m, *info.TargetDate) + } + return m +} + +func (c Track1ReleaseRequests) String() string { + b, _ := json.Marshal(c) + return string(b) +} + +func (c Track1ReleaseRequests) Add(readme, tag string, info ReleaseRequestInfo) { + if !c.Contains(readme) { + c[readme] = TagForRelease{} + } + c[readme].Add(tag, info) +} + +func (c Track1ReleaseRequests) Contains(readme string) bool { + _, ok := c[readme] + return ok +} + +func (c TagForRelease) Add(tag string, info ReleaseRequestInfo) { + if !c.Contains(tag) { + c[tag] = []ReleaseRequestInfo{} + } + c[tag] = append(c[tag], info) +} + +func (c TagForRelease) Contains(tag string) bool { + _, ok := c[tag] + return ok +} diff --git a/tools/generator/config/validate/localValidator.go b/tools/generator/config/validate/localValidator.go new file mode 100644 index 000000000000..c0a15683f279 --- /dev/null +++ b/tools/generator/config/validate/localValidator.go @@ -0,0 +1,91 @@ +package validate + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/config" + "github.com/ahmetb/go-linq/v3" + "github.com/hashicorp/go-multierror" +) + +type localValidator struct { + specRoot string +} + +func (v *localValidator) Validate(cfg config.Config) error { + var errResult error + for readme, infoMap := range cfg.Track1Requests { + if err := v.validateReadmeExistence(readme); err != nil { + errResult = multierror.Append(errResult, err) + continue // readme file cannot pass validation, we just skip the validations + } + // get content of the readme + contentOfReadme, err := v.getReadmeContent(readme) + if err != nil { + errResult = multierror.Append(errResult, fmt.Errorf("cannot get readme.md content: %+v", err)) + continue + } + // validate the existence of readme.go.md + if err := v.validateReadmeExistence(getReadmeGoFromReadme(readme)); err != nil { + errResult = multierror.Append(errResult, err) + continue // readme.go.md is mandatory + } + // get content of the readme.go.md + contentOfReadmeGo, err := v.getReadmeContent(getReadmeGoFromReadme(readme)) + if err != nil { + errResult = multierror.Append(errResult, fmt.Errorf("cannot get readme.go.md content: %+v", err)) + continue + } + // get the keys from infoMap, which is the tags + var tags []string + linq.From(infoMap).Select(func(item interface{}) interface{} { + return item.(linq.KeyValue).Key + }).ToSlice(&tags) + // check the tags one by one + if err := validateTagsInReadme(contentOfReadme, readme, tags...); err != nil { + errResult = multierror.Append(errResult, err) + } + if err := validateTagsInReadmeGo(contentOfReadmeGo, readme, tags...); err != nil { + errResult = multierror.Append(errResult, err) + } + } + return errResult +} + +func (v *localValidator) validateReadmeExistence(readme string) error { + full := filepath.Join(v.specRoot, readme) + if _, err := os.Stat(full); os.IsNotExist(err) { + return fmt.Errorf("readme file %q does not exist", readme) + } + + return nil +} + +func (v *localValidator) getReadmeContent(readme string) ([]byte, error) { + full := filepath.Join(v.specRoot, readme) + return ioutil.ReadFile(full) +} + +func findTagInReadme(content []byte, tag string) bool { + return regexp.MustCompile(fmt.Sprintf(tagDefinedInReadmeRegex, tag)).Match(content) +} + +func findTagInGo(content []byte, tag string) bool { + return regexp.MustCompile(tagInBatchRegex + tag + `\s+`).Match(content) +} + +func getReadmeGoFromReadme(readme string) string { + return strings.ReplaceAll(readme, readmeFilename, goReadmeFilename) +} + +const ( + tagDefinedInReadmeRegex = `\$\(tag\)\s*==\s*'%s'` + tagInBatchRegex = `-\s*tag\s*:\s*` + readmeFilename = "readme.md" + goReadmeFilename = "readme.go.md" +) diff --git a/tools/generator/config/validate/remoteValidator.go b/tools/generator/config/validate/remoteValidator.go new file mode 100644 index 000000000000..b26e9da72bb8 --- /dev/null +++ b/tools/generator/config/validate/remoteValidator.go @@ -0,0 +1,104 @@ +package validate + +import ( + "context" + "fmt" + "strings" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/Azure/azure-sdk-for-go/tools/generator/config" + "github.com/ahmetb/go-linq/v3" + "github.com/google/go-github/v32/github" + "github.com/hashicorp/go-multierror" +) + +type remoteValidator struct { + ctx context.Context + client *query.Client +} + +func (v *remoteValidator) Validate(cfg config.Config) error { + var errResult error + for readme, infoMap := range cfg.Track1Requests { + // first we validate whether the readme exists + file, err := v.validateReadmeExistence(readme) + if err != nil { + errResult = multierror.Append(errResult, err) + continue // readme file does not exist, we could just skip all of the other steps of validations + } + // get content of the readme + contentOfReadme, err := file.GetContent() + if err != nil { + errResult = multierror.Append(errResult, fmt.Errorf("cannot get readme.md content: %+v", err)) + continue + } + // validate the existence of readme.go.md + fileGo, err := v.validateReadmeExistence(getReadmeGoFromReadme(readme)) + if err != nil { + errResult = multierror.Append(errResult, err) + continue // readme.go.md is mandatory + } + // get content of the readme.go.md + contentOfReadmeGo, err := fileGo.GetContent() + if err != nil { + errResult = multierror.Append(errResult, fmt.Errorf("cannot get readme.go.md content: %+v", err)) + continue + } + // get the keys from infoMap, which is the tags + var tags []string + linq.From(infoMap).Select(func(item interface{}) interface{} { + return item.(linq.KeyValue).Key + }).ToSlice(&tags) + // check the tags one by one + if err := validateTagsInReadme([]byte(contentOfReadme), readme, tags...); err != nil { + errResult = multierror.Append(errResult, err) + } + if err := validateTagsInReadmeGo([]byte(contentOfReadmeGo), readme, tags...); err != nil { + errResult = multierror.Append(errResult, err) + } + } + return errResult +} + +func (v *remoteValidator) validateReadmeExistence(readme string) (*github.RepositoryContent, error) { + file, _, _, err := v.client.Repositories.GetContents(v.ctx, SpecOwner, SpecRepo, readme, nil) + if err != nil { + return nil, fmt.Errorf("cannot get readme file '%s' on remote: %+v", readme, err) + } + return file, nil +} + +func validateTagsInReadme(content []byte, readme string, tags ...string) error { + var notFoundTags []string + for _, t := range tags { + if !findTagInReadme(content, t) { + notFoundTags = append(notFoundTags, t) + } + } + + if len(notFoundTags) > 0 { + return fmt.Errorf("%d tag(s) not defined in readme.md '%s': %s", len(notFoundTags), readme, strings.Join(notFoundTags, ", ")) + } + + return nil +} + +func validateTagsInReadmeGo(content []byte, readme string, tags ...string) error { + var notFoundTags []string + for _, t := range tags { + if !findTagInGo(content, t) { + notFoundTags = append(notFoundTags, t) + } + } + + if len(notFoundTags) > 0 { + return fmt.Errorf("%d tag(s) not found in readme.go.md '%s': %s", len(notFoundTags), getReadmeGoFromReadme(readme), strings.Join(notFoundTags, ", ")) + } + + return nil +} + +const ( + SpecOwner = "Azure" + SpecRepo = "azure-rest-api-specs" +) diff --git a/tools/generator/config/validate/validator.go b/tools/generator/config/validate/validator.go new file mode 100644 index 000000000000..b1240f754f12 --- /dev/null +++ b/tools/generator/config/validate/validator.go @@ -0,0 +1,25 @@ +package validate + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/tools/generator/cmd/issue/query" + "github.com/Azure/azure-sdk-for-go/tools/generator/config" +) + +type Validator interface { + Validate(cfg config.Config) error +} + +func NewLocalValidator(specRoot string) Validator { + return &localValidator{ + specRoot: specRoot, + } +} + +func NewRemoteValidator(ctx context.Context, client *query.Client) Validator { + return &remoteValidator{ + ctx: ctx, + client: client, + } +} diff --git a/tools/generator/flags/flags.go b/tools/generator/flags/flags.go new file mode 100644 index 000000000000..aa1cbfc4bbcc --- /dev/null +++ b/tools/generator/flags/flags.go @@ -0,0 +1,47 @@ +package flags + +import ( + "log" + + "github.com/spf13/pflag" +) + +func GetBool(f *pflag.FlagSet, name string) bool { + b, err := f.GetBool(name) + if err != nil { + log.Fatal(err) + } + return b +} + +func GetString(f *pflag.FlagSet, name string) string { + s, err := f.GetString(name) + if err != nil { + log.Fatal(err) + } + return s +} + +func GetStringSlice(f *pflag.FlagSet, name string) []string { + s, err := f.GetStringSlice(name) + if err != nil { + log.Fatal(err) + } + return s +} + +func GetInt(f *pflag.FlagSet, name string) int { + i, err := f.GetInt(name) + if err != nil { + log.Fatal(err) + } + return i +} + +func GetIntSlice(f *pflag.FlagSet, name string) []int { + s, err := f.GetIntSlice(name) + if err != nil { + log.Fatal(err) + } + return s +} diff --git a/tools/generator/go.mod b/tools/generator/go.mod index 57db11777cea..a054e686a516 100644 --- a/tools/generator/go.mod +++ b/tools/generator/go.mod @@ -5,5 +5,10 @@ go 1.13 require ( github.com/Azure/azure-sdk-for-go v54.2.1+incompatible github.com/Azure/azure-sdk-for-go/tools/internal v0.1.0 + github.com/ahmetb/go-linq/v3 v3.2.0 + github.com/google/go-github/v32 v32.1.0 + github.com/hashicorp/go-multierror v1.0.0 github.com/spf13/cobra v1.1.3 + github.com/spf13/pflag v1.0.5 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 ) diff --git a/tools/generator/go.sum b/tools/generator/go.sum index e91d578933ca..68d3611ac154 100644 --- a/tools/generator/go.sum +++ b/tools/generator/go.sum @@ -26,6 +26,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ahmetb/go-linq/v3 v3.2.0 h1:BEuMfp+b59io8g5wYzNoFe9pWPalRklhlhbiU3hYZDE= +github.com/ahmetb/go-linq/v3 v3.2.0/go.mod h1:haQ3JfOeWK8HpVxMtHHEMPVgBKiYyQ+f1/kLZh/cj9U= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -65,11 +67,16 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= +github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -83,10 +90,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -185,6 +194,7 @@ golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -217,9 +227,11 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -270,6 +282,7 @@ google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= From 261794c732b0336bfbb61d6fe01cd081763ba468 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Wed, 2 Jun 2021 14:13:11 +0800 Subject: [PATCH 09/11] add copyright header and comments --- tools/generator/cmd/issue/issueCmd.go | 11 +++++++---- tools/generator/cmd/issue/link/commitLink.go | 3 +++ tools/generator/cmd/issue/link/commitLink_test.go | 3 +++ tools/generator/cmd/issue/link/common.go | 3 +++ tools/generator/cmd/issue/link/common_test.go | 3 +++ tools/generator/cmd/issue/link/consts.go | 3 +++ tools/generator/cmd/issue/link/directoryLink.go | 3 +++ tools/generator/cmd/issue/link/fileLink.go | 3 +++ tools/generator/cmd/issue/link/link.go | 3 +++ tools/generator/cmd/issue/link/pullRequestLink.go | 3 +++ .../generator/cmd/issue/link/pullRequestLink_test.go | 3 +++ tools/generator/cmd/issue/query/login.go | 3 +++ tools/generator/cmd/issue/query/query.go | 3 +++ tools/generator/cmd/issue/request/handlers.go | 3 +++ tools/generator/cmd/issue/request/request.go | 3 +++ tools/generator/cmd/issue/request/request_test.go | 3 +++ 16 files changed, 52 insertions(+), 4 deletions(-) diff --git a/tools/generator/cmd/issue/issueCmd.go b/tools/generator/cmd/issue/issueCmd.go index 71bf0ad8b09f..83e3e30774c2 100644 --- a/tools/generator/cmd/issue/issueCmd.go +++ b/tools/generator/cmd/issue/issueCmd.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package issue import ( @@ -19,8 +22,8 @@ import ( ) const ( - Repo = "sdk-release-request" - Owner = "Azure" + repoName = "sdk-release-request" + repoOwner = "Azure" ) // Command returns the issue command @@ -144,7 +147,7 @@ func (c *commandContext) listOpenIssues() ([]*github.Issue, error) { } var issues []*github.Issue for { - r, resp, err := c.client.Issues.ListByRepo(c.ctx, Owner, Repo, opt) + r, resp, err := c.client.Issues.ListByRepo(c.ctx, repoOwner, repoName, opt) if err != nil { return nil, err } @@ -161,7 +164,7 @@ func (c *commandContext) listOpenIssues() ([]*github.Issue, error) { func (c *commandContext) listSpecifiedIssues(ids []int) ([]*github.Issue, error) { var issues []*github.Issue for _, id := range ids { - issue, _, err := c.client.Issues.Get(c.ctx, Owner, Repo, id) + issue, _, err := c.client.Issues.Get(c.ctx, repoOwner, repoName, id) if err != nil { return nil, err } diff --git a/tools/generator/cmd/issue/link/commitLink.go b/tools/generator/cmd/issue/link/commitLink.go index c7f158e737df..3ff190be44d2 100644 --- a/tools/generator/cmd/issue/link/commitLink.go +++ b/tools/generator/cmd/issue/link/commitLink.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link import ( diff --git a/tools/generator/cmd/issue/link/commitLink_test.go b/tools/generator/cmd/issue/link/commitLink_test.go index dfe126dbc3a1..bc54b23d0a09 100644 --- a/tools/generator/cmd/issue/link/commitLink_test.go +++ b/tools/generator/cmd/issue/link/commitLink_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link_test import ( diff --git a/tools/generator/cmd/issue/link/common.go b/tools/generator/cmd/issue/link/common.go index f91b972ff520..ce0ec77cc4e6 100644 --- a/tools/generator/cmd/issue/link/common.go +++ b/tools/generator/cmd/issue/link/common.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link import ( diff --git a/tools/generator/cmd/issue/link/common_test.go b/tools/generator/cmd/issue/link/common_test.go index 2a979dfe0737..a53226c2b870 100644 --- a/tools/generator/cmd/issue/link/common_test.go +++ b/tools/generator/cmd/issue/link/common_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link_test import ( diff --git a/tools/generator/cmd/issue/link/consts.go b/tools/generator/cmd/issue/link/consts.go index f3c96f22d3d3..41ae704e4152 100644 --- a/tools/generator/cmd/issue/link/consts.go +++ b/tools/generator/cmd/issue/link/consts.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link const ( diff --git a/tools/generator/cmd/issue/link/directoryLink.go b/tools/generator/cmd/issue/link/directoryLink.go index 74f84b2f5f10..fc08eb6cd738 100644 --- a/tools/generator/cmd/issue/link/directoryLink.go +++ b/tools/generator/cmd/issue/link/directoryLink.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link import ( diff --git a/tools/generator/cmd/issue/link/fileLink.go b/tools/generator/cmd/issue/link/fileLink.go index 4bf6e3078af6..d06ce7531676 100644 --- a/tools/generator/cmd/issue/link/fileLink.go +++ b/tools/generator/cmd/issue/link/fileLink.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link import ( diff --git a/tools/generator/cmd/issue/link/link.go b/tools/generator/cmd/issue/link/link.go index 2922a243cd26..cee36ad3fd45 100644 --- a/tools/generator/cmd/issue/link/link.go +++ b/tools/generator/cmd/issue/link/link.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link import ( diff --git a/tools/generator/cmd/issue/link/pullRequestLink.go b/tools/generator/cmd/issue/link/pullRequestLink.go index 77d7565f9704..816ea6cd95fc 100644 --- a/tools/generator/cmd/issue/link/pullRequestLink.go +++ b/tools/generator/cmd/issue/link/pullRequestLink.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link import ( diff --git a/tools/generator/cmd/issue/link/pullRequestLink_test.go b/tools/generator/cmd/issue/link/pullRequestLink_test.go index 608211c3677d..b1e494e97f9a 100644 --- a/tools/generator/cmd/issue/link/pullRequestLink_test.go +++ b/tools/generator/cmd/issue/link/pullRequestLink_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package link_test import ( diff --git a/tools/generator/cmd/issue/query/login.go b/tools/generator/cmd/issue/query/login.go index 55f88a41ad26..17c5a344feb8 100644 --- a/tools/generator/cmd/issue/query/login.go +++ b/tools/generator/cmd/issue/query/login.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package query import ( diff --git a/tools/generator/cmd/issue/query/query.go b/tools/generator/cmd/issue/query/query.go index a478f20dcce1..ca614873343f 100644 --- a/tools/generator/cmd/issue/query/query.go +++ b/tools/generator/cmd/issue/query/query.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package query import ( diff --git a/tools/generator/cmd/issue/request/handlers.go b/tools/generator/cmd/issue/request/handlers.go index 74649d819721..464b182448fe 100644 --- a/tools/generator/cmd/issue/request/handlers.go +++ b/tools/generator/cmd/issue/request/handlers.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package request import ( diff --git a/tools/generator/cmd/issue/request/request.go b/tools/generator/cmd/issue/request/request.go index 56fa53c8ae15..98f800205557 100644 --- a/tools/generator/cmd/issue/request/request.go +++ b/tools/generator/cmd/issue/request/request.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package request import ( diff --git a/tools/generator/cmd/issue/request/request_test.go b/tools/generator/cmd/issue/request/request_test.go index b4b7b9cb44cc..f44ccb820fa7 100644 --- a/tools/generator/cmd/issue/request/request_test.go +++ b/tools/generator/cmd/issue/request/request_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package request_test import ( From 4fa216c6e412c628d0502febbbb00bfa67ad74b9 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Wed, 2 Jun 2021 14:14:31 +0800 Subject: [PATCH 10/11] remove the link to private repo --- tools/generator/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/generator/README.md b/tools/generator/README.md index e168352d69f2..bd9957d27766 100644 --- a/tools/generator/README.md +++ b/tools/generator/README.md @@ -8,7 +8,7 @@ This CLI tool provides 2 commands now: `automation`, `issue`. ### The `issue` command and the configuration file -The `issue` command fetches the release request issues from [GitHub](https://github.com/Azure/sdk-release-request/issues?q=is%3Aissue+is%3Aopen+label%3AGo) and parses them into the configuration that other commands consume. The configuration will output to stdout. +The `issue` command fetches the release request issues from `github.com/Azure/sdk-release-request/issues` and parses them into the configuration that other commands consume. The configuration will output to stdout. The configuration is a JSON string, which has the following pattern: ```json From 49c0492f8bc2f867f7482fd77be8d2d3e22b85c0 Mon Sep 17 00:00:00 2001 From: ArcturusZhang Date: Wed, 2 Jun 2021 14:51:22 +0800 Subject: [PATCH 11/11] add copyright header and gofmt --- tools/generator/cmd/issue/link/link.go | 4 ++-- tools/generator/config/config.go | 3 +++ tools/generator/config/parse.go | 3 +++ tools/generator/config/parse_test.go | 3 +++ tools/generator/config/reader.go | 3 +++ tools/generator/config/refresh.go | 3 +++ tools/generator/config/releaseRequests.go | 3 +++ tools/generator/config/validate/localValidator.go | 3 +++ tools/generator/config/validate/remoteValidator.go | 3 +++ tools/generator/config/validate/validator.go | 3 +++ tools/generator/flags/flags.go | 3 +++ 11 files changed, 32 insertions(+), 2 deletions(-) diff --git a/tools/generator/cmd/issue/link/link.go b/tools/generator/cmd/issue/link/link.go index cee36ad3fd45..38e948487d2e 100644 --- a/tools/generator/cmd/issue/link/link.go +++ b/tools/generator/cmd/issue/link/link.go @@ -42,9 +42,9 @@ type Code string const ( // CodeSuccess marks the resolve is successful - CodeSuccess Code = "Success" + CodeSuccess Code = "Success" // CodeDataPlane marks the resolved readme belongs to a data plane package - CodeDataPlane Code = "DataPlaneRequest" + CodeDataPlane Code = "DataPlaneRequest" // CodePRNotMerged marks the resolve succeeds but the requested PR is not merged yet CodePRNotMerged Code = "PRNotMerged" ) diff --git a/tools/generator/config/config.go b/tools/generator/config/config.go index 247d00378f9f..6102fe173cbd 100644 --- a/tools/generator/config/config.go +++ b/tools/generator/config/config.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package config import ( diff --git a/tools/generator/config/parse.go b/tools/generator/config/parse.go index 230111fabadd..86a99363aade 100644 --- a/tools/generator/config/parse.go +++ b/tools/generator/config/parse.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package config import ( diff --git a/tools/generator/config/parse_test.go b/tools/generator/config/parse_test.go index f8f0af809e14..6857ebc517b7 100644 --- a/tools/generator/config/parse_test.go +++ b/tools/generator/config/parse_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package config_test import ( diff --git a/tools/generator/config/reader.go b/tools/generator/config/reader.go index 74dc7d894cd1..3e3eb552f006 100644 --- a/tools/generator/config/reader.go +++ b/tools/generator/config/reader.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package config import ( diff --git a/tools/generator/config/refresh.go b/tools/generator/config/refresh.go index fb8d1ebd14fd..c5bbf44c9cc0 100644 --- a/tools/generator/config/refresh.go +++ b/tools/generator/config/refresh.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package config import ( diff --git a/tools/generator/config/releaseRequests.go b/tools/generator/config/releaseRequests.go index 8dcbd824f5e2..04fbf4c1db0c 100644 --- a/tools/generator/config/releaseRequests.go +++ b/tools/generator/config/releaseRequests.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package config import ( diff --git a/tools/generator/config/validate/localValidator.go b/tools/generator/config/validate/localValidator.go index c0a15683f279..26d7b72ff202 100644 --- a/tools/generator/config/validate/localValidator.go +++ b/tools/generator/config/validate/localValidator.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package validate import ( diff --git a/tools/generator/config/validate/remoteValidator.go b/tools/generator/config/validate/remoteValidator.go index b26e9da72bb8..32234661a1e9 100644 --- a/tools/generator/config/validate/remoteValidator.go +++ b/tools/generator/config/validate/remoteValidator.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package validate import ( diff --git a/tools/generator/config/validate/validator.go b/tools/generator/config/validate/validator.go index b1240f754f12..4a8df762a2ac 100644 --- a/tools/generator/config/validate/validator.go +++ b/tools/generator/config/validate/validator.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package validate import ( diff --git a/tools/generator/flags/flags.go b/tools/generator/flags/flags.go index aa1cbfc4bbcc..baeba97f847c 100644 --- a/tools/generator/flags/flags.go +++ b/tools/generator/flags/flags.go @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + package flags import (