diff --git a/.travis.yml b/.travis.yml index 675d4834fbbe..a6565aa9529e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,7 @@ before_install: - curl -sSL https://raw.githubusercontent.com/golang/dep/master/install.sh | sh install: - - go get -u github.com/golang/lint/golint + - go get -u golang.org/x/lint/golint - dep ensure script: @@ -37,3 +37,4 @@ script: - go vet $(go list ./... | grep -v vendor) - go test $(sh ./findTestedPackages.sh) - go run ./tools/apidiff/main.go packages ./services FETCH_HEAD~1 FETCH_HEAD --copyrepo --breakingchanges || $IGNORE_BREAKING_CHANGES + - go run ./tools/pkgchk/main.go ./services --exceptions ./tools/pkgchk/exceptions.txt diff --git a/swagger_to_sdk_config.json b/swagger_to_sdk_config.json index 9c4a79e7b249..4b953b9bf90c 100644 --- a/swagger_to_sdk_config.json +++ b/swagger_to_sdk_config.json @@ -7,12 +7,13 @@ "gofmt -w ./services/" ], "autorest_options": { - "use": "@microsoft.azure/autorest.go@~2.1.114", + "use": "@microsoft.azure/autorest.go@~2.1.115", "go": "", "verbose": "", "sdkrel:go-sdk-folder": ".", "multiapi": "", - "use-onever": "" + "use-onever": "", + "preview-chk": "" }, "repotag": "azure-sdk-for-go", "envs": { diff --git a/tools/pkgchk/cmd/root.go b/tools/pkgchk/cmd/root.go new file mode 100644 index 000000000000..d25734ea233b --- /dev/null +++ b/tools/pkgchk/cmd/root.go @@ -0,0 +1,248 @@ +// Copyright 2018 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "bufio" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "unicode" + + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var excepFileFlag string + +var rootCmd = &cobra.Command{ + Use: "pkgchk ", + Short: "Performs package validation tasks against all packages found under the specified directory.", + Long: `This tool will perform various package validation checks against all of the packages +found under the specified directory. Failures can be baselined and thus ignored by +copying the failure text verbatim, pasting it into a text file then specifying that +file via the optional exceptions flag. +`, + Args: func(cmd *cobra.Command, args []string) error { + if err := cobra.ExactArgs(1)(cmd, args); err != nil { + return err + } + return nil + }, + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + return theCommand(args) + }, +} + +func init() { + rootCmd.PersistentFlags().StringVarP(&excepFileFlag, "exceptions", "e", "", "text file containing the list of exceptions") +} + +// Execute executes the specified command. +func Execute() { + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func theCommand(args []string) error { + rootDir := args[0] + if !filepath.IsAbs(rootDir) { + asAbs, err := filepath.Abs(rootDir) + if err != nil { + return errors.Wrap(err, "failed to get absolute path") + } + rootDir = asAbs + } + + pkgs, err := getPkgs(rootDir) + if err != nil { + return errors.Wrap(err, "failed to get packages") + } + + var exceptions []string + if excepFileFlag != "" { + exceptions, err = loadExceptions(excepFileFlag) + if err != nil { + return errors.Wrap(err, "failed to load exceptions") + } + } + verifiers := getVerifiers() + count := 0 + for _, pkg := range pkgs { + for _, v := range verifiers { + if err = v(pkg); err != nil && !contains(exceptions, err.Error()) { + fmt.Fprintln(os.Stderr, err) + count++ + } + } + } + + var res error + if count > 0 { + res = fmt.Errorf("found %d errors", count) + } + return res +} + +func contains(items []string, item string) bool { + if items == nil { + return false + } + for _, i := range items { + if i == item { + return true + } + } + return false +} + +func loadExceptions(excepFile string) ([]string, error) { + f, err := os.Open(excepFile) + if err != nil { + return nil, err + } + defer f.Close() + exceps := []string{} + for scanner := bufio.NewScanner(f); scanner.Scan(); { + exceps = append(exceps, scanner.Text()) + } + return exceps, nil +} + +type pkg struct { + // the directory where the package resides relative to the root dir + d string + + // the AST of the package + p *ast.Package +} + +// returns true if the package directory corresponds to an ARM package +func (p pkg) isARMPkg() bool { + return strings.Index(p.d, "/mgmt/") > -1 +} + +// walks the directory hierarchy from the specified root returning a slice of all the packages found +func getPkgs(rootDir string) ([]pkg, error) { + pkgs := []pkg{} + err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + // check if leaf dir + fi, err := ioutil.ReadDir(path) + if err != nil { + return err + } + hasSubDirs := false + for _, f := range fi { + if f.IsDir() { + hasSubDirs = true + break + } + } + if !hasSubDirs { + fs := token.NewFileSet() + packages, err := parser.ParseDir(fs, path, nil, parser.PackageClauseOnly) + if err != nil { + return err + } + if len(packages) < 1 { + return errors.New("didn't find any packages which is unexpected") + } + if len(packages) > 1 { + return errors.New("found more than one package which is unexpected") + } + var p *ast.Package + for _, pkgs := range packages { + p = pkgs + } + // normalize directory separator to '/' character + pkgs = append(pkgs, pkg{ + d: strings.Replace(path[len(rootDir):], "\\", "/", -1), + p: p, + }) + } + } + return nil + }) + return pkgs, err +} + +type verifier func(p pkg) error + +// returns a list of verifiers to execute +func getVerifiers() []verifier { + return []verifier{ + verifyPkgMatchesDir, + verifyLowerCase, + verifyDirectorySturcture, + } +} + +// ensures that the leaf directory name matches the package name +func verifyPkgMatchesDir(p pkg) error { + leaf := p.d[strings.LastIndex(p.d, "/")+1:] + if strings.Compare(leaf, p.p.Name) != 0 { + return fmt.Errorf("leaf directory of '%s' doesn't match package name '%s'", p.d, p.p.Name) + } + return nil +} + +// ensures that there are no upper-case letters in a package's directory +func verifyLowerCase(p pkg) error { + // walk the package directory looking for upper-case characters + for _, r := range p.d { + if r == '/' { + continue + } + if unicode.IsUpper(r) { + return fmt.Errorf("found upper-case character in directory '%s'", p.d) + } + } + return nil +} + +// ensures that the package's directory hierarchy is properly formed +func verifyDirectorySturcture(p pkg) error { + // for ARM the package directory structure is highly deterministic: + // /redis/mgmt/2015-08-01/redis + // /resources/mgmt/2017-06-01-preview/policy + // /preview/signalr/mgmt/2018-03-01-preview/signalr + if !p.isARMPkg() { + return nil + } + regexStr := strings.Join([]string{ + `^(?:/preview)?`, + `[a-z0-9\-]+`, + `mgmt`, + `\d{4}-\d{2}-\d{2}(?:-preview)?`, + `[a-z0-9]+`, + }, "/") + regex := regexp.MustCompile(regexStr) + if !regex.MatchString(p.d) { + return fmt.Errorf("bad directory structure '%s'", p.d) + } + return nil +} diff --git a/tools/pkgchk/exceptions.txt b/tools/pkgchk/exceptions.txt new file mode 100644 index 000000000000..6b13772390e7 --- /dev/null +++ b/tools/pkgchk/exceptions.txt @@ -0,0 +1,4 @@ +bad directory structure '/datalake/analytics/mgmt/2016-11-01/account' +bad directory structure '/datalake/store/mgmt/2016-11-01/account' +bad directory structure '/preview/datalake/analytics/mgmt/2015-10-01-preview/account' +bad directory structure '/preview/datalake/store/mgmt/2015-10-01-preview/account' diff --git a/tools/pkgchk/main.go b/tools/pkgchk/main.go new file mode 100644 index 000000000000..6d112b92484a --- /dev/null +++ b/tools/pkgchk/main.go @@ -0,0 +1,21 @@ +// Copyright 2018 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "github.com/Azure/azure-sdk-for-go/tools/pkgchk/cmd" + +func main() { + cmd.Execute() +}