From 760a64bda45f9f7665f7d54c85a1ff9ae52414d5 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Thu, 20 Sep 2018 16:41:48 -0700 Subject: [PATCH 01/12] Squash changes --- Gopkg.lock | 99 +++--------------- Gopkg.toml | 15 +-- reposerver/repository/repository.go | 18 ++-- util/ksonnet/ksonnet.go | 150 +++++++++++++++++----------- util/ksonnet/ksonnet_test.go | 13 ++- 5 files changed, 118 insertions(+), 177 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a1482ba0fb3ff..742594dd88dc5 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -9,16 +9,6 @@ revision = "767c40d6a2e058483c25fa193e963a22da17236d" version = "v0.18.0" -[[projects]] - digest = "1:6204a59b379aadf05380cf8cf3ae0f5867588ba028fe84f260312a79ae717272" - name = "github.com/GeertJohan/go.rice" - packages = [ - ".", - "embedded", - ] - pruneopts = "" - revision = "c02ca9a983da5807ddf7d796784928f5be4afd09" - [[projects]] digest = "1:8ec1618fc3ee146af104d6c13be250f25e5976e34557d4afbfe4b28035ce6c05" name = "github.com/Knetic/govaluate" @@ -80,14 +70,6 @@ pruneopts = "" revision = "3a771d992973f24aa725d07868b467d1ddfceafb" -[[projects]] - digest = "1:79421244ba5848aae4b0a5c41e633a04e4894cb0b164a219dc8c15ec7facb7f1" - name = "github.com/blang/semver" - packages = ["."] - pruneopts = "" - revision = "2ee87856327ba09384cabd113bc6b5d174e9ec0f" - version = "v3.5.1" - [[projects]] digest = "1:e04162bd6a6d4950541bae744c968108e14913b1cebccf29f7650b573f44adb3" name = "github.com/casbin/casbin" @@ -122,14 +104,6 @@ pruneopts = "" revision = "1180514eaf4d9f38d0d19eef639a1d695e066e72" -[[projects]] - branch = "master" - digest = "1:5fd5c4d4282935b7a575299494f2c09e9d2cacded7815c83aff7c1602aff3154" - name = "github.com/daaku/go.zipexe" - packages = ["."] - pruneopts = "" - revision = "a5fe2436ffcb3236e175e5149162b41cd28bd27d" - [[projects]] digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" name = "github.com/davecgh/go-spew" @@ -499,14 +473,6 @@ revision = "e7c7f3b33712573affdcc7a107218e7926b9a05b" version = "1.0.6" -[[projects]] - branch = "master" - digest = "1:2c5ad58492804c40bdaf5d92039b0cde8b5becd2b7feeb37d7d1cc36a8aa8dbe" - name = "github.com/kardianos/osext" - packages = ["."] - pruneopts = "" - revision = "ae77be60afb1dcacde03767a8c37337fad28ac14" - [[projects]] digest = "1:41e0bed5df4f9fd04c418bf9b6b7179b3671e416ad6175332601ca1c8dc74606" name = "github.com/kevinburke/ssh_config" @@ -515,44 +481,6 @@ revision = "81db2a75821ed34e682567d48be488a1c3121088" version = "0.5" -[[projects]] - digest = "1:2fe45da14d25bce0a58c5a991967149cc5d07f94be327b928a9fd306466815a3" - name = "github.com/ksonnet/ksonnet" - packages = [ - "metadata/params", - "pkg/app", - "pkg/component", - "pkg/docparser", - "pkg/lib", - "pkg/log", - "pkg/node", - "pkg/params", - "pkg/prototype", - "pkg/schema", - "pkg/util/jsonnet", - "pkg/util/kslib", - "pkg/util/strings", - ] - pruneopts = "" - revision = "e943ae55d4fe256c8330a047ce8426ad9dac110c" - version = "v0.11.0" - -[[projects]] - digest = "1:a345c560e5609bd71b1f54993f3b087ca45eb0e6226886c642ce519de81896cb" - name = "github.com/ksonnet/ksonnet-lib" - packages = [ - "ksonnet-gen/astext", - "ksonnet-gen/jsonnet", - "ksonnet-gen/ksonnet", - "ksonnet-gen/kubespec", - "ksonnet-gen/kubeversion", - "ksonnet-gen/nodemaker", - "ksonnet-gen/printer", - ] - pruneopts = "" - revision = "83f20ee933bcd13fcf4ad1b49a40c92135c5569c" - version = "v0.1.10" - [[projects]] branch = "master" digest = "1:ccc20cacf54eb16464dad02efa1c14fa7c0b9e124639b0d2a51dcc87b0154e4c" @@ -693,11 +621,12 @@ version = "v1.0.0" [[projects]] - digest = "1:c92f01303e3ab3b5da92657841639cb53d1548f0d2733d12ef3b9fd9d47c869e" + digest = "1:3fcbf733a8d810a21265a7f2fe08a3353db2407da052b233f8b204b5afc03d9b" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "" - revision = "ea8897e79973357ba785ac2533559a6297e83c44" + revision = "3e01752db0189b9157070a0e1668a620f9a85da2" + version = "v1.0.6" [[projects]] branch = "master" @@ -715,16 +644,6 @@ revision = "e09e9389d85d8492d313d73d1469c029e710623f" version = "v0.1.4" -[[projects]] - digest = "1:a35a4db30a6094deac33fdb99de9ed99fefc39a7bf06b57d9f04bcaa425bb183" - name = "github.com/spf13/afero" - packages = [ - ".", - "mem", - ] - pruneopts = "" - revision = "9be650865eab0c12963d8753212f4f9c66cdcf12" - [[projects]] digest = "1:2208a80fc3259291e43b30f42f844d18f4218036dff510f42c653ec9890d460a" name = "github.com/spf13/cobra" @@ -867,6 +786,14 @@ pruneopts = "" revision = "cce311a261e6fcf29de72ca96827bdb0b7d9c9e6" +[[projects]] + branch = "master" + digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1" + name = "golang.org/x/sync" + packages = ["errgroup"] + pruneopts = "" + revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + [[projects]] branch = "master" digest = "1:ed900376500543ca05f2a2383e1f541b4606f19cd22f34acb81b17a0b90c7f3e" @@ -1473,8 +1400,6 @@ "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger", "github.com/grpc-ecosystem/grpc-gateway/runtime", "github.com/grpc-ecosystem/grpc-gateway/utilities", - "github.com/ksonnet/ksonnet/pkg/app", - "github.com/ksonnet/ksonnet/pkg/component", "github.com/patrickmn/go-cache", "github.com/pkg/errors", "github.com/prometheus/client_golang/prometheus", @@ -1483,7 +1408,6 @@ "github.com/sirupsen/logrus", "github.com/skratchdot/open-golang/open", "github.com/soheilhy/cmux", - "github.com/spf13/afero", "github.com/spf13/cobra", "github.com/spf13/pflag", "github.com/stretchr/testify/assert", @@ -1496,6 +1420,7 @@ "golang.org/x/crypto/ssh/terminal", "golang.org/x/net/context", "golang.org/x/oauth2", + "golang.org/x/sync/errgroup", "google.golang.org/genproto/googleapis/api/annotations", "google.golang.org/grpc", "google.golang.org/grpc/codes", diff --git a/Gopkg.toml b/Gopkg.toml index 6c4a4fe813dda..b715ff187fca4 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -9,6 +9,7 @@ required = [ "k8s.io/code-generator/cmd/go-to-protobuf", "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway", "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger", + "golang.org/x/sync/errgroup", ] [[constraint]] @@ -37,11 +38,6 @@ required = [ branch = "release-1.10" name = "k8s.io/api" -# override ksonnet dependency -[[override]] - branch = "release-1.10" - name = "k8s.io/apimachinery" - [[constraint]] name = "k8s.io/apiextensions-apiserver" branch = "release-1.10" @@ -58,19 +54,10 @@ required = [ name = "github.com/stretchr/testify" version = "1.2.1" -[[constraint]] - name = "github.com/ksonnet/ksonnet" - version = "v0.11.0" - [[constraint]] name = "github.com/gobuffalo/packr" version = "v1.11.0" -# override ksonnet's logrus dependency -[[override]] - name = "github.com/sirupsen/logrus" - revision = "ea8897e79973357ba785ac2533559a6297e83c44" - [[constraint]] branch = "master" name = "github.com/argoproj/pkg" diff --git a/reposerver/repository/repository.go b/reposerver/repository/repository.go index 36dc2d8b7614c..c9f5d3364286f 100644 --- a/reposerver/repository/repository.go +++ b/reposerver/repository/repository.go @@ -12,7 +12,6 @@ import ( "time" "github.com/google/go-jsonnet" - "github.com/ksonnet/ksonnet/pkg/app" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -196,13 +195,13 @@ func (s *Service) GenerateManifest(c context.Context, q *ManifestRequest) (*Mani func generateManifests(appPath string, q *ManifestRequest) (*ManifestResponse, error) { var targetObjs []*unstructured.Unstructured var params []*v1alpha1.ComponentParameter - var env *app.EnvironmentSpec + var dest *v1alpha1.ApplicationDestination var err error appSourceType := IdentifyAppSourceTypeByAppDir(appPath) switch appSourceType { case AppSourceKsonnet: - targetObjs, params, env, err = ksShow(appPath, q.Environment, q.ComponentParameterOverrides) + targetObjs, params, dest, err = ksShow(appPath, q.Environment, q.ComponentParameterOverrides) case AppSourceHelm: h := helm.NewHelmApp(appPath) err = h.DependencyBuild() @@ -266,9 +265,9 @@ func generateManifests(appPath string, q *ManifestRequest) (*ManifestResponse, e Manifests: manifests, Params: params, } - if env != nil { - res.Namespace = env.Destination.Namespace - res.Server = env.Destination.Server + if dest != nil { + res.Namespace = dest.Namespace + res.Server = dest.Server } return &res, nil } @@ -335,7 +334,7 @@ func getFileCacheKey(commitSHA string, q *GetFileRequest) string { } // ksShow runs `ks show` in an app directory after setting any component parameter overrides -func ksShow(appPath, envName string, overrides []*v1alpha1.ComponentParameter) ([]*unstructured.Unstructured, []*v1alpha1.ComponentParameter, *app.EnvironmentSpec, error) { +func ksShow(appPath, envName string, overrides []*v1alpha1.ComponentParameter) ([]*unstructured.Unstructured, []*v1alpha1.ComponentParameter, *v1alpha1.ApplicationDestination, error) { ksApp, err := ksonnet.NewKsonnetApp(appPath) if err != nil { return nil, nil, nil, status.Errorf(codes.FailedPrecondition, "unable to load application from %s: %v", appPath, err) @@ -352,8 +351,7 @@ func ksShow(appPath, envName string, overrides []*v1alpha1.ComponentParameter) ( } } } - appSpec := ksApp.App() - env, err := appSpec.Environment(envName) + dest, err := ksApp.Destination(envName) if err != nil { return nil, nil, nil, status.Errorf(codes.NotFound, "environment %q does not exist in ksonnet app", envName) } @@ -361,7 +359,7 @@ func ksShow(appPath, envName string, overrides []*v1alpha1.ComponentParameter) ( if err != nil { return nil, nil, nil, err } - return targetObjs, params, env, nil + return targetObjs, params, dest, nil } var manifestFile = regexp.MustCompile(`^.*\.(yaml|yml|json|jsonnet)$`) diff --git a/util/ksonnet/ksonnet.go b/util/ksonnet/ksonnet.go index bb84e2050e9a3..89a50d2823ab3 100644 --- a/util/ksonnet/ksonnet.go +++ b/util/ksonnet/ksonnet.go @@ -1,38 +1,76 @@ package ksonnet import ( + "encoding/json" "fmt" + "io/ioutil" + "os" "os/exec" "path/filepath" "strconv" "strings" - "github.com/ksonnet/ksonnet/pkg/app" - "github.com/ksonnet/ksonnet/pkg/component" + "github.com/ghodss/yaml" log "github.com/sirupsen/logrus" - "github.com/spf13/afero" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" - "github.com/argoproj/argo-cd/util/config" "github.com/argoproj/argo-cd/util/kube" ) +func ksCmd(cwd string, args ...string) (string, error) { + cmd := exec.Command("ks", args...) + cmd.Dir = cwd + + cmdStr := strings.Join(cmd.Args, " ") + log.Debug(cmdStr) + outBytes, err := cmd.Output() + if err != nil { + exErr, ok := err.(*exec.ExitError) + if !ok { + return "", err + } + errOutput := string(exErr.Stderr) + log.Errorf("`%s` failed: %s", cmdStr, errOutput) + return "", fmt.Errorf(strings.TrimSpace(errOutput)) + } + out := string(outBytes) + log.Debug(out) + return out, nil +} + +// Destination returns the deployment destination for an environment in app spec data +func Destination(data []byte, environment string) (*v1alpha1.ApplicationDestination, error) { + var appSpec struct { + Environments map[string]struct { + Destination v1alpha1.ApplicationDestination + } + } + err := yaml.Unmarshal(data, &appSpec) + if err != nil { + return nil, fmt.Errorf("could not unmarshal ksonnet spec app.yaml: %v", err) + } + + envSpec, ok := appSpec.Environments[environment] + if !ok { + return nil, fmt.Errorf("environment '%s' does not exist in ksonnet app", environment) + } + + return &envSpec.Destination, nil +} + // KsonnetApp represents a ksonnet application directory and provides wrapper functionality around // the `ks` command. type KsonnetApp interface { // Root is the root path ksonnet application directory Root() string - // App is the Ksonnet application - App() app.App - - // Spec is the Ksonnet application spec - Spec() *app.Spec - // Show returns a list of unstructured objects that would be applied to an environment Show(environment string) ([]*unstructured.Unstructured, error) + // Destination returns the deployment destination for an environment + Destination(environment string) (*v1alpha1.ApplicationDestination, error) + // ListEnvParams returns list of environment parameters ListEnvParams(environment string) ([]*v1alpha1.ComponentParameter, error) @@ -42,8 +80,7 @@ type KsonnetApp interface { // KsonnetVersion returns the version of ksonnet used when running ksonnet commands func KsonnetVersion() (string, error) { - cmd := exec.Command("ks", "version") - out, err := cmd.Output() + out, err := ksCmd("", "version") if err != nil { return "", fmt.Errorf("unable to determine ksonnet version: %v", err) } @@ -56,66 +93,39 @@ func KsonnetVersion() (string, error) { } type ksonnetApp struct { - app app.App - spec app.Spec + rootDir string } // NewKsonnetApp tries to create a new wrapper to run commands on the `ks` command-line tool. func NewKsonnetApp(path string) (KsonnetApp, error) { - ksApp := ksonnetApp{} - a, err := app.Load(afero.NewOsFs(), path, false) - if err != nil { - return nil, err - } - ksApp.app = a - - var spec app.Spec - err = config.UnmarshalLocalFile(filepath.Join(a.Root(), "app.yaml"), &spec) - if err != nil { + ksApp := ksonnetApp{rootDir: path} + // ensure that the file exists + if _, err := ksApp.appYamlPath(); err != nil { return nil, err } - ksApp.spec = spec return &ksApp, nil } -func (k *ksonnetApp) ksCmd(args ...string) (string, error) { - cmd := exec.Command("ks", args...) - cmd.Dir = k.Root() - - cmdStr := strings.Join(cmd.Args, " ") - log.Debug(cmdStr) - outBytes, err := cmd.Output() - if err != nil { - exErr, ok := err.(*exec.ExitError) - if !ok { - return "", err - } - errOutput := string(exErr.Stderr) - log.Errorf("`%s` failed: %s", cmdStr, errOutput) - return "", fmt.Errorf(strings.TrimSpace(errOutput)) +func (k *ksonnetApp) appYamlPath() (string, error) { + const appYamlName = "app.yaml" + p := filepath.Join(k.rootDir, appYamlName) + if _, err := os.Stat(p); err != nil { + return "", err } - out := string(outBytes) - log.Debug(out) - return out, nil + return p, nil } -func (k *ksonnetApp) Root() string { - return k.app.Root() -} - -// App is the Ksonnet application -func (k *ksonnetApp) App() app.App { - return k.app +func (k *ksonnetApp) ksCmd(args ...string) (string, error) { + return ksCmd(k.rootDir, args...) } -// Spec is the Ksonnet application spec -func (k *ksonnetApp) Spec() *app.Spec { - return &k.spec +func (k *ksonnetApp) Root() string { + return k.rootDir } // Show generates a concatenated list of Kubernetes manifests in the given environment. func (k *ksonnetApp) Show(environment string) ([]*unstructured.Unstructured, error) { - out, err := k.ksCmd("show", environment) + out, err := ksCmd(k.rootDir, "show", environment) if err != nil { return nil, fmt.Errorf("`ks show` failed: %v", err) } @@ -129,19 +139,41 @@ func (k *ksonnetApp) Show(environment string) ([]*unstructured.Unstructured, err return data, err } +// Destination returns the deployment destination for an environment +func (k *ksonnetApp) Destination(environment string) (*v1alpha1.ApplicationDestination, error) { + p, err := k.appYamlPath() + if err != nil { + return nil, err + } + data, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + return Destination(data, environment) +} + // ListEnvParams returns list of environment parameters func (k *ksonnetApp) ListEnvParams(environment string) ([]*v1alpha1.ComponentParameter, error) { log.Infof("listing environment '%s' parameters", environment) - mod, err := component.DefaultManager.Module(k.app, "") + out, err := ksCmd(k.rootDir, "param", "list", "--output", "json", "--env", environment) if err != nil { return nil, err } - ksParams, err := mod.Params(environment) - if err != nil { + + // Auxiliary data to hold unmarshaled JSON output, which may use different field names + var ksParams struct { + Data []struct { + Component string `json:"component"` + Key string `json:"param"` + Value string `json:"value"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(out), &ksParams); err != nil { return nil, err } + var params []*v1alpha1.ComponentParameter - for _, ksParam := range ksParams { + for _, ksParam := range ksParams.Data { value, err := strconv.Unquote(ksParam.Value) if err != nil { value = ksParam.Value @@ -158,6 +190,6 @@ func (k *ksonnetApp) ListEnvParams(environment string) ([]*v1alpha1.ComponentPar // SetComponentParams updates component parameter in specified environment. func (k *ksonnetApp) SetComponentParams(environment string, component string, param string, value string) error { - _, err := k.ksCmd("param", "set", component, param, value, "--env", environment) + _, err := ksCmd(k.rootDir, "param", "set", component, param, value, "--env", environment) return err } diff --git a/util/ksonnet/ksonnet_test.go b/util/ksonnet/ksonnet_test.go index 9e1232b97bac0..b1256f6d31c3b 100644 --- a/util/ksonnet/ksonnet_test.go +++ b/util/ksonnet/ksonnet_test.go @@ -2,7 +2,7 @@ package ksonnet import ( "encoding/json" - "path" + "path/filepath" "reflect" "runtime" "testing" @@ -23,20 +23,19 @@ const ( func init() { _, filename, _, _ := runtime.Caller(0) - testDataDir = path.Join(path.Dir(filename), "testdata") + testDataDir = filepath.Join(filepath.Dir(filename), "testdata") } func TestKsonnet(t *testing.T) { - ksApp, err := NewKsonnetApp(path.Join(testDataDir, testAppName)) + ksApp, err := NewKsonnetApp(filepath.Join(testDataDir, testAppName)) assert.Nil(t, err) - app := ksApp.App() - defaultEnv, err := app.Environment(testEnvName) + defaultEnv, err := ksApp.GetEnvironment(testEnvName) assert.True(t, err == nil) assert.Equal(t, "https://1.2.3.4", defaultEnv.Destination.Server) } func TestShow(t *testing.T) { - ksApp, err := NewKsonnetApp(path.Join(testDataDir, testAppName)) + ksApp, err := NewKsonnetApp(filepath.Join(testDataDir, testAppName)) assert.Nil(t, err) objs, err := ksApp.Show(testEnvName) assert.Nil(t, err) @@ -49,7 +48,7 @@ func TestShow(t *testing.T) { } func TestListEnvParams(t *testing.T) { - ksApp, err := NewKsonnetApp(path.Join(testDataDir, testAppName)) + ksApp, err := NewKsonnetApp(filepath.Join(testDataDir, testAppName)) assert.Nil(t, err) paramPointers, err := ksApp.ListEnvParams(testEnvName) assert.Nil(t, err) From f30de87d3356c0836d4f7eaea1f6592dfd826c04 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Mon, 24 Sep 2018 09:16:02 -0700 Subject: [PATCH 02/12] Update util/argo/argo.go --- util/argo/argo.go | 72 +++++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/util/argo/argo.go b/util/argo/argo.go index f142dcf280417..b270f938f90b9 100644 --- a/util/argo/argo.go +++ b/util/argo/argo.go @@ -9,8 +9,6 @@ import ( "strings" "time" - "github.com/ghodss/yaml" - "github.com/ksonnet/ksonnet/pkg/app" log "github.com/sirupsen/logrus" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -29,6 +27,7 @@ import ( "github.com/argoproj/argo-cd/util" "github.com/argoproj/argo-cd/util/db" "github.com/argoproj/argo-cd/util/git" + "github.com/argoproj/argo-cd/util/ksonnet" ) const ( @@ -223,9 +222,13 @@ func GetSpecErrors( } else { switch appSourceType { case repository.AppSourceKsonnet: - appYamlConditions := verifyAppYAML(ctx, repoRes, spec, repoClient) - if len(appYamlConditions) > 0 { - conditions = append(conditions, appYamlConditions...) + err := verifyAppYAML(ctx, repoRes, spec, repoClient) + if err != nil { + conditions = append(conditions, argoappv1.ApplicationCondition{ + Type: argoappv1.ApplicationConditionInvalidSpecError, + Message: err.Error(), + }) + } case repository.AppSourceHelm: helmConditions := verifyHelmChart(ctx, repoRes, spec, repoClient) @@ -321,7 +324,7 @@ func queryAppSourceType(ctx context.Context, spec *argoappv1.ApplicationSpec, re } // verifyAppYAML verifies that a ksonnet app.yaml is functional -func verifyAppYAML(ctx context.Context, repoRes *argoappv1.Repository, spec *argoappv1.ApplicationSpec, repoClient repository.RepositoryServiceClient) []argoappv1.ApplicationCondition { +func verifyAppYAML(ctx context.Context, repoRes *argoappv1.Repository, spec *argoappv1.ApplicationSpec, repoClient repository.RepositoryServiceClient) error { req := repository.GetFileRequest{ Repo: &argoappv1.Repository{ Repo: spec.Source.RepoURL, @@ -335,47 +338,30 @@ func verifyAppYAML(ctx context.Context, repoRes *argoappv1.Repository, spec *arg req.Repo.SSHPrivateKey = repoRes.SSHPrivateKey } getRes, err := repoClient.GetFile(ctx, &req) - var conditions []argoappv1.ApplicationCondition if err != nil { - conditions = append(conditions, argoappv1.ApplicationCondition{ - Type: argoappv1.ApplicationConditionInvalidSpecError, - Message: fmt.Sprintf("Unable to load app.yaml: %v", err), - }) - } else { - var appSpec app.Spec - err = yaml.Unmarshal(getRes.Data, &appSpec) - if err != nil { - conditions = append(conditions, argoappv1.ApplicationCondition{ - Type: argoappv1.ApplicationConditionInvalidSpecError, - Message: "app.yaml is not a valid ksonnet app spec", - }) - } else { - // Default revision to HEAD if unspecified - if spec.Source.TargetRevision == "" { - spec.Source.TargetRevision = "HEAD" - } + return fmt.Errorf("Unable to load app.yaml: %v", err) + } - // Verify the specified environment is defined in it - envSpec, ok := appSpec.Environments[spec.Source.Environment] - if !ok || envSpec == nil { - conditions = append(conditions, argoappv1.ApplicationCondition{ - Type: argoappv1.ApplicationConditionInvalidSpecError, - Message: fmt.Sprintf("environment '%s' does not exist in ksonnet app", spec.Source.Environment), - }) - } + // Default revision to HEAD if unspecified + if spec.Source.TargetRevision == "" { + spec.Source.TargetRevision = "HEAD" + } - if envSpec != nil { - // If server and namespace are not supplied, pull it from the app.yaml - if spec.Destination.Server == "" { - spec.Destination.Server = envSpec.Destination.Server - } - if spec.Destination.Namespace == "" { - spec.Destination.Namespace = envSpec.Destination.Namespace - } - } - } + // Verify the specified environment is defined in the app spec + dest, err := ksonnet.Destination(getRes.Data, spec.Source.Environment) + if err != nil { + return err } - return conditions + + // If server and namespace are not supplied, pull it from the app.yaml + if spec.Destination.Server == "" { + spec.Destination.Server = dest.Server + } + if spec.Destination.Namespace == "" { + spec.Destination.Namespace = dest.Namespace + } + + return nil } // verifyHelmChart verifies a helm chart is functional From 6704521aed97e24fd8e67903f815c62110b73c4b Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 14:09:06 -0700 Subject: [PATCH 03/12] Fix metrics, stats warnings --- server/metrics/metrics.go | 2 +- util/stats/stats.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/metrics/metrics.go b/server/metrics/metrics.go index 1d2a94c84100a..68241ca9e368a 100644 --- a/server/metrics/metrics.go +++ b/server/metrics/metrics.go @@ -88,7 +88,7 @@ func (c *appCollector) Describe(ch chan<- *prometheus.Desc) { func (c *appCollector) Collect(ch chan<- prometheus.Metric) { apps, err := c.store.List(labels.NewSelector()) if err != nil { - log.Warn("Failed to collect applications: %v", err) + log.Warnf("Failed to collect applications: %v", err) return } for _, app := range apps { diff --git a/util/stats/stats.go b/util/stats/stats.go index 3cf6374719246..ed98aebf36c1c 100644 --- a/util/stats/stats.go +++ b/util/stats/stats.go @@ -45,18 +45,18 @@ func RegisterHeapDumper(filePath string) { if _, err := os.Stat(filePath); err == nil { err = os.Remove(filePath) if err != nil { - log.Warnf("could not delete heap profile file: ", err) + log.Warnf("could not delete heap profile file: %v", err) return } } f, err := os.Create(filePath) if err != nil { - log.Warnf("could not create heap profile file: ", err) + log.Warnf("could not create heap profile file: %v", err) return } if err := pprof.WriteHeapProfile(f); err != nil { - log.Warnf("could not write heap profile: ", err) + log.Warnf("could not write heap profile: %v", err) return } else { log.Infof("dumped heap profile to %s", filePath) From d6671c7b9b66354ffd4a408f6e9fe9af908e6f94 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 14:11:03 -0700 Subject: [PATCH 04/12] Fix test --- util/ksonnet/ksonnet_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/ksonnet/ksonnet_test.go b/util/ksonnet/ksonnet_test.go index b1256f6d31c3b..d8b8c7a4a3bf4 100644 --- a/util/ksonnet/ksonnet_test.go +++ b/util/ksonnet/ksonnet_test.go @@ -29,9 +29,9 @@ func init() { func TestKsonnet(t *testing.T) { ksApp, err := NewKsonnetApp(filepath.Join(testDataDir, testAppName)) assert.Nil(t, err) - defaultEnv, err := ksApp.GetEnvironment(testEnvName) + defaultDest, err := ksApp.Destination(testEnvName) assert.True(t, err == nil) - assert.Equal(t, "https://1.2.3.4", defaultEnv.Destination.Server) + assert.Equal(t, "https://1.2.3.4", defaultDest.Server) } func TestShow(t *testing.T) { From e5e90986002389d1cacc1940c2163cc09fd3bc70 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 14:12:36 -0700 Subject: [PATCH 05/12] Fix warning in server --- server/application/application.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/application/application.go b/server/application/application.go index 7ea29494f86bf..fe555de0db960 100644 --- a/server/application/application.go +++ b/server/application/application.go @@ -809,7 +809,7 @@ func (s *Server) TerminateOperation(ctx context.Context, termOpReq *OperationTer if !apierr.IsConflict(err) { return nil, err } - log.Warnf("Failed to set operation for app '%s' due to update conflict. Retrying again...", termOpReq.Name) + log.Warnf("Failed to set operation for app '%s' due to update conflict. Retrying again...", *termOpReq.Name) time.Sleep(100 * time.Millisecond) a, err = s.appclientset.ArgoprojV1alpha1().Applications(s.ns).Get(*termOpReq.Name, metav1.GetOptions{}) if err != nil { From 54965b541ac210b3630add857acb854780df41c6 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 14:28:26 -0700 Subject: [PATCH 06/12] Trim all quotation marks on Ksonnet output when listing params --- util/ksonnet/ksonnet.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/util/ksonnet/ksonnet.go b/util/ksonnet/ksonnet.go index 89a50d2823ab3..8930fddd9532c 100644 --- a/util/ksonnet/ksonnet.go +++ b/util/ksonnet/ksonnet.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "strconv" "strings" "github.com/ghodss/yaml" @@ -174,10 +173,7 @@ func (k *ksonnetApp) ListEnvParams(environment string) ([]*v1alpha1.ComponentPar var params []*v1alpha1.ComponentParameter for _, ksParam := range ksParams.Data { - value, err := strconv.Unquote(ksParam.Value) - if err != nil { - value = ksParam.Value - } + value := strings.Trim(ksParam.Value, `'"`) componentParam := v1alpha1.ComponentParameter{ Component: ksParam.Component, Name: ksParam.Key, From 547014162bb6512823c0bd9a479c5be0ac5652dc Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 14:43:50 -0700 Subject: [PATCH 07/12] Rm unnecessary conversion --- util/ksonnet/ksonnet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/ksonnet/ksonnet.go b/util/ksonnet/ksonnet.go index 8930fddd9532c..7845fa51a282a 100644 --- a/util/ksonnet/ksonnet.go +++ b/util/ksonnet/ksonnet.go @@ -83,7 +83,7 @@ func KsonnetVersion() (string, error) { if err != nil { return "", fmt.Errorf("unable to determine ksonnet version: %v", err) } - ksonnetVersionStr := strings.Split(string(out), "\n")[0] + ksonnetVersionStr := strings.Split(out, "\n")[0] parts := strings.SplitN(ksonnetVersionStr, ":", 2) if len(parts) != 2 { return "", fmt.Errorf("unexpected version string format: %s", ksonnetVersionStr) From 74027c4c3d40b8f8c98cec076682c887951d7081 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 16:46:55 -0700 Subject: [PATCH 08/12] Rm x/sync/errgroup from Gopkg.toml --- Gopkg.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Gopkg.toml b/Gopkg.toml index b715ff187fca4..8da6d4984569f 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -9,7 +9,6 @@ required = [ "k8s.io/code-generator/cmd/go-to-protobuf", "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway", "github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger", - "golang.org/x/sync/errgroup", ] [[constraint]] From d246fae043cc07c8066baa5a390044744f0cb440 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Tue, 25 Sep 2018 17:14:25 -0700 Subject: [PATCH 09/12] Don't install errgroup --- Gopkg.lock | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 742594dd88dc5..d8df2e2ff09b1 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -481,6 +481,14 @@ revision = "81db2a75821ed34e682567d48be488a1c3121088" version = "0.5" +[[projects]] + branch = "master" + digest = "1:448b4a6e39e46d8740b00dc871f26d58dc39341b160e01267b7917132831a136" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "" + revision = "b729f2633dfe35f4d1d8a32385f6685610ce1cb5" + [[projects]] branch = "master" digest = "1:ccc20cacf54eb16464dad02efa1c14fa7c0b9e124639b0d2a51dcc87b0154e4c" @@ -621,12 +629,12 @@ version = "v1.0.0" [[projects]] - digest = "1:3fcbf733a8d810a21265a7f2fe08a3353db2407da052b233f8b204b5afc03d9b" + digest = "1:01d968ff6535945510c944983eee024e81f1c949043e9bbfe5ab206ebc3588a4" name = "github.com/sirupsen/logrus" packages = ["."] pruneopts = "" - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "a67f783a3814b8729bd2dac5780b5f78f8dbd64d" + version = "v1.1.0" [[projects]] branch = "master" @@ -786,14 +794,6 @@ pruneopts = "" revision = "cce311a261e6fcf29de72ca96827bdb0b7d9c9e6" -[[projects]] - branch = "master" - digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1" - name = "golang.org/x/sync" - packages = ["errgroup"] - pruneopts = "" - revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" - [[projects]] branch = "master" digest = "1:ed900376500543ca05f2a2383e1f541b4606f19cd22f34acb81b17a0b90c7f3e" @@ -1420,7 +1420,6 @@ "golang.org/x/crypto/ssh/terminal", "golang.org/x/net/context", "golang.org/x/oauth2", - "golang.org/x/sync/errgroup", "google.golang.org/genproto/googleapis/api/annotations", "google.golang.org/grpc", "google.golang.org/grpc/codes", From 6365b8bcf552bc6af565de59868f3b9e4f428e78 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Wed, 26 Sep 2018 09:29:31 -0700 Subject: [PATCH 10/12] Don't over-engineer Ksonnet abstractions, thanks @jessesuen --- util/ksonnet/ksonnet.go | 51 +++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/util/ksonnet/ksonnet.go b/util/ksonnet/ksonnet.go index 7845fa51a282a..d828f160a9a82 100644 --- a/util/ksonnet/ksonnet.go +++ b/util/ksonnet/ksonnet.go @@ -17,27 +17,6 @@ import ( "github.com/argoproj/argo-cd/util/kube" ) -func ksCmd(cwd string, args ...string) (string, error) { - cmd := exec.Command("ks", args...) - cmd.Dir = cwd - - cmdStr := strings.Join(cmd.Args, " ") - log.Debug(cmdStr) - outBytes, err := cmd.Output() - if err != nil { - exErr, ok := err.(*exec.ExitError) - if !ok { - return "", err - } - errOutput := string(exErr.Stderr) - log.Errorf("`%s` failed: %s", cmdStr, errOutput) - return "", fmt.Errorf(strings.TrimSpace(errOutput)) - } - out := string(outBytes) - log.Debug(out) - return out, nil -} - // Destination returns the deployment destination for an environment in app spec data func Destination(data []byte, environment string) (*v1alpha1.ApplicationDestination, error) { var appSpec struct { @@ -79,7 +58,8 @@ type KsonnetApp interface { // KsonnetVersion returns the version of ksonnet used when running ksonnet commands func KsonnetVersion() (string, error) { - out, err := ksCmd("", "version") + ksApp := ksonnetApp{} + out, err := ksApp.ksCmd("", "version") if err != nil { return "", fmt.Errorf("unable to determine ksonnet version: %v", err) } @@ -107,7 +87,7 @@ func NewKsonnetApp(path string) (KsonnetApp, error) { func (k *ksonnetApp) appYamlPath() (string, error) { const appYamlName = "app.yaml" - p := filepath.Join(k.rootDir, appYamlName) + p := filepath.Join(k.Root(), appYamlName) if _, err := os.Stat(p); err != nil { return "", err } @@ -115,7 +95,24 @@ func (k *ksonnetApp) appYamlPath() (string, error) { } func (k *ksonnetApp) ksCmd(args ...string) (string, error) { - return ksCmd(k.rootDir, args...) + cmd := exec.Command("ks", args...) + cmd.Dir = k.Root() + + cmdStr := strings.Join(cmd.Args, " ") + log.Debug(cmdStr) + outBytes, err := cmd.Output() + if err != nil { + exErr, ok := err.(*exec.ExitError) + if !ok { + return "", err + } + errOutput := string(exErr.Stderr) + log.Errorf("`%s` failed: %s", cmdStr, errOutput) + return "", fmt.Errorf(strings.TrimSpace(errOutput)) + } + out := string(outBytes) + log.Debug(out) + return out, nil } func (k *ksonnetApp) Root() string { @@ -124,7 +121,7 @@ func (k *ksonnetApp) Root() string { // Show generates a concatenated list of Kubernetes manifests in the given environment. func (k *ksonnetApp) Show(environment string) ([]*unstructured.Unstructured, error) { - out, err := ksCmd(k.rootDir, "show", environment) + out, err := k.ksCmd("show", environment) if err != nil { return nil, fmt.Errorf("`ks show` failed: %v", err) } @@ -154,7 +151,7 @@ func (k *ksonnetApp) Destination(environment string) (*v1alpha1.ApplicationDesti // ListEnvParams returns list of environment parameters func (k *ksonnetApp) ListEnvParams(environment string) ([]*v1alpha1.ComponentParameter, error) { log.Infof("listing environment '%s' parameters", environment) - out, err := ksCmd(k.rootDir, "param", "list", "--output", "json", "--env", environment) + out, err := k.ksCmd("param", "list", "--output", "json", "--env", environment) if err != nil { return nil, err } @@ -186,6 +183,6 @@ func (k *ksonnetApp) ListEnvParams(environment string) ([]*v1alpha1.ComponentPar // SetComponentParams updates component parameter in specified environment. func (k *ksonnetApp) SetComponentParams(environment string, component string, param string, value string) error { - _, err := ksCmd(k.rootDir, "param", "set", component, param, value, "--env", environment) + _, err := k.ksCmd("param", "set", component, param, value, "--env", environment) return err } From ee85d4512bd8eaef35a033081c83ec59b0d8c873 Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Wed, 26 Sep 2018 11:49:02 -0700 Subject: [PATCH 11/12] Upgrade Ksonnet to 0.13.0 with quote fix to support --output --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1db23906bbba3..d107e85227921 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ RUN curl -L -o /usr/local/bin/kubectl -LO https://storage.googleapis.com/kuberne # Option 1: build ksonnet ourselves #RUN go get -v -u github.com/ksonnet/ksonnet && mv ${GOPATH}/bin/ksonnet /usr/local/bin/ks # Option 2: use official tagged ksonnet release -ENV KSONNET_VERSION=0.11.0 +ENV KSONNET_VERSION=0.13.0 RUN wget https://github.com/ksonnet/ksonnet/releases/download/v${KSONNET_VERSION}/ks_${KSONNET_VERSION}_linux_amd64.tar.gz && \ tar -C /tmp/ -xf ks_${KSONNET_VERSION}_linux_amd64.tar.gz && \ mv /tmp/ks_${KSONNET_VERSION}_linux_amd64/ks /usr/local/bin/ks From e2dd26e1161132bb02e1e42c5135be48a5948b5d Mon Sep 17 00:00:00 2001 From: Andrew Merenbach Date: Thu, 27 Sep 2018 09:24:38 -0700 Subject: [PATCH 12/12] Default to HEAD, credit @alexmt --- util/argo/argo.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/util/argo/argo.go b/util/argo/argo.go index b270f938f90b9..0d1a6db04c813 100644 --- a/util/argo/argo.go +++ b/util/argo/argo.go @@ -325,6 +325,11 @@ func queryAppSourceType(ctx context.Context, spec *argoappv1.ApplicationSpec, re // verifyAppYAML verifies that a ksonnet app.yaml is functional func verifyAppYAML(ctx context.Context, repoRes *argoappv1.Repository, spec *argoappv1.ApplicationSpec, repoClient repository.RepositoryServiceClient) error { + // Default revision to HEAD if unspecified + if spec.Source.TargetRevision == "" { + spec.Source.TargetRevision = "HEAD" + } + req := repository.GetFileRequest{ Repo: &argoappv1.Repository{ Repo: spec.Source.RepoURL, @@ -342,11 +347,6 @@ func verifyAppYAML(ctx context.Context, repoRes *argoappv1.Repository, spec *arg return fmt.Errorf("Unable to load app.yaml: %v", err) } - // Default revision to HEAD if unspecified - if spec.Source.TargetRevision == "" { - spec.Source.TargetRevision = "HEAD" - } - // Verify the specified environment is defined in the app spec dest, err := ksonnet.Destination(getRes.Data, spec.Source.Environment) if err != nil {