diff --git a/go.mod b/go.mod index d3b6fcf7b8..5ae99cecd3 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/envoyproxy/protoc-gen-validate v0.1.0 github.com/fsouza/fake-gcs-server v1.21.0 + github.com/goccy/go-yaml v1.8.4 github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.4.2 github.com/gomodule/redigo v2.0.0+incompatible diff --git a/go.sum b/go.sum index afa018c240..f7f128bdc9 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -134,6 +136,14 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= +github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -161,6 +171,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/goccy/go-yaml v1.8.4 h1:AOEdR7aQgbgwHznGe3BLkDQVujxCPUpHOZZcQcp8Y3M= +github.com/goccy/go-yaml v1.8.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -309,10 +321,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= +github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= @@ -568,6 +586,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/yamlprocessor/BUILD.bazel b/pkg/yamlprocessor/BUILD.bazel new file mode 100644 index 0000000000..d5bd463ac4 --- /dev/null +++ b/pkg/yamlprocessor/BUILD.bazel @@ -0,0 +1,21 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["yamlprocessor.go"], + importpath = "github.com/pipe-cd/pipe/pkg/yamlprocessor", + visibility = ["//visibility:public"], + deps = [ + "@com_github_goccy_go_yaml//:go_default_library", + "@com_github_goccy_go_yaml//ast:go_default_library", + "@com_github_goccy_go_yaml//parser:go_default_library", + ], +) + +go_test( + name = "go_default_test", + size = "small", + srcs = ["yamlprocessor_test.go"], + embed = [":go_default_library"], + deps = ["@com_github_stretchr_testify//assert:go_default_library"], +) diff --git a/pkg/yamlprocessor/yamlprocessor.go b/pkg/yamlprocessor/yamlprocessor.go new file mode 100644 index 0000000000..56644d9810 --- /dev/null +++ b/pkg/yamlprocessor/yamlprocessor.go @@ -0,0 +1,97 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yamlprocessor + +import ( + "bytes" + "fmt" + "io" + + goyaml "github.com/goccy/go-yaml" + "github.com/goccy/go-yaml/ast" + "github.com/goccy/go-yaml/parser" +) + +// GetValue gives back the value placed at a given path. +// +// The path requires to start with "$" which represents the root element. +// Available operators are: +// $ : the root object/element +// . : child operator +// .. : recursive descent +// [num] : object/element of array by number +// [*] : all objects/elements for array. +// +// e.g. "$.foo.bar[0].baz" +func GetValue(yml []byte, path string) (interface{}, error) { + if len(yml) == 0 { + return nil, fmt.Errorf("empty yaml given") + } + if path == "" { + return nil, fmt.Errorf("no path given") + } + + p, err := goyaml.PathString(path) + if err != nil { + return nil, err + } + + var value interface{} + if err := p.Read(bytes.NewReader(yml), &value); err != nil { + return nil, err + } + return value, nil +} + +// ReplaceValue replaces the value placed at a given path with +// a given value, and then gives back the new yaml bytes. +// +// For available operators for the path, see GetValue(). +func ReplaceValue(yml []byte, path string, value string) ([]byte, error) { + if len(yml) == 0 { + return nil, fmt.Errorf("empty yaml given") + } + if path == "" { + return nil, fmt.Errorf("no path given") + } + + p, err := goyaml.PathString(path) + if err != nil { + return nil, err + } + file, err := parser.ParseBytes(yml, 0) + if err != nil { + return nil, err + } + + // Retrieve the current node placed at the specified path. + oldNode, err := p.FilterFile(file) + if err != nil { + return nil, err + } + newNode := &ast.StringNode{ + BaseNode: &ast.BaseNode{}, + Token: oldNode.GetToken(), + Value: value, + } + + err = p.ReplaceWithNode(file, newNode) + if err != nil { + return nil, err + } + buf := new(bytes.Buffer) + _, err = io.Copy(buf, file) + return buf.Bytes(), err +} diff --git a/pkg/yamlprocessor/yamlprocessor_test.go b/pkg/yamlprocessor/yamlprocessor_test.go new file mode 100644 index 0000000000..254a68a0bb --- /dev/null +++ b/pkg/yamlprocessor/yamlprocessor_test.go @@ -0,0 +1,212 @@ +// Copyright 2020 The PipeCD Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package yamlprocessor + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetValue(t *testing.T) { + testcases := []struct { + name string + yml string + path string + want interface{} + wantErr bool + }{ + { + name: "empty yaml given", + yml: "", + path: "$.foo", + wantErr: true, + }, + { + name: "empty path given", + yml: "foo: bar", + path: "", + wantErr: true, + }, + { + name: "wrong path given", + yml: "foo: bar", + path: "wrong", + wantErr: true, + }, + { + name: "wrong yaml given", + yml: "::", + path: "$.foo", + wantErr: true, + }, + { + name: "lack of root element", + yml: "foo: bar", + path: "foo", + wantErr: true, + }, + { + name: "given a string path", + yml: "foo: bar", + path: "$.foo", + want: "bar", + wantErr: false, + }, + { + name: "given a bool path", + yml: "foo: true", + path: "$.foo", + want: true, + wantErr: false, + }, + { + name: "given a uint64 path", + yml: "foo: 1", + path: "$.foo", + want: uint64(1), + wantErr: false, + }, + { + name: "given a float64 path", + yml: "foo: 1.5", + path: "$.foo", + want: 1.5, + wantErr: false, + }, + { + name: "given a array path", + yml: ` +foo: +- bar: 1`, + path: "$.foo[0].bar", + want: uint64(1), + wantErr: false, + }, + { + name: "given a array path with wildcard", + yml: ` +foo: +- bar: 1 +- baz: 2`, + path: "$.foo[*]", + want: []interface{}{map[string]interface{}{"bar": uint64(1)}, map[string]interface{}{"baz": uint64(2)}}, + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := GetValue([]byte(tc.yml), tc.path) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestReplaceValue(t *testing.T) { + testcases := []struct { + name string + yml string + path string + value string + want []byte + wantErr bool + }{ + { + name: "empty yaml given", + yml: "", + path: "$.foo", + value: "new-text", + wantErr: true, + }, + { + name: "empty path given", + yml: "foo: bar", + path: "", + value: "new-text", + wantErr: true, + }, + { + name: "wrong path given", + yml: "foo: bar", + path: "wrong", + value: "new-text", + wantErr: true, + }, + { + name: "wrong yaml given", + yml: "::", + path: "$.foo", + value: "new-text", + wantErr: true, + }, + { + name: "empty value given", + yml: "foo: bar", + path: "$.foo", + value: "", + want: []byte("foo: "), + wantErr: false, + }, + { + name: "valid value given", + yml: "foo: bar", + path: "$.foo", + value: "new-text", + want: []byte("foo: new-text"), + wantErr: false, + }, + { + name: "array in block style", + yml: `foo: + - bar + - baz`, + path: "$.foo[0]", + value: "new-text", + want: []byte(`foo: + - new-text + - baz`), + wantErr: false, + }, + { + name: "array in flow style", + yml: `foo: [bar, baz]`, + path: "$.foo[0]", + value: "new-text", + want: []byte(`foo: [new-text, baz]`), + wantErr: false, + }, + { + name: "there is an useless blank line", + yml: ` +foo: + - bar + - baz`, + path: "$.foo[0]", + value: "new-text", + want: []byte(`foo: + - new-text + - baz`), + wantErr: false, + }, + } + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + got, err := ReplaceValue([]byte(tc.yml), tc.path, tc.value) + assert.Equal(t, tc.wantErr, err != nil) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/repositories.bzl b/repositories.bzl index ac9204def9..3a43bfdd97 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -299,8 +299,8 @@ def go_repositories(): go_repository( name = "com_github_fatih_color", importpath = "github.com/fatih/color", - sum = "h1:66qjqZk8kalYAvDRtM1AdAJQI0tj4Wrue3Eq3B3pmFU=", - version = "v1.6.0", + sum = "h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=", + version = "v1.10.0", ) go_repository( @@ -411,6 +411,30 @@ def go_repositories(): sum = "h1:zP3nY8Tk2E6RTkqGYrarZXuzh+ffyLDljLxCy1iJw80=", version = "v0.0.0-20160704191624-1d0bd113de87", ) + go_repository( + name = "com_github_go_playground_assert_v2", + importpath = "github.com/go-playground/assert/v2", + sum = "h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=", + version = "v2.0.1", + ) + go_repository( + name = "com_github_go_playground_locales", + importpath = "github.com/go-playground/locales", + sum = "h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=", + version = "v0.13.0", + ) + go_repository( + name = "com_github_go_playground_universal_translator", + importpath = "github.com/go-playground/universal-translator", + sum = "h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=", + version = "v0.17.0", + ) + go_repository( + name = "com_github_go_playground_validator_v10", + importpath = "github.com/go-playground/validator/v10", + sum = "h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=", + version = "v10.4.1", + ) go_repository( name = "com_github_go_sql_driver_mysql", @@ -557,6 +581,13 @@ def go_repositories(): sum = "h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=", version = "v0.2.3", ) + go_repository( + name = "com_github_goccy_go_yaml", + importpath = "github.com/goccy/go-yaml", + sum = "h1:AOEdR7aQgbgwHznGe3BLkDQVujxCPUpHOZZcQcp8Y3M=", + version = "v1.8.4", + ) + go_repository( name = "com_github_gogo_protobuf", importpath = "github.com/gogo/protobuf", @@ -914,6 +945,12 @@ def go_repositories(): sum = "h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=", version = "v0.1.0", ) + go_repository( + name = "com_github_leodido_go_urn", + importpath = "github.com/leodido/go-urn", + sum = "h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=", + version = "v1.2.0", + ) go_repository( name = "com_github_logrusorgru_aurora", @@ -956,14 +993,14 @@ def go_repositories(): go_repository( name = "com_github_mattn_go_colorable", importpath = "github.com/mattn/go-colorable", - sum = "h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=", - version = "v0.0.9", + sum = "h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=", + version = "v0.1.8", ) go_repository( name = "com_github_mattn_go_isatty", importpath = "github.com/mattn/go-isatty", - sum = "h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=", - version = "v0.0.3", + sum = "h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=", + version = "v0.0.12", ) go_repository( name = "com_github_mattn_goveralls",