Add a package for yaml templating

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/envoyproxy/protoc-gen-validate v0.1.0
 	github.com/fsouza/fake-gcs-server v1.21.0
+	github.com/goccy/go-yaml v1.8.4
 	github.com/golang/mock v1.4.4
 	github.com/golang/protobuf v1.4.2
 	github.com/gomodule/redigo v2.0.0+incompatible

go.sum

@@ -114,6 +114,8 @@ github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
@@ -134,6 +136,14 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -161,6 +171,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
+github.com/goccy/go-yaml v1.8.4 h1:AOEdR7aQgbgwHznGe3BLkDQVujxCPUpHOZZcQcp8Y3M=
+github.com/goccy/go-yaml v1.8.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
@@ -309,10 +321,16 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
+github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
@@ -568,6 +586,7 @@ golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/yamlprocessor/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["yamlprocessor.go"],
+    importpath = "github.com/pipe-cd/pipe/pkg/yamlprocessor",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_goccy_go_yaml//:go_default_library",
+        "@com_github_goccy_go_yaml//ast:go_default_library",
+        "@com_github_goccy_go_yaml//parser:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = ["yamlprocessor_test.go"],
+    embed = [":go_default_library"],
+    deps = ["@com_github_stretchr_testify//assert:go_default_library"],
+)

pkg/yamlprocessor/yamlprocessor.go

@@ -0,0 +1,97 @@
+// Copyright 2020 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package yamlprocessor
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+
+	goyaml "github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	"github.com/goccy/go-yaml/parser"
+)
+
+// GetValue gives back the value placed at a given path.
+//
+// The path requires to start with "$" which represents the root element.
+// Available operators are:
+// $     : the root object/element
+// .     : child operator
+// ..    : recursive descent
+// [num] : object/element of array by number
+// [*]   : all objects/elements for array.
+//
+// e.g. "$.foo.bar[0].baz"
+func GetValue(yml []byte, path string) (interface{}, error) {
+	if len(yml) == 0 {
+		return nil, fmt.Errorf("empty yaml given")
+	}
+	if path == "" {
+		return nil, fmt.Errorf("no path given")
+	}
+
+	p, err := goyaml.PathString(path)
+	if err != nil {
+		return nil, err
+	}
+
+	var value interface{}
+	if err := p.Read(bytes.NewReader(yml), &value); err != nil {
+		return nil, err
+	}
+	return value, nil
+}
+
+// ReplaceValue replaces the value placed at a given path with
+// a given value, and then gives back the new yaml bytes.
+//
+// For available operators for the path, see GetValue().
+func ReplaceValue(yml []byte, path string, value string) ([]byte, error) {
+	if len(yml) == 0 {
+		return nil, fmt.Errorf("empty yaml given")
+	}
+	if path == "" {
+		return nil, fmt.Errorf("no path given")
+	}
+
+	p, err := goyaml.PathString(path)
+	if err != nil {
+		return nil, err
+	}
+	file, err := parser.ParseBytes(yml, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	// Retrieve the current node placed at the specified path.
+	oldNode, err := p.FilterFile(file)
+	if err != nil {
+		return nil, err
+	}
+	newNode := &ast.StringNode{
+		BaseNode: &ast.BaseNode{},
+		Token:    oldNode.GetToken(),
+		Value:    value,
+	}
+
+	err = p.ReplaceWithNode(file, newNode)
+	if err != nil {
+		return nil, err
+	}
+	buf := new(bytes.Buffer)
+	_, err = io.Copy(buf, file)
+	return buf.Bytes(), err
+}

pkg/yamlprocessor/yamlprocessor_test.go

@@ -0,0 +1,212 @@
+// Copyright 2020 The PipeCD Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package yamlprocessor
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetValue(t *testing.T) {
+	testcases := []struct {
+		name    string
+		yml     string
+		path    string
+		want    interface{}
+		wantErr bool
+	}{
+		{
+			name:    "empty yaml given",
+			yml:     "",
+			path:    "$.foo",
+			wantErr: true,
+		},
+		{
+			name:    "empty path given",
+			yml:     "foo: bar",
+			path:    "",
+			wantErr: true,
+		},
+		{
+			name:    "wrong path given",
+			yml:     "foo: bar",
+			path:    "wrong",
+			wantErr: true,
+		},
+		{
+			name:    "wrong yaml given",
+			yml:     "::",
+			path:    "$.foo",
+			wantErr: true,
+		},
+		{
+			name:    "lack of root element",
+			yml:     "foo: bar",
+			path:    "foo",
+			wantErr: true,
+		},
+		{
+			name:    "given a string path",
+			yml:     "foo: bar",
+			path:    "$.foo",
+			want:    "bar",
+			wantErr: false,
+		},
+		{
+			name:    "given a bool path",
+			yml:     "foo: true",
+			path:    "$.foo",
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name:    "given a uint64 path",
+			yml:     "foo: 1",
+			path:    "$.foo",
+			want:    uint64(1),
+			wantErr: false,
+		},
+		{
+			name:    "given a float64 path",
+			yml:     "foo: 1.5",
+			path:    "$.foo",
+			want:    1.5,
+			wantErr: false,
+		},
+		{
+			name: "given a array path",
+			yml: `
+foo:
+- bar: 1`,
+			path:    "$.foo[0].bar",
+			want:    uint64(1),
+			wantErr: false,
+		},
+		{
+			name: "given a array path with wildcard",
+			yml: `
+foo:
+- bar: 1
+- baz: 2`,
+			path:    "$.foo[*]",
+			want:    []interface{}{map[string]interface{}{"bar": uint64(1)}, map[string]interface{}{"baz": uint64(2)}},
+			wantErr: false,
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := GetValue([]byte(tc.yml), tc.path)
+			assert.Equal(t, tc.wantErr, err != nil)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestReplaceValue(t *testing.T) {
+	testcases := []struct {
+		name    string
+		yml     string
+		path    string
+		value   string
+		want    []byte
+		wantErr bool
+	}{
+		{
+			name:    "empty yaml given",
+			yml:     "",
+			path:    "$.foo",
+			value:   "new-text",
+			wantErr: true,
+		},
+		{
+			name:    "empty path given",
+			yml:     "foo: bar",
+			path:    "",
+			value:   "new-text",
+			wantErr: true,
+		},
+		{
+			name:    "wrong path given",
+			yml:     "foo: bar",
+			path:    "wrong",
+			value:   "new-text",
+			wantErr: true,
+		},
+		{
+			name:    "wrong yaml given",
+			yml:     "::",
+			path:    "$.foo",
+			value:   "new-text",
+			wantErr: true,
+		},
+		{
+			name:    "empty value given",
+			yml:     "foo: bar",
+			path:    "$.foo",
+			value:   "",
+			want:    []byte("foo: "),
+			wantErr: false,
+		},
+		{
+			name:    "valid value given",
+			yml:     "foo: bar",
+			path:    "$.foo",
+			value:   "new-text",
+			want:    []byte("foo: new-text"),
+			wantErr: false,
+		},
+		{
+			name: "array in block style",
+			yml: `foo:
+  - bar
+  - baz`,
+			path:  "$.foo[0]",
+			value: "new-text",
+			want: []byte(`foo:
+  - new-text
+  - baz`),
+			wantErr: false,
+		},
+		{
+			name:    "array in flow style",
+			yml:     `foo: [bar, baz]`,
+			path:    "$.foo[0]",
+			value:   "new-text",
+			want:    []byte(`foo: [new-text, baz]`),
+			wantErr: false,
+		},
+		{
+			name: "there is an useless blank line",
+			yml: `
+foo:
+  - bar
+  - baz`,
+			path:  "$.foo[0]",
+			value: "new-text",
+			want: []byte(`foo:
+  - new-text
+  - baz`),
+			wantErr: false,
+		},
+	}
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := ReplaceValue([]byte(tc.yml), tc.path, tc.value)
+			assert.Equal(t, tc.wantErr, err != nil)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}