Skip to content

Commit

Permalink
Merge pull request #22621 from hashicorp/f-lang-funcs-fileset-enhance…
Browse files Browse the repository at this point in the history 
…ments

lang/funcs: Various fileset() function updates for usability and consistency
  • Loading branch information
@bflad
bflad committed Sep 4, 2019
2 parents 3b0de1a + 19cf341 commit d9704d4
Show file tree
Hide file tree
Showing 15 changed files with 863 additions and 32 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ require (
github.com/aws/aws-sdk-go v1.22.0
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
github.com/blang/semver v3.5.1+incompatible
github.com/bmatcuk/doublestar v1.1.5
github.com/boltdb/bolt v1.3.1 // indirect
github.com/chzyer/logex v1.1.10 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,8 @@ github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQ
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk=
github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
Expand Down
39 changes: 27 additions & 12 deletions lang/funcs/filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"path/filepath"
"unicode/utf8"

"github.com/bmatcuk/doublestar"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/hcl2/hcl/hclsyntax"
homedir "github.com/mitchellh/go-homedir"
Expand Down Expand Up @@ -212,27 +213,30 @@ func MakeFileExistsFunc(baseDir string) function.Function {
func MakeFileSetFunc(baseDir string) function.Function {
return function.New(&function.Spec{
Params: []function.Parameter{
{
Name: "path",
Type: cty.String,
},
{
Name: "pattern",
Type: cty.String,
},
},
Type: function.StaticReturnType(cty.Set(cty.String)),
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
pattern := args[0].AsString()
pattern, err := homedir.Expand(pattern)
if err != nil {
return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to expand ~: %s", err)
}
path := args[0].AsString()
pattern := args[1].AsString()

if !filepath.IsAbs(pattern) {
pattern = filepath.Join(baseDir, pattern)
if !filepath.IsAbs(path) {
path = filepath.Join(baseDir, path)
}

// Ensure that the path is canonical for the host OS
pattern = filepath.Clean(pattern)
// Join the path to the glob pattern, while ensuring the full
// pattern is canonical for the host OS. The joined path is
// automatically cleaned during this operation.
pattern = filepath.Join(path, pattern)

matches, err := filepath.Glob(pattern)
matches, err := doublestar.Glob(pattern)
if err != nil {
return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to glob pattern (%s): %s", pattern, err)
}
Expand All @@ -249,6 +253,17 @@ func MakeFileSetFunc(baseDir string) function.Function {
continue
}

// Remove the path and file separator from matches.
match, err = filepath.Rel(path, match)

if err != nil {
return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to trim path of match (%s): %s", match, err)
}

// Replace any remaining file separators with forward slash (/)
// separators for cross-system compatibility.
match = filepath.ToSlash(match)

matchVals = append(matchVals, cty.StringVal(match))
}

Expand Down Expand Up @@ -375,9 +390,9 @@ func FileExists(baseDir string, path cty.Value) (cty.Value, error) {
// The underlying function implementation works relative to a particular base
// directory, so this wrapper takes a base directory string and uses it to
// construct the underlying function before calling it.
func FileSet(baseDir string, pattern cty.Value) (cty.Value, error) {
func FileSet(baseDir string, path, pattern cty.Value) (cty.Value, error) {
fn := MakeFileSetFunc(baseDir)
return fn.Call([]cty.Value{pattern})
return fn.Call([]cty.Value{path, pattern})
}

// FileBase64 reads the contents of the file at the given path.
Expand Down
113 changes: 106 additions & 7 deletions lang/funcs/filesystem_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,57 +226,79 @@ func TestFileExists(t *testing.T) {

func TestFileSet(t *testing.T) {
tests := []struct {
Path cty.Value
Pattern cty.Value
Want cty.Value
Err bool
}{
{
cty.StringVal("testdata/missing"),
cty.StringVal("."),
cty.StringVal("testdata*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata/missing*"),
cty.StringVal("."),
cty.StringVal("testdata"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("*/missing"),
cty.StringVal("."),
cty.StringVal("{testdata,missing}"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("."),
cty.StringVal("testdata/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata*"),
cty.StringVal("."),
cty.StringVal("testdata/missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("**/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello.???"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
Expand All @@ -285,20 +307,32 @@ func TestFileSet(t *testing.T) {
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello.{tmpl,txt}"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
Expand All @@ -307,20 +341,85 @@ func TestFileSet(t *testing.T) {
false,
},
{
cty.StringVal("."),
cty.StringVal("**/hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("**/hello.{tmpl,txt}"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("["),
cty.SetValEmpty(cty.String),
true,
},
{
cty.StringVal("."),
cty.StringVal("\\"),
cty.SetValEmpty(cty.String),
true,
},
{
cty.StringVal("testdata"),
cty.StringVal("missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello.???"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
false,
},
}

for _, test := range tests {
t.Run(fmt.Sprintf("FileSet(\".\", %#v)", test.Pattern), func(t *testing.T) {
got, err := FileSet(".", test.Pattern)
t.Run(fmt.Sprintf("FileSet(\".\", %#v, %#v)", test.Path, test.Pattern), func(t *testing.T) {
got, err := FileSet(".", test.Path, test.Pattern)

if test.Err {
if err == nil {
Expand Down
27 changes: 24 additions & 3 deletions lang/functions_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,10 +281,31 @@ func TestFunctions(t *testing.T) {

"fileset": {
{
`fileset("hello.*")`,
`fileset(".", "*/hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("testdata/functions-test/hello.tmpl"),
cty.StringVal("testdata/functions-test/hello.txt"),
cty.StringVal("subdirectory/hello.tmpl"),
cty.StringVal("subdirectory/hello.txt"),
}),
},
{
`fileset(".", "subdirectory/hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("subdirectory/hello.tmpl"),
cty.StringVal("subdirectory/hello.txt"),
}),
},
{
`fileset(".", "hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
},
{
`fileset("subdirectory", "hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
},
},
Expand Down
1 change: 1 addition & 0 deletions lang/testdata/functions-test/subdirectory/hello.tmpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Hello, ${name}!
1 change: 1 addition & 0 deletions lang/testdata/functions-test/subdirectory/hello.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hello!
32 changes: 32 additions & 0 deletions vendor/github.com/bmatcuk/doublestar/.gitignore
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions vendor/github.com/bmatcuk/doublestar/.travis.yml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit d9704d4

Please sign in to comment.