Skip to content

Commit

Permalink
lang/funcs: Update fileset() function to include path as separate fir…
Browse files Browse the repository at this point in the history 
…st argument, automatically trim the path argument from results, and ensure results are always canonical with forward slash path separators

Reference: #22523 (review)

These changes center around better function usability and consistency with other functions. The function has not yet been released, so these breaking changes can be applied safely.
  • Loading branch information
@bflad
bflad committed Aug 31, 2019
1 parent aa6dca4 commit af7f6ef
Show file tree
Hide file tree
Showing 6 changed files with 135 additions and 24 deletions.
32 changes: 24 additions & 8 deletions lang/funcs/filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"unicode/utf8"

"github.com/hashicorp/hcl2/hcl"
Expand Down Expand Up @@ -212,25 +213,33 @@ func MakeFileExistsFunc(baseDir string) function.Function {
func MakeFileSetFunc(baseDir string) function.Function {
return function.New(&function.Spec{
Params: []function.Parameter{
{
Name: "path",
Type: cty.String,
},
{
Name: "pattern",
Type: cty.String,
},
},
Type: function.StaticReturnType(cty.Set(cty.String)),
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
pattern := args[0].AsString()
pattern, err := homedir.Expand(pattern)
path := args[0].AsString()
pattern := args[1].AsString()

path, err := homedir.Expand(path)
if err != nil {
return cty.UnknownVal(cty.Set(cty.String)), fmt.Errorf("failed to expand ~: %s", err)
}

if !filepath.IsAbs(pattern) {
pattern = filepath.Join(baseDir, pattern)
if !filepath.IsAbs(path) {
path = filepath.Join(baseDir, path)
}

// Ensure that the path is canonical for the host OS
pattern = filepath.Clean(pattern)
// Join the path to the glob pattern, while ensuring the full
// pattern is canonical for the host OS. The joined path is
// automatically cleaned during this operation.
pattern = filepath.Join(path, pattern)

matches, err := filepath.Glob(pattern)
if err != nil {
Expand All @@ -249,6 +258,13 @@ func MakeFileSetFunc(baseDir string) function.Function {
continue
}

// Remove the path and file separator from matches.
match = strings.TrimPrefix(match, path+string(filepath.Separator))

// Return matches with the Terraform canonical pattern
// of forward slashes for cross-system compatibility.
match = filepath.ToSlash(match)

matchVals = append(matchVals, cty.StringVal(match))
}

Expand Down Expand Up @@ -375,9 +391,9 @@ func FileExists(baseDir string, path cty.Value) (cty.Value, error) {
// The underlying function implementation works relative to a particular base
// directory, so this wrapper takes a base directory string and uses it to
// construct the underlying function before calling it.
func FileSet(baseDir string, pattern cty.Value) (cty.Value, error) {
func FileSet(baseDir string, path, pattern cty.Value) (cty.Value, error) {
fn := MakeFileSetFunc(baseDir)
return fn.Call([]cty.Value{pattern})
return fn.Call([]cty.Value{path, pattern})
}

// FileBase64 reads the contents of the file at the given path.
Expand Down
74 changes: 67 additions & 7 deletions lang/funcs/filesystem_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,57 +226,67 @@ func TestFileExists(t *testing.T) {

func TestFileSet(t *testing.T) {
tests := []struct {
Path cty.Value
Pattern cty.Value
Want cty.Value
Err bool
}{
{
cty.StringVal("testdata/missing"),
cty.StringVal("."),
cty.StringVal("testdata*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata/missing*"),
cty.StringVal("."),
cty.StringVal("testdata"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("*/missing"),
cty.StringVal("."),
cty.StringVal("testdata/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("."),
cty.StringVal("testdata/missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata*"),
cty.StringVal("."),
cty.StringVal("*/missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello.???"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("testdata/hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
Expand All @@ -285,20 +295,23 @@ func TestFileSet(t *testing.T) {
false,
},
{
cty.StringVal("."),
cty.StringVal("*/hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.txt"),
}),
false,
},
{
cty.StringVal("."),
cty.StringVal("*/hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("testdata/hello.tmpl"),
Expand All @@ -307,20 +320,67 @@ func TestFileSet(t *testing.T) {
false,
},
{
cty.StringVal("."),
cty.StringVal("["),
cty.SetValEmpty(cty.String),
true,
},
{
cty.StringVal("."),
cty.StringVal("\\"),
cty.SetValEmpty(cty.String),
true,
},
{
cty.StringVal("testdata"),
cty.StringVal("missing"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("missing*"),
cty.SetValEmpty(cty.String),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("*.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello.txt"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello.???"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.txt"),
}),
false,
},
{
cty.StringVal("testdata"),
cty.StringVal("hello*"),
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
false,
},
}

for _, test := range tests {
t.Run(fmt.Sprintf("FileSet(\".\", %#v)", test.Pattern), func(t *testing.T) {
got, err := FileSet(".", test.Pattern)
t.Run(fmt.Sprintf("FileSet(\".\", %#v, %#v)", test.Path, test.Pattern), func(t *testing.T) {
got, err := FileSet(".", test.Path, test.Pattern)

if test.Err {
if err == nil {
Expand Down
27 changes: 24 additions & 3 deletions lang/functions_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,10 +281,31 @@ func TestFunctions(t *testing.T) {

"fileset": {
{
`fileset("hello.*")`,
`fileset(".", "*/hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("testdata/functions-test/hello.tmpl"),
cty.StringVal("testdata/functions-test/hello.txt"),
cty.StringVal("subdirectory/hello.tmpl"),
cty.StringVal("subdirectory/hello.txt"),
}),
},
{
`fileset(".", "subdirectory/hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("subdirectory/hello.tmpl"),
cty.StringVal("subdirectory/hello.txt"),
}),
},
{
`fileset(".", "hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
},
{
`fileset("subdirectory", "hello.*")`,
cty.SetVal([]cty.Value{
cty.StringVal("hello.tmpl"),
cty.StringVal("hello.txt"),
}),
},
},
Expand Down
1 change: 1 addition & 0 deletions lang/testdata/functions-test/subdirectory/hello.tmpl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Hello, ${name}!
1 change: 1 addition & 0 deletions lang/testdata/functions-test/subdirectory/hello.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
hello!
24 changes: 18 additions & 6 deletions website/docs/configuration/functions/fileset.html.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ description: |-
earlier, see
[0.11 Configuration Language: Interpolation Syntax](../../configuration-0-11/interpolation.html).

`fileset` enumerates a set of regular file names given a pattern.
`fileset` enumerates a set of regular file names given a path and pattern.
The path is automatically removed from the resulting set of file names and any
result still containing path separators always returns forward slash (`/`) as
the path separator for cross-system compatibility.

```hcl
fileset(pattern)
fileset(path, pattern)
```

Supported pattern matches:
Expand All @@ -32,16 +35,25 @@ before Terraform takes any actions.
## Examples

```
> fileset("${path.module}/*.txt")
> fileset(path.module, "files/*.txt")
[
"path/to/module/hello.txt",
"path/to/module/world.txt",
"files/hello.txt",
"files/world.txt",
]
> fileset("${path.module}/files", "*.txt")
[
"hello.txt",
"world.txt",
]
```

A common use of `fileset` is to create one resource instance per matched file, using
[the `for_each` meta-argument](/docs/configuration/resources.html#for_each-multiple-resource-instances-defined-by-a-map-or-set-of-strings):

```hcl
resource "example_thing" "example" {
for_each = fileset("${path.module}/files/*")
for_each = fileset(path.module, "files/*")
# other configuration using each.value
}
Expand Down

0 comments on commit af7f6ef

Please sign in to comment.