From 89a0150d6327d5cd33927857ab4ccc1ee6666d58 Mon Sep 17 00:00:00 2001 From: magodo Date: Thu, 12 May 2022 21:20:39 +0800 Subject: [PATCH 1/3] Populate managed resources based on the exported ARM template --- go.mod | 3 + go.sum | 6 ++ internal/armtemplate/armtemplate.go | 67 ++++++++++++--------- internal/armtemplate/armtemplate_hack.go | 70 ++++++++++++++++++++++ internal/armtemplate/armtemplate_test.go | 74 +++++++++++++++++++----- internal/meta/meta.go | 2 +- internal/meta/meta_dummy.go | 4 +- internal/meta/meta_impl.go | 17 ++++-- internal/ui/aztfyclient/client.go | 6 +- main.go | 5 +- 10 files changed, 202 insertions(+), 52 deletions(-) create mode 100644 internal/armtemplate/armtemplate_hack.go diff --git a/go.mod b/go.mod index d765b7f..59508ec 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,9 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect + github.com/tidwall/gjson v1.14.1 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect github.com/vmihailenco/tagparser v0.1.1 // indirect diff --git a/go.sum b/go.sum index 10ee788..a851fa8 100644 --- a/go.sum +++ b/go.sum @@ -261,6 +261,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo= +github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= diff --git a/internal/armtemplate/armtemplate.go b/internal/armtemplate/armtemplate.go index 21b966c..b0a3c96 100644 --- a/internal/armtemplate/armtemplate.go +++ b/internal/armtemplate/armtemplate.go @@ -13,7 +13,8 @@ type Template struct { type Resource struct { ResourceId - DependsOn Dependencies `json:"dependsOn,omitempty"` + Properties interface{} `json:"properties,omitempty"` + DependsOn ResourceIds `json:"dependsOn,omitempty"` } type ResourceId struct { @@ -68,6 +69,32 @@ func NewResourceId(id string) (*ResourceId, error) { }, nil } +func NewResourceIdFromCallExpr(expr string) (*ResourceId, error) { + matches := regexp.MustCompile(`^\[resourceId\(([^,]+), (.+)\)]$`).FindAllStringSubmatch(expr, 1) + if len(matches) == 0 { + return nil, fmt.Errorf("the resourceId call expression %q is not valid (no match)", expr) + } + m := matches[0] + if len(m) != 3 { + return nil, fmt.Errorf("the resourceId call expression %q is not valid (the matched one has invalid form)", expr) + } + + tlit, nlit := m[1], m[2] + + t := strings.Trim(tlit, "' ") + + var names []string + for _, seg := range strings.Split(nlit, ",") { + names = append(names, strings.Trim(seg, "' ")) + } + n := strings.Join(names, "/") + + return &ResourceId{ + Type: t, + Name: n, + }, nil +} + // ID returns the azure resource id func (res ResourceId) ID(sub, rg string) string { typeSegs := strings.Split(res.Type, "/") @@ -87,39 +114,23 @@ func (res ResourceId) ID(sub, rg string) string { return strings.Join(out, "/") } -type Dependencies []ResourceId +type ResourceIds []ResourceId -func (deps *Dependencies) UnmarshalJSON(b []byte) error { - var dependenciesRaw []string - if err := json.Unmarshal(b, &dependenciesRaw); err != nil { +func (resids *ResourceIds) UnmarshalJSON(b []byte) error { + var residExprs []string + if err := json.Unmarshal(b, &residExprs); err != nil { return err } - for _, dep := range dependenciesRaw { - matches := regexp.MustCompile(`^\[resourceId\(([^,]+), (.+)\)]$`).FindAllStringSubmatch(dep, 1) - if len(matches) == 0 { - panic(fmt.Sprintf("the dependency %q is not valid (no match)", dep)) - } - m := matches[0] - if len(m) != 3 { - panic(fmt.Sprintf("the dependency %q is not valid (the matched one has invalid form)", dep)) - } - - tlit, nlit := m[1], m[2] - - t := strings.Trim(tlit, "' ") - - var names []string - for _, seg := range strings.Split(nlit, ",") { - names = append(names, strings.Trim(seg, "' ")) + var ids ResourceIds + for _, residExpr := range residExprs { + id, err := NewResourceIdFromCallExpr(residExpr) + if err != nil { + return err } - n := strings.Join(names, "/") - - *deps = append(*deps, ResourceId{ - Type: t, - Name: n, - }) + ids = append(ids, *id) } + *resids = ids return nil } diff --git a/internal/armtemplate/armtemplate_hack.go b/internal/armtemplate/armtemplate_hack.go new file mode 100644 index 0000000..08fb158 --- /dev/null +++ b/internal/armtemplate/armtemplate_hack.go @@ -0,0 +1,70 @@ +package armtemplate + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/tidwall/gjson" +) + +// PopulateManagedResources populate extra resources that are missing from ARM template due to they are exclusively managed by another resource. +// E.g. managed disk resource id is not returned via ARM template as it is exclusively managed by VM. +// Terraform models the resources differently than ARM template and needs to import those managed resources separately. +func (tpl *Template) PopulateManagedResources() error { + oldResources := make([]Resource, len(tpl.Resources)) + copy(oldResources, tpl.Resources) + for _, res := range oldResources { + switch res.Type { + case "Microsoft.Compute/virtualMachines": + resources, err := populateManagedResources(res.Properties, "storageProfile.dataDisks.#.managedDisk.id") + if err != nil { + return fmt.Errorf(`populating managed resources for "Microsoft.Compute/virtualMachines": %v`, err) + } + tpl.Resources = append(tpl.Resources, resources...) + } + } + return nil +} + +func populateManagedResources(props interface{}, paths ...string) ([]Resource, error) { + b, err := json.Marshal(props) + if err != nil { + return nil, fmt.Errorf("marshaling %v: %v", props, err) + } + var resources []Resource + for _, path := range paths { + result := gjson.GetBytes(b, path) + if !result.Exists() { + continue + } + + for _, exprResult := range result.Array() { + // ARM template export ids in two forms: + // - Call expression: [resourceids(type, args)]. This is for resources within current export scope. + // - Id literal: This is for resources beyond current export scope . + if !strings.HasPrefix(exprResult.String(), "[") { + continue + } + id, err := NewResourceIdFromCallExpr(exprResult.String()) + if err != nil { + return nil, err + } + + // Ideally, we should recursively export ARM template for this resource, fill in its properties + // and populate any managed resources within it, unless it has already exported. + // But here, as we explicitly pick up the managed resource to be populated, which means it is rarely possible that + // these resource are exported by the ARM template. + // TODO: needs to recursively populate these resources? + res := Resource{ + ResourceId: ResourceId{ + Type: id.Type, + Name: id.Name, + }, + DependsOn: []ResourceId{}, + } + resources = append(resources, res) + } + } + return resources, nil +} diff --git a/internal/armtemplate/armtemplate_test.go b/internal/armtemplate/armtemplate_test.go index 20f9e07..710e1c1 100644 --- a/internal/armtemplate/armtemplate_test.go +++ b/internal/armtemplate/armtemplate_test.go @@ -47,7 +47,7 @@ func TestUnmarshalTemplate(t *testing.T) { Type: "Microsoft.Storage/storageAccounts/fileServices", Name: "a/default", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Storage/storageAccounts", Name: "a", @@ -108,7 +108,7 @@ func TestUnmarshalTemplate(t *testing.T) { Type: "Microsoft.Network/networkInterfaces", Name: "nic", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Network/publicIPAddresses", Name: "pip", @@ -128,7 +128,7 @@ func TestUnmarshalTemplate(t *testing.T) { Type: "Microsoft.Network/virtualNetworks/subnets", Name: "vnet/subnet", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Network/virtualNetworks", Name: "vnet", @@ -144,7 +144,7 @@ func TestUnmarshalTemplate(t *testing.T) { Type: "Microsoft.Network/networkSecurityGroups/securityRules", Name: "nsg/nsr", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Network/networkSecurityGroups", Name: "nsg", @@ -196,7 +196,7 @@ func TestDependencyInfo(t *testing.T) { Type: "Microsoft.Network/networkInterfaces", Name: "nic", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Network/publicIPAddresses", Name: "pip", @@ -216,7 +216,7 @@ func TestDependencyInfo(t *testing.T) { Type: "Microsoft.Network/virtualNetworks/subnets", Name: "vnet/subnet", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Network/virtualNetworks", Name: "vnet", @@ -232,7 +232,7 @@ func TestDependencyInfo(t *testing.T) { Type: "Microsoft.Network/networkSecurityGroups/securityRules", Name: "nsg/nsr", }, - DependsOn: armtemplate.Dependencies{ + DependsOn: armtemplate.ResourceIds{ { Type: "Microsoft.Network/networkSecurityGroups", Name: "nsg", @@ -260,7 +260,7 @@ func TestDependencyInfo(t *testing.T) { }, }, expect: map[armtemplate.ResourceId][]armtemplate.ResourceId{ - armtemplate.ResourceId{ + { Type: "Microsoft.Network/networkInterfaces", Name: "nic", }: { @@ -277,7 +277,7 @@ func TestDependencyInfo(t *testing.T) { Name: "nsg", }, }, - armtemplate.ResourceId{ + { Type: "Microsoft.Network/virtualNetworks/subnets", Name: "vnet/subnet", }: { @@ -290,7 +290,7 @@ func TestDependencyInfo(t *testing.T) { Name: "nsg", }, }, - armtemplate.ResourceId{ + { Type: "Microsoft.Network/networkSecurityGroups/securityRules", Name: "nsg/nsr", }: { @@ -299,19 +299,19 @@ func TestDependencyInfo(t *testing.T) { Name: "nsg", }, }, - armtemplate.ResourceId{ + { Type: "Microsoft.Network/networkSecurityGroups", Name: "nsg", }: { armtemplate.ResourceGroupId, }, - armtemplate.ResourceId{ + { Type: "Microsoft.Network/virtualNetworks", Name: "vnet", }: { armtemplate.ResourceGroupId, }, - armtemplate.ResourceId{ + { Type: "Microsoft.Network/publicIPAddresses", Name: "pip", }: { @@ -394,3 +394,51 @@ func TestNewResourceId(t *testing.T) { require.Equal(t, c.expect, *output, c.name) } } + +func TestNewResourceIdFromCallExpr(t *testing.T) { + cases := []struct { + name string + expr string + expect armtemplate.ResourceId + error bool + }{ + { + name: "empty", + expr: "", + error: true, + }, + { + name: "no args", + expr: "[resourceId()]", + error: true, + }, + { + name: "one level", + expr: "[resourceId('Microsoft.Storage/storageAccounts', 'a')]", + expect: armtemplate.ResourceId{ + Type: "Microsoft.Storage/storageAccounts", + Name: "a", + }, + error: false, + }, + { + name: "two levels", + expr: "[resourceId('Microsoft.Storage/storageAccounts/services', 'a', 'b')]", + expect: armtemplate.ResourceId{ + Type: "Microsoft.Storage/storageAccounts/services", + Name: "a/b", + }, + error: false, + }, + } + + for _, c := range cases { + output, err := armtemplate.NewResourceIdFromCallExpr(c.expr) + if c.error { + require.Error(t, err, c.name) + continue + } + require.NoError(t, err, c.name) + require.Equal(t, c.expect, *output, c.name) + } +} diff --git a/internal/meta/meta.go b/internal/meta/meta.go index d9caa88..3d89bc8 100644 --- a/internal/meta/meta.go +++ b/internal/meta/meta.go @@ -8,7 +8,7 @@ type Meta interface { Init() error ResourceGroupName() string Workspace() string - ListResource() ImportList + ListResource() (ImportList, error) CleanTFState(addr string) Import(item *ImportItem) GenerateCfg(l ImportList) error diff --git a/internal/meta/meta_dummy.go b/internal/meta/meta_dummy.go index ff4053b..1e6c05d 100644 --- a/internal/meta/meta_dummy.go +++ b/internal/meta/meta_dummy.go @@ -25,7 +25,7 @@ func (m MetaDummy) Workspace() string { return "example-workspace" } -func (m MetaDummy) ListResource() ImportList { +func (m MetaDummy) ListResource() (ImportList, error) { time.Sleep(500 * time.Millisecond) return ImportList{ ImportItem{ @@ -43,7 +43,7 @@ func (m MetaDummy) ListResource() ImportList { ImportItem{ ResourceID: "/subscriptions/0000000-0000-0000-0000-00000000000/resourceGroups/example-rg", }, - } + }, nil } func (m MetaDummy) CleanTFState(_ string) { diff --git a/internal/meta/meta_impl.go b/internal/meta/meta_impl.go index 67ad4db..af07ed2 100644 --- a/internal/meta/meta_impl.go +++ b/internal/meta/meta_impl.go @@ -173,15 +173,16 @@ func (meta *MetaImpl) Init() error { if err := meta.initProvider(ctx); err != nil { return err } + return nil +} + +func (meta *MetaImpl) ListResource() (ImportList, error) { + ctx := context.TODO() - // Export ARM template if err := meta.exportArmTemplate(ctx); err != nil { - return err + return nil, err } - return nil -} -func (meta MetaImpl) ListResource() ImportList { var ids []string for _, res := range meta.armTemplate.Resources { ids = append(ids, res.ID(meta.subscriptionId, meta.resourceGroup)) @@ -207,7 +208,7 @@ func (meta MetaImpl) ListResource() ImportList { } l = append(l, item) } - return l + return l, nil } func (meta *MetaImpl) CleanTFState(addr string) { @@ -334,6 +335,10 @@ func (meta *MetaImpl) exportArmTemplate(ctx context.Context) error { return fmt.Errorf("unmarshalling the template: %w", err) } + if err := meta.armTemplate.PopulateManagedResources(); err != nil { + return fmt.Errorf("populating managed resources in the ARM template: %v", err) + } + return nil } diff --git a/internal/ui/aztfyclient/client.go b/internal/ui/aztfyclient/client.go index 7f04d05..96debe5 100644 --- a/internal/ui/aztfyclient/client.go +++ b/internal/ui/aztfyclient/client.go @@ -66,7 +66,11 @@ func Init(c meta.Meta) tea.Cmd { func ListResource(c meta.Meta) tea.Cmd { return func() tea.Msg { - return ListResourceDoneMsg{List: c.ListResource()} + list, err := c.ListResource() + if err != nil { + return ErrMsg(err) + } + return ListResourceDoneMsg{List: list} } } diff --git a/main.go b/main.go index e0d7f82..88a9e1e 100644 --- a/main.go +++ b/main.go @@ -195,7 +195,10 @@ func batchImport(cfg config.Config, continueOnError bool) error { } fmt.Println("List resources...") - list := c.ListResource() + list, err := c.ListResource() + if err != nil { + return err + } fmt.Println("Import resources...") for i := range list { From d217d2f4e638997c1522ad145f926e2813bbae7c Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 13 May 2022 15:40:31 +0800 Subject: [PATCH 2/3] Add dependency for the managing resource on the managed resource --- internal/armtemplate/armtemplate_hack.go | 41 +++++++++++++++--------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/internal/armtemplate/armtemplate_hack.go b/internal/armtemplate/armtemplate_hack.go index 08fb158..cf9ba99 100644 --- a/internal/armtemplate/armtemplate_hack.go +++ b/internal/armtemplate/armtemplate_hack.go @@ -12,25 +12,35 @@ import ( // E.g. managed disk resource id is not returned via ARM template as it is exclusively managed by VM. // Terraform models the resources differently than ARM template and needs to import those managed resources separately. func (tpl *Template) PopulateManagedResources() error { - oldResources := make([]Resource, len(tpl.Resources)) - copy(oldResources, tpl.Resources) - for _, res := range oldResources { - switch res.Type { - case "Microsoft.Compute/virtualMachines": - resources, err := populateManagedResources(res.Properties, "storageProfile.dataDisks.#.managedDisk.id") + knownManagedResourceTypes := map[string][]string{ + "Microsoft.Compute/virtualMachines": { + "storageProfile.dataDisks.#.managedDisk.id", + }, + } + + newResoruces := []Resource{} + for _, res := range tpl.Resources { + if paths, ok := knownManagedResourceTypes[res.Type]; ok { + res, resources, err := populateManagedResources(res, paths...) if err != nil { - return fmt.Errorf(`populating managed resources for "Microsoft.Compute/virtualMachines": %v`, err) + return fmt.Errorf(`populating managed resources for %q: %v`, res.Type, err) } - tpl.Resources = append(tpl.Resources, resources...) + newResoruces = append(newResoruces, *res) + newResoruces = append(newResoruces, resources...) + } else { + newResoruces = append(newResoruces, res) } } + tpl.Resources = newResoruces return nil } -func populateManagedResources(props interface{}, paths ...string) ([]Resource, error) { - b, err := json.Marshal(props) +// populateManagedResources populate the managed resources in the specified paths. +// It will also update the specified resource's dependency accordingly. +func populateManagedResources(res Resource, paths ...string) (*Resource, []Resource, error) { + b, err := json.Marshal(res.Properties) if err != nil { - return nil, fmt.Errorf("marshaling %v: %v", props, err) + return nil, nil, fmt.Errorf("marshaling %v: %v", res.Properties, err) } var resources []Resource for _, path := range paths { @@ -48,7 +58,7 @@ func populateManagedResources(props interface{}, paths ...string) ([]Resource, e } id, err := NewResourceIdFromCallExpr(exprResult.String()) if err != nil { - return nil, err + return nil, nil, err } // Ideally, we should recursively export ARM template for this resource, fill in its properties @@ -56,15 +66,16 @@ func populateManagedResources(props interface{}, paths ...string) ([]Resource, e // But here, as we explicitly pick up the managed resource to be populated, which means it is rarely possible that // these resource are exported by the ARM template. // TODO: needs to recursively populate these resources? - res := Resource{ + mres := Resource{ ResourceId: ResourceId{ Type: id.Type, Name: id.Name, }, DependsOn: []ResourceId{}, } - resources = append(resources, res) + res.DependsOn = append(res.DependsOn, mres.ResourceId) + resources = append(resources, mres) } } - return resources, nil + return &res, resources, nil } From 518d5cdc3aea5ed1ed69281db3bcff4ebaeeedd2 Mon Sep 17 00:00:00 2001 From: magodo Date: Fri, 13 May 2022 16:16:08 +0800 Subject: [PATCH 3/3] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 59508ec..b2fcd0d 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.0 github.com/muesli/reflow v0.3.0 github.com/stretchr/testify v1.7.0 + github.com/tidwall/gjson v1.14.1 ) require ( @@ -64,7 +65,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/sahilm/fuzzy v0.1.0 // indirect - github.com/tidwall/gjson v1.14.1 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect