Post-ARM template export step for fixing some known ARM template issues

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/mitchellh/go-wordwrap v1.0.0
 	github.com/muesli/reflow v0.3.0
 	github.com/stretchr/testify v1.7.0
+	github.com/tidwall/gjson v1.14.1
 )
 
 require (
@@ -64,6 +65,8 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/sahilm/fuzzy v0.1.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	github.com/vmihailenco/msgpack/v4 v4.3.12 // indirect
 	github.com/vmihailenco/tagparser v0.1.1 // indirect

go.sum

@@ -261,6 +261,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/gjson v1.14.1 h1:iymTbGkQBhveq21bEvAQ81I0LEBork8BFe1CUZXdyuo=
+github.com/tidwall/gjson v1.14.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=

internal/armtemplate/armtemplate.go

@@ -13,7 +13,8 @@ type Template struct {
 
 type Resource struct {
 	ResourceId
-	DependsOn Dependencies `json:"dependsOn,omitempty"`
+	Properties interface{} `json:"properties,omitempty"`
+	DependsOn  ResourceIds `json:"dependsOn,omitempty"`
 }
 
 type ResourceId struct {
@@ -68,6 +69,32 @@ func NewResourceId(id string) (*ResourceId, error) {
 	}, nil
 }
 
+func NewResourceIdFromCallExpr(expr string) (*ResourceId, error) {
+	matches := regexp.MustCompile(`^\[resourceId\(([^,]+), (.+)\)]$`).FindAllStringSubmatch(expr, 1)
+	if len(matches) == 0 {
+		return nil, fmt.Errorf("the resourceId call expression %q is not valid (no match)", expr)
+	}
+	m := matches[0]
+	if len(m) != 3 {
+		return nil, fmt.Errorf("the resourceId call expression %q is not valid (the matched one has invalid form)", expr)
+	}
+
+	tlit, nlit := m[1], m[2]
+
+	t := strings.Trim(tlit, "' ")
+
+	var names []string
+	for _, seg := range strings.Split(nlit, ",") {
+		names = append(names, strings.Trim(seg, "' "))
+	}
+	n := strings.Join(names, "/")
+
+	return &ResourceId{
+		Type: t,
+		Name: n,
+	}, nil
+}
+
 // ID returns the azure resource id
 func (res ResourceId) ID(sub, rg string) string {
 	typeSegs := strings.Split(res.Type, "/")
@@ -87,39 +114,23 @@ func (res ResourceId) ID(sub, rg string) string {
 	return strings.Join(out, "/")
 }
 
-type Dependencies []ResourceId
+type ResourceIds []ResourceId
 
-func (deps *Dependencies) UnmarshalJSON(b []byte) error {
-	var dependenciesRaw []string
-	if err := json.Unmarshal(b, &dependenciesRaw); err != nil {
+func (resids *ResourceIds) UnmarshalJSON(b []byte) error {
+	var residExprs []string
+	if err := json.Unmarshal(b, &residExprs); err != nil {
 		return err
 	}
 
-	for _, dep := range dependenciesRaw {
-		matches := regexp.MustCompile(`^\[resourceId\(([^,]+), (.+)\)]$`).FindAllStringSubmatch(dep, 1)
-		if len(matches) == 0 {
-			panic(fmt.Sprintf("the dependency %q is not valid (no match)", dep))
-		}
-		m := matches[0]
-		if len(m) != 3 {
-			panic(fmt.Sprintf("the dependency %q is not valid (the matched one has invalid form)", dep))
-		}
-
-		tlit, nlit := m[1], m[2]
-
-		t := strings.Trim(tlit, "' ")
-
-		var names []string
-		for _, seg := range strings.Split(nlit, ",") {
-			names = append(names, strings.Trim(seg, "' "))
+	var ids ResourceIds
+	for _, residExpr := range residExprs {
+		id, err := NewResourceIdFromCallExpr(residExpr)
+		if err != nil {
+			return err
 		}
-		n := strings.Join(names, "/")
-
-		*deps = append(*deps, ResourceId{
-			Type: t,
-			Name: n,
-		})
+		ids = append(ids, *id)
 	}
+	*resids = ids
 	return nil
 }
 

internal/armtemplate/armtemplate_hack.go

@@ -0,0 +1,81 @@
+package armtemplate
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/tidwall/gjson"
+)
+
+// PopulateManagedResources populate extra resources that are missing from ARM template due to they are exclusively managed by another resource.
+// E.g. managed disk resource id is not returned via ARM template as it is exclusively managed by VM.
+// Terraform models the resources differently than ARM template and needs to import those managed resources separately.
+func (tpl *Template) PopulateManagedResources() error {
+	knownManagedResourceTypes := map[string][]string{
+		"Microsoft.Compute/virtualMachines": {
+			"storageProfile.dataDisks.#.managedDisk.id",
+		},
+	}
+
+	newResoruces := []Resource{}
+	for _, res := range tpl.Resources {
+		if paths, ok := knownManagedResourceTypes[res.Type]; ok {
+			res, resources, err := populateManagedResources(res, paths...)
+			if err != nil {
+				return fmt.Errorf(`populating managed resources for %q: %v`, res.Type, err)
+			}
+			newResoruces = append(newResoruces, *res)
+			newResoruces = append(newResoruces, resources...)
+		} else {
+			newResoruces = append(newResoruces, res)
+		}
+	}
+	tpl.Resources = newResoruces
+	return nil
+}
+
+// populateManagedResources populate the managed resources in the specified paths.
+// It will also update the specified resource's dependency accordingly.
+func populateManagedResources(res Resource, paths ...string) (*Resource, []Resource, error) {
+	b, err := json.Marshal(res.Properties)
+	if err != nil {
+		return nil, nil, fmt.Errorf("marshaling %v: %v", res.Properties, err)
+	}
+	var resources []Resource
+	for _, path := range paths {
+		result := gjson.GetBytes(b, path)
+		if !result.Exists() {
+			continue
+		}
+
+		for _, exprResult := range result.Array() {
+			// ARM template export ids in two forms:
+			// - Call expression: [resourceids(type, args)]. This is for resources within current export scope.
+			// - Id literal: This is for resources beyond current export scope .
+			if !strings.HasPrefix(exprResult.String(), "[") {
+				continue
+			}
+			id, err := NewResourceIdFromCallExpr(exprResult.String())
+			if err != nil {
+				return nil, nil, err
+			}
+
+			// Ideally, we should recursively export ARM template for this resource, fill in its properties
+			// and populate any managed resources within it, unless it has already exported.
+			// But here, as we explicitly pick up the managed resource to be populated, which means it is rarely possible that
+			// these resource are exported by the ARM template.
+			// TODO: needs to recursively populate these resources?
+			mres := Resource{
+				ResourceId: ResourceId{
+					Type: id.Type,
+					Name: id.Name,
+				},
+				DependsOn: []ResourceId{},
+			}
+			res.DependsOn = append(res.DependsOn, mres.ResourceId)
+			resources = append(resources, mres)
+		}
+	}
+	return &res, resources, nil
+}

internal/armtemplate/armtemplate_test.go

@@ -47,7 +47,7 @@ func TestUnmarshalTemplate(t *testing.T) {
 							Type: "Microsoft.Storage/storageAccounts/fileServices",
 							Name: "a/default",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Storage/storageAccounts",
 								Name: "a",
@@ -108,7 +108,7 @@ func TestUnmarshalTemplate(t *testing.T) {
 							Type: "Microsoft.Network/networkInterfaces",
 							Name: "nic",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Network/publicIPAddresses",
 								Name: "pip",
@@ -128,7 +128,7 @@ func TestUnmarshalTemplate(t *testing.T) {
 							Type: "Microsoft.Network/virtualNetworks/subnets",
 							Name: "vnet/subnet",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Network/virtualNetworks",
 								Name: "vnet",
@@ -144,7 +144,7 @@ func TestUnmarshalTemplate(t *testing.T) {
 							Type: "Microsoft.Network/networkSecurityGroups/securityRules",
 							Name: "nsg/nsr",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Network/networkSecurityGroups",
 								Name: "nsg",
@@ -196,7 +196,7 @@ func TestDependencyInfo(t *testing.T) {
 							Type: "Microsoft.Network/networkInterfaces",
 							Name: "nic",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Network/publicIPAddresses",
 								Name: "pip",
@@ -216,7 +216,7 @@ func TestDependencyInfo(t *testing.T) {
 							Type: "Microsoft.Network/virtualNetworks/subnets",
 							Name: "vnet/subnet",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Network/virtualNetworks",
 								Name: "vnet",
@@ -232,7 +232,7 @@ func TestDependencyInfo(t *testing.T) {
 							Type: "Microsoft.Network/networkSecurityGroups/securityRules",
 							Name: "nsg/nsr",
 						},
-						DependsOn: armtemplate.Dependencies{
+						DependsOn: armtemplate.ResourceIds{
 							{
 								Type: "Microsoft.Network/networkSecurityGroups",
 								Name: "nsg",
@@ -260,7 +260,7 @@ func TestDependencyInfo(t *testing.T) {
 				},
 			},
 			expect: map[armtemplate.ResourceId][]armtemplate.ResourceId{
-				armtemplate.ResourceId{
+				{
 					Type: "Microsoft.Network/networkInterfaces",
 					Name: "nic",
 				}: {
@@ -277,7 +277,7 @@ func TestDependencyInfo(t *testing.T) {
 						Name: "nsg",
 					},
 				},
-				armtemplate.ResourceId{
+				{
 					Type: "Microsoft.Network/virtualNetworks/subnets",
 					Name: "vnet/subnet",
 				}: {
@@ -290,7 +290,7 @@ func TestDependencyInfo(t *testing.T) {
 						Name: "nsg",
 					},
 				},
-				armtemplate.ResourceId{
+				{
 					Type: "Microsoft.Network/networkSecurityGroups/securityRules",
 					Name: "nsg/nsr",
 				}: {
@@ -299,19 +299,19 @@ func TestDependencyInfo(t *testing.T) {
 						Name: "nsg",
 					},
 				},
-				armtemplate.ResourceId{
+				{
 					Type: "Microsoft.Network/networkSecurityGroups",
 					Name: "nsg",
 				}: {
 					armtemplate.ResourceGroupId,
 				},
-				armtemplate.ResourceId{
+				{
 					Type: "Microsoft.Network/virtualNetworks",
 					Name: "vnet",
 				}: {
 					armtemplate.ResourceGroupId,
 				},
-				armtemplate.ResourceId{
+				{
 					Type: "Microsoft.Network/publicIPAddresses",
 					Name: "pip",
 				}: {
@@ -394,3 +394,51 @@ func TestNewResourceId(t *testing.T) {
 		require.Equal(t, c.expect, *output, c.name)
 	}
 }
+
+func TestNewResourceIdFromCallExpr(t *testing.T) {
+	cases := []struct {
+		name   string
+		expr   string
+		expect armtemplate.ResourceId
+		error  bool
+	}{
+		{
+			name:  "empty",
+			expr:  "",
+			error: true,
+		},
+		{
+			name:  "no args",
+			expr:  "[resourceId()]",
+			error: true,
+		},
+		{
+			name: "one level",
+			expr: "[resourceId('Microsoft.Storage/storageAccounts', 'a')]",
+			expect: armtemplate.ResourceId{
+				Type: "Microsoft.Storage/storageAccounts",
+				Name: "a",
+			},
+			error: false,
+		},
+		{
+			name: "two levels",
+			expr: "[resourceId('Microsoft.Storage/storageAccounts/services', 'a', 'b')]",
+			expect: armtemplate.ResourceId{
+				Type: "Microsoft.Storage/storageAccounts/services",
+				Name: "a/b",
+			},
+			error: false,
+		},
+	}
+
+	for _, c := range cases {
+		output, err := armtemplate.NewResourceIdFromCallExpr(c.expr)
+		if c.error {
+			require.Error(t, err, c.name)
+			continue
+		}
+		require.NoError(t, err, c.name)
+		require.Equal(t, c.expect, *output, c.name)
+	}
+}

internal/meta/meta.go

@@ -8,7 +8,7 @@ type Meta interface {
 	Init() error
 	ResourceGroupName() string
 	Workspace() string
-	ListResource() ImportList
+	ListResource() (ImportList, error)
 	CleanTFState(addr string)
 	Import(item *ImportItem)
 	GenerateCfg(l ImportList) error

internal/meta/meta_dummy.go

@@ -25,7 +25,7 @@ func (m MetaDummy) Workspace() string {
 	return "example-workspace"
 }
 
-func (m MetaDummy) ListResource() ImportList {
+func (m MetaDummy) ListResource() (ImportList, error) {
 	time.Sleep(500 * time.Millisecond)
 	return ImportList{
 		ImportItem{
@@ -43,7 +43,7 @@ func (m MetaDummy) ListResource() ImportList {
 		ImportItem{
 			ResourceID: "/subscriptions/0000000-0000-0000-0000-00000000000/resourceGroups/example-rg",
 		},
-	}
+	}, nil
 }
 
 func (m MetaDummy) CleanTFState(_ string) {