diff --git a/.changes/v1.14/NEW FEATURES-20250829-183404.yaml b/.changes/v1.14/NEW FEATURES-20250829-183404.yaml new file mode 100644 index 000000000000..e0751ba14201 --- /dev/null +++ b/.changes/v1.14/NEW FEATURES-20250829-183404.yaml @@ -0,0 +1,3 @@ +kind: NEW FEATURES +body: "**List Resources**: List resources can be defined in `*.tfquery.hcl` files and allow querying and filterting existing infrastructure." +time: 2025-08-29T18:34:04.250038+02:00 diff --git a/.changes/v1.14/NEW FEATURES-20250829-184206.yaml b/.changes/v1.14/NEW FEATURES-20250829-184206.yaml new file mode 100644 index 000000000000..ae79073e2f9a --- /dev/null +++ b/.changes/v1.14/NEW FEATURES-20250829-184206.yaml @@ -0,0 +1,3 @@ +kind: NEW FEATURES +body: "A new Terraform command `terraform query`: Executes list operations against existing infrastructure and displays the results. The command can optionally generate configuration for importing results into Terraform." +time: 2025-08-29T18:42:06.659172+02:00 diff --git a/commands.go b/commands.go index 79424845043b..b8489519b40b 100644 --- a/commands.go +++ b/commands.go @@ -276,6 +276,12 @@ func initCommands( }, nil }, + "query": func() (cli.Command, error) { + return &command.QueryCommand{ + Meta: meta, + }, nil + }, + "refresh": func() (cli.Command, error) { return &command.RefreshCommand{ Meta: meta, @@ -451,12 +457,6 @@ func initCommands( }, nil } - Commands["query"] = func() (cli.Command, error) { - return &command.QueryCommand{ - Meta: meta, - }, nil - } - Commands["test cleanup"] = func() (cli.Command, error) { return &command.TestCleanupCommand{ Meta: meta, diff --git a/internal/cloud/backend_query.go b/internal/cloud/backend_query.go index 3fa78d4bfa8f..a8759c5f5cb9 100644 --- a/internal/cloud/backend_query.go +++ b/internal/cloud/backend_query.go @@ -20,6 +20,7 @@ import ( "github.com/hashicorp/terraform/internal/genconfig" "github.com/hashicorp/terraform/internal/terraform" "github.com/hashicorp/terraform/internal/tfdiags" + "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" ) @@ -276,7 +277,7 @@ func (b *Cloud) cancelQueryRun(cancelCtx context.Context, op *backendrun.Operati // formatIdentity formats the identity map into a string representation. // It flattens the map into a string of key=value pairs, separated by commas. func formatIdentity(identity map[string]json.RawMessage) string { - parts := make([]string, 0, len(identity)) + ctyObj := make(map[string]cty.Value, len(identity)) for key, value := range identity { ty, err := ctyjson.ImpliedType(value) if err != nil { @@ -286,9 +287,9 @@ func formatIdentity(identity map[string]json.RawMessage) string { if err != nil { continue } - parts = append(parts, fmt.Sprintf("%s=%s", key, tfdiags.ValueToString(v))) + ctyObj[key] = v } - return strings.Join(parts, ",") + return tfdiags.ObjectToString(cty.ObjectVal(ctyObj)) } const queryDefaultHeader = ` diff --git a/internal/cloud/backend_query_test.go b/internal/cloud/backend_query_test.go index 31fab4db2402..83feb260a6dc 100644 --- a/internal/cloud/backend_query_test.go +++ b/internal/cloud/backend_query_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/google/go-cmp/cmp" "github.com/hashicorp/cli" "github.com/hashicorp/terraform/internal/addrs" "github.com/hashicorp/terraform/internal/backend/backendrun" @@ -121,8 +122,15 @@ func TestCloud_queryJSONBasic(t *testing.T) { outp := close(t) gotOut := outp.Stdout() - if !strings.Contains(gotOut, "list.concept_pet.pets id=complete-gannet,legs=6 This is a complete-gannet") { - t.Fatalf("expected query results in output: %s", gotOut) + expectedOut := `list.concept_pet.pets id=large-roughy,legs=2 This is a large-roughy +list.concept_pet.pets id=able-werewolf,legs=5 This is a able-werewolf +list.concept_pet.pets id=complete-gannet,legs=6 This is a complete-gannet +list.concept_pet.pets id=charming-beagle,legs=3 This is a charming-beagle +list.concept_pet.pets id=legal-lamprey,legs=2 This is a legal-lamprey + +` + if diff := cmp.Diff(expectedOut, gotOut); diff != "" { + t.Fatalf("expected query results output to be %s, got %s: diff: %s", expectedOut, gotOut, diff) } stateMgr, _ := b.StateMgr(testBackendSingleWorkspaceName) diff --git a/internal/command/e2etest/providers_schema_test.go b/internal/command/e2etest/providers_schema_test.go index 7a5d7dea8d47..582a21710c21 100644 --- a/internal/command/e2etest/providers_schema_test.go +++ b/internal/command/e2etest/providers_schema_test.go @@ -134,6 +134,21 @@ func TestProvidersSchema(t *testing.T) { } } }, + "list_resource_schemas": { + "simple_resource": { + "version": 0, + "block": { + "attributes": { + "value": { + "type": "string", + "description_kind": "plain", + "optional": true + } + }, + "description_kind": "plain" + } + } + }, "resource_identity_schemas": { "simple_resource": { "version": 0, @@ -213,6 +228,21 @@ func TestProvidersSchema(t *testing.T) { } } }, + "list_resource_schemas": { + "simple_resource": { + "version": 0, + "block": { + "attributes": { + "value": { + "type": "string", + "description_kind": "plain", + "optional": true + } + }, + "description_kind": "plain" + } + } + }, "functions": { "noop": { "description": "noop takes any single argument and returns the same value", diff --git a/internal/command/jsonformat/plan_test.go b/internal/command/jsonformat/plan_test.go index f971c08d02af..5f99c2fd9e7e 100644 --- a/internal/command/jsonformat/plan_test.go +++ b/internal/command/jsonformat/plan_test.go @@ -8071,7 +8071,7 @@ func runTestCases(t *testing.T, testCases map[string]testCase) { return } - jsonschemas := jsonprovider.MarshalForRenderer(tfschemas, false) + jsonschemas := jsonprovider.MarshalForRenderer(tfschemas) change := structured.FromJsonChange(jsonchanges[0].Change, attribute_path.AlwaysMatcher()) renderer := Renderer{Colorize: color} diff := diff{ @@ -8421,7 +8421,7 @@ func TestResourceChange_deferredActions(t *testing.T) { } renderer := Renderer{Colorize: color} - jsonschemas := jsonprovider.MarshalForRenderer(fullSchema, false) + jsonschemas := jsonprovider.MarshalForRenderer(fullSchema) diffs := precomputeDiffs(Plan{ DeferredChanges: deferredChanges, ProviderSchemas: jsonschemas, @@ -8714,7 +8714,7 @@ func TestResourceChange_actions(t *testing.T) { }, }, } - jsonschemas := jsonprovider.MarshalForRenderer(fullSchema, false) + jsonschemas := jsonprovider.MarshalForRenderer(fullSchema) diffs := precomputeDiffs(Plan{ ResourceChanges: []jsonplan.ResourceChange{defaultResourceChange}, ActionInvocations: tc.actionInvocations, diff --git a/internal/command/jsonformat/state_test.go b/internal/command/jsonformat/state_test.go index 9c4eafa50b23..b8350f29bbbb 100644 --- a/internal/command/jsonformat/state_test.go +++ b/internal/command/jsonformat/state_test.go @@ -86,7 +86,7 @@ func TestState(t *testing.T) { RootModule: root, RootModuleOutputs: outputs, ProviderFormatVersion: jsonprovider.FormatVersion, - ProviderSchemas: jsonprovider.MarshalForRenderer(tt.Schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(tt.Schemas), }) result := done(t).All() diff --git a/internal/command/jsonprovider/provider.go b/internal/command/jsonprovider/provider.go index 0ebb76f82b91..10fdcf7831f4 100644 --- a/internal/command/jsonprovider/provider.go +++ b/internal/command/jsonprovider/provider.go @@ -45,22 +45,22 @@ func newProviders() *Providers { // schema into the public structured JSON versions. // // This is a format that can be read by the structured plan renderer. -func MarshalForRenderer(s *terraform.Schemas, includeExperimentalSchemas bool) map[string]*Provider { +func MarshalForRenderer(s *terraform.Schemas) map[string]*Provider { schemas := make(map[string]*Provider, len(s.Providers)) for k, v := range s.Providers { - schemas[k.String()] = marshalProvider(v, includeExperimentalSchemas) + schemas[k.String()] = marshalProvider(v) } return schemas } -func Marshal(s *terraform.Schemas, includeExperimentalSchemas bool) ([]byte, error) { +func Marshal(s *terraform.Schemas) ([]byte, error) { providers := newProviders() - providers.Schemas = MarshalForRenderer(s, includeExperimentalSchemas) + providers.Schemas = MarshalForRenderer(s) ret, err := json.Marshal(providers) return ret, err } -func marshalProvider(tps providers.ProviderSchema, includeExperimentalSchemas bool) *Provider { +func marshalProvider(tps providers.ProviderSchema) *Provider { p := &Provider{ Provider: marshalSchema(tps.Provider), ResourceSchemas: marshalSchemas(tps.ResourceTypes), @@ -71,20 +71,18 @@ func marshalProvider(tps providers.ProviderSchema, includeExperimentalSchemas bo ActionSchemas: marshalActionSchemas(tps.Actions), } - if includeExperimentalSchemas { - // List resource schemas are nested under a "config" block, so we need to - // extract that block to get the actual provider schema for the list resource. - // When getting the provider schemas, Terraform adds this extra level to - // better match the actual configuration structure. - listSchemas := make(map[string]providers.Schema, len(tps.ListResourceTypes)) - for k, v := range tps.ListResourceTypes { - listSchemas[k] = providers.Schema{ - Body: &v.Body.BlockTypes["config"].Block, - Version: v.Version, - } + // List resource schemas are nested under a "config" block, so we need to + // extract that block to get the actual provider schema for the list resource. + // When getting the provider schemas, Terraform adds this extra level to + // better match the actual configuration structure. + listSchemas := make(map[string]providers.Schema, len(tps.ListResourceTypes)) + for k, v := range tps.ListResourceTypes { + listSchemas[k] = providers.Schema{ + Body: &v.Body.BlockTypes["config"].Block, + Version: v.Version, } - p.ListResourceSchemas = marshalSchemas(listSchemas) } + p.ListResourceSchemas = marshalSchemas(listSchemas) return p } diff --git a/internal/command/jsonprovider/provider_test.go b/internal/command/jsonprovider/provider_test.go index e65e4867869b..bb9ca97486b1 100644 --- a/internal/command/jsonprovider/provider_test.go +++ b/internal/command/jsonprovider/provider_test.go @@ -20,25 +20,23 @@ var cmpOpts = cmpopts.IgnoreUnexported(Provider{}) func TestMarshalProvider(t *testing.T) { tests := []struct { - Input providers.ProviderSchema - IncludeExperimental bool - Want *Provider + Input providers.ProviderSchema + Want *Provider }{ { providers.ProviderSchema{}, - false, &Provider{ Provider: &Schema{}, ResourceSchemas: map[string]*Schema{}, DataSourceSchemas: map[string]*Schema{}, EphemeralResourceSchemas: map[string]*Schema{}, ResourceIdentitySchemas: map[string]*IdentitySchema{}, + ListResourceSchemas: map[string]*Schema{}, ActionSchemas: map[string]*ActionSchema{}, }, }, { testProvider(), - false, &Provider{ Provider: &Schema{ Block: &Block{ @@ -212,53 +210,6 @@ func TestMarshalProvider(t *testing.T) { }, }, }, - ResourceIdentitySchemas: map[string]*IdentitySchema{}, - ActionSchemas: map[string]*ActionSchema{}, - }, - }, - { - providers.ProviderSchema{ - ListResourceTypes: map[string]providers.Schema{ - "test_list_resource": { - Version: 1, - Body: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "data": { - Type: cty.DynamicPseudoType, - Computed: true, - }, - }, - BlockTypes: map[string]*configschema.NestedBlock{ - "config": { - Block: configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "filter": {Type: cty.String, Optional: true}, - "items": {Type: cty.List(cty.String), Required: true}, - }, - }, - Nesting: configschema.NestingSingle, - }, - }, - }, - }, - }, - Actions: map[string]providers.ActionSchema{ - "test_action": { - ConfigSchema: &configschema.Block{ - Attributes: map[string]*configschema.Attribute{ - "opt_attr": {Type: cty.String, Optional: true}, - "req_attr": {Type: cty.List(cty.String), Required: true}, - }, - }, - }, - }, - }, - true, - &Provider{ - Provider: &Schema{}, - ResourceSchemas: map[string]*Schema{}, - DataSourceSchemas: map[string]*Schema{}, - EphemeralResourceSchemas: map[string]*Schema{}, ListResourceSchemas: map[string]*Schema{ "test_list_resource": { Version: 1, @@ -305,7 +256,7 @@ func TestMarshalProvider(t *testing.T) { for i, test := range tests { t.Run(fmt.Sprint(i), func(t *testing.T) { - got := marshalProvider(test.Input, test.IncludeExperimental) + got := marshalProvider(test.Input) if diff := cmp.Diff(test.Want, got, cmpOpts); diff != "" { t.Fatalf("wrong result:\n %s\n", diff) } @@ -431,5 +382,15 @@ func testProvider() providers.ProviderSchema { }, }, }, + Actions: map[string]providers.ActionSchema{ + "test_action": { + ConfigSchema: &configschema.Block{ + Attributes: map[string]*configschema.Attribute{ + "opt_attr": {Type: cty.String, Optional: true}, + "req_attr": {Type: cty.List(cty.String), Required: true}, + }, + }, + }, + }, } } diff --git a/internal/command/providers_schema.go b/internal/command/providers_schema.go index 0d51769133d3..919e1a57078c 100644 --- a/internal/command/providers_schema.go +++ b/internal/command/providers_schema.go @@ -107,7 +107,7 @@ func (c *ProvidersSchemaCommand) Run(args []string) int { return 1 } - jsonSchemas, err := jsonprovider.Marshal(schemas, c.AllowExperimentalFeatures) + jsonSchemas, err := jsonprovider.Marshal(schemas) if err != nil { c.Ui.Error(fmt.Sprintf("Failed to marshal provider schemas to json: %s", err)) return 1 diff --git a/internal/command/state_show.go b/internal/command/state_show.go index 4ef3bd80977c..b76d398ccd45 100644 --- a/internal/command/state_show.go +++ b/internal/command/state_show.go @@ -155,7 +155,7 @@ func (c *StateShowCommand) Run(args []string) int { ProviderFormatVersion: jsonprovider.FormatVersion, RootModule: root, RootModuleOutputs: outputs, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), } renderer := jsonformat.Renderer{ diff --git a/internal/command/views/operation.go b/internal/command/views/operation.go index ae9beca1b190..3b132991f9fa 100644 --- a/internal/command/views/operation.go +++ b/internal/command/views/operation.go @@ -110,7 +110,7 @@ func (v *OperationHuman) Plan(plan *plans.Plan, schemas *terraform.Schemas) { OutputChanges: outputs, ResourceChanges: changed, ResourceDrift: drift, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), RelevantAttributes: attrs, ActionInvocations: actions, } diff --git a/internal/command/views/show.go b/internal/command/views/show.go index 50429aa315f8..7eb76bf17e3d 100644 --- a/internal/command/views/show.go +++ b/internal/command/views/show.go @@ -83,7 +83,7 @@ func (v *ShowHuman) Display(config *configs.Config, plan *plans.Plan, planJSON * OutputChanges: outputs, ResourceChanges: changed, ResourceDrift: drift, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), RelevantAttributes: attrs, ActionInvocations: actions, } @@ -113,7 +113,7 @@ func (v *ShowHuman) Display(config *configs.Config, plan *plans.Plan, planJSON * ProviderFormatVersion: jsonprovider.FormatVersion, RootModule: root, RootModuleOutputs: outputs, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), } renderer.RenderHumanState(jstate) diff --git a/internal/command/views/test.go b/internal/command/views/test.go index 0feb2bab780d..e95e6afdc62a 100644 --- a/internal/command/views/test.go +++ b/internal/command/views/test.go @@ -203,7 +203,7 @@ func (t *TestHuman) Run(run *moduletest.Run, file *moduletest.File, progress mod ProviderFormatVersion: jsonprovider.FormatVersion, RootModule: root, RootModuleOutputs: outputs, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), } t.view.streams.Println() // Separate the state from any previous statements. @@ -225,7 +225,7 @@ func (t *TestHuman) Run(run *moduletest.Run, file *moduletest.File, progress mod OutputChanges: outputs, ResourceChanges: changed, ResourceDrift: drift, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), RelevantAttributes: attrs, ActionInvocations: actions, } @@ -568,7 +568,7 @@ func (t *TestJSON) Run(run *moduletest.Run, file *moduletest.File, progress modu ProviderFormatVersion: jsonprovider.FormatVersion, RootModule: root, RootModuleOutputs: outputs, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), } t.view.log.Info( @@ -592,7 +592,7 @@ func (t *TestJSON) Run(run *moduletest.Run, file *moduletest.File, progress modu OutputChanges: outputs, ResourceChanges: changed, ResourceDrift: drift, - ProviderSchemas: jsonprovider.MarshalForRenderer(schemas, false), + ProviderSchemas: jsonprovider.MarshalForRenderer(schemas), RelevantAttributes: attrs, ActionInvocations: actions, } diff --git a/internal/tfdiags/object.go b/internal/tfdiags/object.go index 56da886c67e7..8fb115673e3d 100644 --- a/internal/tfdiags/object.go +++ b/internal/tfdiags/object.go @@ -4,6 +4,7 @@ package tfdiags import ( "fmt" + "slices" "strings" "github.com/zclconf/go-cty/cty" @@ -23,29 +24,39 @@ func ObjectToString(obj cty.Value) string { return "" } - if obj.Type().IsObjectType() { - result := "" - it := obj.ElementIterator() - for it.Next() { - key, val := it.Element() - keyStr := key.AsString() + if !obj.Type().IsObjectType() { + panic("not an object") + } - if result != "" { - result += "," - } + it := obj.ElementIterator() + keys := make([]string, 0, obj.LengthInt()) + objMap := make(map[string]cty.Value) + result := "" + // store the keys for the object, and sort them + // before appending to the result so that the final value is deterministic. + for it.Next() { + key, val := it.Element() + keyStr := key.AsString() + keys = append(keys, keyStr) + objMap[keyStr] = val + } - if val.IsNull() { - result += fmt.Sprintf("%s=", keyStr) - continue - } + slices.Sort(keys) + for _, key := range keys { + val := objMap[key] + if result != "" { + result += "," + } - result += fmt.Sprintf("%s=%s", keyStr, ValueToString(val)) + if val.IsNull() { + result += fmt.Sprintf("%s=", key) + continue } - return result + result += fmt.Sprintf("%s=%s", key, ValueToString(val)) } - panic("not an object") + return result } func ValueToString(val cty.Value) string {