Backport of search: Drop identity from state if no identity schema is present into v1.14

.changes/v1.14/BUG FIXES-20250924-110416.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: 'console and test: return explicit diagnostics when referencing resources that were not included in the most recent operation.'
+time: 2025-09-24T11:04:16.860364+02:00
+custom:
+    Issue: "37663"

.changes/v1.14/BUG FIXES-20250926-113318.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: 'query: generate unique resource identifiers for results of expanded list resources'
+time: 2025-09-26T11:33:18.241184+02:00
+custom:
+    Issue: "37681"

.changes/v1.14/BUGFIX-20250927-184134.yaml

@@ -0,0 +1,5 @@
+kind: BUGFIX
+body: The CLI now summarizes the number of actions invoked during `terraform apply`, matching the plan output.
+time: 2025-09-27T18:41:34.771437+02:00
+custom:
+    Issue: "37689"

.changes/v1.14/ENHANCEMENTS-20250919-115253.yaml

@@ -0,0 +1,5 @@
+kind: ENHANCEMENTS
+body: '`terraform stacks` command support for `-help` flag'
+time: 2025-09-19T11:52:53.923764-04:00
+custom:
+    Issue: "37645"

.changes/v1.14/ENHANCEMENTS-20250925-151237.yaml

@@ -0,0 +1,5 @@
+kind: ENHANCEMENTS
+body: "query: support offline validation of query files via -query flag in the validate command"
+time: 2025-09-25T15:12:37.198573+02:00
+custom:
+  Issue: "37671"

.changes/v1.14/NEW FEATURES-20250829-183404.yaml

@@ -0,0 +1,3 @@
+kind: NEW FEATURES
+body: "**List Resources**: List resources can be defined in `*.tfquery.hcl` files and allow querying and filterting existing infrastructure."
+time: 2025-08-29T18:34:04.250038+02:00

.changes/v1.14/NEW FEATURES-20250829-184206.yaml

@@ -0,0 +1,3 @@
+kind: NEW FEATURES
+body: "A new Terraform command `terraform query`: Executes list operations against existing infrastructure and displays the results. The command can optionally generate configuration for importing results into Terraform."
+time: 2025-08-29T18:42:06.659172+02:00

CHANGELOG.md

@@ -3,22 +3,32 @@
 
 NEW FEATURES:
 
+* **List Resources**: List resources can be defined in `*.tfquery.hcl` files and allow querying and filterting existing infrastructure.
+
+* A new Terraform command `terraform query`: Executes list operations against existing infrastructure and displays the results. The command can optionally generate configuration for importing results into Terraform.
+
 * A new GenerateResourceConfiguration RPC allows providers to create more precise configuration values during import. ([#37515](https://github.com/hashicorp/terraform/issues/37515))
 
+* New top-level Actions block: Actions are provider defined and meant to codify use cases outside the normal CRUD model in your Terraform configuration. Providers can define Actions like `aws_lambda_invoke` or `aws_cloudfront_create_invalidation` that do something imparative outside of Terraforms normal CRUD model. You can configure such a side-effect with an action block and have actions triggered through the lifecycle of a resource or through passing the `-invoke` CLI flag. ([#37553](https://github.com/hashicorp/terraform/issues/37553))
+
 
 ENHANCEMENTS:
 
 * terraform test: expected diagnostics will be included in test output when running in verbose mode" ([#37362](https://github.com/hashicorp/terraform/issues/37362))
 
 * terraform test: ignore prevent_destroy attribute during when cleaning up tests" ([#37364](https://github.com/hashicorp/terraform/issues/37364))
 
+* `terraform stacks` command support for `-help` flag ([#37645](https://github.com/hashicorp/terraform/issues/37645))
+
 
 BUG FIXES:
 
 * Retrieve all workspace variables while doing a `terraform import`, include variables inherited from variable sets but not overwritten by the workspace. ([#37241](https://github.com/hashicorp/terraform/issues/37241))
 
 * Fix OSS backend proxy support by adding a proxy layer for OSS backend operations. Resolves hashicorp/terraform#36897. ([#36897](https://github.com/hashicorp/terraform/issues/36897))
 
+* console and test: return explicit diagnostics when referencing resources that were not included in the most recent operation. ([#37663](https://github.com/hashicorp/terraform/issues/37663))
+
 
 UPGRADE NOTES:
 

CODEOWNERS

@@ -32,6 +32,3 @@
 builtin/provisioners/file               @hashicorp/terraform-core
 builtin/provisioners/local-exec         @hashicorp/terraform-core
 builtin/provisioners/remote-exec        @hashicorp/terraform-core
-
-# temporary go.mod protection during freeze
-/go.mod      @tommyokeefe

commands.go

@@ -276,6 +276,12 @@ func initCommands(
 			}, nil
 		},
 
+		"query": func() (cli.Command, error) {
+			return &command.QueryCommand{
+				Meta: meta,
+			}, nil
+		},
+
 		"refresh": func() (cli.Command, error) {
 			return &command.RefreshCommand{
 				Meta: meta,
@@ -451,12 +457,6 @@ func initCommands(
 			}, nil
 		}
 
-		Commands["query"] = func() (cli.Command, error) {
-			return &command.QueryCommand{
-				Meta: meta,
-			}, nil
-		}
-
 		Commands["test cleanup"] = func() (cli.Command, error) {
 			return &command.TestCleanupCommand{
 				Meta: meta,

internal/backend/local/backend_local.go

@@ -132,7 +132,9 @@ func (b *Local) localRun(op *backendrun.Operation) (*backendrun.LocalRun, *confi
 		// If validation is enabled, validate
 		if b.OpValidation {
 			log.Printf("[TRACE] backend/local: running validation operation")
-			validateDiags := ret.Core.Validate(ret.Config, nil)
+			// TODO: Implement query validate command. op.Query is false when running the command "terraform validate"
+			opts := &terraform.ValidateOpts{Query: op.Query}
+			validateDiags := ret.Core.Validate(ret.Config, opts)
 			diags = diags.Append(validateDiags)
 		}
 	}

internal/cloud/backend_query.go

@@ -20,6 +20,7 @@ import (
 	"github.com/hashicorp/terraform/internal/genconfig"
 	"github.com/hashicorp/terraform/internal/terraform"
 	"github.com/hashicorp/terraform/internal/tfdiags"
+	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
 )
 
@@ -276,7 +277,7 @@ func (b *Cloud) cancelQueryRun(cancelCtx context.Context, op *backendrun.Operati
 // formatIdentity formats the identity map into a string representation.
 // It flattens the map into a string of key=value pairs, separated by commas.
 func formatIdentity(identity map[string]json.RawMessage) string {
-	parts := make([]string, 0, len(identity))
+	ctyObj := make(map[string]cty.Value, len(identity))
 	for key, value := range identity {
 		ty, err := ctyjson.ImpliedType(value)
 		if err != nil {
@@ -286,9 +287,9 @@ func formatIdentity(identity map[string]json.RawMessage) string {
 		if err != nil {
 			continue
 		}
-		parts = append(parts, fmt.Sprintf("%s=%s", key, tfdiags.ValueToString(v)))
+		ctyObj[key] = v
 	}
-	return strings.Join(parts, ",")
+	return tfdiags.ObjectToString(cty.ObjectVal(ctyObj))
 }
 
 const queryDefaultHeader = `

internal/cloud/backend_query_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/hashicorp/cli"
 	"github.com/hashicorp/terraform/internal/addrs"
 	"github.com/hashicorp/terraform/internal/backend/backendrun"
@@ -121,8 +122,15 @@ func TestCloud_queryJSONBasic(t *testing.T) {
 	outp := close(t)
 	gotOut := outp.Stdout()
 
-	if !strings.Contains(gotOut, "list.concept_pet.pets   id=complete-gannet,legs=6   This is a complete-gannet") {
-		t.Fatalf("expected query results in output: %s", gotOut)
+	expectedOut := `list.concept_pet.pets   id=large-roughy,legs=2      This is a large-roughy
+list.concept_pet.pets   id=able-werewolf,legs=5     This is a able-werewolf
+list.concept_pet.pets   id=complete-gannet,legs=6   This is a complete-gannet
+list.concept_pet.pets   id=charming-beagle,legs=3   This is a charming-beagle
+list.concept_pet.pets   id=legal-lamprey,legs=2     This is a legal-lamprey
+
+`
+	if diff := cmp.Diff(expectedOut, gotOut); diff != "" {
+		t.Fatalf("expected query results output to be %s, got %s: diff: %s", expectedOut, gotOut, diff)
 	}
 
 	stateMgr, _ := b.StateMgr(testBackendSingleWorkspaceName)

internal/cloud/testdata/query-json-basic/main.tfquery.hcl

@@ -1 +1,3 @@
-list "concept_pet" "pets" {}
+list "concept_pet" "pets" {
+  provider = concept
+}

internal/cloud/testdata/query/main.tfquery.hcl

@@ -1 +1,3 @@
-list "null_resource" "foo" {}
+list "null_resource" "foo" {
+  provider = null
+}

internal/command/arguments/validate.go

@@ -24,6 +24,9 @@ type Validate struct {
 
 	// ViewType specifies which output format to use: human, JSON, or "raw".
 	ViewType ViewType
+
+	// Query indicates that Terraform should also validate .tfquery files.
+	Query bool
 }
 
 // ParseValidate processes CLI arguments, returning a Validate value and errors.
@@ -40,6 +43,7 @@ func ParseValidate(args []string) (*Validate, tfdiags.Diagnostics) {
 	cmdFlags.BoolVar(&jsonOutput, "json", false, "json")
 	cmdFlags.StringVar(&validate.TestDirectory, "test-directory", "tests", "test-directory")
 	cmdFlags.BoolVar(&validate.NoTests, "no-tests", false, "no-tests")
+	cmdFlags.BoolVar(&validate.Query, "query", false, "query")
 
 	if err := cmdFlags.Parse(args); err != nil {
 		diags = diags.Append(tfdiags.Sourceless(

internal/command/e2etest/providers_schema_test.go

@@ -134,6 +134,21 @@ func TestProvidersSchema(t *testing.T) {
                     }
                 }
             },
+            "list_resource_schemas": {
+                "simple_resource": {
+                    "version": 0,
+                    "block": {
+                        "attributes": {
+                            "value": {
+                                "type": "string",
+                                "description_kind": "plain",
+                                "optional": true
+                            }
+                        },
+                        "description_kind": "plain"
+                    }
+                }
+            },
             "resource_identity_schemas": {
                 "simple_resource": {
                     "version": 0,
@@ -213,6 +228,21 @@ func TestProvidersSchema(t *testing.T) {
                     }
                 }
             },
+            "list_resource_schemas": {
+                "simple_resource": {
+                    "version": 0,
+                    "block": {
+                        "attributes": {
+                            "value": {
+                                "type": "string",
+                                "description_kind": "plain",
+                                "optional": true
+                            }
+                        },
+                        "description_kind": "plain"
+                    }
+                }
+            },
             "functions": {
                 "noop": {
                     "description": "noop takes any single argument and returns the same value",

internal/command/jsonformat/plan_test.go

@@ -8071,7 +8071,7 @@ func runTestCases(t *testing.T, testCases map[string]testCase) {
 				return
 			}
 
-			jsonschemas := jsonprovider.MarshalForRenderer(tfschemas, false)
+			jsonschemas := jsonprovider.MarshalForRenderer(tfschemas)
 			change := structured.FromJsonChange(jsonchanges[0].Change, attribute_path.AlwaysMatcher())
 			renderer := Renderer{Colorize: color}
 			diff := diff{
@@ -8421,7 +8421,7 @@ func TestResourceChange_deferredActions(t *testing.T) {
 			}
 
 			renderer := Renderer{Colorize: color}
-			jsonschemas := jsonprovider.MarshalForRenderer(fullSchema, false)
+			jsonschemas := jsonprovider.MarshalForRenderer(fullSchema)
 			diffs := precomputeDiffs(Plan{
 				DeferredChanges: deferredChanges,
 				ProviderSchemas: jsonschemas,
@@ -8714,7 +8714,7 @@ func TestResourceChange_actions(t *testing.T) {
 					},
 				},
 			}
-			jsonschemas := jsonprovider.MarshalForRenderer(fullSchema, false)
+			jsonschemas := jsonprovider.MarshalForRenderer(fullSchema)
 			diffs := precomputeDiffs(Plan{
 				ResourceChanges:   []jsonplan.ResourceChange{defaultResourceChange},
 				ActionInvocations: tc.actionInvocations,

internal/command/jsonformat/state_test.go

@@ -86,7 +86,7 @@ func TestState(t *testing.T) {
 				RootModule:            root,
 				RootModuleOutputs:     outputs,
 				ProviderFormatVersion: jsonprovider.FormatVersion,
-				ProviderSchemas:       jsonprovider.MarshalForRenderer(tt.Schemas, false),
+				ProviderSchemas:       jsonprovider.MarshalForRenderer(tt.Schemas),
 			})
 
 			result := done(t).All()

internal/command/jsonprovider/provider.go

@@ -45,22 +45,22 @@ func newProviders() *Providers {
 // schema into the public structured JSON versions.
 //
 // This is a format that can be read by the structured plan renderer.
-func MarshalForRenderer(s *terraform.Schemas, includeExperimentalSchemas bool) map[string]*Provider {
+func MarshalForRenderer(s *terraform.Schemas) map[string]*Provider {
 	schemas := make(map[string]*Provider, len(s.Providers))
 	for k, v := range s.Providers {
-		schemas[k.String()] = marshalProvider(v, includeExperimentalSchemas)
+		schemas[k.String()] = marshalProvider(v)
 	}
 	return schemas
 }
 
-func Marshal(s *terraform.Schemas, includeExperimentalSchemas bool) ([]byte, error) {
+func Marshal(s *terraform.Schemas) ([]byte, error) {
 	providers := newProviders()
-	providers.Schemas = MarshalForRenderer(s, includeExperimentalSchemas)
+	providers.Schemas = MarshalForRenderer(s)
 	ret, err := json.Marshal(providers)
 	return ret, err
 }
 
-func marshalProvider(tps providers.ProviderSchema, includeExperimentalSchemas bool) *Provider {
+func marshalProvider(tps providers.ProviderSchema) *Provider {
 	p := &Provider{
 		Provider:                 marshalSchema(tps.Provider),
 		ResourceSchemas:          marshalSchemas(tps.ResourceTypes),
@@ -71,20 +71,18 @@ func marshalProvider(tps providers.ProviderSchema, includeExperimentalSchemas bo
 		ActionSchemas:            marshalActionSchemas(tps.Actions),
 	}
 
-	if includeExperimentalSchemas {
-		// List resource schemas are nested under a "config" block, so we need to
-		// extract that block to get the actual provider schema for the list resource.
-		// When getting the provider schemas, Terraform adds this extra level to
-		// better match the actual configuration structure.
-		listSchemas := make(map[string]providers.Schema, len(tps.ListResourceTypes))
-		for k, v := range tps.ListResourceTypes {
-			listSchemas[k] = providers.Schema{
-				Body:    &v.Body.BlockTypes["config"].Block,
-				Version: v.Version,
-			}
+	// List resource schemas are nested under a "config" block, so we need to
+	// extract that block to get the actual provider schema for the list resource.
+	// When getting the provider schemas, Terraform adds this extra level to
+	// better match the actual configuration structure.
+	listSchemas := make(map[string]providers.Schema, len(tps.ListResourceTypes))
+	for k, v := range tps.ListResourceTypes {
+		listSchemas[k] = providers.Schema{
+			Body:    &v.Body.BlockTypes["config"].Block,
+			Version: v.Version,
 		}
-		p.ListResourceSchemas = marshalSchemas(listSchemas)
 	}
+	p.ListResourceSchemas = marshalSchemas(listSchemas)
 
 	return p
 }