Merge branch 'hashicorp:main' into main

.changelog/23964.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+metrics: introduce client config to include alloc metadata as part of the base labels
+```

.changelog/24007.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+bug: Allow client template config block to be parsed when using json config
+```

.changelog/24065.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+scaling: Fixed a bug where scaling policies would not get created during job submission unless namespace field was set in jobspec
+```

.changelog/24073.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+ui: Fixes an issue where variables paths would not let namespaced users write variables unless they also had wildcard namespace variable write permissions
+```

.changelog/24125.txt

@@ -0,0 +1,3 @@
+```release-note:security
+security: Fixed a bug in client FS API where the check to prevent reads from the secrets dir could be bypassed on case-insensitive file systems
+```

.changelog/24127.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+state: Fixed a bug where compatibility updates for node topology for nodes older than 1.7.0 were not being correctly applied
+```

.github/workflows/actionlint.yml

@@ -10,6 +10,6 @@ jobs:
   actionlint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - name: "Check workflow files"
         uses: docker://docker.mirror.hashicorp.services/rhysd/actionlint:latest

.github/workflows/backport.yml

@@ -50,7 +50,7 @@ jobs:
     if: always() && (needs.backport.result == 'failure' || needs.backport-ent.result == 'failure')
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "type=m7a.2xlarge;m6a.2xlarge"]') || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: ./.github/actions/vault-secrets
         with:
           paths: |-

.github/workflows/build.yml

@@ -28,7 +28,7 @@ jobs:
     outputs:
       go-version: ${{ steps.get-go-version.outputs.go-version }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.inputs.build-ref }}
       - name: Determine Go version
@@ -43,7 +43,7 @@ jobs:
     outputs:
       product-version: ${{ steps.get-product-version.outputs.product-version }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.inputs.build-ref }}
       - name: get product version
@@ -58,7 +58,7 @@ jobs:
       filepath: ${{ steps.generate-metadata-file.outputs.filepath }}
     steps:
       - name: "Checkout directory"
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.inputs.build-ref }}
       - name: Generate metadata file
@@ -86,7 +86,7 @@ jobs:
     name: Go ${{ needs.get-go-version.outputs.go-version }} ${{ matrix.goos }} ${{ matrix.goarch }} build
 
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.inputs.build-ref }}
       - name: Setup go
@@ -138,7 +138,7 @@ jobs:
     name: Go ${{ needs.get-go-version.outputs.go-version }} ${{ matrix.goos }} ${{ matrix.goarch }} build
 
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.inputs.build-ref }}
       - name: Setup go
@@ -245,7 +245,7 @@ jobs:
     name: Go ${{ needs.get-go-version.outputs.go-version }} ${{ matrix.goos }} ${{ matrix.goarch }} build
 
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.inputs.build-ref }}
 
@@ -307,7 +307,7 @@ jobs:
       version: ${{needs.get-product-version.outputs.product-version}}
       revision: ${{github.sha}}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - name: Set revision
         if: "${{ github.event.inputs.build-ref != '' }}"
         run: |

.github/workflows/checks.yaml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux", "disk_gb=255", "type=m7a.2xlarge;m6a.2xlarge"]') || 'custom-linux-xl-nomad-22.04' }}
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           fetch-depth: 0 # needs tags for checkproto
       - uses: ./.github/actions/vault-secrets

.github/workflows/copywrite.yml

@@ -7,7 +7,7 @@ jobs:
   copywrite:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: hashicorp/setup-copywrite@32638da2d4e81d56a0764aa1547882fc4d209636 # v1.1.3
         name: Setup Copywrite
         with:

.github/workflows/ember-test-audit.yml

@@ -15,7 +15,7 @@ jobs:
   time-base:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           ref: ${{ github.event.pull_request.base.sha }}
       - uses: nanasess/setup-chromedriver@42cc2998329f041de87dc3cfa33a930eacd57eaa # v2.2.2
@@ -34,7 +34,7 @@ jobs:
   time-pr:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: nanasess/setup-chromedriver@42cc2998329f041de87dc3cfa33a930eacd57eaa # v2.2.2
       - name: Use Node.js
         uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4

.github/workflows/release.yml

@@ -52,7 +52,7 @@ jobs:
             echo "::error::Version ${{ github.event.inputs.version }} is invalid"
             exit 1
           fi
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: ./.github/actions/vault-secrets
         with:
           paths: |-

.github/workflows/security-scan.yml

@@ -36,15 +36,15 @@ jobs:
       && (github.actor != 'dependabot[bot]') && (github.actor != 'hc-github-team-nomad-core') }}
 
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           cache: ${{ contains(runner.name, 'Github Actions') }}
           go-version-file: .go-version
           cache-dependency-path: '**/go.sum'
 
       - name: Clone Security Scanner repo
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
         with:
           repository: hashicorp/security-scanner
           token: ${{ secrets.PRODSEC_SCANNER_READ_ONLY }}

.github/workflows/semgrep.yml

@@ -16,7 +16,7 @@ jobs:
     # Skip any PR created by dependabot to avoid permission issues
     if: (github.actor != 'dependabot[bot]')
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - run: semgrep ci --config=.semgrep/
 permissions:
   contents: read

.github/workflows/test-core.yaml

@@ -58,7 +58,7 @@ jobs:
     runs-on: ${{matrix.os}}
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           cache: ${{ contains(runner.name, 'Github Actions') }}
@@ -73,7 +73,7 @@ jobs:
     runs-on: custom-linux-xl-nomad-22.04
     timeout-minutes: 8
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           cache: true
@@ -101,7 +101,7 @@ jobs:
           - drivers
           - quick
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
           cache: ${{ contains(runner.name, 'Github Actions') }}

.github/workflows/test-e2e.yml

@@ -38,7 +38,7 @@ jobs:
   test-e2e-vault:
     runs-on: ${{ endsWith(github.repository, '-enterprise') && fromJSON('["self-hosted", "ondemand", "linux"]') || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: ./.github/actions/vault-secrets
         with:
           paths: |-
@@ -59,7 +59,7 @@ jobs:
   test-e2e-consul:
     runs-on: 'ubuntu-22.04' # this job requires sudo, so not currently suitable for self-hosted runners
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - name: Git config token
         if: endsWith(github.repository, '-enterprise')
         run: git config --global url.'https://${{ secrets.ELEVATED_GITHUB_TOKEN }}@github.com'.insteadOf 'https://github.com'

.github/workflows/test-ui.yml

@@ -21,7 +21,7 @@ jobs:
     outputs:
       nonce: ${{ steps.nonce.outputs.nonce }}
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: ./.github/actions/setup-js
       - name: lint:js
         run: yarn run lint:js
@@ -45,7 +45,7 @@ jobs:
         partition: [1, 2, 3, 4]
         split: [4]
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: ./.github/actions/setup-js
       - uses: browser-actions/setup-chrome@facf10a55b9caf92e0cc749b4f82bf8220989148 # v1.7.2
       - uses: ./.github/actions/vault-secrets
@@ -68,7 +68,7 @@ jobs:
       run:
         working-directory: ui
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - uses: ./.github/actions/setup-js
       - uses: ./.github/actions/vault-secrets
         with:

.github/workflows/test-windows.yml

@@ -50,7 +50,7 @@ jobs:
       - name: Docker Info
         run: docker version
       - run: git config --global core.autocrlf false
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
       - name: Setup go
         uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
         with:
@@ -90,6 +90,7 @@ jobs:
             github.com/hashicorp/nomad/client/lib/fifo \
             github.com/hashicorp/nomad/client/logmon \
             github.com/hashicorp/nomad/client/allocrunner/taskrunner/template \
+            github.com/hashicorp/nomad/client/allocdir \
             github.com/hashicorp/nomad/plugins/base
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:

.go-version

@@ -1 +1 @@
-1.23.1
+1.23.2

client/allocdir/alloc_dir.go

@@ -471,11 +471,11 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) {
 	// Check if it is trying to read into a secret directory
 	d.mu.RLock()
 	for _, dir := range d.TaskDirs {
-		if filepath.HasPrefix(p, dir.SecretsDir) {
+		if caseInsensitiveHasPrefix(p, dir.SecretsDir) {
 			d.mu.RUnlock()
 			return nil, fmt.Errorf("Reading secret file prohibited: %s", path)
 		}
-		if filepath.HasPrefix(p, dir.PrivateDir) {
+		if caseInsensitiveHasPrefix(p, dir.PrivateDir) {
 			d.mu.RUnlock()
 			return nil, fmt.Errorf("Reading private file prohibited: %s", path)
 		}
@@ -492,6 +492,11 @@ func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) {
 	return f, nil
 }
 
+// CaseInsensitiveHasPrefix checks if the prefix is a case-insensitive prefix.
+func caseInsensitiveHasPrefix(s, prefix string) bool {
+	return strings.HasPrefix(strings.ToLower(s), strings.ToLower(prefix))
+}
+
 // BlockUntilExists blocks until the passed file relative the allocation
 // directory exists. The block can be cancelled with the passed context.
 func (d *AllocDir) BlockUntilExists(ctx context.Context, path string) (chan error, error) {

client/allocdir/alloc_dir_nonlinux_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+//go:build !linux
+// +build !linux
+
+package allocdir
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/plugins/drivers/fsisolation"
+	"github.com/shoenig/test/must"
+)
+
+func TestAllocDir_ReadAt_CaseInsensitiveSecretDir(t *testing.T) {
+	ci.Parallel(t)
+
+	// On macOS, os.TempDir returns a symlinked path under /var which
+	// is outside of the directories shared into the VM used for Docker.
+	// Expand the symlink to get the real path in /private, which is ok.
+	tmp, err := filepath.EvalSymlinks(t.TempDir())
+	must.NoError(t, err)
+
+	d := NewAllocDir(testlog.HCLogger(t), tmp, tmp, "test")
+	must.NoError(t, d.Build())
+	defer func() { _ = d.Destroy() }()
+
+	td := d.NewTaskDir(t1Windows)
+	must.NoError(t, td.Build(fsisolation.None, nil, "nobody"))
+
+	target := filepath.Join(t1Windows.Name, TaskSecrets, "test_file")
+
+	full := filepath.Join(d.AllocDir, target)
+	must.NoError(t, os.WriteFile(full, []byte("hi"), 0o600))
+
+	targetCaseInsensitive := filepath.Join(t1Windows.Name, "sEcReTs", "test_file")
+
+	_, err = d.ReadAt(targetCaseInsensitive, 0)
+	must.EqError(t, err, "Reading secret file prohibited: "+targetCaseInsensitive)
+}
+
+var (
+	t1Windows = &structs.Task{
+		Name:   "web",
+		Driver: "exec",
+		Config: map[string]interface{}{
+			"command": "/bin/date",
+			"args":    "+%s",
+		},
+		Resources: &structs.Resources{
+			DiskMB: 1,
+		},
+	}
+)

client/allocdir/alloc_dir_test.go

@@ -1,6 +1,9 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
+//go:build !windows
+// +build !windows
+
 package allocdir
 
 import (

client/allocdir/fs_linux_test.go

@@ -1,6 +1,9 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
+//go:build !windows
+// +build !windows
+
 package allocdir
 
 import (

client/allocdir/task_dir_test.go

@@ -1,6 +1,9 @@
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
+//go:build !windows
+// +build !windows
+
 package allocdir
 
 import (