From 990ff2c0a4574ac85321ddd81a179968c43960fd Mon Sep 17 00:00:00 2001 From: Tim Gross Date: Wed, 4 Mar 2026 15:39:17 -0500 Subject: [PATCH] batch: reduce server overhead when dispatching jobs from CLI When dispatching parameterized jobs or forcing periodic jobs to run via the CLI, we do a prefix lookup as we do with most other commands. But in this case we end up getting a potentially very large set of jobs back from the server, even if we have an exact match for the prefix. This can cause excess CPU/memory load in the RPC and HTTP API handlers as we have to serialize these large sets just to report the error to the user. Update the CLI so that it uses a go-bexpr filter to filter down to the set of jobs we need for these operations. This requires an update to go-bexpr to support nil checking on pointers in structs. Also add a page size to the list results to reduce the load for all commands that need to do a prefix lookup on jobs. Ref: https://hashicorp.atlassian.net/browse/NMD-941 Fixes: https://github.com/hashicorp/nomad/issues/26653 --- .changelog/27631.txt | 7 +++++++ command/alloc_exec.go | 2 +- command/alloc_fs.go | 2 +- command/alloc_logs.go | 2 +- command/job_action.go | 6 +++--- command/job_allocs.go | 2 +- command/job_deployments.go | 2 +- command/job_dispatch.go | 11 +++++++---- command/job_eval.go | 2 +- command/job_history.go | 2 +- command/job_inspect.go | 2 +- command/job_periodic_force.go | 4 +--- command/job_promote.go | 2 +- command/job_restart.go | 2 +- command/job_revert.go | 2 +- command/job_scale.go | 2 +- command/job_scaling_events.go | 2 +- command/job_start.go | 2 +- command/job_status.go | 2 +- command/job_stop.go | 2 +- command/job_tag_apply.go | 2 +- command/job_tag_unset.go | 2 +- command/meta.go | 33 ++++++++++++++------------------- command/meta_test.go | 10 ++++------ go.mod | 2 +- go.sum | 4 ++-- 26 files changed, 57 insertions(+), 56 deletions(-) create mode 100644 .changelog/27631.txt diff --git a/.changelog/27631.txt b/.changelog/27631.txt new file mode 100644 index 00000000000..182a7dc8e1b --- /dev/null +++ b/.changelog/27631.txt @@ -0,0 +1,7 @@ +```release-note:improvement +cli: Reduced server overhead when dispatching jobs or forcing periodic jobs from the CLI +``` + +```release-note:improvement +cli: Truncate results when job commands return a large set of jobs that match the provided ID prefix +``` diff --git a/command/alloc_exec.go b/command/alloc_exec.go index 57a247094ae..2b1f8312c5b 100644 --- a/command/alloc_exec.go +++ b/command/alloc_exec.go @@ -167,7 +167,7 @@ func (l *AllocExecCommand) Run(args []string) int { var allocStub *api.AllocationListStub if job { - jobID, ns, err := l.JobIDByPrefix(client, args[0], nil) + jobID, ns, err := l.JobIDByPrefix(client, args[0], "") if err != nil { l.Ui.Error(err.Error()) return 1 diff --git a/command/alloc_fs.go b/command/alloc_fs.go index aefab09f867..6ed902a51fd 100644 --- a/command/alloc_fs.go +++ b/command/alloc_fs.go @@ -171,7 +171,7 @@ func (f *AllocFSCommand) Run(args []string) int { // If -job is specified, use random allocation, otherwise use provided allocation allocID := args[0] if job { - jobID, ns, err := f.JobIDByPrefix(client, args[0], nil) + jobID, ns, err := f.JobIDByPrefix(client, args[0], "") if err != nil { f.Ui.Error(err.Error()) return 1 diff --git a/command/alloc_logs.go b/command/alloc_logs.go index a8b2e01435b..0599b40a3d0 100644 --- a/command/alloc_logs.go +++ b/command/alloc_logs.go @@ -172,7 +172,7 @@ func (l *AllocLogsCommand) Run(args []string) int { // If -job is specified, use random allocation, otherwise use provided allocation allocID := args[0] if l.job { - jobID, ns, err := l.JobIDByPrefix(client, args[0], nil) + jobID, ns, err := l.JobIDByPrefix(client, args[0], "") if err != nil { l.Ui.Error(err.Error()) return 1 diff --git a/command/job_action.go b/command/job_action.go index 5052d60a661..86ccad11941 100644 --- a/command/job_action.go +++ b/command/job_action.go @@ -32,10 +32,10 @@ Usage: nomad job action [options] Perform a predefined command inside the environment of the given allocation and job, or given task, group and job. - + Either an allocation or a task and group must be provided; for example, either of the following will work: - + nomad job action -alloc= -job= nomad job action -task= -group= -job= @@ -182,7 +182,7 @@ func (c *JobActionCommand) Run(args []string) int { return 1 } - jobID, ns, err := c.JobIDByPrefix(client, job, nil) + jobID, ns, err := c.JobIDByPrefix(client, job, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_allocs.go b/command/job_allocs.go index 034a5cb2fd9..eddedd4a27a 100644 --- a/command/job_allocs.go +++ b/command/job_allocs.go @@ -99,7 +99,7 @@ func (c *JobAllocsCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_deployments.go b/command/job_deployments.go index 5704129445d..6b0d703838f 100644 --- a/command/job_deployments.go +++ b/command/job_deployments.go @@ -104,7 +104,7 @@ func (c *JobDeploymentsCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_dispatch.go b/command/job_dispatch.go index 238b92a0c9d..5a4436d8e7a 100644 --- a/command/job_dispatch.go +++ b/command/job_dispatch.go @@ -4,6 +4,7 @@ package command import ( + "errors" "fmt" "io" "net/url" @@ -189,12 +190,14 @@ func (c *JobDispatchCommand) Run(args []string) int { return 1 } - // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, func(j *api.JobListStub) bool { - return j.ParameterizedJob - }) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, + `ParentID == "" and ParameterizedJob is not nil`) if err != nil { + var noPrefixErr *NoJobWithPrefixError + if errors.As(err, &noPrefixErr) { + err = fmt.Errorf("No parameterized job(s) with prefix or ID %q found", jobIDPrefix) + } c.Ui.Error(err.Error()) return 1 } diff --git a/command/job_eval.go b/command/job_eval.go index ca786bdc872..3d18dd078df 100644 --- a/command/job_eval.go +++ b/command/job_eval.go @@ -107,7 +107,7 @@ func (c *JobEvalCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_history.go b/command/job_history.go index fb499942bc0..a06a757f1cc 100644 --- a/command/job_history.go +++ b/command/job_history.go @@ -150,7 +150,7 @@ func (c *JobHistoryCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_inspect.go b/command/job_inspect.go index 7569084f8f5..9cc08823dea 100644 --- a/command/job_inspect.go +++ b/command/job_inspect.go @@ -133,7 +133,7 @@ func (c *JobInspectCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_periodic_force.go b/command/job_periodic_force.go index 9cf42813f4a..60e296ac398 100644 --- a/command/job_periodic_force.go +++ b/command/job_periodic_force.go @@ -120,9 +120,7 @@ func (c *JobPeriodicForceCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, func(j *api.JobListStub) bool { - return j.Periodic - }) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "Periodic is not nil") if err != nil { var noPrefixErr *NoJobWithPrefixError if errors.As(err, &noPrefixErr) { diff --git a/command/job_promote.go b/command/job_promote.go index 113e4940c88..fa076cc5a36 100644 --- a/command/job_promote.go +++ b/command/job_promote.go @@ -111,7 +111,7 @@ func (c *JobPromoteCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_restart.go b/command/job_restart.go index f623b39a1a7..6b6de048901 100644 --- a/command/job_restart.go +++ b/command/job_restart.go @@ -256,7 +256,7 @@ func (c *JobRestartCommand) Run(args []string) int { } // Use prefix matching to find job. - job, err := c.JobByPrefix(c.client, c.jobID, nil) + job, err := c.JobByPrefix(c.client, c.jobID, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_revert.go b/command/job_revert.go index 0eb3096a3e0..37520da206b 100644 --- a/command/job_revert.go +++ b/command/job_revert.go @@ -117,7 +117,7 @@ func (c *JobRevertCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_scale.go b/command/job_scale.go index 8619278ae56..dd16b3a48a8 100644 --- a/command/job_scale.go +++ b/command/job_scale.go @@ -128,7 +128,7 @@ func (j *JobScaleCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { j.Ui.Error(err.Error()) return 1 diff --git a/command/job_scaling_events.go b/command/job_scaling_events.go index f04b08c693c..b98a09b6f46 100644 --- a/command/job_scaling_events.go +++ b/command/job_scaling_events.go @@ -90,7 +90,7 @@ func (j *JobScalingEventsCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := j.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { j.Ui.Error(err.Error()) return 1 diff --git a/command/job_start.go b/command/job_start.go index 7600001226d..56f15a7625f 100644 --- a/command/job_start.go +++ b/command/job_start.go @@ -104,7 +104,7 @@ func (c *JobStartCommand) Run(args []string) int { length = fullId } - job, err := c.JobByPrefix(client, jobIDPrefix, nil) + job, err := c.JobByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_status.go b/command/job_status.go index 59cca020ea5..5406c3d8e55 100644 --- a/command/job_status.go +++ b/command/job_status.go @@ -206,7 +206,7 @@ func (c *JobStatusCommand) Run(args []string) int { // Try querying the job jobIDPrefix := strings.TrimSpace(args[0]) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_stop.go b/command/job_stop.go index 51b905a53e1..69dd85e3249 100644 --- a/command/job_stop.go +++ b/command/job_stop.go @@ -148,7 +148,7 @@ func (c *JobStopCommand) Run(args []string) int { } // Check if the job exists - job, err := c.JobByPrefix(client, jobID, nil) + job, err := c.JobByPrefix(client, jobID, "") if err != nil { c.Ui.Error(err.Error()) statusCh <- 1 diff --git a/command/job_tag_apply.go b/command/job_tag_apply.go index e126bb75278..5079313e4a6 100644 --- a/command/job_tag_apply.go +++ b/command/job_tag_apply.go @@ -114,7 +114,7 @@ func (c *JobTagApplyCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(job) - jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, namespace, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/job_tag_unset.go b/command/job_tag_unset.go index 9663bcedd91..f9da8a4b350 100644 --- a/command/job_tag_unset.go +++ b/command/job_tag_unset.go @@ -94,7 +94,7 @@ func (c *JobTagUnsetCommand) Run(args []string) int { // Check if the job exists jobIDPrefix := strings.TrimSpace(job) - jobID, _, err := c.JobIDByPrefix(client, jobIDPrefix, nil) + jobID, _, err := c.JobIDByPrefix(client, jobIDPrefix, "") if err != nil { c.Ui.Error(err.Error()) return 1 diff --git a/command/meta.go b/command/meta.go index 427435fe779..b9e51cb2907 100644 --- a/command/meta.go +++ b/command/meta.go @@ -311,10 +311,6 @@ func (m *Meta) FormatWarnings(header string, warnings string) string { )) } -// JobByPrefixFilterFunc is a function used to filter jobs when performing a -// prefix match. Only jobs that return true are included in the prefix match. -type JobByPrefixFilterFunc func(*api.JobListStub) bool - // NoJobWithPrefixError is the error returned when the job prefix doesn't // return any matches. type NoJobWithPrefixError struct { @@ -328,7 +324,7 @@ func (e *NoJobWithPrefixError) Error() string { // JobByPrefix returns the job that best matches the given prefix. Returns an // error if there are no matches or if there are more than one exact match // across namespaces. -func (m *Meta) JobByPrefix(client *api.Client, prefix string, filter JobByPrefixFilterFunc) (*api.Job, error) { +func (m *Meta) JobByPrefix(client *api.Client, prefix string, filter string) (*api.Job, error) { jobID, namespace, err := m.JobIDByPrefix(client, prefix, filter) if err != nil { return nil, err @@ -348,9 +344,13 @@ func (m *Meta) JobByPrefix(client *api.Client, prefix string, filter JobByPrefix // Returns the prefix itself if job prefix search is not allowed and an error // if there are no matches or if there are more than one exact match across // namespaces. -func (m *Meta) JobIDByPrefix(client *api.Client, prefix string, filter JobByPrefixFilterFunc) (string, string, error) { - // Search job by prefix. Return an error if there is not an exact match. - jobs, _, err := client.Jobs().PrefixList(prefix) +func (m *Meta) JobIDByPrefix(client *api.Client, prefix string, filter string) (string, string, error) { + maxResults := 20 // reduce load for large sets + jobs, _, err := client.Jobs().ListOptions(nil, &api.QueryOptions{ + Prefix: prefix, + Filter: filter, + PerPage: int32(maxResults), + }) if err != nil { if strings.Contains(err.Error(), api.PermissionDeniedErrorContent) { return prefix, "", nil @@ -358,27 +358,22 @@ func (m *Meta) JobIDByPrefix(client *api.Client, prefix string, filter JobByPref return "", "", fmt.Errorf("Error querying job prefix %q: %s", prefix, err) } - if filter != nil { - var filtered []*api.JobListStub - for _, j := range jobs { - if filter(j) { - filtered = append(filtered, j) - } - } - jobs = filtered - } - if len(jobs) == 0 { return "", "", &NoJobWithPrefixError{Prefix: prefix} } if len(jobs) > 1 { exactMatch := prefix == jobs[0].ID matchInMultipleNamespaces := m.allNamespaces() && jobs[0].ID == jobs[1].ID + truncatedMsg := "" + if len(jobs) >= maxResults { + truncatedMsg = "\n(results may be truncated)" + } if !exactMatch || matchInMultipleNamespaces { return "", "", fmt.Errorf( - "Prefix %q matched multiple jobs\n\n%s", + "Prefix %q matched multiple jobs\n\n%s%s", prefix, createStatusListOutput(jobs, m.allNamespaces()), + truncatedMsg, ) } } diff --git a/command/meta_test.go b/command/meta_test.go index 4434c218f80..7bc6e0b1bc6 100644 --- a/command/meta_test.go +++ b/command/meta_test.go @@ -202,7 +202,7 @@ func TestMeta_JobByPrefix(t *testing.T) { testCases := []struct { name string prefix string - filterFunc JobByPrefixFilterFunc + filter string expectedError string }{ { @@ -216,10 +216,8 @@ func TestMeta_JobByPrefix(t *testing.T) { { name: "match with filter", prefix: "job-", - filterFunc: func(j *api.JobListStub) bool { - // Filter out jobs with "job-" so that only "job-2" matches. - return j.ID == "job-2" - }, + // Filter out jobs so that only "job-2" matches. + filter: `ID == "job-2"`, }, { name: "multiple matches", @@ -240,7 +238,7 @@ func TestMeta_JobByPrefix(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - job, err := meta.JobByPrefix(client, tc.prefix, tc.filterFunc) + job, err := meta.JobByPrefix(client, tc.prefix, tc.filter) if tc.expectedError != "" { must.Nil(t, job) must.ErrorContains(t, err, tc.expectedError) diff --git a/go.mod b/go.mod index d1ddd4b883a..b6d7b61cf78 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/hashicorp/consul/api v1.33.4 github.com/hashicorp/consul/sdk v0.17.2 github.com/hashicorp/cronexpr v1.1.3 - github.com/hashicorp/go-bexpr v0.1.15 + github.com/hashicorp/go-bexpr v0.1.16 github.com/hashicorp/go-checkpoint v0.5.0 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-connlimit v0.3.1 diff --git a/go.sum b/go.sum index c2e23663036..300c90a36a3 100644 --- a/go.sum +++ b/go.sum @@ -434,8 +434,8 @@ github.com/hashicorp/cronexpr v1.1.3/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-bexpr v0.1.15 h1:flTYJAqZAlK+t8ezezb6WQGlRO1D4+GEF/HmH+xZo5k= -github.com/hashicorp/go-bexpr v0.1.15/go.mod h1:HGKbAByHn2aJWUV47gL7+IjLK79iU3EZIbOwCXJZLoE= +github.com/hashicorp/go-bexpr v0.1.16 h1:D+fKoGyUzXVS0FdjOX1ws3vIck8DVtBqQ0tsusmYDR8= +github.com/hashicorp/go-bexpr v0.1.16/go.mod h1:HGKbAByHn2aJWUV47gL7+IjLK79iU3EZIbOwCXJZLoE= github.com/hashicorp/go-checkpoint v0.5.0 h1:MFYpPZCnQqQTE18jFwSII6eUQrD/oxMFp3mlgcqk5mU= github.com/hashicorp/go-checkpoint v0.5.0/go.mod h1:7nfLNL10NsxqO4iWuW6tWW0HjZuDrwkBuEQsVcpCOgg= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=