From 89bd9e843c1586908bb2a930f4e7948d480041be Mon Sep 17 00:00:00 2001 From: Chris Martin Date: Thu, 25 Jul 2024 16:22:14 +0100 Subject: [PATCH] Remove priority from PodRequirements (#3808) * wip Signed-off-by: Chris Martin * fix some tests Signed-off-by: Chris Martin * fix some tests Signed-off-by: Chris Martin * fix some tests Signed-off-by: Chris Martin * lint Signed-off-by: Chris Martin * update proto Signed-off-by: Chris Martin --------- Signed-off-by: Chris Martin Co-authored-by: Chris Martin --- config/scheduleringester/config.yaml | 5 - .../event/conversion/conversions_test.go | 1 - internal/common/ingest/testfixtures/event.go | 5 - internal/scheduler/adapters/adapters.go | 41 +-- internal/scheduler/adapters/adapters_test.go | 146 +-------- internal/scheduler/api_test.go | 1 - .../scheduler/database/job_repository_test.go | 1 - internal/scheduler/jobdb/jobdb_test.go | 27 -- internal/scheduler/nodedb/nodedb.go | 5 +- internal/scheduler/nodedb/nodedb_test.go | 6 +- .../scheduler/nodedb/nodematching_test.go | 10 +- internal/scheduler/scheduler.go | 1 - internal/scheduler/scheduler_test.go | 102 ++----- .../schedulerobjects/podutils_test.go | 2 - .../schedulerobjects/schedulerobjects.pb.go | 282 ++++++++---------- .../schedulerobjects/schedulerobjects.proto | 5 - internal/scheduler/scheduling_algo_test.go | 4 +- internal/scheduler/simulator/simulator.go | 2 +- .../scheduler/testfixtures/testfixtures.go | 6 +- internal/scheduleringester/config.go | 3 - internal/scheduleringester/ingester.go | 2 +- internal/scheduleringester/instructions.go | 18 +- .../scheduleringester/instructions_test.go | 3 +- 23 files changed, 179 insertions(+), 499 deletions(-) diff --git a/config/scheduleringester/config.yaml b/config/scheduleringester/config.yaml index 9e4ea45f13f..6ae53c03965 100644 --- a/config/scheduleringester/config.yaml +++ b/config/scheduleringester/config.yaml @@ -16,10 +16,5 @@ pulsar: subscriptionName: "scheduler-ingester" batchSize: 10000 batchDuration: 500ms -priorityClasses: - armada-default: - priority: 1000 - armada-preemptible: - priority: 1000 diff --git a/internal/armada/event/conversion/conversions_test.go b/internal/armada/event/conversion/conversions_test.go index 26dc7738699..fa183866dc0 100644 --- a/internal/armada/event/conversion/conversions_test.go +++ b/internal/armada/event/conversion/conversions_test.go @@ -252,7 +252,6 @@ func TestConvertLeased(t *testing.T) { Effect: v1.TaintEffectNoSchedule, }, }, - Priority: 1000, }, }, }, diff --git a/internal/common/ingest/testfixtures/event.go b/internal/common/ingest/testfixtures/event.go index 1fd83b738fa..1a2636bb60b 100644 --- a/internal/common/ingest/testfixtures/event.go +++ b/internal/common/ingest/testfixtures/event.go @@ -10,7 +10,6 @@ import ( "github.com/armadaproject/armada/internal/armada/configuration" protoutil "github.com/armadaproject/armada/internal/common/proto" - "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/internal/scheduler/testfixtures" "github.com/armadaproject/armada/pkg/armadaevents" @@ -31,8 +30,6 @@ var ( RunIdUuid = armadaevents.UuidFromProtoUuid(RunIdProto) PartitionMarkerGroupIdUuid = armadaevents.UuidFromProtoUuid(PartitionMarkerGroupIdProto) PriorityClassName = "test-priority" - PriorityClassValue = int32(100) - PriorityClasses = map[string]types.PriorityClass{PriorityClassName: {Priority: PriorityClassValue}} Groups = []string{"group1", "group2"} NodeSelector = map[string]string{"foo": "bar"} Affinity = &v1.Affinity{ @@ -227,7 +224,6 @@ var Leased = &armadaevents.EventSequence_Event{ Effect: v1.TaintEffectNoSchedule, }, }, - Priority: 15, }, }, }, @@ -347,7 +343,6 @@ var JobRequeued = &armadaevents.EventSequence_Event{ NodeSelector: NodeSelector, Tolerations: Tolerations, PreemptionPolicy: "PreemptLowerPriority", - Priority: PriorityClassValue, Affinity: Affinity, ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ diff --git a/internal/scheduler/adapters/adapters.go b/internal/scheduler/adapters/adapters.go index 19b314c1413..68a08d58c66 100644 --- a/internal/scheduler/adapters/adapters.go +++ b/internal/scheduler/adapters/adapters.go @@ -1,13 +1,9 @@ package adapters import ( - "github.com/pkg/errors" - "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" k8sResource "k8s.io/apimachinery/pkg/api/resource" - "github.com/armadaproject/armada/internal/common/logging" - "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" "github.com/armadaproject/armada/pkg/api" ) @@ -15,14 +11,7 @@ import ( // PodRequirementsFromPodSpec function returns *schedulerobjects.PodRequirements for podSpec. // An error is logged if the podSpec uses an unknown priority class. // This function may mutate podSpec. -func PodRequirementsFromPodSpec(podSpec *v1.PodSpec, priorityByPriorityClassName map[string]types.PriorityClass) *schedulerobjects.PodRequirements { - priority, ok := PriorityFromPodSpec(podSpec, priorityByPriorityClassName) - if priorityByPriorityClassName != nil && !ok { - // Ignore this error if priorityByPriorityClassName is explicitly set to nil. - // We assume that in this case the caller is sure the priority does not need to be set. - err := errors.Errorf("unknown priorityClassName %s", podSpec.PriorityClassName) - logging.WithStacktrace(logrus.NewEntry(logrus.StandardLogger()), err).Error("failed to get priority from priorityClassName") - } +func PodRequirementsFromPodSpec(podSpec *v1.PodSpec) *schedulerobjects.PodRequirements { preemptionPolicy := string(v1.PreemptLowerPriority) if podSpec.PreemptionPolicy != nil { preemptionPolicy = string(*podSpec.PreemptionPolicy) @@ -31,39 +20,11 @@ func PodRequirementsFromPodSpec(podSpec *v1.PodSpec, priorityByPriorityClassName NodeSelector: podSpec.NodeSelector, Affinity: podSpec.Affinity, Tolerations: podSpec.Tolerations, - Priority: priority, PreemptionPolicy: preemptionPolicy, ResourceRequirements: api.SchedulingResourceRequirementsFromPodSpec(podSpec), } } -// PriorityFromPodSpec returns the priority in a pod spec. -// If priority is set directly, that value is returned. -// Otherwise, it returns the value of the key podSpec. -// In both cases the value along with true boolean is returned. -// PriorityClassName in priorityByPriorityClassName map. -// If no priority is set for the pod spec, 0 along with a false boolean would be returned -func PriorityFromPodSpec(podSpec *v1.PodSpec, priorityClasses map[string]types.PriorityClass) (int32, bool) { - // If there's no podspec there's nothing we can do - if podSpec == nil { - return 0, false - } - - // If a priority is directly specified, use that - if podSpec.Priority != nil { - return *podSpec.Priority, true - } - - // If we find a priority class use that - priorityClass, ok := priorityClasses[podSpec.PriorityClassName] - if ok { - return priorityClass.Priority, true - } - - // Couldn't find anything - return 0, false -} - func K8sResourceListToMap(resources v1.ResourceList) map[string]k8sResource.Quantity { if resources == nil { return nil diff --git a/internal/scheduler/adapters/adapters_test.go b/internal/scheduler/adapters/adapters_test.go index 520892a50d2..5891d0a34c0 100644 --- a/internal/scheduler/adapters/adapters_test.go +++ b/internal/scheduler/adapters/adapters_test.go @@ -1,36 +1,14 @@ package adapters import ( - "io" - "os" "testing" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/utils/pointer" - - "github.com/armadaproject/armada/internal/common/types" - "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" -) - -const ( - PriorityClass0 = "priority-0" - PriorityClass1 = "priority-1" - PriorityClass2 = "priority-2" - PriorityClass3 = "priority-3" ) var ( - priorityByPriorityClassName = map[string]types.PriorityClass{ - PriorityClass0: {Priority: 0, Preemptible: true}, - PriorityClass1: {Priority: 1, Preemptible: true}, - PriorityClass2: {Priority: 2, Preemptible: true}, - PriorityClass3: {Priority: 3, Preemptible: false}, - } - priority int32 = 1 containerObj = []v1.Container{ @@ -47,87 +25,8 @@ var ( }, }, } - - expectedResourceRequirement = v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceName("cpu"): *resource.NewMilliQuantity(5300, resource.DecimalSI), - v1.ResourceName("memory"): *resource.NewQuantity(5*1024*1024*1024, resource.BinarySI), - }, - Requests: v1.ResourceList{ - v1.ResourceName("cpu"): *resource.NewMilliQuantity(300, resource.DecimalSI), - v1.ResourceName("memory"): *resource.NewQuantity(2*1024*1024*1024, resource.BinarySI), - }, - } - expectedScheduler = &schedulerobjects.PodRequirements{ - ResourceRequirements: expectedResourceRequirement, - PreemptionPolicy: string(v1.PreemptLowerPriority), - } ) -func TestPodRequirementsFromPodSpecPriorityByPriorityClassName(t *testing.T) { - tests := []struct { - name string - podspec v1.PodSpec - priorityByPriorityClassName map[string]types.PriorityClass - loggedError bool - priority int32 - }{ - { - name: "PriorityClassName not present in priorityByPriorityClassName map", - podspec: v1.PodSpec{ - PriorityClassName: "priority-8", - Containers: containerObj, - }, - priorityByPriorityClassName: priorityByPriorityClassName, - loggedError: true, - priority: 0, - }, - { - name: "PriorityByPriorityClassName map is nil", - podspec: v1.PodSpec{ - PriorityClassName: "priority-3", - Containers: containerObj, - }, - priorityByPriorityClassName: nil, - loggedError: false, - priority: 0, - }, - { - name: "Priority is set directly on podspec", - podspec: v1.PodSpec{ - Priority: &priority, - Containers: containerObj, - }, - priorityByPriorityClassName: priorityByPriorityClassName, - loggedError: false, - priority: priority, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - r, w, _ := os.Pipe() - // Stderr from this function would be written to file w - logrus.SetOutput(w) - scheduler := PodRequirementsFromPodSpec(&test.podspec, test.priorityByPriorityClassName) - // Closing file, w - err := w.Close() - require.NoError(t, err) - // Reading from file - out, _ := io.ReadAll(r) - expectedScheduler.Priority = test.priority - assert.Equal(t, scheduler, expectedScheduler) - // if loggedError is true, bytes should be written to stderr, - // Otherwise, no byte is expected to be written to stderr - if test.loggedError { - assert.NotEqual(t, len(out), 0) - } else { - assert.Equal(t, len(out), 0) - } - }) - } -} - func TestPodRequirementsFromPodSpecPreemptionPolicy(t *testing.T) { preemptNever := v1.PreemptNever tests := []struct { @@ -157,55 +56,12 @@ func TestPodRequirementsFromPodSpecPreemptionPolicy(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - scheduler := PodRequirementsFromPodSpec(&test.podspec, priorityByPriorityClassName) + scheduler := PodRequirementsFromPodSpec(&test.podspec) assert.Equal(t, scheduler.PreemptionPolicy, string(test.preemptionpolicy)) }) } } -func TestPriorityFromPodSpec(t *testing.T) { - tests := map[string]struct { - podSpec *v1.PodSpec - expectedPriority int32 - expectedOk bool - }{ - "nil podSpec": { - podSpec: nil, - expectedPriority: 0, - expectedOk: false, - }, - "priority already set": { - podSpec: &v1.PodSpec{ - Priority: pointer.Int32(1), - PriorityClassName: PriorityClass2, - }, - expectedPriority: 1, - expectedOk: true, - }, - "existing priorityClass": { - podSpec: &v1.PodSpec{ - PriorityClassName: PriorityClass2, - }, - expectedPriority: 2, - expectedOk: true, - }, - "non-existing priorityClass": { - podSpec: &v1.PodSpec{ - PriorityClassName: "does not exist", - }, - expectedPriority: 0, - expectedOk: false, - }, - } - for name, tc := range tests { - t.Run(name, func(t *testing.T) { - p, ok := PriorityFromPodSpec(tc.podSpec, priorityByPriorityClassName) - assert.Equal(t, tc.expectedPriority, p) - assert.Equal(t, tc.expectedOk, ok) - }) - } -} - func TestK8sResourceListToMap(t *testing.T) { result := K8sResourceListToMap(v1.ResourceList{ "one": resource.MustParse("1"), diff --git a/internal/scheduler/api_test.go b/internal/scheduler/api_test.go index fee87828929..3098ea745d1 100644 --- a/internal/scheduler/api_test.go +++ b/internal/scheduler/api_test.go @@ -166,7 +166,6 @@ func TestExecutorApi_LeaseJobRuns(t *testing.T) { &schedulerobjects.PodRequirements{ Tolerations: tolerations, Annotations: map[string]string{"runtime_gang_cardinality": "3"}, - Priority: 1000, }, ), } diff --git a/internal/scheduler/database/job_repository_test.go b/internal/scheduler/database/job_repository_test.go index 63d8b7ce335..519371258f3 100644 --- a/internal/scheduler/database/job_repository_test.go +++ b/internal/scheduler/database/job_repository_test.go @@ -426,7 +426,6 @@ func TestFetchJobRunLeases(t *testing.T) { Effect: v1.TaintEffectNoSchedule, }, }, - Priority: 1000, }, ), }, diff --git a/internal/scheduler/jobdb/jobdb_test.go b/internal/scheduler/jobdb/jobdb_test.go index 46656e2aa24..d86a245ec36 100644 --- a/internal/scheduler/jobdb/jobdb_test.go +++ b/internal/scheduler/jobdb/jobdb_test.go @@ -259,7 +259,6 @@ func TestJobDb_TestBatchDelete(t *testing.T) { func TestJobDb_SchedulingKeyIsPopulated(t *testing.T) { podRequirements := &schedulerobjects.PodRequirements{ NodeSelector: map[string]string{"foo": "bar"}, - Priority: 2, } jobSchedulingInfo := &schedulerobjects.JobSchedulingInfo{ PriorityClassName: "foo", @@ -303,7 +302,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { "fish": "chips", "salt": "pepper", }, - Priority: 1, PreemptionPolicy: "abc", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -334,7 +332,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { "foo": "bar", "fish": "chips", }, - Priority: 1, PreemptionPolicy: "abc", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -364,7 +361,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, PreemptionPolicy: "abc", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -391,7 +387,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, PreemptionPolicy: "abcdef", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -421,7 +416,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, PreemptionPolicy: "abc", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -448,7 +442,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, PreemptionPolicy: "abcdef", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -478,7 +471,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -499,7 +491,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 2, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -523,7 +514,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -544,7 +534,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -569,7 +558,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -591,7 +579,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -614,7 +601,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -635,7 +621,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -671,7 +656,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -692,7 +676,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -715,7 +698,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -736,7 +718,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -759,7 +740,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -780,7 +760,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -803,7 +782,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -824,7 +802,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d-2", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -847,7 +824,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -868,7 +844,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(2), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -892,7 +867,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "cpu": resource.MustParse("4"), @@ -913,7 +887,6 @@ func TestJobDb_SchedulingKey(t *testing.T) { Effect: "d", TolerationSeconds: pointer.Int64(1), }}, - Priority: 1, ResourceRequirements: v1.ResourceRequirements{ Requests: map[v1.ResourceName]resource.Quantity{ "memory": resource.MustParse("5"), diff --git a/internal/scheduler/nodedb/nodedb.go b/internal/scheduler/nodedb/nodedb.go index 1132dca7e5d..14f03bebb80 100644 --- a/internal/scheduler/nodedb/nodedb.go +++ b/internal/scheduler/nodedb/nodedb.go @@ -498,14 +498,13 @@ func deleteEvictedJobSchedulingContextIfExistsWithTxn(txn *memdb.Txn, jobId stri // SelectNodeForJobWithTxn selects a node on which the job can be scheduled. func (nodeDb *NodeDb) SelectNodeForJobWithTxn(txn *memdb.Txn, jctx *schedulercontext.JobSchedulingContext) (*internaltypes.Node, error) { - req := jctx.PodRequirements priorityClass := jctx.Job.PriorityClass() // If the job has already been scheduled, get the priority at which it was scheduled. // Otherwise, get the original priority the job was submitted with. priority, ok := nodeDb.GetScheduledAtPriority(jctx.JobId) if !ok { - priority = req.Priority + priority = jctx.Job.PriorityClass().Priority } pctx := &schedulercontext.PodSchedulingContext{ Created: time.Now(), @@ -873,7 +872,7 @@ func (nodeDb *NodeDb) selectNodeForJobWithFairPreemption(txn *memdb.Txn, jctx *s priority, ok := nodeDb.GetScheduledAtPriority(evictedJctx.JobId) if !ok { - priority = evictedJctx.PodRequirements.Priority + priority = evictedJctx.Job.PriorityClass().Priority } if priority > maxPriority { maxPriority = priority diff --git a/internal/scheduler/nodedb/nodedb_test.go b/internal/scheduler/nodedb/nodedb_test.go index a42608d372a..2ddbd5693a6 100644 --- a/internal/scheduler/nodedb/nodedb_test.go +++ b/internal/scheduler/nodedb/nodedb_test.go @@ -130,7 +130,7 @@ func TestNodeBindingEvictionUnbinding(t *testing.T) { jobId := job.Id() - boundNode, err := nodeDb.bindJobToNode(entry, job, job.PodRequirements().Priority) + boundNode, err := nodeDb.bindJobToNode(entry, job, job.PriorityClass().Priority) require.NoError(t, err) unboundNode, err := nodeDb.UnbindJobFromNode(job, boundNode) @@ -146,7 +146,7 @@ func TestNodeBindingEvictionUnbinding(t *testing.T) { evictedUnboundNode, err := nodeDb.UnbindJobFromNode(job, evictedNode) require.NoError(t, err) - evictedBoundNode, err := nodeDb.bindJobToNode(evictedNode, job, job.PodRequirements().Priority) + evictedBoundNode, err := nodeDb.bindJobToNode(evictedNode, job, job.PriorityClass().Priority) require.NoError(t, err) _, _, err = nodeDb.EvictJobsFromNode(jobFilter, []*jobdb.Job{job}, entry) @@ -155,7 +155,7 @@ func TestNodeBindingEvictionUnbinding(t *testing.T) { _, err = nodeDb.UnbindJobFromNode(job, entry) require.NoError(t, err) - _, err = nodeDb.bindJobToNode(boundNode, job, job.PodRequirements().Priority) + _, err = nodeDb.bindJobToNode(boundNode, job, job.PriorityClass().Priority) require.Error(t, err) _, _, err = nodeDb.EvictJobsFromNode(jobFilter, []*jobdb.Job{job}, evictedNode) diff --git a/internal/scheduler/nodedb/nodematching_test.go b/internal/scheduler/nodedb/nodematching_test.go index d84a5bfd3ce..b70bafd417f 100644 --- a/internal/scheduler/nodedb/nodematching_test.go +++ b/internal/scheduler/nodedb/nodematching_test.go @@ -19,6 +19,7 @@ func TestNodeSchedulingRequirementsMet(t *testing.T) { tests := map[string]struct { node *internaltypes.Node req *schedulerobjects.PodRequirements + priority int32 expectSuccess bool }{ "nil taints and labels": { @@ -287,8 +288,8 @@ func TestNodeSchedulingRequirementsMet(t *testing.T) { "cpu": resource.MustParse("1"), }, }, - Priority: 0, }, + priority: 0, expectSuccess: true, }, "insufficient cpu": { @@ -312,8 +313,8 @@ func TestNodeSchedulingRequirementsMet(t *testing.T) { "cpu": resource.MustParse("1"), }, }, - Priority: 0, }, + priority: 0, expectSuccess: false, }, "sufficient cpu at priority": { @@ -343,8 +344,8 @@ func TestNodeSchedulingRequirementsMet(t *testing.T) { "cpu": resource.MustParse("1"), }, }, - Priority: 1, }, + priority: 1, expectSuccess: true, }, "insufficient cpu at priority": { @@ -374,7 +375,6 @@ func TestNodeSchedulingRequirementsMet(t *testing.T) { "cpu": resource.MustParse("1"), }, }, - Priority: 0, }, expectSuccess: false, }, @@ -383,7 +383,7 @@ func TestNodeSchedulingRequirementsMet(t *testing.T) { t.Run(name, func(t *testing.T) { matches, reason, err := JobRequirementsMet( tc.node, - tc.req.Priority, + tc.priority, // TODO(albin): Define a jctx in the test case instead. &schedulercontext.JobSchedulingContext{ PodRequirements: tc.req, diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index acbc375cee6..e1ff9ea01de 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -598,7 +598,6 @@ func AppendEventSequencesFromScheduledJobs(eventSequences []*armadaevents.EventS PodRequirementsOverlay: &schedulerobjects.PodRequirements{ Tolerations: jctx.AdditionalTolerations, Annotations: additionalAnnotationsByJobId[job.Id()], - Priority: scheduledAtPriority, }, Pool: run.Pool(), }, diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index f0be5629d0c..d4b9de21097 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -45,7 +45,8 @@ const ( var ( failFastSchedulingInfo = &schedulerobjects.JobSchedulingInfo{ - AtMostOnce: true, + AtMostOnce: true, + PriorityClassName: testfixtures.PriorityClass2NonPreemptible, ObjectRequirements: []*schedulerobjects.ObjectRequirements{ { Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ @@ -53,7 +54,6 @@ var ( Annotations: map[string]string{ apiconfig.FailFastAnnotation: "true", }, - Priority: int32(10), }, }, }, @@ -68,22 +68,19 @@ var ( ObjectRequirements: []*schedulerobjects.ObjectRequirements{ { Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ - PodRequirements: &schedulerobjects.PodRequirements{ - Priority: int32(10), - }, + PodRequirements: &schedulerobjects.PodRequirements{}, }, }, }, Version: 1, } schedulingInfo = &schedulerobjects.JobSchedulingInfo{ - AtMostOnce: true, + AtMostOnce: true, + PriorityClassName: testfixtures.PriorityClass2NonPreemptible, ObjectRequirements: []*schedulerobjects.ObjectRequirements{ { Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ - PodRequirements: &schedulerobjects.PodRequirements{ - Priority: int32(10), - }, + PodRequirements: &schedulerobjects.PodRequirements{}, }, }, }, @@ -95,9 +92,7 @@ var ( ObjectRequirements: []*schedulerobjects.ObjectRequirements{ { Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ - PodRequirements: &schedulerobjects.PodRequirements{ - Priority: int32(10), - }, + PodRequirements: &schedulerobjects.PodRequirements{}, }, }, }, @@ -105,13 +100,12 @@ var ( } updatedSchedulingInfoBytes = protoutil.MustMarshall(updatedSchedulingInfo) schedulingInfoWithUpdatedPriority = &schedulerobjects.JobSchedulingInfo{ - AtMostOnce: true, + AtMostOnce: true, + PriorityClassName: testfixtures.PriorityClass2NonPreemptible, ObjectRequirements: []*schedulerobjects.ObjectRequirements{ { Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ - PodRequirements: &schedulerobjects.PodRequirements{ - Priority: int32(20), - }, + PodRequirements: &schedulerobjects.PodRequirements{}, }, }, }, @@ -276,7 +270,7 @@ var ( testNode = "test-node" testPool = "test-pool" testNodeId = api.NodeIdFromExecutorAndNodeName(testExecutor, testNode) - scheduledAtPriority = int32(10) + scheduledAtPriority = int32(2) requeuedJobId = util.NewULID() requeuedJob = testfixtures.NewJob( requeuedJobId, @@ -1399,16 +1393,12 @@ func (t *testSchedulingAlgo) Schedule(_ *armadacontext.Context, txn *jobdb.Txn) if !job.Queued() { return nil, errors.Errorf("was asked to lease %s but job is not queued", job.Id()) } - priority := int32(0) - if req := job.PodRequirements(); req != nil { - priority = req.Priority - } job = job.WithQueuedVersion(job.QueuedVersion()+1).WithQueued(false).WithNewRun( testExecutor, testNodeId, testNode, testPool, - priority, + job.PriorityClass().Priority, ) scheduledJobs = append(scheduledJobs, job) } @@ -1532,16 +1522,6 @@ var ( Validated: true, Serial: 0, } - runningJobWithUpdatedPriorityA = &database.Job{ - JobID: queuedJobA.JobID, - JobSet: "testJobSet", - Queue: "testQueue", - QueuedVersion: 1, - SchedulingInfo: schedulingInfoWithUpdatedPriorityBytes, - SchedulingInfoVersion: int32(schedulingInfoWithUpdatedPriority.Version), - Validated: true, - Serial: 1, - } failedJobA = &database.Job{ JobID: queuedJobA.JobID, JobSet: "testJobSet", @@ -1844,7 +1824,7 @@ func TestCycleConsistency(t *testing.T) { }, expectedJobDbCycleOne: []*jobdb.Job{ func() *jobdb.Job { - job := jobDbJobFromDbJob(resourceListFactory, runningCancelRequestedJobA).WithCancelled(true).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10) + job := jobDbJobFromDbJob(resourceListFactory, runningCancelRequestedJobA).WithCancelled(true).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2) return job.WithUpdatedRun(job.LatestRun().WithCancelled(true).WithAttempted(true)) }(), }, @@ -1890,7 +1870,7 @@ func TestCycleConsistency(t *testing.T) { }, expectedJobDbCycleOne: []*jobdb.Job{ func() *jobdb.Job { - job := jobDbJobFromDbJob(resourceListFactory, runningCancelByJobSetRequestedJobA).WithCancelled(true).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10) + job := jobDbJobFromDbJob(resourceListFactory, runningCancelByJobSetRequestedJobA).WithCancelled(true).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2) return job.WithUpdatedRun(job.LatestRun().WithCancelled(true).WithAttempted(true)) }(), }, @@ -1942,13 +1922,13 @@ func TestCycleConsistency(t *testing.T) { }, idsOfJobsToSchedule: []string{queuedJobA.JobID}, expectedJobDbCycleOne: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedJobDbCycleTwo: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedJobDbCycleThree: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedEventSequencesCycleThree: []*armadaevents.EventSequence{ { @@ -1967,8 +1947,8 @@ func TestCycleConsistency(t *testing.T) { NodeId: testNode, UpdateSequenceNumber: 1, HasScheduledAtPriority: true, - ScheduledAtPriority: 10, - PodRequirementsOverlay: &schedulerobjects.PodRequirements{Priority: 10}, + ScheduledAtPriority: 2, + PodRequirementsOverlay: &schedulerobjects.PodRequirements{}, Pool: testPool, }, }, @@ -1990,11 +1970,11 @@ func TestCycleConsistency(t *testing.T) { }, idsOfJobsToSchedule: []string{queuedJobA.JobID}, expectedJobDbCycleOne: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedJobDbCycleTwo: []*jobdb.Job{ func() *jobdb.Job { - job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10).WithSucceeded(true) + job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2).WithSucceeded(true) return job.WithUpdatedRun(job.LatestRun().WithSucceeded(true).WithAttempted(true)) }(), }, @@ -2016,8 +1996,8 @@ func TestCycleConsistency(t *testing.T) { NodeId: testNode, UpdateSequenceNumber: 1, HasScheduledAtPriority: true, - ScheduledAtPriority: 10, - PodRequirementsOverlay: &schedulerobjects.PodRequirements{Priority: 10}, + ScheduledAtPriority: 2, + PodRequirementsOverlay: &schedulerobjects.PodRequirements{}, Pool: testPool, }, }, @@ -2057,11 +2037,11 @@ func TestCycleConsistency(t *testing.T) { }, idsOfJobsToSchedule: []string{queuedJobA.JobID}, expectedJobDbCycleOne: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedJobDbCycleTwo: []*jobdb.Job{ func() *jobdb.Job { - job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10).WithFailed(true) + job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2).WithFailed(true) return job.WithUpdatedRun(job.LatestRun().WithFailed(true).WithAttempted(true)) }(), }, @@ -2083,8 +2063,8 @@ func TestCycleConsistency(t *testing.T) { NodeId: testNode, UpdateSequenceNumber: 1, HasScheduledAtPriority: true, - ScheduledAtPriority: 10, - PodRequirementsOverlay: &schedulerobjects.PodRequirements{Priority: 10}, + ScheduledAtPriority: 2, + PodRequirementsOverlay: &schedulerobjects.PodRequirements{}, Pool: testPool, }, }, @@ -2134,11 +2114,11 @@ func TestCycleConsistency(t *testing.T) { }, idsOfJobsToSchedule: []string{queuedJobA.JobID}, expectedJobDbCycleOne: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedJobDbCycleTwo: []*jobdb.Job{ func() *jobdb.Job { - job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10).WithFailed(true) + job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2).WithFailed(true) // TODO(albin): RunAttempted is implicitly set to true for failed runs with error other than PodLeaseReturned. // See func (c *InstructionConverter) handleJobRunErrors. return job.WithUpdatedRun(job.LatestRun().WithFailed(true).WithAttempted(true)) @@ -2162,11 +2142,11 @@ func TestCycleConsistency(t *testing.T) { }, idsOfJobsToSchedule: []string{queuedJobA.JobID}, expectedJobDbCycleOne: []*jobdb.Job{ - jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10), + jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2), }, expectedJobDbCycleTwo: []*jobdb.Job{ func() *jobdb.Job { - job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 10).WithFailed(true) + job := jobDbJobFromDbJob(resourceListFactory, queuedJobA).WithQueued(false).WithQueuedVersion(1).WithNewRun(testExecutor, testNodeId, testNode, testPool, 2).WithFailed(true) return job.WithUpdatedRun(job.LatestRun().WithFailed(true).WithAttempted(true).WithReturned(true)) }(), }, @@ -2323,21 +2303,6 @@ func TestCycleConsistency(t *testing.T) { }, }, }, - "Running job is re-prioritised": { - firstSchedulerDbUpdate: schedulerDbUpdate{ - jobUpdates: []*database.Job{ - runningJobA, - }, - runUpdates: []*database.Run{ - newRunA, - }, - }, - secondSchedulerDbUpdate: schedulerDbUpdate{ - jobUpdates: []*database.Job{ - runningJobWithUpdatedPriorityA, - }, - }, - }, } for name, tc := range tests { t.Run(name, func(t *testing.T) { @@ -2355,10 +2320,7 @@ func TestCycleConsistency(t *testing.T) { queueByJobId[jobUpdate.JobID] = jobUpdate.Queue jobSetByJobId[jobUpdate.JobID] = jobUpdate.JobSet } - instructionConverter, err := scheduleringester.NewInstructionConverter( - nil, - testfixtures.TestPriorityClasses, - ) + instructionConverter, err := scheduleringester.NewInstructionConverter(nil) require.NoError(t, err) // Helper function for creating new schedulers for use in tests. diff --git a/internal/scheduler/schedulerobjects/podutils_test.go b/internal/scheduler/schedulerobjects/podutils_test.go index a1de25d2c2b..45822c4cfa4 100644 --- a/internal/scheduler/schedulerobjects/podutils_test.go +++ b/internal/scheduler/schedulerobjects/podutils_test.go @@ -209,7 +209,6 @@ func getBenchmarkJobSchedulingSchedulingInfo() *JobSchedulingInfo { "fish": "chips", "salt": "pepper", }, - Priority: 1, PreemptionPolicy: "abc", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ @@ -319,7 +318,6 @@ func getBenchmarkJobSchedulingSchedulingInfoWithAffinity() *JobSchedulingInfo { "fish": "chips", "salt": "pepper", }, - Priority: 1, PreemptionPolicy: "abc", ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ diff --git a/internal/scheduler/schedulerobjects/schedulerobjects.pb.go b/internal/scheduler/schedulerobjects/schedulerobjects.pb.go index 37fe2e6ba47..34b1747edf6 100644 --- a/internal/scheduler/schedulerobjects/schedulerobjects.pb.go +++ b/internal/scheduler/schedulerobjects/schedulerobjects.pb.go @@ -769,11 +769,6 @@ type PodRequirements struct { Tolerations []v1.Toleration `protobuf:"bytes,3,rep,name=tolerations,proto3" json:"tolerations"` // Kubernetes annotations. Included here since we use annotations with special meaning. Annotations map[string]string `protobuf:"bytes,7,rep,name=annotations,proto3" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - // Priority class priority of the pod as submitted. Should be mapped from the priority class name of the submitted pod. - // - // During scheduling, the priority stored on the podSchedulingContext should be used instead, - // since a pod may be scheduled at a priority different from the priority it was submitted with. - Priority int32 `protobuf:"varint,4,opt,name=priority,proto3" json:"priority,omitempty"` // One of Never, PreemptLowerPriority. // Defaults to PreemptLowerPriority if unset. PreemptionPolicy string `protobuf:"bytes,5,opt,name=preemptionPolicy,proto3" json:"preemptionPolicy,omitempty"` @@ -842,13 +837,6 @@ func (m *PodRequirements) GetAnnotations() map[string]string { return nil } -func (m *PodRequirements) GetPriority() int32 { - if m != nil { - return m.Priority - } - return 0 -} - func (m *PodRequirements) GetPreemptionPolicy() string { if m != nil { return m.PreemptionPolicy @@ -957,132 +945,131 @@ func init() { } var fileDescriptor_97dadc5fbd620721 = []byte{ - // 1987 bytes of a gzipped FileDescriptorProto + // 1978 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x59, 0x4f, 0x73, 0x1b, 0x49, - 0x15, 0xf7, 0xc8, 0xb2, 0x2c, 0xb7, 0x1c, 0x5b, 0x6e, 0x3b, 0xc9, 0x58, 0x49, 0x34, 0x5a, 0x6d, + 0x15, 0xf7, 0xc8, 0xb6, 0x2c, 0xb5, 0x1c, 0x5b, 0x6e, 0x3b, 0xc9, 0x58, 0x49, 0x34, 0x5a, 0x6d, 0xd8, 0x32, 0xec, 0xae, 0xc4, 0x7a, 0xa9, 0x22, 0x15, 0xb8, 0x48, 0xb6, 0x21, 0x32, 0x89, 0xec, 0x8c, 0x6d, 0x28, 0xa8, 0x62, 0xa7, 0x5a, 0x9a, 0xb6, 0x32, 0xeb, 0x51, 0xb7, 0x32, 0xd3, 0x13, 0x56, 0x9c, 0xe1, 0x40, 0x51, 0xb5, 0x6c, 0x51, 0x0b, 0x6c, 0x15, 0x14, 0xd4, 0x7e, 0x09, 0x38, - 0xf0, 0x05, 0x72, 0xdc, 0x23, 0x27, 0x41, 0x25, 0x37, 0x5d, 0xf9, 0x02, 0x54, 0x77, 0xcf, 0x48, - 0xad, 0x19, 0xc9, 0xf2, 0xb2, 0x84, 0x9c, 0xac, 0x7e, 0x7f, 0x7f, 0xef, 0xf5, 0xeb, 0xd7, 0xaf, - 0xc7, 0xe0, 0xbe, 0x43, 0x18, 0xf6, 0x08, 0x72, 0xab, 0x7e, 0xfb, 0x09, 0xb6, 0x03, 0x17, 0x7b, - 0xe3, 0x5f, 0xb4, 0xf5, 0x21, 0x6e, 0x33, 0x3f, 0x41, 0xa8, 0xf4, 0x3c, 0xca, 0x28, 0xcc, 0xc7, - 0xe9, 0x05, 0xa3, 0x43, 0x69, 0xc7, 0xc5, 0x55, 0xc1, 0x6f, 0x05, 0xe7, 0x55, 0xe6, 0x74, 0xb1, - 0xcf, 0x50, 0xb7, 0x27, 0x55, 0x0a, 0xe5, 0x8b, 0x7b, 0x7e, 0xc5, 0xa1, 0x55, 0xd4, 0x73, 0xaa, - 0x6d, 0xea, 0xe1, 0xea, 0xb3, 0xf7, 0xaa, 0x1d, 0x4c, 0xb0, 0x87, 0x18, 0xb6, 0x43, 0x99, 0x6f, - 0x8d, 0x65, 0xba, 0xa8, 0xfd, 0xc4, 0x21, 0xd8, 0xeb, 0x57, 0x7b, 0x17, 0x1d, 0xa1, 0xe4, 0x61, - 0x9f, 0x06, 0x5e, 0x1b, 0x27, 0xb4, 0xde, 0xed, 0x38, 0xec, 0x49, 0xd0, 0xaa, 0xb4, 0x69, 0xb7, - 0xda, 0xa1, 0x1d, 0x3a, 0xc6, 0xc0, 0x57, 0x62, 0x21, 0x7e, 0x49, 0xf1, 0xf2, 0xf3, 0x14, 0xc8, - 0x1e, 0x7c, 0x84, 0xdb, 0x01, 0xa3, 0x1e, 0x2c, 0x81, 0x94, 0x63, 0xeb, 0x5a, 0x49, 0xdb, 0x59, - 0xa9, 0xe7, 0x87, 0x03, 0x63, 0xd5, 0xb1, 0xdf, 0xa1, 0x5d, 0x87, 0xe1, 0x6e, 0x8f, 0xf5, 0xcd, - 0x94, 0x63, 0xc3, 0xb7, 0x40, 0xba, 0x47, 0xa9, 0xab, 0xa7, 0x84, 0x0c, 0x1c, 0x0e, 0x8c, 0x35, - 0xbe, 0x56, 0xa4, 0x04, 0x1f, 0xd6, 0xc0, 0x12, 0xa1, 0x36, 0xf6, 0xf5, 0xc5, 0xd2, 0xe2, 0x4e, - 0x6e, 0xf7, 0x46, 0x25, 0x91, 0xba, 0x26, 0xb5, 0x71, 0x7d, 0x73, 0x38, 0x30, 0xd6, 0x85, 0xa0, - 0x62, 0x41, 0x6a, 0xc2, 0x0f, 0xc0, 0x9a, 0x8b, 0x7c, 0x76, 0xd6, 0xb3, 0x11, 0xc3, 0xa7, 0x4e, - 0x17, 0xeb, 0x4b, 0x25, 0x6d, 0x27, 0xb7, 0x5b, 0xa8, 0xc8, 0xe4, 0x56, 0xa2, 0xc0, 0x2a, 0xa7, - 0x51, 0x72, 0xeb, 0x85, 0xe7, 0x03, 0x63, 0x81, 0x83, 0x9a, 0xd4, 0xfc, 0xe4, 0x9f, 0x86, 0x66, - 0xc6, 0x68, 0xf0, 0x08, 0x6c, 0x06, 0x04, 0xf9, 0xbe, 0xd3, 0x21, 0xd8, 0xb6, 0x3e, 0xa4, 0x2d, - 0xcb, 0x0b, 0x88, 0xaf, 0xaf, 0x94, 0x16, 0x77, 0x56, 0xea, 0xc6, 0x70, 0x60, 0xdc, 0x1a, 0xb3, - 0x0f, 0x69, 0xcb, 0x0c, 0x88, 0x0a, 0x72, 0x23, 0xc1, 0x2c, 0xff, 0x69, 0x1d, 0xa4, 0x79, 0x54, - 0x57, 0x4b, 0x23, 0x41, 0x5d, 0xac, 0xaf, 0x8e, 0xd3, 0xc8, 0xd7, 0x6a, 0x1a, 0xf9, 0x1a, 0xee, - 0x82, 0x2c, 0x0e, 0x37, 0x47, 0xdf, 0x14, 0xb2, 0x37, 0x86, 0x03, 0x03, 0x46, 0x34, 0x45, 0x7e, - 0x24, 0x07, 0x1f, 0x81, 0x15, 0x1e, 0xa9, 0xe5, 0x63, 0x4c, 0xc4, 0x3e, 0x5d, 0x9e, 0xb2, 0xad, - 0x30, 0x65, 0x59, 0xae, 0x74, 0x82, 0x31, 0x11, 0xc9, 0x1a, 0xad, 0x60, 0x0d, 0x64, 0x18, 0x72, - 0x08, 0xf3, 0xf5, 0x25, 0xb1, 0x95, 0xdb, 0x15, 0x59, 0x96, 0x15, 0xd4, 0x73, 0x2a, 0xbc, 0x74, - 0x2b, 0xcf, 0xde, 0xab, 0x9c, 0x72, 0x89, 0xfa, 0x5a, 0x68, 0x2a, 0x54, 0x30, 0xc3, 0xbf, 0xf0, - 0x18, 0x64, 0x5c, 0xd4, 0xc2, 0xae, 0xaf, 0x67, 0x84, 0x89, 0xf2, 0xf4, 0x6a, 0xa8, 0x3c, 0x14, - 0x42, 0x07, 0x84, 0x79, 0xfd, 0xfa, 0xd6, 0x70, 0x60, 0xe4, 0xa5, 0x96, 0x12, 0x65, 0x68, 0x07, - 0x5a, 0x60, 0x9d, 0x51, 0x86, 0x5c, 0x2b, 0x3a, 0x06, 0xbe, 0xbe, 0x2c, 0x22, 0x2d, 0x26, 0x4d, - 0x9b, 0xa1, 0xc8, 0x43, 0xc7, 0x67, 0xf5, 0x1b, 0x51, 0x81, 0x08, 0xf5, 0x88, 0xe5, 0x9b, 0xb1, - 0x35, 0xfc, 0xab, 0x06, 0xee, 0x22, 0xd7, 0xa5, 0x6d, 0xc4, 0x50, 0xcb, 0xc5, 0x56, 0xab, 0x6f, - 0xf5, 0x3c, 0x87, 0x7a, 0x0e, 0xeb, 0x5b, 0x88, 0xd8, 0x23, 0xbf, 0x7a, 0x56, 0x44, 0xf4, 0xdd, - 0x19, 0x11, 0xd5, 0xc6, 0x26, 0xea, 0xfd, 0xe3, 0xd0, 0x40, 0x8d, 0xd8, 0x91, 0x23, 0x19, 0xeb, - 0x4e, 0x08, 0xaa, 0x84, 0xe6, 0x88, 0x9b, 0x73, 0x25, 0xa0, 0x07, 0x36, 0x7d, 0x86, 0x98, 0x40, - 0x1c, 0xd6, 0xb4, 0xe5, 0xd8, 0xa2, 0xaa, 0x73, 0xbb, 0x6f, 0xcf, 0x80, 0x79, 0xc2, 0x35, 0xea, - 0x7d, 0x59, 0xc8, 0x0d, 0x5b, 0xa2, 0xba, 0x19, 0xa2, 0x5a, 0xf7, 0x27, 0xb9, 0x66, 0x9c, 0x00, - 0x3f, 0xd7, 0x40, 0x91, 0x50, 0x62, 0x21, 0xaf, 0x8b, 0x6c, 0x64, 0x85, 0x18, 0xb1, 0xad, 0xec, - 0xce, 0x35, 0xe1, 0xff, 0xdb, 0x33, 0xfc, 0x37, 0x29, 0xa9, 0x09, 0xdd, 0x5a, 0xa4, 0x3a, 0xda, - 0x09, 0x89, 0xe5, 0xcd, 0x10, 0xcb, 0x2d, 0x32, 0x5b, 0xd2, 0xbc, 0x8c, 0x09, 0x6b, 0xe0, 0x5a, - 0x40, 0x42, 0xef, 0x3c, 0x7b, 0xfa, 0x7a, 0x49, 0xdb, 0xc9, 0xd6, 0x6f, 0x0d, 0x07, 0xc6, 0xcd, - 0x09, 0x86, 0x52, 0x6d, 0x93, 0x1a, 0xf0, 0xd7, 0x1a, 0xb8, 0x19, 0x45, 0x64, 0x05, 0x3e, 0xea, - 0x88, 0x24, 0x3f, 0x0d, 0x70, 0x80, 0xf5, 0xbc, 0x88, 0xef, 0x9b, 0x33, 0xe2, 0x8b, 0x60, 0x9c, - 0x71, 0xa5, 0x7a, 0xff, 0x31, 0x57, 0x91, 0x81, 0x95, 0x87, 0x03, 0xa3, 0xe8, 0x4d, 0x61, 0x2b, - 0x30, 0xb6, 0xa6, 0xf1, 0x79, 0xfb, 0xf2, 0x70, 0x8f, 0x7a, 0xcc, 0x21, 0x1d, 0x8b, 0x77, 0x4c, - 0x8b, 0xf5, 0x7b, 0x58, 0xdf, 0x10, 0x5d, 0x42, 0xb4, 0xaf, 0x11, 0x9b, 0x63, 0x38, 0xed, 0xf7, - 0x54, 0x9b, 0x1b, 0x09, 0xe6, 0xa8, 0xb5, 0xc3, 0xcb, 0x5b, 0x7b, 0x01, 0x81, 0x9c, 0x72, 0x50, - 0xe1, 0x9b, 0x60, 0xf1, 0x02, 0xf7, 0xc3, 0x6e, 0xb7, 0x31, 0x1c, 0x18, 0xd7, 0x2e, 0x70, 0x5f, - 0x51, 0xe2, 0x5c, 0xf8, 0x75, 0xb0, 0xf4, 0x0c, 0xb9, 0x01, 0x0e, 0xef, 0x0d, 0xd1, 0xf6, 0x05, - 0x41, 0x6d, 0xfb, 0x82, 0x70, 0x3f, 0x75, 0x4f, 0x2b, 0xfc, 0x51, 0x03, 0x5f, 0xbb, 0xd2, 0xd1, - 0x51, 0xbd, 0x2f, 0xcd, 0xf4, 0xde, 0x50, 0xbd, 0xcf, 0xef, 0x11, 0xf3, 0xd0, 0xfd, 0x4a, 0x03, - 0x5b, 0xd3, 0x4e, 0xcc, 0xd5, 0x52, 0xf1, 0x40, 0x05, 0xb3, 0xb6, 0x7b, 0x27, 0x09, 0x46, 0x1a, - 0x95, 0x1e, 0xe6, 0x61, 0xf9, 0xbd, 0x06, 0x4a, 0xf3, 0x4e, 0xcf, 0x6b, 0x49, 0xd2, 0x6f, 0x34, - 0xb0, 0x3d, 0xb3, 0xec, 0xaf, 0x96, 0xa9, 0xff, 0x2d, 0xa2, 0xf2, 0x9f, 0xd3, 0x20, 0x3b, 0x2a, - 0xf6, 0x12, 0x48, 0x35, 0xe4, 0x15, 0x9d, 0x96, 0x57, 0x74, 0x63, 0xe2, 0x8a, 0x6e, 0xd8, 0xca, - 0xbd, 0x97, 0xfa, 0x6f, 0xef, 0xbd, 0xd3, 0xd1, 0xbd, 0x27, 0xa7, 0xa0, 0xb7, 0xa6, 0xb7, 0x07, - 0x0e, 0xe8, 0x4b, 0xdc, 0x7d, 0xbf, 0xd0, 0x00, 0x0c, 0x88, 0x8f, 0x59, 0x83, 0xd8, 0xf8, 0x23, - 0x6c, 0x4b, 0x4d, 0x3d, 0x2d, 0x5c, 0xec, 0x5e, 0xe2, 0xe2, 0x2c, 0xa1, 0x24, 0xdd, 0x95, 0x86, - 0x03, 0xe3, 0x76, 0xd2, 0xa2, 0xe2, 0x7a, 0x8a, 0xbf, 0xff, 0x47, 0x1b, 0xe8, 0x82, 0x9b, 0x33, - 0x30, 0xbf, 0x0a, 0x77, 0xe5, 0xe7, 0x19, 0xb0, 0x2d, 0x6a, 0x74, 0xcf, 0x0d, 0x7c, 0x86, 0xbd, - 0x89, 0xf2, 0x85, 0x0d, 0xb0, 0xdc, 0xf6, 0x30, 0x3f, 0x5d, 0xc2, 0xeb, 0xe5, 0x43, 0xd5, 0x66, - 0x58, 0x11, 0x91, 0x8a, 0x98, 0xa9, 0xa2, 0x05, 0xc7, 0x25, 0x6f, 0x0d, 0x05, 0xd7, 0xd3, 0x58, - 0xd3, 0x97, 0x12, 0xf0, 0x1e, 0x00, 0xd1, 0x60, 0xd7, 0xb0, 0xf5, 0x45, 0x21, 0xaf, 0x0f, 0x07, - 0xc6, 0xd6, 0x98, 0xaa, 0x28, 0x29, 0xb2, 0xf0, 0x77, 0x1a, 0xbf, 0x20, 0xc2, 0x3e, 0x30, 0xee, - 0xa0, 0x61, 0x9d, 0xec, 0x27, 0xeb, 0x64, 0x66, 0xe8, 0xa3, 0x63, 0xa6, 0x98, 0x91, 0x95, 0x73, - 0x27, 0x0c, 0x73, 0xaa, 0x23, 0xcd, 0x9c, 0x46, 0x86, 0x7f, 0xd3, 0xc0, 0xed, 0x29, 0xf4, 0x3d, - 0x17, 0xf9, 0x7e, 0x13, 0x89, 0x29, 0x9f, 0x03, 0x7c, 0xf4, 0x15, 0x01, 0x8e, 0xec, 0x49, 0xa4, - 0x77, 0x43, 0xa4, 0x97, 0xba, 0x36, 0x2f, 0xe5, 0x16, 0x3e, 0xd6, 0x80, 0x3e, 0x2b, 0x15, 0xaf, - 0xa5, 0xc7, 0xfe, 0x41, 0x03, 0x6f, 0xcc, 0x0d, 0xfd, 0xb5, 0xf4, 0xda, 0xbf, 0x2f, 0x82, 0xc2, - 0xb4, 0x9d, 0x32, 0xc5, 0xd4, 0x31, 0x1a, 0x35, 0xb4, 0x39, 0xaf, 0x48, 0xe5, 0xcc, 0xa5, 0xbe, - 0xe2, 0x99, 0xfb, 0x58, 0x03, 0x79, 0x65, 0x77, 0x45, 0x2d, 0x85, 0x6d, 0xb9, 0x9e, 0x0c, 0x76, - 0x36, 0x76, 0xb5, 0xd6, 0x94, 0x39, 0xae, 0x38, 0x1c, 0x18, 0x85, 0xb8, 0x7d, 0x25, 0x9e, 0x84, - 0xef, 0xc2, 0x67, 0x1a, 0xb8, 0x3e, 0xd5, 0xd6, 0xd5, 0x36, 0xec, 0x87, 0x93, 0x1b, 0xf6, 0xf6, - 0x97, 0x38, 0x2e, 0x73, 0x77, 0xef, 0x97, 0x29, 0xb0, 0xaa, 0x6e, 0x37, 0xfc, 0x00, 0xac, 0x8c, - 0x47, 0x79, 0x4d, 0x24, 0xed, 0xdd, 0xcb, 0x2b, 0xa4, 0x12, 0x1b, 0xe0, 0x37, 0xc2, 0xcd, 0x19, - 0xdb, 0x31, 0xc7, 0x3f, 0x0b, 0x9f, 0x6a, 0x60, 0x6d, 0xf6, 0xcc, 0x32, 0x3b, 0x09, 0x3f, 0x9e, - 0x4c, 0x42, 0x45, 0xb9, 0xa2, 0x47, 0x5f, 0x4c, 0x2a, 0xbd, 0x8b, 0x8e, 0xb8, 0xb3, 0x23, 0x77, - 0x95, 0xc7, 0x01, 0x22, 0xcc, 0x61, 0xfd, 0xb9, 0x79, 0xf8, 0x74, 0x09, 0x6c, 0x1c, 0xd2, 0xd6, - 0x89, 0x0c, 0xd4, 0x21, 0x9d, 0x06, 0x39, 0xa7, 0xfc, 0x4d, 0xee, 0x3a, 0xe7, 0x98, 0x39, 0x5d, - 0x2c, 0xe0, 0x5d, 0x93, 0x6f, 0xf2, 0x88, 0xa6, 0xbe, 0xc9, 0x23, 0x1a, 0xbc, 0x0f, 0x56, 0x11, - 0xb3, 0xba, 0xd4, 0x67, 0x16, 0x25, 0x6d, 0x89, 0x37, 0x2b, 0x1b, 0x39, 0x62, 0x8f, 0xa8, 0xcf, - 0x8e, 0x48, 0x5b, 0xd5, 0x04, 0x63, 0x2a, 0xfc, 0x0e, 0xc8, 0xf5, 0x3c, 0xcc, 0xe9, 0x0e, 0x7f, - 0xb7, 0x2c, 0x0a, 0xd5, 0xed, 0xe1, 0xc0, 0xb8, 0xae, 0x90, 0x15, 0x5d, 0x55, 0x1a, 0x3e, 0x00, - 0xf9, 0x36, 0x25, 0xed, 0xc0, 0xf3, 0x30, 0x69, 0xf7, 0x2d, 0x1f, 0x9d, 0x63, 0x3d, 0x2d, 0x2c, - 0xdc, 0x19, 0x0e, 0x8c, 0x6d, 0x85, 0x77, 0x82, 0xce, 0x55, 0x2b, 0xeb, 0x31, 0x16, 0x7f, 0x6f, - 0x8c, 0x5e, 0xc0, 0x6d, 0xde, 0x61, 0x2c, 0xf1, 0x05, 0x23, 0x33, 0x7e, 0x6f, 0xf4, 0xe2, 0xfd, - 0x47, 0x7d, 0x6f, 0x24, 0x98, 0xf0, 0x04, 0xe4, 0xfc, 0xa0, 0xd5, 0x75, 0x98, 0x25, 0x52, 0xb9, - 0x3c, 0xf7, 0x80, 0x47, 0x6f, 0x77, 0x20, 0xd5, 0x46, 0x1f, 0x76, 0x94, 0x35, 0xdf, 0x9c, 0xc8, - 0x93, 0x9e, 0x1d, 0x6f, 0x4e, 0x44, 0x53, 0x37, 0x27, 0xa2, 0xc1, 0x9f, 0x81, 0x4d, 0x59, 0xc2, - 0x96, 0x87, 0x9f, 0x06, 0x8e, 0x87, 0xbb, 0x78, 0xfc, 0xb9, 0xe3, 0x6e, 0xb2, 0xce, 0x8f, 0xc4, - 0x5f, 0x53, 0x91, 0x95, 0x23, 0x14, 0x4d, 0xd0, 0xd5, 0x11, 0x2a, 0xc9, 0x85, 0x55, 0xb0, 0xfc, - 0x0c, 0x7b, 0xbe, 0x43, 0x89, 0xbe, 0x22, 0xb0, 0x5e, 0x1f, 0x0e, 0x8c, 0x8d, 0x90, 0xa4, 0xe8, - 0x46, 0x52, 0xf7, 0xd3, 0x9f, 0x7d, 0x6e, 0x68, 0xe5, 0xdf, 0x6a, 0x00, 0x26, 0x31, 0x40, 0x17, - 0xac, 0xf7, 0xa8, 0xad, 0x92, 0xc2, 0x41, 0xe5, 0x8d, 0x64, 0x08, 0xc7, 0x93, 0x82, 0xb2, 0x18, - 0x62, 0xda, 0x63, 0x00, 0x0f, 0x16, 0xcc, 0xb8, 0xe9, 0xfa, 0x1a, 0x58, 0x55, 0xb3, 0x55, 0xfe, - 0x77, 0x06, 0xac, 0xc7, 0xac, 0x42, 0x1f, 0xac, 0xf2, 0x87, 0xe9, 0x09, 0x76, 0x71, 0x9b, 0x51, - 0x2f, 0xec, 0x1c, 0xef, 0xcf, 0x85, 0x23, 0x46, 0xd6, 0x48, 0x4b, 0xf6, 0x8f, 0xc2, 0x70, 0x60, - 0xdc, 0x50, 0x8d, 0x29, 0xe9, 0x99, 0x70, 0x02, 0x8f, 0x41, 0x16, 0x9d, 0x9f, 0x3b, 0x84, 0x57, - 0x80, 0x6c, 0x0b, 0xb7, 0xa7, 0x4d, 0xee, 0xb5, 0x50, 0x46, 0xd6, 0x47, 0xa4, 0xa1, 0xd6, 0x47, - 0x44, 0x83, 0x67, 0x20, 0xc7, 0xa8, 0x8b, 0x3d, 0xc4, 0x1c, 0x4a, 0xa2, 0x59, 0xbe, 0x38, 0xf5, - 0x39, 0x30, 0x12, 0x1b, 0xdd, 0x46, 0xaa, 0xaa, 0xa9, 0x2e, 0x20, 0x05, 0x39, 0x44, 0x08, 0x65, - 0xa1, 0xd9, 0xe5, 0x59, 0xf3, 0x7b, 0x3c, 0x39, 0xb5, 0xb1, 0x92, 0xcc, 0x8d, 0xe8, 0x05, 0x8a, - 0x29, 0xb5, 0x17, 0x28, 0xe4, 0x89, 0xb3, 0x91, 0x16, 0x73, 0xca, 0xfc, 0xb3, 0x71, 0x08, 0xf2, - 0x51, 0x3b, 0xa1, 0xe4, 0x98, 0xba, 0x4e, 0xbb, 0x2f, 0x3e, 0xc3, 0xae, 0xc8, 0x1b, 0x2f, 0xce, - 0x53, 0x6f, 0xbc, 0x38, 0x0f, 0xfe, 0x1c, 0x8c, 0xbe, 0x64, 0x4c, 0x54, 0x69, 0x46, 0xec, 0xd2, - 0xce, 0xb4, 0x84, 0x9a, 0x53, 0xe4, 0xeb, 0xb7, 0xc3, 0xd4, 0x4e, 0xb5, 0x66, 0x4e, 0xa5, 0x16, - 0x3a, 0x60, 0x23, 0x51, 0x54, 0xaf, 0xe4, 0xcd, 0x72, 0x0e, 0xf2, 0xf1, 0x0d, 0x7a, 0x15, 0x7e, - 0x0e, 0xd3, 0xd9, 0x6c, 0x7e, 0xa5, 0xfc, 0x17, 0x0d, 0x6c, 0x1f, 0x07, 0xae, 0x8f, 0xbc, 0x93, - 0xa8, 0x6c, 0x0e, 0x69, 0x6b, 0x1f, 0x33, 0xe4, 0xb8, 0x3e, 0x37, 0x79, 0x48, 0x5b, 0x8d, 0xe8, - 0x53, 0xb4, 0x30, 0x29, 0x08, 0xaa, 0x49, 0x41, 0xe0, 0xa2, 0x8f, 0xe3, 0x4f, 0x92, 0xf8, 0x0c, - 0x23, 0x25, 0xe0, 0x3b, 0x20, 0xc3, 0x2f, 0x45, 0xcc, 0xc2, 0xe7, 0x88, 0x78, 0xad, 0x4a, 0x8a, - 0xfa, 0x5a, 0x95, 0x94, 0x6f, 0x1c, 0x81, 0x9c, 0xf2, 0x3d, 0x03, 0xe6, 0xc0, 0xf2, 0x59, 0xf3, - 0x07, 0xcd, 0xa3, 0x1f, 0x35, 0xf3, 0x0b, 0x7c, 0x71, 0x7c, 0xd0, 0xdc, 0x6f, 0x34, 0xbf, 0x9f, - 0xd7, 0xf8, 0xc2, 0x3c, 0x6b, 0x36, 0xf9, 0x22, 0x05, 0xaf, 0x81, 0x95, 0x93, 0xb3, 0xbd, 0xbd, - 0x83, 0x83, 0xfd, 0x83, 0xfd, 0xfc, 0x22, 0x04, 0x20, 0xf3, 0xbd, 0x5a, 0xe3, 0xe1, 0xc1, 0x7e, - 0x3e, 0x5d, 0xff, 0xe9, 0xf3, 0x17, 0x45, 0xed, 0x8b, 0x17, 0x45, 0xed, 0x5f, 0x2f, 0x8a, 0xda, - 0x27, 0x2f, 0x8b, 0x0b, 0x5f, 0xbc, 0x2c, 0x2e, 0xfc, 0xe3, 0x65, 0x71, 0xe1, 0x27, 0x7b, 0xca, - 0x7f, 0x3e, 0xe4, 0xa7, 0xc8, 0x9e, 0x47, 0xf9, 0x19, 0x0a, 0x57, 0xd5, 0x2b, 0xfc, 0x8b, 0xa7, - 0x95, 0x11, 0x17, 0xcf, 0xfb, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0xb9, 0xfb, 0x69, 0x28, 0x10, - 0x1a, 0x00, 0x00, + 0x70, 0xe0, 0x9a, 0xe3, 0x1e, 0x39, 0x09, 0x2a, 0xb9, 0xe9, 0x53, 0x50, 0xdd, 0x3d, 0x23, 0xb5, + 0x66, 0x24, 0xcb, 0xcb, 0x12, 0x72, 0xb2, 0xfa, 0xfd, 0xfd, 0xbd, 0xd7, 0xaf, 0x5f, 0xbf, 0x1e, + 0x83, 0xfb, 0x0e, 0x61, 0xd8, 0x23, 0xc8, 0xad, 0xfa, 0xed, 0x27, 0xd8, 0x0e, 0x5c, 0xec, 0x8d, + 0x7f, 0xd1, 0xd6, 0x87, 0xb8, 0xcd, 0xfc, 0x04, 0xa1, 0xd2, 0xf3, 0x28, 0xa3, 0x30, 0x1f, 0xa7, + 0x17, 0x8c, 0x0e, 0xa5, 0x1d, 0x17, 0x57, 0x05, 0xbf, 0x15, 0x9c, 0x57, 0x99, 0xd3, 0xc5, 0x3e, + 0x43, 0xdd, 0x9e, 0x54, 0x29, 0x94, 0x2f, 0xee, 0xf9, 0x15, 0x87, 0x56, 0x51, 0xcf, 0xa9, 0xb6, + 0xa9, 0x87, 0xab, 0xcf, 0xde, 0xab, 0x76, 0x30, 0xc1, 0x1e, 0x62, 0xd8, 0x0e, 0x65, 0xbe, 0x35, + 0x96, 0xe9, 0xa2, 0xf6, 0x13, 0x87, 0x60, 0xaf, 0x5f, 0xed, 0x5d, 0x74, 0x84, 0x92, 0x87, 0x7d, + 0x1a, 0x78, 0x6d, 0x9c, 0xd0, 0x7a, 0xb7, 0xe3, 0xb0, 0x27, 0x41, 0xab, 0xd2, 0xa6, 0xdd, 0x6a, + 0x87, 0x76, 0xe8, 0x18, 0x03, 0x5f, 0x89, 0x85, 0xf8, 0x25, 0xc5, 0xcb, 0xcf, 0x53, 0x20, 0x73, + 0xf0, 0x11, 0x6e, 0x07, 0x8c, 0x7a, 0xb0, 0x04, 0x52, 0x8e, 0xad, 0x6b, 0x25, 0x6d, 0x27, 0x5b, + 0xcf, 0x0f, 0x07, 0xc6, 0xaa, 0x63, 0xbf, 0x43, 0xbb, 0x0e, 0xc3, 0xdd, 0x1e, 0xeb, 0x9b, 0x29, + 0xc7, 0x86, 0x6f, 0x81, 0xa5, 0x1e, 0xa5, 0xae, 0x9e, 0x12, 0x32, 0x70, 0x38, 0x30, 0xd6, 0xf8, + 0x5a, 0x91, 0x12, 0x7c, 0x58, 0x03, 0xcb, 0x84, 0xda, 0xd8, 0xd7, 0x17, 0x4b, 0x8b, 0x3b, 0xb9, + 0xdd, 0x1b, 0x95, 0x44, 0xea, 0x9a, 0xd4, 0xc6, 0xf5, 0xcd, 0xe1, 0xc0, 0x58, 0x17, 0x82, 0x8a, + 0x05, 0xa9, 0x09, 0x3f, 0x00, 0x6b, 0x2e, 0xf2, 0xd9, 0x59, 0xcf, 0x46, 0x0c, 0x9f, 0x3a, 0x5d, + 0xac, 0x2f, 0x97, 0xb4, 0x9d, 0xdc, 0x6e, 0xa1, 0x22, 0x93, 0x5b, 0x89, 0x02, 0xab, 0x9c, 0x46, + 0xc9, 0xad, 0x17, 0x9e, 0x0f, 0x8c, 0x05, 0x0e, 0x6a, 0x52, 0xf3, 0x93, 0x7f, 0x19, 0x9a, 0x19, + 0xa3, 0xc1, 0x23, 0xb0, 0x19, 0x10, 0xe4, 0xfb, 0x4e, 0x87, 0x60, 0xdb, 0xfa, 0x90, 0xb6, 0x2c, + 0x2f, 0x20, 0xbe, 0x9e, 0x2d, 0x2d, 0xee, 0x64, 0xeb, 0xc6, 0x70, 0x60, 0xdc, 0x1a, 0xb3, 0x0f, + 0x69, 0xcb, 0x0c, 0x88, 0x0a, 0x72, 0x23, 0xc1, 0x2c, 0xff, 0x69, 0x1d, 0x2c, 0xf1, 0xa8, 0xae, + 0x96, 0x46, 0x82, 0xba, 0x58, 0x5f, 0x1d, 0xa7, 0x91, 0xaf, 0xd5, 0x34, 0xf2, 0x35, 0xdc, 0x05, + 0x19, 0x1c, 0x6e, 0x8e, 0xbe, 0x29, 0x64, 0x6f, 0x0c, 0x07, 0x06, 0x8c, 0x68, 0x8a, 0xfc, 0x48, + 0x0e, 0x3e, 0x02, 0x59, 0x1e, 0xa9, 0xe5, 0x63, 0x4c, 0xc4, 0x3e, 0x5d, 0x9e, 0xb2, 0xad, 0x30, + 0x65, 0x19, 0xae, 0x74, 0x82, 0x31, 0x11, 0xc9, 0x1a, 0xad, 0x60, 0x0d, 0xa4, 0x19, 0x72, 0x08, + 0xf3, 0xf5, 0x65, 0xb1, 0x95, 0xdb, 0x15, 0x59, 0x96, 0x15, 0xd4, 0x73, 0x2a, 0xbc, 0x74, 0x2b, + 0xcf, 0xde, 0xab, 0x9c, 0x72, 0x89, 0xfa, 0x5a, 0x68, 0x2a, 0x54, 0x30, 0xc3, 0xbf, 0xf0, 0x18, + 0xa4, 0x5d, 0xd4, 0xc2, 0xae, 0xaf, 0xa7, 0x85, 0x89, 0xf2, 0xf4, 0x6a, 0xa8, 0x3c, 0x14, 0x42, + 0x07, 0x84, 0x79, 0xfd, 0xfa, 0xd6, 0x70, 0x60, 0xe4, 0xa5, 0x96, 0x12, 0x65, 0x68, 0x07, 0x5a, + 0x60, 0x9d, 0x51, 0x86, 0x5c, 0x2b, 0x3a, 0x06, 0xbe, 0xbe, 0x22, 0x22, 0x2d, 0x26, 0x4d, 0x9b, + 0xa1, 0xc8, 0x43, 0xc7, 0x67, 0xf5, 0x1b, 0x51, 0x81, 0x08, 0xf5, 0x88, 0xe5, 0x9b, 0xb1, 0x35, + 0xfc, 0xab, 0x06, 0xee, 0x22, 0xd7, 0xa5, 0x6d, 0xc4, 0x50, 0xcb, 0xc5, 0x56, 0xab, 0x6f, 0xf5, + 0x3c, 0x87, 0x7a, 0x0e, 0xeb, 0x5b, 0x88, 0xd8, 0x23, 0xbf, 0x7a, 0x46, 0x44, 0xf4, 0xdd, 0x19, + 0x11, 0xd5, 0xc6, 0x26, 0xea, 0xfd, 0xe3, 0xd0, 0x40, 0x8d, 0xd8, 0x91, 0x23, 0x19, 0xeb, 0x4e, + 0x08, 0xaa, 0x84, 0xe6, 0x88, 0x9b, 0x73, 0x25, 0xa0, 0x07, 0x36, 0x7d, 0x86, 0x98, 0x40, 0x1c, + 0xd6, 0xb4, 0xe5, 0xd8, 0xa2, 0xaa, 0x73, 0xbb, 0x6f, 0xcf, 0x80, 0x79, 0xc2, 0x35, 0xea, 0x7d, + 0x59, 0xc8, 0x0d, 0x5b, 0xa2, 0xba, 0x19, 0xa2, 0x5a, 0xf7, 0x27, 0xb9, 0x66, 0x9c, 0x00, 0x3f, + 0xd7, 0x40, 0x91, 0x50, 0x62, 0x21, 0xaf, 0x8b, 0x6c, 0x64, 0x85, 0x18, 0xb1, 0xad, 0xec, 0xce, + 0x35, 0xe1, 0xff, 0xdb, 0x33, 0xfc, 0x37, 0x29, 0xa9, 0x09, 0xdd, 0x5a, 0xa4, 0x3a, 0xda, 0x09, + 0x89, 0xe5, 0xcd, 0x10, 0xcb, 0x2d, 0x32, 0x5b, 0xd2, 0xbc, 0x8c, 0x09, 0x6b, 0xe0, 0x5a, 0x40, + 0x42, 0xef, 0x3c, 0x7b, 0xfa, 0x7a, 0x49, 0xdb, 0xc9, 0xd4, 0x6f, 0x0d, 0x07, 0xc6, 0xcd, 0x09, + 0x86, 0x52, 0x6d, 0x93, 0x1a, 0xf0, 0xd7, 0x1a, 0xb8, 0x19, 0x45, 0x64, 0x05, 0x3e, 0xea, 0x88, + 0x24, 0x3f, 0x0d, 0x70, 0x80, 0xf5, 0xbc, 0x88, 0xef, 0x9b, 0x33, 0xe2, 0x8b, 0x60, 0x9c, 0x71, + 0xa5, 0x7a, 0xff, 0x31, 0x57, 0x91, 0x81, 0x95, 0x87, 0x03, 0xa3, 0xe8, 0x4d, 0x61, 0x2b, 0x30, + 0xb6, 0xa6, 0xf1, 0x79, 0xfb, 0xf2, 0x70, 0x8f, 0x7a, 0xcc, 0x21, 0x1d, 0x8b, 0x77, 0x4c, 0x8b, + 0xf5, 0x7b, 0x58, 0xdf, 0x10, 0x5d, 0x42, 0xb4, 0xaf, 0x11, 0x9b, 0x63, 0x38, 0xed, 0xf7, 0x54, + 0x9b, 0x1b, 0x09, 0xe6, 0xa8, 0xb5, 0xc3, 0xcb, 0x5b, 0x7b, 0x01, 0x81, 0x9c, 0x72, 0x50, 0xe1, + 0x9b, 0x60, 0xf1, 0x02, 0xf7, 0xc3, 0x6e, 0xb7, 0x31, 0x1c, 0x18, 0xd7, 0x2e, 0x70, 0x5f, 0x51, + 0xe2, 0x5c, 0xf8, 0x75, 0xb0, 0xfc, 0x0c, 0xb9, 0x01, 0x0e, 0xef, 0x0d, 0xd1, 0xf6, 0x05, 0x41, + 0x6d, 0xfb, 0x82, 0x70, 0x3f, 0x75, 0x4f, 0x2b, 0xfc, 0x51, 0x03, 0x5f, 0xbb, 0xd2, 0xd1, 0x51, + 0xbd, 0x2f, 0xcf, 0xf4, 0xde, 0x50, 0xbd, 0xcf, 0xef, 0x11, 0xf3, 0xd0, 0xfd, 0x4a, 0x03, 0x5b, + 0xd3, 0x4e, 0xcc, 0xd5, 0x52, 0xf1, 0x40, 0x05, 0xb3, 0xb6, 0x7b, 0x27, 0x09, 0x46, 0x1a, 0x95, + 0x1e, 0xe6, 0x61, 0xf9, 0xbd, 0x06, 0x4a, 0xf3, 0x4e, 0xcf, 0x6b, 0x49, 0xd2, 0x6f, 0x34, 0xb0, + 0x3d, 0xb3, 0xec, 0xaf, 0x96, 0xa9, 0xff, 0x2d, 0xa2, 0xf2, 0x9f, 0x97, 0x40, 0x66, 0x54, 0xec, + 0x25, 0x90, 0x6a, 0xc8, 0x2b, 0x7a, 0x49, 0x5e, 0xd1, 0x8d, 0x89, 0x2b, 0xba, 0x61, 0x2b, 0xf7, + 0x5e, 0xea, 0xbf, 0xbd, 0xf7, 0x4e, 0x47, 0xf7, 0x9e, 0x9c, 0x82, 0xde, 0x9a, 0xde, 0x1e, 0x38, + 0xa0, 0x2f, 0x71, 0xf7, 0xfd, 0x42, 0x03, 0x30, 0x20, 0x3e, 0x66, 0x0d, 0x62, 0xe3, 0x8f, 0xb0, + 0x2d, 0x35, 0xf5, 0x25, 0xe1, 0x62, 0xf7, 0x12, 0x17, 0x67, 0x09, 0x25, 0xe9, 0xae, 0x34, 0x1c, + 0x18, 0xb7, 0x93, 0x16, 0x15, 0xd7, 0x53, 0xfc, 0xfd, 0x3f, 0xda, 0x40, 0x17, 0xdc, 0x9c, 0x81, + 0xf9, 0x55, 0xb8, 0x2b, 0x3f, 0x4f, 0x83, 0x6d, 0x51, 0xa3, 0x7b, 0x6e, 0xe0, 0x33, 0xec, 0x4d, + 0x94, 0x2f, 0x6c, 0x80, 0x95, 0xb6, 0x87, 0xf9, 0xe9, 0x12, 0x5e, 0x2f, 0x1f, 0xaa, 0x36, 0xc3, + 0x8a, 0x88, 0x54, 0xc4, 0x4c, 0x15, 0x2d, 0x38, 0x2e, 0x79, 0x6b, 0x28, 0xb8, 0x9e, 0xc6, 0x9a, + 0xbe, 0x94, 0x80, 0xf7, 0x00, 0x88, 0x06, 0xbb, 0x86, 0xad, 0x2f, 0x0a, 0x79, 0x7d, 0x38, 0x30, + 0xb6, 0xc6, 0x54, 0x45, 0x49, 0x91, 0x85, 0xbf, 0xd3, 0xf8, 0x05, 0x11, 0xf6, 0x81, 0x71, 0x07, + 0x0d, 0xeb, 0x64, 0x3f, 0x59, 0x27, 0x33, 0x43, 0x1f, 0x1d, 0x33, 0xc5, 0x8c, 0xac, 0x9c, 0x3b, + 0x61, 0x98, 0x53, 0x1d, 0x69, 0xe6, 0x34, 0x32, 0xfc, 0x9b, 0x06, 0x6e, 0x4f, 0xa1, 0xef, 0xb9, + 0xc8, 0xf7, 0x9b, 0x48, 0x4c, 0xf9, 0x1c, 0xe0, 0xa3, 0xaf, 0x08, 0x70, 0x64, 0x4f, 0x22, 0xbd, + 0x1b, 0x22, 0xbd, 0xd4, 0xb5, 0x79, 0x29, 0xb7, 0xf0, 0xb1, 0x06, 0xf4, 0x59, 0xa9, 0x78, 0x2d, + 0x3d, 0xf6, 0x0f, 0x1a, 0x78, 0x63, 0x6e, 0xe8, 0xaf, 0xa5, 0xd7, 0xfe, 0x7d, 0x11, 0x14, 0xa6, + 0xed, 0x94, 0x29, 0xa6, 0x8e, 0xd1, 0xa8, 0xa1, 0xcd, 0x79, 0x45, 0x2a, 0x67, 0x2e, 0xf5, 0x15, + 0xcf, 0xdc, 0xc7, 0x1a, 0xc8, 0x2b, 0xbb, 0x2b, 0x6a, 0x29, 0x6c, 0xcb, 0xf5, 0x64, 0xb0, 0xb3, + 0xb1, 0xab, 0xb5, 0xa6, 0xcc, 0x71, 0xc5, 0xe1, 0xc0, 0x28, 0xc4, 0xed, 0x2b, 0xf1, 0x24, 0x7c, + 0x17, 0x3e, 0xd3, 0xc0, 0xf5, 0xa9, 0xb6, 0xae, 0xb6, 0x61, 0x3f, 0x9c, 0xdc, 0xb0, 0xb7, 0xbf, + 0xc4, 0x71, 0x99, 0xbb, 0x7b, 0xbf, 0x4c, 0x81, 0x55, 0x75, 0xbb, 0xe1, 0x07, 0x20, 0x3b, 0x1e, + 0xe5, 0x35, 0x91, 0xb4, 0x77, 0x2f, 0xaf, 0x90, 0x4a, 0x6c, 0x80, 0xdf, 0x08, 0x37, 0x67, 0x6c, + 0xc7, 0x1c, 0xff, 0x2c, 0x7c, 0xaa, 0x81, 0xb5, 0xd9, 0x33, 0xcb, 0xec, 0x24, 0xfc, 0x78, 0x32, + 0x09, 0x15, 0xe5, 0x8a, 0x1e, 0x7d, 0x31, 0xa9, 0xf4, 0x2e, 0x3a, 0xe2, 0xce, 0x8e, 0xdc, 0x55, + 0x1e, 0x07, 0x88, 0x30, 0x87, 0xf5, 0xe7, 0xe6, 0xe1, 0xd3, 0x65, 0xb0, 0x71, 0x48, 0x5b, 0x27, + 0x32, 0x50, 0x87, 0x74, 0x1a, 0xe4, 0x9c, 0xf2, 0x37, 0xb9, 0xeb, 0x9c, 0x63, 0xe6, 0x74, 0xb1, + 0x80, 0x77, 0x4d, 0xbe, 0xc9, 0x23, 0x9a, 0xfa, 0x26, 0x8f, 0x68, 0xf0, 0x3e, 0x58, 0x45, 0xcc, + 0xea, 0x52, 0x9f, 0x59, 0x94, 0xb4, 0x25, 0xde, 0x8c, 0x6c, 0xe4, 0x88, 0x3d, 0xa2, 0x3e, 0x3b, + 0x22, 0x6d, 0x55, 0x13, 0x8c, 0xa9, 0xf0, 0x3b, 0x20, 0xd7, 0xf3, 0x30, 0xa7, 0x3b, 0xfc, 0xdd, + 0xb2, 0x28, 0x54, 0xb7, 0x87, 0x03, 0xe3, 0xba, 0x42, 0x56, 0x74, 0x55, 0x69, 0xf8, 0x00, 0xe4, + 0xdb, 0x94, 0xb4, 0x03, 0xcf, 0xc3, 0xa4, 0xdd, 0xb7, 0x7c, 0x74, 0x8e, 0xf5, 0x25, 0x61, 0xe1, + 0xce, 0x70, 0x60, 0x6c, 0x2b, 0xbc, 0x13, 0x74, 0xae, 0x5a, 0x59, 0x8f, 0xb1, 0xf8, 0x7b, 0x63, + 0xf4, 0x02, 0x6e, 0xf3, 0x0e, 0x63, 0x89, 0x2f, 0x18, 0xe9, 0xf1, 0x7b, 0xa3, 0x17, 0xef, 0x3f, + 0xea, 0x7b, 0x23, 0xc1, 0x84, 0x27, 0x20, 0xe7, 0x07, 0xad, 0xae, 0xc3, 0x2c, 0x91, 0xca, 0x95, + 0xb9, 0x07, 0x3c, 0x7a, 0xbb, 0x03, 0xa9, 0x36, 0xfa, 0xb0, 0xa3, 0xac, 0xf9, 0xe6, 0x44, 0x9e, + 0xf4, 0xcc, 0x78, 0x73, 0x22, 0x9a, 0xba, 0x39, 0x11, 0x0d, 0xfe, 0x0c, 0x6c, 0xca, 0x12, 0xb6, + 0x3c, 0xfc, 0x34, 0x70, 0x3c, 0xdc, 0xc5, 0xe3, 0xcf, 0x1d, 0x77, 0x93, 0x75, 0x7e, 0x24, 0xfe, + 0x9a, 0x8a, 0xac, 0x1c, 0xa1, 0x68, 0x82, 0xae, 0x8e, 0x50, 0x49, 0x2e, 0xac, 0x82, 0x95, 0x67, + 0xd8, 0xf3, 0x1d, 0x4a, 0xf4, 0xac, 0xc0, 0x7a, 0x7d, 0x38, 0x30, 0x36, 0x42, 0x92, 0xa2, 0x1b, + 0x49, 0xdd, 0x5f, 0xfa, 0xec, 0x73, 0x43, 0x2b, 0xff, 0x56, 0x03, 0x30, 0x89, 0x01, 0xba, 0x60, + 0xbd, 0x47, 0x6d, 0x95, 0x14, 0x0e, 0x2a, 0x6f, 0x24, 0x43, 0x38, 0x9e, 0x14, 0x94, 0xc5, 0x10, + 0xd3, 0x1e, 0x03, 0x78, 0xb0, 0x60, 0xc6, 0x4d, 0xd7, 0xd7, 0xc0, 0xaa, 0x9a, 0xad, 0xf2, 0x3f, + 0xd2, 0x60, 0x3d, 0x66, 0x15, 0xfa, 0x60, 0x95, 0x3f, 0x4c, 0x4f, 0xb0, 0x8b, 0xdb, 0x8c, 0x7a, + 0x61, 0xe7, 0x78, 0x7f, 0x2e, 0x1c, 0x31, 0xb2, 0x46, 0x5a, 0xb2, 0x7f, 0x14, 0x86, 0x03, 0xe3, + 0x86, 0x6a, 0x4c, 0x49, 0xcf, 0x84, 0x13, 0x78, 0x0c, 0x32, 0xe8, 0xfc, 0xdc, 0x21, 0xbc, 0x02, + 0x64, 0x5b, 0xb8, 0x3d, 0x6d, 0x72, 0xaf, 0x85, 0x32, 0xb2, 0x3e, 0x22, 0x0d, 0xb5, 0x3e, 0x22, + 0x1a, 0x3c, 0x03, 0x39, 0x46, 0x5d, 0xec, 0x21, 0xe6, 0x50, 0x12, 0xcd, 0xf2, 0xc5, 0xa9, 0xcf, + 0x81, 0x91, 0xd8, 0xe8, 0x36, 0x52, 0x55, 0x4d, 0x75, 0x01, 0x29, 0xc8, 0x21, 0x42, 0x28, 0x0b, + 0xcd, 0xae, 0xcc, 0x9a, 0xdf, 0xe3, 0xc9, 0xa9, 0x8d, 0x95, 0x64, 0x6e, 0x44, 0x2f, 0x50, 0x4c, + 0xa9, 0xbd, 0x40, 0x21, 0xc3, 0x43, 0x90, 0x8f, 0x5a, 0x03, 0x25, 0xc7, 0xd4, 0x75, 0xda, 0x7d, + 0xf1, 0x49, 0x35, 0x2b, 0x6f, 0xaf, 0x38, 0x4f, 0xbd, 0xbd, 0xe2, 0x3c, 0xf8, 0x73, 0x30, 0xfa, + 0x2a, 0x31, 0x51, 0x71, 0x69, 0x91, 0xf1, 0x9d, 0x69, 0xc9, 0x31, 0xa7, 0xc8, 0xd7, 0x6f, 0x87, + 0x69, 0x9a, 0x6a, 0xcd, 0x9c, 0x4a, 0x2d, 0x74, 0xc0, 0x46, 0xa2, 0x40, 0x5e, 0xc9, 0xfb, 0xe3, + 0x1c, 0xe4, 0xe3, 0xc9, 0x7e, 0x15, 0x7e, 0x0e, 0x97, 0x32, 0x99, 0x7c, 0xb6, 0xfc, 0x17, 0x0d, + 0x6c, 0x1f, 0x07, 0xae, 0x8f, 0xbc, 0x93, 0xa8, 0x04, 0x0e, 0x69, 0x6b, 0x1f, 0x33, 0xe4, 0xb8, + 0x3e, 0x37, 0x79, 0x48, 0x5b, 0x8d, 0xe8, 0xb3, 0xb2, 0x30, 0x29, 0x08, 0xaa, 0x49, 0x41, 0xe0, + 0xa2, 0x8f, 0xe3, 0xcf, 0x8b, 0xf8, 0x3c, 0x22, 0x25, 0xe0, 0x3b, 0x20, 0xcd, 0x2f, 0x38, 0xcc, + 0xc2, 0xa7, 0x85, 0x78, 0x79, 0x4a, 0x8a, 0xfa, 0xf2, 0x94, 0x94, 0x6f, 0x1c, 0x81, 0x9c, 0xf2, + 0x6d, 0x02, 0xe6, 0xc0, 0xca, 0x59, 0xf3, 0x07, 0xcd, 0xa3, 0x1f, 0x35, 0xf3, 0x0b, 0x7c, 0x71, + 0x7c, 0xd0, 0xdc, 0x6f, 0x34, 0xbf, 0x9f, 0xd7, 0xf8, 0xc2, 0x3c, 0x6b, 0x36, 0xf9, 0x22, 0x05, + 0xaf, 0x81, 0xec, 0xc9, 0xd9, 0xde, 0xde, 0xc1, 0xc1, 0xfe, 0xc1, 0x7e, 0x7e, 0x11, 0x02, 0x90, + 0xfe, 0x5e, 0xad, 0xf1, 0xf0, 0x60, 0x3f, 0xbf, 0x54, 0xff, 0xe9, 0xf3, 0x17, 0x45, 0xed, 0x8b, + 0x17, 0x45, 0xed, 0xdf, 0x2f, 0x8a, 0xda, 0x27, 0x2f, 0x8b, 0x0b, 0x5f, 0xbc, 0x2c, 0x2e, 0xfc, + 0xf3, 0x65, 0x71, 0xe1, 0x27, 0x7b, 0xca, 0x7f, 0x31, 0xe4, 0x67, 0xc5, 0x9e, 0x47, 0xf9, 0x79, + 0x08, 0x57, 0xd5, 0x2b, 0xfc, 0xbb, 0xa6, 0x95, 0x16, 0x97, 0xc8, 0xfb, 0xff, 0x09, 0x00, 0x00, + 0xff, 0xff, 0xad, 0xb8, 0xe9, 0xc4, 0xdc, 0x19, 0x00, 0x00, } func (m *Executor) Marshal() (dAtA []byte, err error) { @@ -1855,11 +1842,6 @@ func (m *PodRequirements) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x2a } - if m.Priority != 0 { - i = encodeVarintSchedulerobjects(dAtA, i, uint64(m.Priority)) - i-- - dAtA[i] = 0x20 - } if len(m.Tolerations) > 0 { for iNdEx := len(m.Tolerations) - 1; iNdEx >= 0; iNdEx-- { { @@ -2287,9 +2269,6 @@ func (m *PodRequirements) Size() (n int) { n += 1 + l + sovSchedulerobjects(uint64(l)) } } - if m.Priority != 0 { - n += 1 + sovSchedulerobjects(uint64(m.Priority)) - } l = len(m.PreemptionPolicy) if l > 0 { n += 1 + l + sovSchedulerobjects(uint64(l)) @@ -5224,25 +5203,6 @@ func (m *PodRequirements) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Priority", wireType) - } - m.Priority = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowSchedulerobjects - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Priority |= int32(b&0x7F) << shift - if b < 0x80 { - break - } - } case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field PreemptionPolicy", wireType) diff --git a/internal/scheduler/schedulerobjects/schedulerobjects.proto b/internal/scheduler/schedulerobjects/schedulerobjects.proto index d86f91be750..c7197ccc08b 100644 --- a/internal/scheduler/schedulerobjects/schedulerobjects.proto +++ b/internal/scheduler/schedulerobjects/schedulerobjects.proto @@ -143,11 +143,6 @@ message PodRequirements { repeated k8s.io.api.core.v1.Toleration tolerations = 3 [(gogoproto.nullable) = false]; // Kubernetes annotations. Included here since we use annotations with special meaning. map annotations = 7; - // Priority class priority of the pod as submitted. Should be mapped from the priority class name of the submitted pod. - // - // During scheduling, the priority stored on the podSchedulingContext should be used instead, - // since a pod may be scheduled at a priority different from the priority it was submitted with. - int32 priority = 4; // One of Never, PreemptLowerPriority. // Defaults to PreemptLowerPriority if unset. string preemptionPolicy = 5; diff --git a/internal/scheduler/scheduling_algo_test.go b/internal/scheduler/scheduling_algo_test.go index e26c633e511..0b9ef94e5a1 100644 --- a/internal/scheduler/scheduling_algo_test.go +++ b/internal/scheduler/scheduling_algo_test.go @@ -434,7 +434,7 @@ func TestSchedule(t *testing.T) { for nodeIndex, existingJobs := range existingJobsByExecutorNodeIndex { node := executor.Nodes[nodeIndex] for jobIndex, job := range existingJobs.jobs { - job = job.WithQueued(false).WithNewRun(executor.Id, node.Id, node.Name, node.Pool, job.PodRequirements().Priority) + job = job.WithQueued(false).WithNewRun(executor.Id, node.Id, node.Name, node.Pool, job.PriorityClass().Priority) if existingJobs.acknowledged { run := job.LatestRun() node.StateByJobRunId[run.Id().String()] = schedulerobjects.JobRunState_RUNNING @@ -559,7 +559,7 @@ func BenchmarkNodeDbConstruction(b *testing.B) { nodes := testfixtures.N32CpuNodes(numNodes, testfixtures.TestPriorities) for i, node := range nodes { for j := 32 * i; j < 32*(i+1); j++ { - jobs[j] = jobs[j].WithNewRun("executor-01", node.Id, node.Name, node.Pool, jobs[j].PodRequirements().Priority) + jobs[j] = jobs[j].WithNewRun("executor-01", node.Id, node.Name, node.Pool, jobs[j].PriorityClass().Priority) } } armadaslices.Shuffle(jobs) diff --git a/internal/scheduler/simulator/simulator.go b/internal/scheduler/simulator/simulator.go index d2caab2248f..b6bd84858c1 100644 --- a/internal/scheduler/simulator/simulator.go +++ b/internal/scheduler/simulator/simulator.go @@ -689,7 +689,7 @@ func (s *Simulator) handleEventSequence(ctx *armadacontext.Context, es *armadaev } func (s *Simulator) handleSubmitJob(txn *jobdb.Txn, e *armadaevents.SubmitJob, time time.Time, eventSequence *armadaevents.EventSequence) (*jobdb.Job, bool, error) { - schedulingInfo, err := scheduleringester.SchedulingInfoFromSubmitJob(e, time, s.schedulingConfig.PriorityClasses) + schedulingInfo, err := scheduleringester.SchedulingInfoFromSubmitJob(e, time) if err != nil { return nil, false, err } diff --git a/internal/scheduler/testfixtures/testfixtures.go b/internal/scheduler/testfixtures/testfixtures.go index 27212a07b69..2fe70108674 100644 --- a/internal/scheduler/testfixtures/testfixtures.go +++ b/internal/scheduler/testfixtures/testfixtures.go @@ -555,7 +555,6 @@ func N1GpuPodReqs(queue string, priority int32, n int) []*schedulerobjects.PodRe func TestPodReqs(queue string, jobId ulid.ULID, priority int32, requests v1.ResourceList) *schedulerobjects.PodRequirements { return &schedulerobjects.PodRequirements{ - Priority: priority, ResourceRequirements: v1.ResourceRequirements{Requests: requests}, Annotations: make(map[string]string), NodeSelector: make(map[string]string), @@ -644,9 +643,8 @@ func Test1GpuPodReqs(queue string, jobId ulid.ULID, priority int32) *schedulerob return req } -func TestUnitReqs(priority int32) *schedulerobjects.PodRequirements { +func TestUnitReqs() *schedulerobjects.PodRequirements { return &schedulerobjects.PodRequirements{ - Priority: priority, ResourceRequirements: v1.ResourceRequirements{ Requests: v1.ResourceList{ "cpu": resource.MustParse("1"), @@ -858,7 +856,7 @@ func TestQueuedJobDbJob() *jobdb.Job { ObjectRequirements: []*schedulerobjects.ObjectRequirements{ { Requirements: &schedulerobjects.ObjectRequirements_PodRequirements{ - PodRequirements: TestUnitReqs(1), + PodRequirements: TestUnitReqs(), }, }, }, diff --git a/internal/scheduleringester/config.go b/internal/scheduleringester/config.go index 2e1f945afbc..51a60ed2dd0 100644 --- a/internal/scheduleringester/config.go +++ b/internal/scheduleringester/config.go @@ -5,7 +5,6 @@ import ( "github.com/armadaproject/armada/internal/armada/configuration" profilingconfig "github.com/armadaproject/armada/internal/common/profiling/configuration" - "github.com/armadaproject/armada/internal/common/types" ) type Configuration struct { @@ -15,8 +14,6 @@ type Configuration struct { MetricsPort uint16 // General Pulsar configuration Pulsar configuration.PulsarConfig - // Map of allowed priority classes by name - PriorityClasses map[string]types.PriorityClass // Pulsar subscription name SubscriptionName string // Number of event messages that will be batched together before being inserted into the database diff --git a/internal/scheduleringester/ingester.go b/internal/scheduleringester/ingester.go index cc4ae584716..d5ced39d209 100644 --- a/internal/scheduleringester/ingester.go +++ b/internal/scheduleringester/ingester.go @@ -27,7 +27,7 @@ func Run(config Configuration) error { } schedulerDb := NewSchedulerDb(db, svcMetrics, 100*time.Millisecond, 60*time.Second, 5*time.Second) - converter, err := NewInstructionConverter(svcMetrics, config.PriorityClasses) + converter, err := NewInstructionConverter(svcMetrics) if err != nil { return err } diff --git a/internal/scheduleringester/instructions.go b/internal/scheduleringester/instructions.go index de82d765b49..9cb973d6853 100644 --- a/internal/scheduleringester/instructions.go +++ b/internal/scheduleringester/instructions.go @@ -15,7 +15,6 @@ import ( "github.com/armadaproject/armada/internal/common/ingest" "github.com/armadaproject/armada/internal/common/ingest/metrics" protoutil "github.com/armadaproject/armada/internal/common/proto" - "github.com/armadaproject/armada/internal/common/types" "github.com/armadaproject/armada/internal/scheduler/adapters" schedulerdb "github.com/armadaproject/armada/internal/scheduler/database" "github.com/armadaproject/armada/internal/scheduler/schedulerobjects" @@ -30,23 +29,20 @@ type eventSequenceCommon struct { } type InstructionConverter struct { - metrics *metrics.Metrics - priorityClasses map[string]types.PriorityClass - compressor compress.Compressor + metrics *metrics.Metrics + compressor compress.Compressor } func NewInstructionConverter( metrics *metrics.Metrics, - priorityClasses map[string]types.PriorityClass, ) (*InstructionConverter, error) { compressor, err := compress.NewZlibCompressor(1024) if err != nil { return nil, errors.WithMessage(err, "failed to create compressor") } return &InstructionConverter{ - metrics: metrics, - priorityClasses: priorityClasses, - compressor: compressor, + metrics: metrics, + compressor: compressor, }, nil } @@ -423,11 +419,11 @@ func (c *InstructionConverter) handleJobValidated(checked *armadaevents.JobValid // schedulingInfoFromSubmitJob returns a minimal representation of a job containing only the info needed by the scheduler. func (c *InstructionConverter) schedulingInfoFromSubmitJob(submitJob *armadaevents.SubmitJob, submitTime time.Time) (*schedulerobjects.JobSchedulingInfo, error) { - return SchedulingInfoFromSubmitJob(submitJob, submitTime, c.priorityClasses) + return SchedulingInfoFromSubmitJob(submitJob, submitTime) } // SchedulingInfoFromSubmitJob returns a minimal representation of a job containing only the info needed by the scheduler. -func SchedulingInfoFromSubmitJob(submitJob *armadaevents.SubmitJob, submitTime time.Time, priorityClasses map[string]types.PriorityClass) (*schedulerobjects.JobSchedulingInfo, error) { +func SchedulingInfoFromSubmitJob(submitJob *armadaevents.SubmitJob, submitTime time.Time) (*schedulerobjects.JobSchedulingInfo, error) { // Component common to all jobs. schedulingInfo := &schedulerobjects.JobSchedulingInfo{ Lifetime: submitJob.Lifetime, @@ -444,7 +440,7 @@ func SchedulingInfoFromSubmitJob(submitJob *armadaevents.SubmitJob, submitTime t case *armadaevents.KubernetesMainObject_PodSpec: podSpec := object.PodSpec.PodSpec schedulingInfo.PriorityClassName = podSpec.PriorityClassName - podRequirements := adapters.PodRequirementsFromPodSpec(podSpec, priorityClasses) + podRequirements := adapters.PodRequirementsFromPodSpec(podSpec) if submitJob.ObjectMeta != nil { podRequirements.Annotations = maps.Clone(submitJob.ObjectMeta.Annotations) } diff --git a/internal/scheduleringester/instructions_test.go b/internal/scheduleringester/instructions_test.go index e092f404f73..7df26d75009 100644 --- a/internal/scheduleringester/instructions_test.go +++ b/internal/scheduleringester/instructions_test.go @@ -241,7 +241,7 @@ func TestConvertSequence(t *testing.T) { for name, tc := range tests { t.Run(name, func(t *testing.T) { - converter := InstructionConverter{m, f.PriorityClasses, compressor} + converter := InstructionConverter{m, compressor} es := f.NewEventSequence(tc.events...) results := converter.dbOperationsFromEventSequence(es) assertOperationsEqual(t, tc.expected, results) @@ -329,7 +329,6 @@ func getExpectedSubmitMessageSchedulingInfo(t *testing.T) *schedulerobjects.JobS NodeSelector: f.NodeSelector, Tolerations: f.Tolerations, PreemptionPolicy: "PreemptLowerPriority", - Priority: f.PriorityClassValue, ResourceRequirements: v1.ResourceRequirements{ Limits: map[v1.ResourceName]resource.Quantity{ "memory": resource.MustParse("64Mi"),