From 5752903d73dee1a5ab4574a2254722299f4978b2 Mon Sep 17 00:00:00 2001 From: i583051 Date: Wed, 22 Nov 2023 22:39:08 +0200 Subject: [PATCH 1/2] Add memory boundaries to executor conf --- depot/containerstore/containerstore.go | 3 +++ depot/containerstore/storenode.go | 17 ++++++++++++++++- initializer/initializer.go | 4 ++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/depot/containerstore/containerstore.go b/depot/containerstore/containerstore.go index 8e13e48e..072f77c7 100644 --- a/depot/containerstore/containerstore.go +++ b/depot/containerstore/containerstore.go @@ -55,6 +55,9 @@ type ContainerConfig struct { MaxCPUShares uint64 SetCPUWeight bool + MinInstanceMmemoryMB int + MaxInstanceMmemoryMB int + ReservedExpirationTime time.Duration ReapInterval time.Duration MaxLogLinesPerSecond int diff --git a/depot/containerstore/storenode.go b/depot/containerstore/storenode.go index 4c267761..9df3a1e0 100644 --- a/depot/containerstore/storenode.go +++ b/depot/containerstore/storenode.go @@ -386,6 +386,9 @@ func (n *storeNode) createGardenContainer(logger lager.Logger, traceID string, i return nil, err } + cpuWeight := n.calculateCpuWeight(info.MemoryMB) + logger.Info("setting-cpu-weight", lager.Data{"cpu-weight": cpuWeight}) + containerSpec := garden.ContainerSpec{ Handle: info.Guid, Privileged: info.Privileged, @@ -409,7 +412,7 @@ func (n *storeNode) createGardenContainer(logger lager.Logger, traceID string, i Max: uint64(info.MaxPids), }, CPU: garden.CPULimits{ - LimitInShares: uint64(float64(n.config.MaxCPUShares) * float64(info.CPUWeight) / 100.0), + LimitInShares: uint64(float64(n.config.MaxCPUShares) * float64(cpuWeight) / 100.0), }, }, Properties: gardenProperties, @@ -883,3 +886,15 @@ func createContainer(logger lager.Logger, spec garden.ContainerSpec, client gard } return container, nil } + +func (n *storeNode) calculateCpuWeight(memoryMB int) int { + if memoryMB > n.config.MaxInstanceMmemoryMB { + return 100 + } + + if memoryMB < n.config.MinInstanceMmemoryMB { + memoryMB = n.config.MinInstanceMmemoryMB + } + + return 100 * memoryMB / n.config.MinInstanceMmemoryMB +} diff --git a/initializer/initializer.go b/initializer/initializer.go index 8ba7723f..feb12fcc 100644 --- a/initializer/initializer.go +++ b/initializer/initializer.go @@ -144,6 +144,8 @@ type ExecutorConfig struct { ReadWorkPoolSize int `json:"read_work_pool_size,omitempty"` ReservedExpirationTime durationjson.Duration `json:"reserved_expiration_time,omitempty"` SetCPUWeight bool `json:"set_cpu_weight,omitempty"` + MinInstanceMmemoryMB int `json:"min_instance_memory_mb,omitempty"` + MaxInstanceMmemoryMB int `json:"max_instance_memory_mb,omitempty"` SkipCertVerify bool `json:"skip_cert_verify,omitempty"` TempDir string `json:"temp_dir,omitempty"` TrustedSystemCertificatesPath string `json:"trusted_system_certificates_path"` @@ -270,6 +272,8 @@ func Initialize(logger lager.Logger, config ExecutorConfig, cellID, zone string, INodeLimit: config.ContainerInodeLimit, MaxCPUShares: config.ContainerMaxCpuShares, SetCPUWeight: config.SetCPUWeight, + MinInstanceMmemoryMB: config.MinInstanceMmemoryMB, + MaxInstanceMmemoryMB: config.MaxInstanceMmemoryMB, ReservedExpirationTime: time.Duration(config.ReservedExpirationTime), ReapInterval: time.Duration(config.ContainerReapInterval), MaxLogLinesPerSecond: config.MaxLogLinesPerSecond, From 3b81c9ee4e51b4e521e12ecefca51db9d30c955f Mon Sep 17 00:00:00 2001 From: i583051 Date: Wed, 29 Nov 2023 21:51:37 +0200 Subject: [PATCH 2/2] Add tests --- depot/containerstore/containerstore.go | 1 + depot/containerstore/containerstore_test.go | 109 +++++++++++++++++++- depot/containerstore/storenode.go | 7 +- 3 files changed, 111 insertions(+), 6 deletions(-) diff --git a/depot/containerstore/containerstore.go b/depot/containerstore/containerstore.go index 072f77c7..8cd4f5aa 100644 --- a/depot/containerstore/containerstore.go +++ b/depot/containerstore/containerstore.go @@ -54,6 +54,7 @@ type ContainerConfig struct { INodeLimit uint64 MaxCPUShares uint64 SetCPUWeight bool + CPUWeight int MinInstanceMmemoryMB int MaxInstanceMmemoryMB int diff --git a/depot/containerstore/containerstore_test.go b/depot/containerstore/containerstore_test.go index f4a006ca..f1bf258e 100644 --- a/depot/containerstore/containerstore_test.go +++ b/depot/containerstore/containerstore_test.go @@ -139,6 +139,9 @@ var _ = Describe("Container Store", func() { OwnerName: ownerName, INodeLimit: iNodeLimit, MaxCPUShares: maxCPUShares, + CPUWeight: 12, + MinInstanceMmemoryMB: 128, + MaxInstanceMmemoryMB: 8192, ReapInterval: 20 * time.Millisecond, ReservedExpirationTime: 20 * time.Millisecond, MetricReportInterval: 30 * time.Second, @@ -507,7 +510,7 @@ var _ = Describe("Container Store", func() { Expect(int(containerSpec.Limits.Pid.Max)).To(Equal(resource.MaxPids)) - expectedCPUShares := uint64(float64(maxCPUShares) * float64(runReq.CPUWeight) / 100.0) + expectedCPUShares := uint64(float64(maxCPUShares) * float64(containerConfig.CPUWeight) / 100.0) Expect(containerSpec.Limits.CPU.LimitInShares).To(Equal(expectedCPUShares)) expectedCPUWeight := uint64(resource.MemoryMB) @@ -515,6 +518,108 @@ var _ = Describe("Container Store", func() { }) }) + Context("when the requested memory is more than MaxInstanceMmemoryMB", func() { + BeforeEach(func() { + allocationReq.Resource.MemoryMB = 10240 + resource.MemoryMB = 10240 + + containerConfig.CPUWeight = 100 + containerStore = containerstore.New( + containerConfig, + &totalCapacity, + gardenClientFactory, + dependencyManager, + volumeManager, + credManager, + logManager, + clock, + eventEmitter, + megatron, + "/var/vcap/data/cf-system-trusted-certs", + metronClient, + rootFSSizer, + false, + "/var/vcap/packages/healthcheck", + proxyManager, + cellID, + true, + advertisePreferenceForInstanceAddress, + json.Marshal, + ) + }) + + It("creates the container in garden with the correct limits ", func() { + _, err := containerStore.Create(logger, "some-trace-id", containerGuid) + Expect(err).NotTo(HaveOccurred()) + + Expect(gardenClient.CreateCallCount()).To(Equal(1)) + containerSpec := gardenClient.CreateArgsForCall(0) + Expect(containerSpec.Limits.Memory.LimitInBytes).To(BeEquivalentTo(resource.MemoryMB * 1024 * 1024)) + + Expect(containerSpec.Limits.Disk.Scope).To(Equal(garden.DiskLimitScopeTotal)) + Expect(containerSpec.Limits.Disk.ByteHard).To(BeEquivalentTo((resource.DiskMB * 1024 * 1024) + 1000)) + Expect(containerSpec.Limits.Disk.InodeHard).To(Equal(iNodeLimit)) + + Expect(int(containerSpec.Limits.Pid.Max)).To(Equal(resource.MaxPids)) + + expectedCPUShares := uint64(float64(maxCPUShares) * float64(containerConfig.CPUWeight) / 100.0) + Expect(containerSpec.Limits.CPU.LimitInShares).To(Equal(expectedCPUShares)) + Expect(containerSpec.Limits.CPU.Weight).To(BeZero()) + + }) + }) + + Context("when the requested memory is less than MaxInstanceMmemoryMB", func() { + BeforeEach(func() { + allocationReq.Resource.MemoryMB = 100 + resource.MemoryMB = 100 + + containerConfig.CPUWeight = 1 + containerStore = containerstore.New( + containerConfig, + &totalCapacity, + gardenClientFactory, + dependencyManager, + volumeManager, + credManager, + logManager, + clock, + eventEmitter, + megatron, + "/var/vcap/data/cf-system-trusted-certs", + metronClient, + rootFSSizer, + false, + "/var/vcap/packages/healthcheck", + proxyManager, + cellID, + true, + advertisePreferenceForInstanceAddress, + json.Marshal, + ) + }) + + It("creates the container in garden with the correct limits", func() { + _, err := containerStore.Create(logger, "some-trace-id", containerGuid) + Expect(err).NotTo(HaveOccurred()) + + Expect(gardenClient.CreateCallCount()).To(Equal(1)) + containerSpec := gardenClient.CreateArgsForCall(0) + Expect(containerSpec.Limits.Memory.LimitInBytes).To(BeEquivalentTo(resource.MemoryMB * 1024 * 1024)) + + Expect(containerSpec.Limits.Disk.Scope).To(Equal(garden.DiskLimitScopeTotal)) + Expect(containerSpec.Limits.Disk.ByteHard).To(BeEquivalentTo((resource.DiskMB * 1024 * 1024) + 1000)) + Expect(containerSpec.Limits.Disk.InodeHard).To(Equal(iNodeLimit)) + + Expect(int(containerSpec.Limits.Pid.Max)).To(Equal(resource.MaxPids)) + + expectedCPUShares := uint64(float64(maxCPUShares) * float64(containerConfig.CPUWeight) / 100.0) + Expect(containerSpec.Limits.CPU.LimitInShares).To(Equal(expectedCPUShares)) + Expect(containerSpec.Limits.CPU.Weight).To(BeZero()) + + }) + }) + It("creates the container in garden with the correct limits", func() { _, err := containerStore.Create(logger, "some-trace-id", containerGuid) Expect(err).NotTo(HaveOccurred()) @@ -529,7 +634,7 @@ var _ = Describe("Container Store", func() { Expect(int(containerSpec.Limits.Pid.Max)).To(Equal(resource.MaxPids)) - expectedCPUShares := uint64(float64(maxCPUShares) * float64(runReq.CPUWeight) / 100.0) + expectedCPUShares := uint64(float64(maxCPUShares) * float64(containerConfig.CPUWeight) / 100.0) Expect(containerSpec.Limits.CPU.LimitInShares).To(Equal(expectedCPUShares)) Expect(containerSpec.Limits.CPU.Weight).To(BeZero()) }) diff --git a/depot/containerstore/storenode.go b/depot/containerstore/storenode.go index 9df3a1e0..1a769045 100644 --- a/depot/containerstore/storenode.go +++ b/depot/containerstore/storenode.go @@ -386,8 +386,7 @@ func (n *storeNode) createGardenContainer(logger lager.Logger, traceID string, i return nil, err } - cpuWeight := n.calculateCpuWeight(info.MemoryMB) - logger.Info("setting-cpu-weight", lager.Data{"cpu-weight": cpuWeight}) + n.config.CPUWeight = n.calculateCpuWeight(info.MemoryMB) containerSpec := garden.ContainerSpec{ Handle: info.Guid, @@ -412,7 +411,7 @@ func (n *storeNode) createGardenContainer(logger lager.Logger, traceID string, i Max: uint64(info.MaxPids), }, CPU: garden.CPULimits{ - LimitInShares: uint64(float64(n.config.MaxCPUShares) * float64(cpuWeight) / 100.0), + LimitInShares: uint64(float64(n.config.MaxCPUShares) * float64(n.config.CPUWeight) / 100.0), }, }, Properties: gardenProperties, @@ -896,5 +895,5 @@ func (n *storeNode) calculateCpuWeight(memoryMB int) int { memoryMB = n.config.MinInstanceMmemoryMB } - return 100 * memoryMB / n.config.MinInstanceMmemoryMB + return 100 * memoryMB / n.config.MaxInstanceMmemoryMB }