From b98999122c1e70dd6173ec1c38c1ed5a4ebeea3b Mon Sep 17 00:00:00 2001 From: curl-li <48984720+Curl-Li@users.noreply.github.com> Date: Sat, 10 Sep 2022 17:34:02 +0800 Subject: [PATCH] support cgroup v2 --- consts.go | 6 - doc/zh.md | 2 +- internal/cg/cgroups/cgroup.go | 74 +++++++ internal/cg/cgroups/cgroup_test.go | 122 ++++++++++++ internal/cg/cgroups/cgroups.go | 109 +++++++++++ internal/cg/cgroups/cgroups2.go | 164 ++++++++++++++++ internal/cg/cgroups/cgroups2_test.go | 152 +++++++++++++++ internal/cg/cgroups/cgroups_test.go | 120 ++++++++++++ internal/cg/cgroups/load.go | 72 +++++++ internal/cg/cgroups/load_test.go | 137 +++++++++++++ internal/cg/cgroups/mountpoint.go | 180 ++++++++++++++++++ internal/cg/cgroups/mountpoint_test.go | 179 +++++++++++++++++ internal/cg/cgroups/subsys.go | 107 +++++++++++ internal/cg/cgroups/subsys_test.go | 99 ++++++++++ .../cgroups/v1/cpu/empty/cpu.cfs_quota_us | 0 .../cgroups/v1/cpu/invalid/cpu.cfs_quota_us | 1 + .../cgroups/v1/cpu/set/cpu.cfs_period_us | 1 + .../cgroups/v1/cpu/set/cpu.cfs_quota_us | 1 + .../v1/cpu/undefined-period/cpu.cfs_quota_us | 1 + .../v1/cpu/undefined/cpu.cfs_period_us | 1 + .../cgroups/v1/cpu/undefined/cpu.cfs_quota_us | 1 + .../v1/memory/set/memory.limit_in_bytes | 1 + .../v1/memory/undefined/memory.limit_in_bytes | 1 + .../cgroups/testdata/cgroups/v2/empty/cpu.max | 0 .../testdata/cgroups/v2/empty/memory.max | 0 .../testdata/cgroups/v2/invalid-max/cpu.max | 1 + .../cgroups/v2/invalid-max/memory.max | 1 + .../cgroups/v2/invalid-period/cpu.max | 1 + .../testdata/cgroups/v2/only-max/cpu.max | 1 + .../cgroups/testdata/cgroups/v2/set/cpu.max | 1 + .../testdata/cgroups/v2/set/memory.max | 1 + .../cgroups/v2/too-few-fields/cpu.max | 1 + .../cgroups/v2/too-many-fields/cpu.max | 1 + .../cgroups/v2/too-many-fields/memory.max | 1 + .../cgroups/testdata/cgroups/v2/unset/cpu.max | 1 + .../testdata/cgroups/v2/unset/memory.max | 1 + .../cg/cgroups/testdata/proc/cgroup/cgroup | 3 + .../testdata/proc/cgroup/cgroup-no-match | 1 + .../cgroups/testdata/proc/cgroup/cgroup-root | 1 + .../testdata/proc/cgroup/cgroup-subdir | 1 + .../cg/cgroups/testdata/proc/cgroup/invalid | 2 + .../testdata/proc/cgroup/untranslatable | 2 + .../cgroups/testdata/proc/mountinfo/invalid | 1 + .../cgroups/testdata/proc/mountinfo/mountinfo | 8 + .../testdata/proc/mountinfo/mountinfo-v2 | 2 + .../testdata/proc/mountinfo/nonexistent | 0 .../testdata/proc/mountinfo/untranslatable | 2 + internal/cg/cgroups/util_test.go | 39 ++++ internal/cg/quota_linux.go | 57 ++++++ internal/cg/quota_unsupported.go | 31 +++ readme.md | 2 +- util.go | 55 ++---- 52 files changed, 1699 insertions(+), 50 deletions(-) create mode 100644 internal/cg/cgroups/cgroup.go create mode 100644 internal/cg/cgroups/cgroup_test.go create mode 100644 internal/cg/cgroups/cgroups.go create mode 100644 internal/cg/cgroups/cgroups2.go create mode 100644 internal/cg/cgroups/cgroups2_test.go create mode 100644 internal/cg/cgroups/cgroups_test.go create mode 100644 internal/cg/cgroups/load.go create mode 100644 internal/cg/cgroups/load_test.go create mode 100644 internal/cg/cgroups/mountpoint.go create mode 100644 internal/cg/cgroups/mountpoint_test.go create mode 100644 internal/cg/cgroups/subsys.go create mode 100644 internal/cg/cgroups/subsys_test.go create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/empty/cpu.cfs_quota_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/invalid/cpu.cfs_quota_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_period_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_quota_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined-period/cpu.cfs_quota_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_period_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_quota_us create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/memory/set/memory.limit_in_bytes create mode 100644 internal/cg/cgroups/testdata/cgroups/v1/memory/undefined/memory.limit_in_bytes create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/empty/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/empty/memory.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/invalid-max/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/invalid-max/memory.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/invalid-period/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/only-max/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/set/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/set/memory.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/too-few-fields/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/memory.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/unset/cpu.max create mode 100644 internal/cg/cgroups/testdata/cgroups/v2/unset/memory.max create mode 100644 internal/cg/cgroups/testdata/proc/cgroup/cgroup create mode 100644 internal/cg/cgroups/testdata/proc/cgroup/cgroup-no-match create mode 100644 internal/cg/cgroups/testdata/proc/cgroup/cgroup-root create mode 100644 internal/cg/cgroups/testdata/proc/cgroup/cgroup-subdir create mode 100644 internal/cg/cgroups/testdata/proc/cgroup/invalid create mode 100644 internal/cg/cgroups/testdata/proc/cgroup/untranslatable create mode 100644 internal/cg/cgroups/testdata/proc/mountinfo/invalid create mode 100644 internal/cg/cgroups/testdata/proc/mountinfo/mountinfo create mode 100644 internal/cg/cgroups/testdata/proc/mountinfo/mountinfo-v2 create mode 100644 internal/cg/cgroups/testdata/proc/mountinfo/nonexistent create mode 100644 internal/cg/cgroups/testdata/proc/mountinfo/untranslatable create mode 100644 internal/cg/cgroups/util_test.go create mode 100644 internal/cg/quota_linux.go create mode 100644 internal/cg/quota_unsupported.go diff --git a/consts.go b/consts.go index 8648bf6..3354c62 100644 --- a/consts.go +++ b/consts.go @@ -92,12 +92,6 @@ var check2name = map[configureType]string{ gcHeap: "GCHeap", } -const ( - cgroupMemLimitPath = "/sys/fs/cgroup/memory/memory.limit_in_bytes" - cgroupCpuQuotaPath = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us" - cgroupCpuPeriodPath = "/sys/fs/cgroup/cpu/cpu.cfs_period_us" -) - const minCollectCyclesBeforeDumpStart = 10 const ( diff --git a/doc/zh.md b/doc/zh.md index 2580aa8..371ef21 100644 --- a/doc/zh.md +++ b/doc/zh.md @@ -235,7 +235,7 @@ h, _ := holmes.New( ``` ### 在docker 或者cgroup环境下运行 holmes - +支持 Cgroup V1, V2 两个版本, 但暂不支持混用。 ```go h, _ := holmes.New( holmes.WithCollectInterval("5s"), diff --git a/internal/cg/cgroups/cgroup.go b/internal/cg/cgroups/cgroup.go new file mode 100644 index 0000000..6fc9fd7 --- /dev/null +++ b/internal/cg/cgroups/cgroup.go @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "bufio" + "io" + "os" + "path/filepath" + "strconv" +) + +// CGroup represents the data structure for a Linux control group. +type CGroup struct { + path string +} + +// NewCGroup returns a new *CGroup from a given path. +func NewCGroup(path string) CGroup { + return CGroup{path: path} +} + +// Path returns the path of the CGroup*. +func (cg *CGroup) Path() string { + return cg.path +} + +// ParamPath returns the path of the given cgroup param under itself. +func (cg *CGroup) ParamPath(param string) string { + return filepath.Join(cg.path, param) +} + +// readFirstLine reads the first line from a cgroup param file. +func (cg *CGroup) readFirstLine(param string) (string, error) { + paramFile, err := os.Open(cg.ParamPath(param)) + if err != nil { + return "", err + } + defer paramFile.Close() // nolint: errcheck + + scanner := bufio.NewScanner(paramFile) + if scanner.Scan() { + return scanner.Text(), nil + } + if err := scanner.Err(); err != nil { + return "", err + } + return "", io.ErrUnexpectedEOF +} + +// readInt parses the first line from a cgroup param file as int. +func (cg *CGroup) readInt(param string) (int, error) { + text, err := cg.readFirstLine(param) + if err != nil { + return 0, err + } + return strconv.Atoi(text) +} diff --git a/internal/cg/cgroups/cgroup_test.go b/internal/cg/cgroups/cgroup_test.go new file mode 100644 index 0000000..e6e4a04 --- /dev/null +++ b/internal/cg/cgroups/cgroup_test.go @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCGroupParamPath(t *testing.T) { + cgroup := NewCGroup("/sys/fs/cgroup/cpu") + assert.Equal(t, "/sys/fs/cgroup/cpu", cgroup.Path()) + assert.Equal(t, "/sys/fs/cgroup/cpu/cpu.cfs_quota_us", cgroup.ParamPath("cpu.cfs_quota_us")) +} + +func TestCGroupReadFirstLine(t *testing.T) { + testTable := []struct { + name string + paramName string + expectedContent string + shouldHaveError bool + }{ + { + name: "set", + paramName: "cpu.cfs_period_us", + expectedContent: "100000", + shouldHaveError: false, + }, + { + name: "absent", + paramName: "cpu.stat", + expectedContent: "", + shouldHaveError: true, + }, + { + name: "empty", + paramName: "cpu.cfs_quota_us", + expectedContent: "", + shouldHaveError: true, + }, + } + + for _, tt := range testTable { + cgroupPath := filepath.Join(testDataCGroupsPath, "v1", "cpu", tt.name) + cgroup := NewCGroup(cgroupPath) + + content, err := cgroup.readFirstLine(tt.paramName) + assert.Equal(t, tt.expectedContent, content, tt.name) + + if tt.shouldHaveError { + assert.Error(t, err, tt.name) + } else { + assert.NoError(t, err, tt.name) + } + } +} + +func TestCGroupReadInt(t *testing.T) { + testTable := []struct { + name string + paramName string + expectedValue int + shouldHaveError bool + }{ + { + name: "set", + paramName: "cpu.cfs_period_us", + expectedValue: 100000, + shouldHaveError: false, + }, + { + name: "empty", + paramName: "cpu.cfs_quota_us", + expectedValue: 0, + shouldHaveError: true, + }, + { + name: "invalid", + paramName: "cpu.cfs_quota_us", + expectedValue: 0, + shouldHaveError: true, + }, + { + name: "absent", + paramName: "cpu.cfs_quota_us", + expectedValue: 0, + shouldHaveError: true, + }, + } + + for _, tt := range testTable { + cgroupPath := filepath.Join(testDataCGroupsPath, "v1", "cpu", tt.name) + cgroup := NewCGroup(cgroupPath) + + value, err := cgroup.readInt(tt.paramName) + assert.Equal(t, tt.expectedValue, value, "%s/%s", tt.name, tt.paramName) + + if tt.shouldHaveError { + assert.Error(t, err, tt.name) + } else { + assert.NoError(t, err, tt.name) + } + } +} diff --git a/internal/cg/cgroups/cgroups.go b/internal/cg/cgroups/cgroups.go new file mode 100644 index 0000000..60c2bd0 --- /dev/null +++ b/internal/cg/cgroups/cgroups.go @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +const ( + // _cgroupFSType is the Linux CGroup file system type used in + // `/proc/$PID/mountinfo`. + _cgroupFSType = "cgroup" + // _cgroupSubsysCPU is the CPU CGroup subsystem. + _cgroupSubsysCPU = "cpu" + // _cgroupSubsysCPUAcct is the CPU accounting CGroup subsystem. + _cgroupSubsysCPUAcct = "cpuacct" // nolint:unused + // _cgroupSubsysCPUSet is the CPUSet CGroup subsystem. + _cgroupSubsysCPUSet = "cpuset" // nolint:unused + // _cgroupSubsysMemory is the Memory CGroup subsystem. + _cgroupSubsysMemory = "memory" + + // _cgroupCPUCFSQuotaUsParam is the file name for the CGroup CFS quota + // parameter. + _cgroupCPUCFSQuotaUsParam = "cpu.cfs_quota_us" + // _cgroupCPUCFSPeriodUsParam is the file name for the CGroup CFS period + // parameter. + _cgroupCPUCFSPeriodUsParam = "cpu.cfs_period_us" + // _cgroupMemLimitParam is the file name for the CGroup CFS memory + // parameter. + _cgroupMemLimitParam = "memory.limit_in_bytes" +) + +// CGroups is a map that associates each CGroup with its subsystem name. +type CGroups map[string]CGroup + +func newCGroups(mountInfo []*MountPoint, subsystems map[string]*CGroupSubsys) (CGroups, error) { + cgroups := make(CGroups) + for _, mp := range mountInfo { + if mp.FSType != _cgroupFSType { + continue + } + for _, opt := range mp.SuperOptions { + subsys, exists := subsystems[opt] + if !exists { + continue + } + + cgroupPath, err := mp.Translate(subsys.Name) + if err != nil { + return nil, err + } + cgroups[opt] = NewCGroup(cgroupPath) + } + + } + return cgroups, nil +} + +// MemLimit returns the memory limit with memory cgroup controller. +// `memory.max` wat not set, the method returns `(-1, nil)` +func (cg CGroups) MemLimit() (int, bool, error) { + memCGroup, ok := cg[_cgroupSubsysMemory] + if !ok { + return -1, false, nil + } + memLimit, err := memCGroup.readInt(_cgroupMemLimitParam) + if defined := memLimit > 0; err != nil || !defined { + return -1, defined, err + } + return memLimit, true, nil +} + +// CPUQuota returns the CPU quota applied with the CPU cgroup controller. +// It is a result of `cpu.cfs_quota_us / cpu.cfs_period_us`. If the value of +// `cpu.cfs_quota_us` was not set (-1), the method returns `(-1, nil)`. +func (cg CGroups) CPUQuota() (float64, bool, error) { + cpuCGroup, exists := cg[_cgroupSubsysCPU] + if !exists { + return -1, false, nil + } + + cfsQuotaUs, err := cpuCGroup.readInt(_cgroupCPUCFSQuotaUsParam) + if defined := cfsQuotaUs > 0; err != nil || !defined { + return -1, defined, err + } + + cfsPeriodUs, err := cpuCGroup.readInt(_cgroupCPUCFSPeriodUsParam) + if err != nil { + return -1, false, err + } + + return float64(cfsQuotaUs) / float64(cfsPeriodUs), true, nil +} + +func (cg CGroups) Version() string { + return _cgroupFSType +} diff --git a/internal/cg/cgroups/cgroups2.go b/internal/cg/cgroups/cgroups2.go new file mode 100644 index 0000000..e4ca4b7 --- /dev/null +++ b/internal/cg/cgroups/cgroups2.go @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path" + "strconv" + "strings" +) + +const ( + // _cgroupv2CPUMax is the file name for the CGroup-V2 CPU max and period + // parameter. + _cgroupv2CPUMax = "cpu.max" + _cgroupv2MEMMax = "memory.max" + // _cgroupFSType is the Linux CGroup-V2 file system type used in + // `/proc/$PID/mountinfo`. + _cgroupv2FSType = "cgroup2" + + _cgroupV2CPUMaxDefaultPeriod = 100000 + _cgroupV2CPUMaxQuotaMax = "max" + _cgroupV2MEMMaxDefault = "max" +) + +const ( + _cgroupv2CPUMaxQuotaIndex = iota + _cgroupv2CPUMaxPeriodIndex +) + +// ErrNotV2 indicates that the system is not using cgroups2. +var ErrNotV2 = errors.New("not using cgroups2") + +// CGroups2 provides access to cgroups data for systems using cgroups2. +type CGroups2 struct { + mountPoint string + groupPath string +} + +func newCGroups2(mountInfo *MountPoint, subsystems map[string]*CGroupSubsys) (*CGroups2, error) { + // Find v2 subsystem by looking for the `0` id + var v2subsys *CGroupSubsys + for _, subsys := range subsystems { + if subsys.ID == 0 { + v2subsys = subsys + break + } + } + + if v2subsys == nil { + return nil, ErrNotV2 + } + + return &CGroups2{ + mountPoint: mountInfo.MountPoint, + groupPath: v2subsys.Name, + }, nil +} + +// MemLimit returns the memory limit with memory cgroup controller. +// `memory.max` wat not set, the method returns `(-1, nil)` +func (cg *CGroups2) MemLimit() (int, bool, error) { + memMaxParams, err := os.Open(path.Join(cg.mountPoint, cg.groupPath, _cgroupv2MEMMax)) + if err != nil { + if os.IsNotExist(err) { + return -1, false, nil + } + return -1, false, err + } + defer memMaxParams.Close() // nolint: errcheck + + scanner := bufio.NewScanner(memMaxParams) + if scanner.Scan() { + text := scanner.Text() + if text == _cgroupV2MEMMaxDefault { + return -1, false, nil + } + max, err := strconv.Atoi(scanner.Text()) + if err != nil { + return -1, false, fmt.Errorf("parse max memory failed, invalid format. %w", err) + } + return max, true, nil + } + if err = scanner.Err(); err != nil { + return -1, false, err + } + + return 0, false, io.ErrUnexpectedEOF +} + +// CPUQuota returns the CPU quota applied with the CPU cgroup2 controller. +// It is a result of reading cpu quota and period from cpu.max file. +// It will return `cpu.max / cpu.period`. If cpu.max is set to max, it returns +// (-1, false, nil) +func (cg *CGroups2) CPUQuota() (float64, bool, error) { + cpuMaxParams, err := os.Open(path.Join(cg.mountPoint, cg.groupPath, _cgroupv2CPUMax)) + if err != nil { + if os.IsNotExist(err) { + return -1, false, nil + } + return -1, false, err + } + defer cpuMaxParams.Close() // nolint: errcheck + + scanner := bufio.NewScanner(cpuMaxParams) + if scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) == 0 || len(fields) > 2 { + return -1, false, fmt.Errorf("invalid format") + } + + if fields[_cgroupv2CPUMaxQuotaIndex] == _cgroupV2CPUMaxQuotaMax { + return -1, false, nil + } + + max, err := strconv.Atoi(fields[_cgroupv2CPUMaxQuotaIndex]) + if err != nil { + return -1, false, err + } + + var period int + if len(fields) == 1 { + period = _cgroupV2CPUMaxDefaultPeriod + } else { + period, err = strconv.Atoi(fields[_cgroupv2CPUMaxPeriodIndex]) + if err != nil { + return -1, false, err + } + } + + return float64(max) / float64(period), true, nil + } + + if err = scanner.Err(); err != nil { + return -1, false, err + } + + return 0, false, io.ErrUnexpectedEOF +} + +// Version return version of cgroupfs. +func (cg *CGroups2) Version() string { + return _cgroupv2FSType +} diff --git a/internal/cg/cgroups/cgroups2_test.go b/internal/cg/cgroups/cgroups2_test.go new file mode 100644 index 0000000..e83e8ee --- /dev/null +++ b/internal/cg/cgroups/cgroups2_test.go @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCGroupsCPUQuotaV2(t *testing.T) { + tests := []struct { + name string + want float64 + wantOK bool + wantErr string + }{ + { + name: "set", + want: 2.5, + wantOK: true, + }, + { + name: "unset", + want: -1.0, + wantOK: false, + }, + { + name: "only-max", + want: 5.0, + wantOK: true, + }, + { + name: "invalid-max", + wantErr: `parsing "asdf": invalid syntax`, + }, + { + name: "invalid-period", + wantErr: `parsing "njn": invalid syntax`, + }, + { + name: "nonexistent", + want: -1.0, + wantOK: false, + }, + { + name: "empty", + wantErr: "unexpected EOF", + }, + { + name: "too-few-fields", + wantErr: "invalid format", + }, + { + name: "too-many-fields", + wantErr: "invalid format", + }, + } + + mountPoint := filepath.Join(testDataCGroupsPath, "v2") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + quota, defined, err := (&CGroups2{ + mountPoint: mountPoint, + groupPath: tt.name, + }).CPUQuota() + + if len(tt.wantErr) > 0 { + require.Error(t, err, tt.name) + assert.Contains(t, err.Error(), tt.wantErr) + } else { + require.NoError(t, err, tt.name) + assert.Equal(t, tt.want, quota, tt.name) + assert.Equal(t, tt.wantOK, defined, tt.name) + } + }) + } +} + +func TestCGroupsMemLimitV2(t *testing.T) { + tests := []struct { + name string + want int + wantOK bool + wantErr string + }{ + { + name: "set", + want: 2147483648, + wantOK: true, + }, + { + name: "unset", + want: -1, + wantOK: false, + }, + { + name: "invalid-max", + wantErr: `parsing "asdf": invalid syntax`, + }, + { + name: "nonexistent", + want: -1, + wantOK: false, + }, + { + name: "empty", + wantErr: "unexpected EOF", + }, + { + name: "too-many-fields", + wantErr: `parsing "250000 100000 100": invalid syntax`, + }, + } + + mountPoint := filepath.Join(testDataCGroupsPath, "v2") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + memLimit, defined, err := (&CGroups2{ + mountPoint: mountPoint, + groupPath: tt.name, + }).MemLimit() + + if len(tt.wantErr) > 0 { + require.Error(t, err, tt.name) + assert.Contains(t, err.Error(), tt.wantErr) + } else { + require.NoError(t, err, tt.name) + assert.Equal(t, tt.want, memLimit, tt.name) + assert.Equal(t, tt.wantOK, defined, tt.name) + } + }) + } +} diff --git a/internal/cg/cgroups/cgroups_test.go b/internal/cg/cgroups/cgroups_test.go new file mode 100644 index 0000000..fe531f0 --- /dev/null +++ b/internal/cg/cgroups/cgroups_test.go @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCGroupsCPUQuota(t *testing.T) { + testTable := []struct { + name string + expectedQuota float64 + expectedDefined bool + shouldHaveError bool + }{ + { + name: "set", + expectedQuota: 6.0, + expectedDefined: true, + shouldHaveError: false, + }, + { + name: "undefined", + expectedQuota: -1.0, + expectedDefined: false, + shouldHaveError: false, + }, + { + name: "undefined-period", + expectedQuota: -1.0, + expectedDefined: false, + shouldHaveError: true, + }, + } + + cgroups := make(CGroups) + + quota, defined, err := cgroups.CPUQuota() + assert.Equal(t, -1.0, quota, "nonexistent") + assert.Equal(t, false, defined, "nonexistent") + assert.NoError(t, err, "nonexistent") + + for _, tt := range testTable { + cgroupPath := filepath.Join(testDataCGroupsPath, "v1", "cpu", tt.name) + cgroups[_cgroupSubsysCPU] = NewCGroup(cgroupPath) + + quota, defined, err := cgroups.CPUQuota() + assert.Equal(t, tt.expectedQuota, quota, tt.name) + assert.Equal(t, tt.expectedDefined, defined, tt.name) + + if tt.shouldHaveError { + assert.Error(t, err, tt.name) + } else { + assert.NoError(t, err, tt.name) + } + } +} + +func TestCGroupsMemLimit(t *testing.T) { + testTable := []struct { + name string + expectedLimit int + expectedDefined bool + shouldHaveError bool + }{ + { + name: "set", + expectedLimit: 2147483648, + expectedDefined: true, + shouldHaveError: false, + }, + { + name: "undefined", + expectedLimit: -1, + expectedDefined: false, + shouldHaveError: false, + }, + } + + cgroups := make(CGroups) + + quota, defined, err := cgroups.MemLimit() + assert.Equal(t, -1, quota, "nonexistent") + assert.Equal(t, false, defined, "nonexistent") + assert.NoError(t, err, "nonexistent") + + for _, tt := range testTable { + cgroupPath := filepath.Join(testDataCGroupsPath, "v1", "memory", tt.name) + cgroups[_cgroupSubsysMemory] = NewCGroup(cgroupPath) + + quota, defined, err := cgroups.MemLimit() + assert.Equal(t, tt.expectedLimit, quota, tt.name) + assert.Equal(t, tt.expectedDefined, defined, tt.name) + + if tt.shouldHaveError { + assert.Error(t, err, tt.name) + } else { + assert.NoError(t, err, tt.name) + } + } +} diff --git a/internal/cg/cgroups/load.go b/internal/cg/cgroups/load.go new file mode 100644 index 0000000..2a04f41 --- /dev/null +++ b/internal/cg/cgroups/load.go @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "errors" + "fmt" +) + +const ( + _procPathCGroup = "/proc/self/cgroup" + _procPathMountInfo = "/proc/self/mountinfo" +) + +// ErrCGroupFSNotFound indicates that the system is not using cgroups. +var ErrCGroupFSNotFound = errors.New("cgroupfs not found") + +type ICGroups interface { + // CPUQuota returns the CPU quota applied with the CPU cgroup controller. + // `cpu.cfs_quota_us` was not set, the method returns `(-1, nil)`. + CPUQuota() (float64, bool, error) + // MemLimit returns the memory limit with memory cgroup controller. + // `memory.max` wat not set, the method returns `(-1, nil)` + MemLimit() (int, bool, error) + // Version returns CGroup version. + Version() string +} + +func LoadCGroupsForCurrentProcess() (ICGroups, error) { + return loadCGroups(_procPathMountInfo, _procPathCGroup) +} + +func LoadCGroups(pid int) (ICGroups, error) { + return loadCGroups(fmt.Sprintf("/proc/%d/mountinfo", pid), fmt.Sprintf("/proc/%d/cgroup", pid)) +} + +func loadCGroups(mountInfoPath, procCGroupPath string) (ICGroups, error) { + mps, err := parseMountInfo(mountInfoPath) + if err != nil { + return nil, err + } + subsystems, err := parseCGroupSubsystems(procCGroupPath) + if err != nil { + return nil, err + } + + for _, mp := range mps { + if mp.FSType == _cgroupv2FSType { + return newCGroups2(mp, subsystems) + } + if mp.FSType == _cgroupFSType { + return newCGroups(mps, subsystems) + } + } + return nil, ErrCGroupFSNotFound +} diff --git a/internal/cg/cgroups/load_test.go b/internal/cg/cgroups/load_test.go new file mode 100644 index 0000000..81fad22 --- /dev/null +++ b/internal/cg/cgroups/load_test.go @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadCGroups(t *testing.T) { + cgroupsProcCGroupPath := filepath.Join(testDataProcPath, "cgroup", "cgroup") + cgroupsProcMountInfoPath := filepath.Join(testDataProcPath, "mountinfo", "mountinfo") + + testTable := []struct { + subsys string + path string + }{ + {_cgroupSubsysCPU, "/sys/fs/cgroup/cpu,cpuacct"}, + {_cgroupSubsysCPUAcct, "/sys/fs/cgroup/cpu,cpuacct"}, + {_cgroupSubsysCPUSet, "/sys/fs/cgroup/cpuset"}, + {_cgroupSubsysMemory, "/sys/fs/cgroup/memory/large"}, + } + + icg, err := loadCGroups(cgroupsProcMountInfoPath, cgroupsProcCGroupPath) + assert.NoError(t, err) + assert.Equal(t, _cgroupFSType, icg.Version()) + + cgroups, ok := icg.(CGroups) + assert.True(t, ok) + assert.Equal(t, len(testTable), len(cgroups)) + assert.NoError(t, err) + + for _, tt := range testTable { + cgroup, exists := cgroups[tt.subsys] + assert.Equal(t, true, exists, "%q expected to present in `cgroups`", tt.subsys) + assert.Equal(t, tt.path, cgroup.path, "%q expected for `cgroups[%q].path`, got %q", tt.path, tt.subsys, cgroup.path) + } +} + +func TestLoadCGroups2(t *testing.T) { + tests := []struct { + procCgroup string + wantPath string + wantError error + }{ + { + procCgroup: "cgroup-no-match", + wantError: ErrNotV2, + }, + { + procCgroup: "cgroup-root", + wantPath: "/", + }, + { + procCgroup: "cgroup-subdir", + wantPath: "/Example", + }, + } + + for _, tt := range tests { + t.Run(tt.procCgroup, func(t *testing.T) { + mountInfoPath := filepath.Join(testDataProcPath, "mountinfo", "mountinfo-v2") + procCgroupPath := filepath.Join(testDataProcPath, "cgroup", tt.procCgroup) + cg, err := loadCGroups(mountInfoPath, procCgroupPath) + if tt.wantError == nil { + require.NoError(t, err) + assert.Equal(t, _cgroupv2FSType, cg.Version()) + cgroups, ok := cg.(*CGroups2) + assert.True(t, ok) + assert.Equal(t, tt.wantPath, cgroups.groupPath) + } else { + assert.ErrorIs(t, err, tt.wantError) + } + + }) + } +} + +func TestLoadCGroupsWrongFile(t *testing.T) { + tests := []struct { + name string + mountInfoPath string + procCGroupPath string + }{ + { + name: "mountInfoNotFound", + mountInfoPath: "non-existing-file", + procCGroupPath: "/dev/null", + }, + { + name: "procCGroupNotFound", + mountInfoPath: "/dev/null", + procCGroupPath: "non-existing-file", + }, + { + name: "invalid-cgroup", + mountInfoPath: "/dev/null", + procCGroupPath: filepath.Join(testDataProcPath, "cgroup", "invalid"), + }, + { + name: "invalid-mountinfo", + mountInfoPath: filepath.Join(testDataProcPath, "mountinfo", "invalid"), + procCGroupPath: "/dev/null", + }, + { + name: "untranslatable", + mountInfoPath: filepath.Join(testDataProcPath, "mountinfo", "untranslatable"), + procCGroupPath: filepath.Join(testDataProcPath, "cgroup", "untranslatable"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cg, err := loadCGroups(tt.mountInfoPath, tt.procCGroupPath) + assert.Nil(t, cg) + assert.Error(t, err) + }) + } +} diff --git a/internal/cg/cgroups/mountpoint.go b/internal/cg/cgroups/mountpoint.go new file mode 100644 index 0000000..f4ffdb9 --- /dev/null +++ b/internal/cg/cgroups/mountpoint.go @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" +) + +type mountPointFormatInvalidError struct { + line string +} + +func (err mountPointFormatInvalidError) Error() string { + return fmt.Sprintf("invalid format for MountPoint: %q", err.line) +} + +type pathNotExposedFromMountPointError struct { + mountPoint string + root string + path string +} + +func (err pathNotExposedFromMountPointError) Error() string { + return fmt.Sprintf("path %q is not a descendant of mount point root %q and cannot be exposed from %q", err.path, err.root, err.mountPoint) +} + +const ( + _mountInfoSep = " " + _mountInfoOptsSep = "," + _mountInfoOptionalFieldsSep = "-" +) + +const ( + _miFieldIDMountID = iota + _miFieldIDParentID + _miFieldIDDeviceID + _miFieldIDRoot + _miFieldIDMountPoint + _miFieldIDOptions + _miFieldIDOptionalFields + + _miFieldCountFirstHalf +) + +const ( + _miFieldOffsetFSType = iota + _miFieldOffsetMountSource + _miFieldOffsetSuperOptions + + _miFieldCountSecondHalf +) + +const _miFieldCountMin = _miFieldCountFirstHalf + _miFieldCountSecondHalf + +// MountPoint is the data structure for the mount points in +// `/proc/$PID/mountinfo`. See also proc(5) for more information. +type MountPoint struct { + MountID int + ParentID int + DeviceID string + Root string + MountPoint string + Options []string + OptionalFields []string + FSType string + MountSource string + SuperOptions []string +} + +// NewMountPointFromLine parses a line read from `/proc/$PID/mountinfo` and +// returns a new *MountPoint. +func NewMountPointFromLine(line string) (*MountPoint, error) { + fields := strings.Split(line, _mountInfoSep) + + if len(fields) < _miFieldCountMin { + return nil, mountPointFormatInvalidError{line} + } + + mountID, err := strconv.Atoi(fields[_miFieldIDMountID]) + if err != nil { + return nil, err + } + + parentID, err := strconv.Atoi(fields[_miFieldIDParentID]) + if err != nil { + return nil, err + } + + for i, field := range fields[_miFieldIDOptionalFields:] { + if field == _mountInfoOptionalFieldsSep { + fsTypeStart := _miFieldIDOptionalFields + i + 1 + + if len(fields) != fsTypeStart+_miFieldCountSecondHalf { + return nil, mountPointFormatInvalidError{line} + } + + miFieldIDFSType := _miFieldOffsetFSType + fsTypeStart + miFieldIDMountSource := _miFieldOffsetMountSource + fsTypeStart + miFieldIDSuperOptions := _miFieldOffsetSuperOptions + fsTypeStart + + return &MountPoint{ + MountID: mountID, + ParentID: parentID, + DeviceID: fields[_miFieldIDDeviceID], + Root: fields[_miFieldIDRoot], + MountPoint: fields[_miFieldIDMountPoint], + Options: strings.Split(fields[_miFieldIDOptions], _mountInfoOptsSep), + OptionalFields: fields[_miFieldIDOptionalFields:(fsTypeStart - 1)], + FSType: fields[miFieldIDFSType], + MountSource: fields[miFieldIDMountSource], + SuperOptions: strings.Split(fields[miFieldIDSuperOptions], _mountInfoOptsSep), + }, nil + } + } + + return nil, mountPointFormatInvalidError{line} +} + +// Translate converts an absolute path inside the *MountPoint's file system to +// the host file system path in the mount namespace the *MountPoint belongs to. +func (mp *MountPoint) Translate(absPath string) (string, error) { + relPath, err := filepath.Rel(mp.Root, absPath) + + if err != nil { + return "", err + } + if relPath == ".." || strings.HasPrefix(relPath, "../") { + return "", pathNotExposedFromMountPointError{ + mountPoint: mp.MountPoint, + root: mp.Root, + path: absPath, + } + } + + return filepath.Join(mp.MountPoint, relPath), nil +} + +// parseMountInfo parses procPathMountInfo (usually at `/proc/$PID/mountinfo`) +// and yields parsed *MountPoint into newMountPoint. +func parseMountInfo(procPathMountInfo string) ([]*MountPoint, error) { + mountInfoFile, err := os.Open(procPathMountInfo) + if err != nil { + return nil, err + } + defer mountInfoFile.Close() // nolint: errcheck + + scanner := bufio.NewScanner(mountInfoFile) + + mps := make([]*MountPoint, 0, 10) + for scanner.Scan() { + mountPoint, err := NewMountPointFromLine(scanner.Text()) + if err != nil { + return nil, err + } + mps = append(mps, mountPoint) + } + + return mps, nil +} diff --git a/internal/cg/cgroups/mountpoint_test.go b/internal/cg/cgroups/mountpoint_test.go new file mode 100644 index 0000000..4b8926b --- /dev/null +++ b/internal/cg/cgroups/mountpoint_test.go @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewMountPointFromLine(t *testing.T) { + testTable := []struct { + name string + line string + expected *MountPoint + }{ + { + name: "root", + line: "1 0 252:0 / / rw,noatime - ext4 /dev/dm-0 rw,errors=remount-ro,data=ordered", + expected: &MountPoint{ + MountID: 1, + ParentID: 0, + DeviceID: "252:0", + Root: "/", + MountPoint: "/", + Options: []string{"rw", "noatime"}, + OptionalFields: []string{}, + FSType: "ext4", + MountSource: "/dev/dm-0", + SuperOptions: []string{"rw", "errors=remount-ro", "data=ordered"}, + }, + }, + { + name: "cgroup", + line: "31 23 0:24 /docker /sys/fs/cgroup/cpu rw,nosuid,nodev,noexec,relatime shared:1 - cgroup cgroup rw,cpu", + expected: &MountPoint{ + MountID: 31, + ParentID: 23, + DeviceID: "0:24", + Root: "/docker", + MountPoint: "/sys/fs/cgroup/cpu", + Options: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}, + OptionalFields: []string{"shared:1"}, + FSType: "cgroup", + MountSource: "cgroup", + SuperOptions: []string{"rw", "cpu"}, + }, + }, + } + + for _, tt := range testTable { + mountPoint, err := NewMountPointFromLine(tt.line) + assert.Equal(t, tt.expected, mountPoint, tt.name) + assert.NoError(t, err, tt.name) + } +} + +func TestNewMountPointFromLineErr(t *testing.T) { + linesWithInvalidIDs := []string{ + "invalidMountID 0 252:0 / / rw,noatime - ext4 /dev/dm-0 rw,errors=remount-ro,data=ordered", + "1 invalidParentID 252:0 / / rw,noatime - ext4 /dev/dm-0 rw,errors=remount-ro,data=ordered", + "invalidMountID invalidParentID 252:0 / / rw,noatime - ext4 /dev/dm-0 rw,errors=remount-ro,data=ordered", + } + + for i, line := range linesWithInvalidIDs { + mountPoint, err := NewMountPointFromLine(line) + assert.Nil(t, mountPoint, "[%d] %q", i, line) + assert.Error(t, err, line) + } + + linesWithInvalidFields := []string{ + "1 0 252:0 / / rw,noatime ext4 /dev/dm-0 rw,errors=remount-ro,data=ordered", + "1 0 252:0 / / rw,noatime shared:1 - ext4 /dev/dm-0", + "1 0 252:0 / / rw,noatime shared:1 ext4 - /dev/dm-0 rw,errors=remount-ro,data=ordered", + "1 0 252:0 / / rw,noatime shared:1 ext4 /dev/dm-0 rw,errors=remount-ro,data=ordered", + "random line", + } + + for i, line := range linesWithInvalidFields { + mountPoint, err := NewMountPointFromLine(line) + errExpected := mountPointFormatInvalidError{line} + + assert.Nil(t, mountPoint, "[%d] %q", i, line) + assert.Equal(t, err, errExpected, "[%d] %q", i, line) + } +} + +func TestMountPointTranslate(t *testing.T) { + line := "31 23 0:24 /docker/0123456789abcdef /sys/fs/cgroup/cpu rw,nosuid,nodev,noexec,relatime shared:1 - cgroup cgroup rw,cpu" + cgroupMountPoint, err := NewMountPointFromLine(line) + + assert.NotNil(t, cgroupMountPoint) + assert.NoError(t, err) + + testTable := []struct { + name string + pathToTranslate string + pathTranslated string + }{ + { + name: "root", + pathToTranslate: "/docker/0123456789abcdef", + pathTranslated: "/sys/fs/cgroup/cpu", + }, + { + name: "root-with-extra-slash", + pathToTranslate: "/docker/0123456789abcdef/", + pathTranslated: "/sys/fs/cgroup/cpu", + }, + { + name: "descendant-from-root", + pathToTranslate: "/docker/0123456789abcdef/large/cpu.cfs_quota_us", + pathTranslated: "/sys/fs/cgroup/cpu/large/cpu.cfs_quota_us", + }, + } + + for _, tt := range testTable { + path, err := cgroupMountPoint.Translate(tt.pathToTranslate) + assert.Equal(t, tt.pathTranslated, path, tt.name) + assert.NoError(t, err, tt.name) + } +} + +func TestMountPointTranslateError(t *testing.T) { + line := "31 23 0:24 /docker/0123456789abcdef /sys/fs/cgroup/cpu rw,nosuid,nodev,noexec,relatime shared:1 - cgroup cgroup rw,cpu" + cgroupMountPoint, err := NewMountPointFromLine(line) + + assert.NotNil(t, cgroupMountPoint) + assert.NoError(t, err) + + inaccessiblePaths := []string{ + "/", + "/docker", + "/docker/0123456789abcdef-let-me-hack-this-path", + "/docker/0123456789abcde/abc/../../def", + "/system.slice/docker.service", + } + + for i, path := range inaccessiblePaths { + translated, err := cgroupMountPoint.Translate(path) + errExpected := pathNotExposedFromMountPointError{ + mountPoint: cgroupMountPoint.MountPoint, + root: cgroupMountPoint.Root, + path: path, + } + + assert.Equal(t, "", translated, "inaccessiblePaths[%d] == %q", i, path) + assert.Equal(t, errExpected, err, "inaccessiblePaths[%d] == %q", i, path) + } + + relPaths := []string{ + "docker", + "docker/0123456789abcde/large", + "system.slice/docker.service", + } + + for i, path := range relPaths { + translated, err := cgroupMountPoint.Translate(path) + + assert.Equal(t, "", translated, "relPaths[%d] == %q", i, path) + assert.Error(t, err, path) + } +} diff --git a/internal/cg/cgroups/subsys.go b/internal/cg/cgroups/subsys.go new file mode 100644 index 0000000..bf9fb7c --- /dev/null +++ b/internal/cg/cgroups/subsys.go @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" +) + +const ( + _cgroupSep = ":" + _cgroupSubsysSep = "," +) + +const ( + _csFieldIDID = iota + _csFieldIDSubsystems + _csFieldIDName + _csFieldCount +) + +type cgroupSubsysFormatInvalidError struct { + line string +} + +func (err cgroupSubsysFormatInvalidError) Error() string { + return fmt.Sprintf("invalid format for CGroupSubsys: %q", err.line) +} + +// CGroupSubsys represents the data structure for entities in +// `/proc/$PID/cgroup`. See also proc(5) for more information. +type CGroupSubsys struct { + ID int + Subsystems []string + Name string +} + +// NewCGroupSubsysFromLine returns a new *CGroupSubsys by parsing a string in +// the format of `/proc/$PID/cgroup` +func NewCGroupSubsysFromLine(line string) (*CGroupSubsys, error) { + fields := strings.SplitN(line, _cgroupSep, _csFieldCount) + + if len(fields) != _csFieldCount { + return nil, cgroupSubsysFormatInvalidError{line} + } + + id, err := strconv.Atoi(fields[_csFieldIDID]) + if err != nil { + return nil, err + } + + cgroup := &CGroupSubsys{ + ID: id, + Subsystems: strings.Split(fields[_csFieldIDSubsystems], _cgroupSubsysSep), + Name: fields[_csFieldIDName], + } + + return cgroup, nil +} + +// parseCGroupSubsystems parses procPathCGroup (usually at `/proc/$PID/cgroup`) +// and returns a new map[string]*CGroupSubsys. +func parseCGroupSubsystems(procPathCGroup string) (map[string]*CGroupSubsys, error) { + cgroupFile, err := os.Open(procPathCGroup) + if err != nil { + return nil, err + } + defer cgroupFile.Close() // nolint: errcheck + + scanner := bufio.NewScanner(cgroupFile) + subsystems := make(map[string]*CGroupSubsys) + + for scanner.Scan() { + cgroup, err := NewCGroupSubsysFromLine(scanner.Text()) + if err != nil { + return nil, err + } + for _, subsys := range cgroup.Subsystems { + subsystems[subsys] = cgroup + } + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return subsystems, nil +} diff --git a/internal/cg/cgroups/subsys_test.go b/internal/cg/cgroups/subsys_test.go new file mode 100644 index 0000000..5ad0bae --- /dev/null +++ b/internal/cg/cgroups/subsys_test.go @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "strconv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewCGroupSubsysFromLine(t *testing.T) { + testTable := []struct { + name string + line string + expectedSubsys *CGroupSubsys + }{ + { + name: "single-subsys", + line: "1:cpu:/", + expectedSubsys: &CGroupSubsys{ + ID: 1, + Subsystems: []string{"cpu"}, + Name: "/", + }, + }, + { + name: "multi-subsys", + line: "8:cpu,cpuacct,cpuset:/docker/1234567890abcdef", + expectedSubsys: &CGroupSubsys{ + ID: 8, + Subsystems: []string{"cpu", "cpuacct", "cpuset"}, + Name: "/docker/1234567890abcdef", + }, + }, + { + name: "multi-subsys", + line: "12:cpu,cpuacct:/system.slice/containerd.service/kubepods-besteffort-podb41662f7_b03a_4c65_8ef9_6e4e55c3cf27.slice:cri-containerd:1753b7cbbf62734d812936961224d5bc0cf8f45214e0d5cdd1a781a053e7c48f", + expectedSubsys: &CGroupSubsys{ + ID: 12, + Subsystems: []string{"cpu", "cpuacct"}, + Name: "/system.slice/containerd.service/kubepods-besteffort-podb41662f7_b03a_4c65_8ef9_6e4e55c3cf27.slice:cri-containerd:1753b7cbbf62734d812936961224d5bc0cf8f45214e0d5cdd1a781a053e7c48f", + }, + }, + } + + for _, tt := range testTable { + subsys, err := NewCGroupSubsysFromLine(tt.line) + assert.Equal(t, tt.expectedSubsys, subsys, tt.name) + assert.NoError(t, err, tt.name) + } +} + +func TestNewCGroupSubsysFromLineErr(t *testing.T) { + lines := []string{ + "1:cpu", + "not-a-number:cpu:/", + } + _, parseError := strconv.Atoi("not-a-number") + + testTable := []struct { + name string + line string + expectedError error + }{ + { + name: "fewer-fields", + line: lines[0], + expectedError: cgroupSubsysFormatInvalidError{lines[0]}, + }, + { + name: "illegal-id", + line: lines[1], + expectedError: parseError, + }, + } + + for _, tt := range testTable { + subsys, err := NewCGroupSubsysFromLine(tt.line) + assert.Nil(t, subsys, tt.name) + assert.Equal(t, tt.expectedError, err, tt.name) + } +} diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/empty/cpu.cfs_quota_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/empty/cpu.cfs_quota_us new file mode 100644 index 0000000..e69de29 diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/invalid/cpu.cfs_quota_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/invalid/cpu.cfs_quota_us new file mode 100644 index 0000000..f43dfb1 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/cpu/invalid/cpu.cfs_quota_us @@ -0,0 +1 @@ +non-an-integer diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_period_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_period_us new file mode 100644 index 0000000..f7393e8 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_quota_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_quota_us new file mode 100644 index 0000000..26f3b3d --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/cpu/set/cpu.cfs_quota_us @@ -0,0 +1 @@ +600000 diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined-period/cpu.cfs_quota_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined-period/cpu.cfs_quota_us new file mode 100644 index 0000000..959e88a --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined-period/cpu.cfs_quota_us @@ -0,0 +1 @@ +800000 diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_period_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_period_us new file mode 100644 index 0000000..f7393e8 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_quota_us b/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_quota_us new file mode 100644 index 0000000..3a2e3f4 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/cpu/undefined/cpu.cfs_quota_us @@ -0,0 +1 @@ +-1 diff --git a/internal/cg/cgroups/testdata/cgroups/v1/memory/set/memory.limit_in_bytes b/internal/cg/cgroups/testdata/cgroups/v1/memory/set/memory.limit_in_bytes new file mode 100644 index 0000000..1ed5441 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/memory/set/memory.limit_in_bytes @@ -0,0 +1 @@ +2147483648 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v1/memory/undefined/memory.limit_in_bytes b/internal/cg/cgroups/testdata/cgroups/v1/memory/undefined/memory.limit_in_bytes new file mode 100644 index 0000000..d7d17fc --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v1/memory/undefined/memory.limit_in_bytes @@ -0,0 +1 @@ +-1 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/empty/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/empty/cpu.max new file mode 100644 index 0000000..e69de29 diff --git a/internal/cg/cgroups/testdata/cgroups/v2/empty/memory.max b/internal/cg/cgroups/testdata/cgroups/v2/empty/memory.max new file mode 100644 index 0000000..e69de29 diff --git a/internal/cg/cgroups/testdata/cgroups/v2/invalid-max/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/invalid-max/cpu.max new file mode 100644 index 0000000..abb2a1c --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/invalid-max/cpu.max @@ -0,0 +1 @@ +asdf 100000 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/invalid-max/memory.max b/internal/cg/cgroups/testdata/cgroups/v2/invalid-max/memory.max new file mode 100644 index 0000000..5e40c08 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/invalid-max/memory.max @@ -0,0 +1 @@ +asdf \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/invalid-period/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/invalid-period/cpu.max new file mode 100644 index 0000000..f15ab25 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/invalid-period/cpu.max @@ -0,0 +1 @@ +500000 njn \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/only-max/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/only-max/cpu.max new file mode 100644 index 0000000..516a58a --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/only-max/cpu.max @@ -0,0 +1 @@ +500000 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/set/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/set/cpu.max new file mode 100644 index 0000000..fa494ca --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/set/cpu.max @@ -0,0 +1 @@ +250000 100000 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/set/memory.max b/internal/cg/cgroups/testdata/cgroups/v2/set/memory.max new file mode 100644 index 0000000..1ed5441 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/set/memory.max @@ -0,0 +1 @@ +2147483648 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/too-few-fields/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/too-few-fields/cpu.max new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/too-few-fields/cpu.max @@ -0,0 +1 @@ + diff --git a/internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/cpu.max new file mode 100644 index 0000000..320baa1 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/cpu.max @@ -0,0 +1 @@ +250000 100000 100 diff --git a/internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/memory.max b/internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/memory.max new file mode 100644 index 0000000..320baa1 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/too-many-fields/memory.max @@ -0,0 +1 @@ +250000 100000 100 diff --git a/internal/cg/cgroups/testdata/cgroups/v2/unset/cpu.max b/internal/cg/cgroups/testdata/cgroups/v2/unset/cpu.max new file mode 100644 index 0000000..f293e9b --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/unset/cpu.max @@ -0,0 +1 @@ +max 100000 \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/cgroups/v2/unset/memory.max b/internal/cg/cgroups/testdata/cgroups/v2/unset/memory.max new file mode 100644 index 0000000..7c35932 --- /dev/null +++ b/internal/cg/cgroups/testdata/cgroups/v2/unset/memory.max @@ -0,0 +1 @@ +max \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/proc/cgroup/cgroup b/internal/cg/cgroups/testdata/proc/cgroup/cgroup new file mode 100644 index 0000000..1724dc8 --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/cgroup/cgroup @@ -0,0 +1,3 @@ +3:memory:/docker/large +2:cpu,cpuacct:/docker +1:cpuset:/ diff --git a/internal/cg/cgroups/testdata/proc/cgroup/cgroup-no-match b/internal/cg/cgroups/testdata/proc/cgroup/cgroup-no-match new file mode 100644 index 0000000..537b6fa --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/cgroup/cgroup-no-match @@ -0,0 +1 @@ +1::/ diff --git a/internal/cg/cgroups/testdata/proc/cgroup/cgroup-root b/internal/cg/cgroups/testdata/proc/cgroup/cgroup-root new file mode 100644 index 0000000..62bb6b6 --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/cgroup/cgroup-root @@ -0,0 +1 @@ +0::/ \ No newline at end of file diff --git a/internal/cg/cgroups/testdata/proc/cgroup/cgroup-subdir b/internal/cg/cgroups/testdata/proc/cgroup/cgroup-subdir new file mode 100644 index 0000000..58d5e10 --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/cgroup/cgroup-subdir @@ -0,0 +1 @@ +0::/Example diff --git a/internal/cg/cgroups/testdata/proc/cgroup/invalid b/internal/cg/cgroups/testdata/proc/cgroup/invalid new file mode 100644 index 0000000..6d9b22b --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/cgroup/invalid @@ -0,0 +1,2 @@ +1:cpu:/cpu +invalid-line: diff --git a/internal/cg/cgroups/testdata/proc/cgroup/untranslatable b/internal/cg/cgroups/testdata/proc/cgroup/untranslatable new file mode 100644 index 0000000..4451966 --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/cgroup/untranslatable @@ -0,0 +1,2 @@ +1:cpu:/docker +2:cpuacct:/docker diff --git a/internal/cg/cgroups/testdata/proc/mountinfo/invalid b/internal/cg/cgroups/testdata/proc/mountinfo/invalid new file mode 100644 index 0000000..3c8dabe --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/mountinfo/invalid @@ -0,0 +1 @@ +1 0 8:1 / / rw,noatime shared:1 - ext4 /dev/sda1 diff --git a/internal/cg/cgroups/testdata/proc/mountinfo/mountinfo b/internal/cg/cgroups/testdata/proc/mountinfo/mountinfo new file mode 100644 index 0000000..e68af08 --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/mountinfo/mountinfo @@ -0,0 +1,8 @@ +1 0 8:1 / / rw,noatime shared:1 - ext4 /dev/sda1 rw,errors=remount-ro,data=reordered +2 1 0:1 / /dev rw,relatime shared:2 - devtmpfs udev rw,size=10240k,nr_inodes=16487629,mode=755 +3 1 0:2 / /proc rw,nosuid,nodev,noexec,relatime shared:3 - proc proc rw +4 1 0:3 / /sys rw,nosuid,nodev,noexec,relatime shared:4 - sysfs sysfs rw +5 4 0:4 / /sys/fs/cgroup ro,nosuid,nodev,noexec shared:5 - tmpfs tmpfs ro,mode=755 +6 5 0:5 / /sys/fs/cgroup/cpuset rw,nosuid,nodev,noexec,relatime shared:6 - cgroup cgroup rw,cpuset +7 5 0:6 /docker /sys/fs/cgroup/cpu,cpuacct rw,nosuid,nodev,noexec,relatime shared:7 - cgroup cgroup rw,cpu,cpuacct +8 5 0:7 /docker /sys/fs/cgroup/memory rw,nosuid,nodev,noexec,relatime shared:8 - cgroup cgroup rw,memory diff --git a/internal/cg/cgroups/testdata/proc/mountinfo/mountinfo-v2 b/internal/cg/cgroups/testdata/proc/mountinfo/mountinfo-v2 new file mode 100644 index 0000000..c4d6116 --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/mountinfo/mountinfo-v2 @@ -0,0 +1,2 @@ +34 33 0:29 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:10 - cgroup2 cgroup rw,nsdelegate +34 33 0:29 / /sys/fs/foo rw,nosuid,nodev,noexec,relatime shared:10 - foo cgroup rw,nsdelegate diff --git a/internal/cg/cgroups/testdata/proc/mountinfo/nonexistent b/internal/cg/cgroups/testdata/proc/mountinfo/nonexistent new file mode 100644 index 0000000..e69de29 diff --git a/internal/cg/cgroups/testdata/proc/mountinfo/untranslatable b/internal/cg/cgroups/testdata/proc/mountinfo/untranslatable new file mode 100644 index 0000000..245daae --- /dev/null +++ b/internal/cg/cgroups/testdata/proc/mountinfo/untranslatable @@ -0,0 +1,2 @@ +31 23 0:24 / /sys/fs/cgroup/cpu rw,nosuid,nodev,noexec,relatime shared:1 - cgroup cgroup rw,cpu +32 23 0:25 /docker/0123456789abcdef /sys/fs/cgroup/cpuacct rw,nosuid,nodev,noexec,relatime shared:2 - cgroup cgroup rw,cpuacct diff --git a/internal/cg/cgroups/util_test.go b/internal/cg/cgroups/util_test.go new file mode 100644 index 0000000..e80feb1 --- /dev/null +++ b/internal/cg/cgroups/util_test.go @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cgroups + +import ( + "os" + "path/filepath" +) + +var ( + pwd = mustGetWd() + testDataPath = filepath.Join(pwd, "testdata") + testDataCGroupsPath = filepath.Join(testDataPath, "cgroups") + testDataProcPath = filepath.Join(testDataPath, "proc") +) + +func mustGetWd() string { + pwd, err := os.Getwd() + if err != nil { + panic(err) + } + return pwd +} diff --git a/internal/cg/quota_linux.go b/internal/cg/quota_linux.go new file mode 100644 index 0000000..a989c2e --- /dev/null +++ b/internal/cg/quota_linux.go @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux +// +build linux + +package cg + +import ( + "sync" + + "mosn.io/holmes/internal/cg/cgroups" +) + +var ( + globalCG cgroups.ICGroups + once sync.Once +) + +func GetCPUCore() (float64, error) { + if globalCG == nil { + if err := initCG(); err != nil { + return 0, err + } + } + quota, _, err := globalCG.CPUQuota() + return quota, err +} + +func GetMemoryLimit() (int, error) { + if globalCG == nil { + if err := initCG(); err != nil { + return 0, err + } + } + limit, _, err := globalCG.MemLimit() + return limit, err +} + +func initCG() (err error) { + once.Do(func() { + globalCG, err = cgroups.LoadCGroupsForCurrentProcess() + }) + return +} diff --git a/internal/cg/quota_unsupported.go b/internal/cg/quota_unsupported.go new file mode 100644 index 0000000..4e5a344 --- /dev/null +++ b/internal/cg/quota_unsupported.go @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !linux +// +build !linux + +package cg + +import "errors" + +var ErrUnsupported = errors.New("cgroups only supported in linux. ") + +func GetCPUCore() (float64, error) { + return 0, ErrUnsupported +} + +func GetMemoryLimit() (int, error) { + return 0, ErrUnsupported +} diff --git a/readme.md b/readme.md index 1f12a5c..53dcf5a 100644 --- a/readme.md +++ b/readme.md @@ -217,7 +217,7 @@ h, _ := holmes.New( ``` ### Running in docker or other cgroup limited environment - +Support both CGroup V1 and V2 versions, but don't support mixed-use. ```go h, _ := holmes.New( holmes.WithCollectInterval("5s"), diff --git a/util.go b/util.go index 4ad8bb4..e2d0221 100644 --- a/util.go +++ b/util.go @@ -20,48 +20,20 @@ package holmes import ( "bytes" "fmt" - "io/ioutil" "math" "os" "path" "runtime" "runtime/pprof" - "strconv" "strings" "time" + "mosn.io/holmes/internal/cg" + mem_util "github.com/shirou/gopsutil/mem" "github.com/shirou/gopsutil/process" ) -// copied from https://github.com/containerd/cgroups/blob/318312a373405e5e91134d8063d04d59768a1bff/utils.go#L251 -func parseUint(s string, base, bitSize int) (uint64, error) { - v, err := strconv.ParseUint(s, base, bitSize) - if err != nil { - intValue, intErr := strconv.ParseInt(s, base, bitSize) - // 1. Handle negative values greater than MinInt64 (and) - // 2. Handle negative values lesser than MinInt64 - if intErr == nil && intValue < 0 { - return 0, nil - } else if intErr != nil && - intErr.(*strconv.NumError).Err == strconv.ErrRange && - intValue < 0 { - return 0, nil - } - return 0, err - } - return v, nil -} - -// copied from https://github.com/containerd/cgroups/blob/318312a373405e5e91134d8063d04d59768a1bff/utils.go#L243 -func readUint(path string) (uint64, error) { - v, err := ioutil.ReadFile(path) - if err != nil { - return 0, err - } - return parseUint(strings.TrimSpace(string(v)), 10, 64) -} - // only reserve the top n. func trimResultTop(buffer bytes.Buffer) []byte { index := TrimResultTopN @@ -111,30 +83,29 @@ func getUsage() (float64, uint64, int, int, error) { // get cpu core number limited by CGroup. func getCGroupCPUCore() (float64, error) { - var cpuQuota uint64 - - cpuPeriod, err := readUint(cgroupCpuPeriodPath) - if cpuPeriod == 0 || err != nil { + quota, err := cg.GetCPUCore() + if err != nil { return 0, err } - - if cpuQuota, err = readUint(cgroupCpuQuotaPath); err != nil { - return 0, err + if quota == -1 { + quota = float64(runtime.NumCPU()) } - - return float64(cpuQuota) / float64(cpuPeriod), nil + return quota, nil } func getCGroupMemoryLimit() (uint64, error) { - usage, err := readUint(cgroupMemLimitPath) + usage, err := cg.GetMemoryLimit() if err != nil { return 0, err } - machineMemory, err := mem_util.VirtualMemory() + machineMemory, err := getNormalMemoryLimit() if err != nil { return 0, err } - limit := uint64(math.Min(float64(usage), float64(machineMemory.Total))) + if usage == -1 { + return machineMemory, nil + } + limit := uint64(math.Min(float64(usage), float64(machineMemory))) return limit, nil }