cpusetinfo: introduce pkg and cmd

Add a package, and its command frontend, to detect the allowed cpuset
for a process from its pid. This reads the cpuset from the host side.

Signed-off-by: Francesco Romani <[email protected]>

.github/workflows/go.yml

@@ -0,0 +1,27 @@
+name: CI Go
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: set up golang
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.16
+
+    - name: build
+      run: make all
+
+    - name: Test
+      run: make test-unit

Makefile

@@ -16,6 +16,10 @@ binaries: outdir
 	# go flags are set in here
 	./hack/build-binaries.sh
 
+.PHONY: test-unit
+test-unit:
+	go test ./pkg/...
+
 clean:
 	rm -rf _output
 
@@ -28,3 +32,8 @@ image: binaries
 push: image
 	@echo "pushing image"
 	$(RUNTIME) push quay.io/$(REPOOWNER)/$(IMAGENAME):$(IMAGETAG)
+
+.PHONY: gofmt
+gofmt:
+	@echo "Running gofmt"
+	gofmt -s -w `find . -path ./vendor -prune -o -type f -name '*.go' -print`

cmd/cpusetinfo/main.go

@@ -0,0 +1,82 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2020 Red Hat, Inc.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strconv"
+
+	flag "github.com/spf13/pflag"
+
+	"github.com/fromanirh/numalign/pkg/cpusetinfo"
+)
+
+type result struct {
+	Aligned        bool  `json:"aligned"`
+	CPUsAllowed    []int `json:"cpus_allowed"`
+	CPUsMisaligned []int `json:"cpus_misaligned"`
+	Pid            int   `json:"pid"`
+}
+
+func main() {
+	flag.Parse()
+	pids := flag.Args()
+
+	if len(pids) != 0 && len(pids) != 1 {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	pid := 0
+	if len(pids) == 1 && pids[0] != "self" {
+		v, err := strconv.Atoi(pids[0])
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "bad argument %q: %v\n", pids[0], err)
+			os.Exit(2)
+		}
+		pid = v
+	}
+
+	fsh := cpusetinfo.FSHandle{}
+
+	cpus, err := cpusetinfo.GetCPUSetForPID(fsh, pid)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "cannot fetch the cpuset for pid %d: %v\n", pid, err)
+		os.Exit(2)
+	}
+
+	tsm := cpusetinfo.NewThreadSiblingMap(fsh)
+
+	misaligned, err := tsm.CheckCPUSetAligned(cpus)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "cannot check the cpuset for pid %d: %v\n", pid, err)
+		os.Exit(4)
+	}
+
+	err = json.NewEncoder(os.Stdout).Encode(result{
+		Aligned:        misaligned.Size() == 0,
+		CPUsAllowed:    cpus.ToSlice(),
+		CPUsMisaligned: misaligned.ToSlice(),
+		Pid:            pid,
+	})
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "cannot encode the result: %v\n", err)
+		os.Exit(8)
+	}
+}

pkg/cpusetinfo/cpusetinfo.go

@@ -0,0 +1,202 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright 2021 Red Hat, Inc.
+ */
+
+package cpusetinfo
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
+)
+
+const (
+	DefaultCGroupsMountPoint string = "/sys/fs/cgroup"
+	DefaultProcMountPoint    string = "/proc"
+	DefaultSysMountPoint     string = "/sys"
+)
+
+const (
+	onlineCPUsPath            string = "devices/system/cpu/online"
+	cpusetFile                string = "cpuset.cpus"
+	threadSiblingListTmplPath string = "devices/system/cpu/cpu%d/topology/thread_siblings_list"
+)
+
+const (
+	PIDSelf int = 0
+)
+
+const (
+	cgroupV1 string = "v1"
+)
+
+// GetCPUSetForPID retrieves the cpuset allowed for a process, given its pid
+func GetCPUSetForPID(fsh FSHandle, pid int) (cpuset.CPUSet, error) {
+	cgroupsFile, err := os.Open(cGroupsFileForPID(fsh, pid))
+	if err != nil {
+		return cpuset.CPUSet{}, err
+	}
+	defer cgroupsFile.Close()
+
+	cpusPath := ""
+	subPath, version := GetCPUSetCGroupPathFromReader(cgroupsFile)
+	if subPath == "" {
+		cpusPath = filepath.Join(fsh.GetSysMountPoint(), onlineCPUsPath)
+	} else {
+		switch version {
+		case cgroupV1:
+			cpusPath = filepath.Join(fsh.GetCGroupsMountPoint(), "cpuset", subPath, cpusetFile)
+		default:
+			return cpuset.CPUSet{}, fmt.Errorf("detected unsupported cgroup version: %q", version)
+		}
+	}
+
+	return parseCPUSetFile(cpusPath)
+}
+
+type ThreadSiblingMap struct {
+	siblings map[int][]int
+	fsh      FSHandle
+}
+
+func NewThreadSiblingMap(fsh FSHandle) *ThreadSiblingMap {
+	return &ThreadSiblingMap{
+		siblings: make(map[int][]int),
+		fsh:      fsh,
+	}
+}
+
+func (tsm *ThreadSiblingMap) SetCPUSiblings(cpu int, siblings []int) *ThreadSiblingMap {
+	tsm.siblings[cpu] = siblings
+	return tsm
+}
+
+func (tsm *ThreadSiblingMap) ForCPU(cpu int) ([]int, error) {
+	if val, ok := tsm.siblings[cpu]; ok {
+		return val, nil
+	}
+
+	ts, err := parseCPUSetFile(filepath.Join(tsm.fsh.GetSysMountPoint(), fmt.Sprintf(threadSiblingListTmplPath, cpu)))
+	if err != nil {
+		return []int{}, err
+	}
+	val := ts.ToSliceNoSort()
+	tsm.siblings[cpu] = val
+	return val, nil
+}
+
+// CheckCPUSetIsSiblingAligned tells if a given cpuset is composed only by thread siblings sets,
+// IOW if core-level noisy neighbours are possible or not. Returns the misaligned CPU IDs.
+func (tsm ThreadSiblingMap) CheckCPUSetAligned(cpus cpuset.CPUSet) (cpuset.CPUSet, error) {
+	misaligned := cpuset.CPUSet{}
+	reconstructed := cpuset.NewBuilder()
+	for _, cpuId := range cpus.ToSliceNoSort() {
+		ts, err := tsm.ForCPU(cpuId)
+		if err != nil {
+			return misaligned, err
+		}
+		reconstructed.Add(ts...)
+	}
+
+	found := reconstructed.Result()
+	if found.Equals(cpus) {
+		return misaligned, nil
+	}
+
+	builder := cpuset.NewBuilder()
+	for _, extraCpu := range found.Difference(cpus).ToSliceNoSort() {
+		cpuIds, err := tsm.ForCPU(extraCpu)
+		if err != nil {
+			return misaligned, err
+		}
+		for _, cpuId := range cpuIds {
+			if !cpus.Contains(cpuId) {
+				continue
+			}
+			builder.Add(cpuId)
+		}
+	}
+	return builder.Result(), nil
+}
+
+func GetCPUSetCGroupPathFromReader(r io.Reader) (string, string) {
+	return getCPUSetCGroupPathFromReaderV1(r), cgroupV1
+}
+
+func getCPUSetCGroupPathFromReaderV1(r io.Reader) string {
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		entry := strings.TrimSpace(scanner.Text())
+		if !strings.Contains(entry, "cpuset") {
+			continue
+		}
+		// entry format is "number:name:path"
+		items := strings.Split(entry, ":")
+		if len(items) != 3 {
+			// how come?
+			continue
+		}
+		return items[2]
+	}
+	return ""
+}
+
+type FSHandle struct {
+	CGroupsMountPoint string
+	ProcMountPoint    string
+	SysMountPoint     string
+}
+
+func (fsh FSHandle) GetCGroupsMountPoint() string {
+	if fsh.CGroupsMountPoint == "" {
+		return DefaultCGroupsMountPoint
+	}
+	return fsh.CGroupsMountPoint
+}
+
+func (fsh FSHandle) GetProcMountPoint() string {
+	if fsh.ProcMountPoint == "" {
+		return DefaultProcMountPoint
+	}
+	return fsh.ProcMountPoint
+}
+
+func (fsh FSHandle) GetSysMountPoint() string {
+	if fsh.SysMountPoint == "" {
+		return DefaultSysMountPoint
+	}
+	return fsh.SysMountPoint
+}
+
+func cGroupsFileForPID(fsh FSHandle, pid int) string {
+	pidStr := "self"
+	if pid > 0 && pid != PIDSelf {
+		pidStr = fmt.Sprintf("%d", pid)
+	}
+	return filepath.Join(fsh.GetProcMountPoint(), pidStr, "cgroup")
+}
+
+func parseCPUSetFile(cpusPath string) (cpuset.CPUSet, error) {
+	data, err := os.ReadFile(cpusPath)
+	if err != nil {
+		return cpuset.CPUSet{}, err
+	}
+	return cpuset.Parse(strings.TrimSpace(string(data)))
+}