add numa_stat for cgroup v2

Author: vickiegpt

libcontainer/cgroups/fs2/fs2.go

@@ -13,6 +13,10 @@ import (
 
 type parseError = fscommon.ParseError
 
+func malformedLine(path, file, line string) error {
+	return &parseError{Path: path, File: file, Err: fmt.Errorf("malformed line: %s", line)}
+}
+
 type Manager struct {
 	config *configs.Cgroup
 	// dirPath is like "/sys/fs/cgroup/user.slice/user-1001.slice/session-1.scope"

libcontainer/cgroups/fs2/memory.go

@@ -3,6 +3,7 @@ package fs2
 import (
 	"bufio"
 	"errors"
+	"io/ioutil"
 	"math"
 	"os"
 	"strconv"
@@ -102,6 +103,12 @@ func statMemory(dirPath string, stats *cgroups.Stats) error {
 	// cgroup v2 is always hierarchical.
 	stats.MemoryStats.UseHierarchy = true
 
+	pagesByNUMA, err := getPageUsageByNUMAV2(dirPath)
+	if err != nil {
+		return err
+	}
+	stats.MemoryStats.PageUsageByNUMA = pagesByNUMA
+
 	memoryUsage, err := getMemoryDataV2(dirPath, "")
 	if err != nil {
 		if errors.Is(err, unix.ENOENT) && dirPath == UnifiedMountpoint {
@@ -124,7 +131,9 @@ func statMemory(dirPath string, stats *cgroups.Stats) error {
 		swapUsage.Limit += memoryUsage.Limit
 	}
 	stats.MemoryStats.SwapUsage = swapUsage
-
+	if stats.MemoryStats.PageUsageByNUMA.Hierarchical.Total.Total != 0 {
+		stats.MemoryStats.UseHierarchy = true
+	}
 	return nil
 }
 
@@ -219,3 +228,147 @@ func statsFromMeminfo(stats *cgroups.Stats) error {
 
 	return nil
 }
+
+func getPageUsageByNUMAV2(path string) (cgroups.PageUsageByNUMA, error) {
+	const (
+		maxColumns = math.MaxUint8 + 1
+		file       = "memory.numa_stat"
+	)
+	stats := cgroups.PageUsageByNUMA{}
+
+	fd, err := cgroups.OpenFile(path, file, os.O_RDONLY)
+	if os.IsNotExist(err) {
+		return stats, nil
+	} else if err != nil {
+		return stats, err
+	}
+	defer fd.Close()
+
+	// https://docs.kernel.org/admin-guide/cgroup-v2.html
+	// anon N0=<> N1=<> # The Anon page size in byte which equals to page_num * page_size
+	// file N0=<> N1=0 # The File page size in byte which equals to file_mmaped_page_num * page_size
+	// kernel_stack N0=<> N1=0 # The Kernel's stack occupation
+	// pagetables N0=<> N1=0 # The total number of pagetable entry been occupied
+	// sec_pagetables N0=<> N1=<>
+	// shmem N0=<> N1=<>
+	// file_mapped N0=<> N1=<> # file page breakdown
+	// file_dirty N0=<> N1=<> # file page breakdown
+	// file_writeback N0=<> N1=<> # file page breakdown
+	// swapcached N0=<> N1=<>
+	// anon_thp N0=<> N1=<> # The transparent huge page occupation
+	// file_thp N0=<> N1=<> # The transparent huge page occupation
+	// shmem_thp N0=<> N1=<> # The transparent huge page occupation
+	// inactive_anon N0=<> N1=<>
+	// active_anon N0=<> N1=<>
+	// inactive_file N0=<> N1=<>
+	// active_file N0=<> N1=<>
+	// unevictable N0=<> N1=<>
+	// slab_reclaimable N0=<> N1=<>
+	// slab_unreclaimable N0=<> N1=<>
+	// workingset_refault_anon N0=<> N1=<>
+	// workingset_refault_file N0=<> N1=<>
+	// workingset_activate_anon N0=<> N1=<>
+	// workingset_activate_file N0=<> N1=<>
+	// workingset_restore_anon N0=<> N1=<>
+	// workingset_restore_file N0=<> N1=<>
+	// workingset_nodereclaim N0=<> N1=<>
+
+	scanner := bufio.NewScanner(fd)
+	for scanner.Scan() {
+		var field *cgroups.PageStats
+
+		line := scanner.Text()
+		columns := strings.SplitN(line, " ", maxColumns)
+		for i, column := range columns {
+			byNode := strings.SplitN(column, "=", 2)
+			key := byNode[0]
+			if i == 0 { // First column: key is name, val is total.
+				field = getNUMAFieldV2(&stats, key)
+				if field == nil { // unknown field (new kernel?)
+					break
+				}
+				field.Nodes = map[uint8]uint64{}
+			} else { // Subsequent columns: key is N<id>, val is usage.
+				if len(byNode) != 2 {
+					// This is definitely an error.
+					return stats, malformedLine(path, file, line)
+				}
+				val := byNode[1]
+				if len(key) < 2 || key[0] != 'N' {
+					// This is definitely an error.
+					return stats, malformedLine(path, file, line)
+				}
+
+				n, err := strconv.ParseUint(key[1:], 10, 8)
+				if err != nil {
+					return stats, &parseError{Path: path, File: file, Err: err}
+				}
+
+				usage, err := strconv.ParseUint(val, 10, 64)
+				if err != nil {
+					return stats, &parseError{Path: path, File: file, Err: err}
+				}
+				field.Nodes[uint8(n)] += usage
+				field.Total += usage
+			}
+
+		}
+		stats.Total.Total = stats.File.Total + stats.Anon.Total
+		stats.Total.Nodes = map[uint8]uint64{}
+		for k, v := range stats.File.Nodes {
+			stats.Total.Nodes[k] = v + stats.Anon.Nodes[k]
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return cgroups.PageUsageByNUMA{}, &parseError{Path: path, File: file, Err: err}
+	}
+
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		return stats, err
+	}
+	// hierarchical stats in subdirectory
+	for _, file := range files {
+		if file.IsDir() {
+			stat_tmp, err := getPageUsageByNUMAV2(path + "/" + file.Name())
+			if err != nil {
+				return stats, err
+			}
+			if stats.Hierarchical.Total.Total == 0 {
+				stats.Hierarchical.Total.Nodes = map[uint8]uint64{}
+				stats.Hierarchical.Anon.Nodes = map[uint8]uint64{}
+				stats.Hierarchical.File.Nodes = map[uint8]uint64{}
+				stats.Hierarchical.Unevictable.Nodes = map[uint8]uint64{}
+			}
+			stats.Hierarchical.Total.Total += stat_tmp.Total.Total
+			stats.Hierarchical.Anon.Total += stat_tmp.Anon.Total
+			stats.Hierarchical.File.Total += stat_tmp.File.Total
+			stats.Hierarchical.Unevictable.Total += stat_tmp.Unevictable.Total
+			for k, v := range stat_tmp.Total.Nodes {
+				stats.Hierarchical.Total.Nodes[k] += v
+			}
+			for k, v := range stat_tmp.Anon.Nodes {
+				stats.Hierarchical.Anon.Nodes[k] += v
+			}
+			for k, v := range stat_tmp.File.Nodes {
+				stats.Hierarchical.File.Nodes[k] += v
+			}
+			for k, v := range stat_tmp.Unevictable.Nodes {
+				stats.Hierarchical.Unevictable.Nodes[k] += v
+			}
+		}
+	}
+	return stats, nil
+}
+
+func getNUMAFieldV2(stats *cgroups.PageUsageByNUMA, name string) *cgroups.PageStats {
+	switch name {
+	case "anon":
+		return &stats.Anon
+	case "file":
+		return &stats.File
+	case "unevictable":
+		return &stats.Unevictable
+	}
+	return nil
+}