process_collector: fill in virtual and resident memory values on macOS (prometheus#1600)

Unfortunately, these values aren't available from getrusage(2), or any other
builtin Go API.  Using cgo is one alternative.  It's possible to conditionalize
everything such that cgo can remain disabled on non-Darwin platforms, or even
when cross-compiling Darwin executables on a non-Darwin platform (and stub in
code that causes the metrics to not be exported).  `CGO_ENABLED=1` is set by
default on macOS, but unfortunately is off for the non-host architecture, even
when gcc supports cross-compiling.  (e.g. building with GOARCH=amd on an M2 mac
skipped the cgo code.)  I think that's too subtle of a distinction to rely on
cgo.

There's no builtin equivalent of `syscall.NewLazyDLL()` and `.NewProc()` on
macOS that Go provides for Windows, so we're stuck with a 3rd party dependency.
But it seems stable, maintained, ang getting a fair amount of usage.  I'm
avoiding their struct deserialization because these native structs are packed
differently than the equivalent Go structs, which was causing bad values to be
returned.

The code is heavy with inline comments, and I tried keeping the type names the
same as the C code to make it easier to search for them.  I'm not sure that we
need to do the `mach_vm_region()` call to adjust the `task_info()` values,
because I've never seen that conditional evaluate to True on either amd64,
arm64, or when amd64 is run under Rosetta.  But this is what `ps(1)` does, and
I think it's reasonable to try to match that unless somebody knows it's dead
code.

Signed-off-by: Matt Harbison <[email protected]>

Author: mharbison72

go.mod

@@ -17,6 +17,7 @@ require (
 )
 
 require (
+	github.com/ebitengine/purego v0.7.1 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go.sum

@@ -6,6 +6,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/ebitengine/purego v0.7.1 h1:6/55d26lG3o9VCZX8lping+bZcmShseiqlh2bnUDiPA=
+github.com/ebitengine/purego v0.7.1/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=

prometheus/process_collector_amd64_darwin.go

@@ -0,0 +1,54 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build amd64
+
+package prometheus
+
+// These are macros in xnu/osfmk/mach/shared_memory_server.h
+const (
+	globalSharedTextSegment uint64 = 0x90000000
+
+	sharedTextRegionSize uint64 = 0x10000000
+	sharedDataRegionSize uint64 = 0x10000000
+)
+
+const (
+	machTaskBasicInfoSizeOf   = 48 // sizeof(struct mach_task_basic_info)
+	vmRegionBasicInfo64SizeOf = 36 // sizeof(struct vm_region_basic_info_64)
+)
+
+// Fundamental mach types defined in xnu/osfmk/mach/i386/vm_types.h.  The integer_t and
+// __darwin_natural_t types are explicitly 32-bit here to help decoding into a structure,
+// where 'int' types are not supported.  The __darwin_natural_t type is defined in
+// xnu/bsd/i386/_types.h.
+type (
+	__darwin_natural_t = uint32             // typedef unsigned int __darwin_natural_t;
+	integer_t          = int32              // typedef int integer_t;
+	natural_t          = __darwin_natural_t // typedef __darwin_natural_t natural_t;
+
+	mach_vm_offset_t = uint64 // typedef uint64_t mach_vm_offset_t __kernel_ptr_semantics;
+	mach_vm_size_t   = uint64 // typedef uint64_t mach_vm_size_t;
+)
+
+// Defined in xnu/osfmk/mach/i386/boolean.h, and explicitly 32-bit here to help decoding
+// into a structure, where 'int' types are not supported.
+type (
+	boolean_t = uint32 // typedef unsigned int boolean_t;
+)
+
+// Defined in xnu/osfmk/mach/i386/kern_return.h; see xnu/osfmk/mach/kern_return.h for
+// possible values.
+type (
+	kern_return_t = int // typedef int kern_return_t;
+)

prometheus/process_collector_arm64_darwin.go

@@ -0,0 +1,55 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build arm64
+
+package prometheus
+
+// These are macros in xnu/osfmk/mach/shared_memory_server.h.  Note that __arm__
+// is not defined for 64-bit arm.
+const (
+	globalSharedTextSegment uint64 = 0x90000000
+
+	sharedTextRegionSize uint64 = 0x10000000
+	sharedDataRegionSize uint64 = 0x10000000
+)
+
+const (
+	machTaskBasicInfoSizeOf   = 48 // sizeof(struct mach_task_basic_info)
+	vmRegionBasicInfo64SizeOf = 36 // sizeof(struct vm_region_basic_info_64)
+)
+
+// Fundamental mach types defined in xnu/osfmk/mach/arm/vm_types.h.  The integer_t and
+// __darwin_natural_t types are explicitly 32-bit here to help decoding into a structure,
+// where 'int' types are not supported.  The __darwin_natural_t type is defined in
+// xnu/bsd/arm/_types.h.
+type (
+	__darwin_natural_t = uint32             // typedef unsigned int __darwin_natural_t;
+	integer_t          = int32              // typedef int integer_t;
+	natural_t          = __darwin_natural_t // typedef __darwin_natural_t natural_t;
+
+	mach_vm_offset_t = uint64 // typedef uint64_t mach_vm_offset_t __kernel_ptr_semantics;
+	mach_vm_size_t   = uint64 // typedef uint64_t mach_vm_size_t;
+)
+
+// Defined in xnu/osfmk/mach/arm/boolean.h, and explicitly 32-bit here to help decoding
+// into a structure, where 'int' types are not supported.
+type (
+	boolean_t = int32 // typedef int boolean_t;
+)
+
+// Defined in xnu/osfmk/mach/arm/kern_return.h; see xnu/osfmk/mach/kern_return.h for
+// possible values.
+type (
+	kern_return_t = int // typedef int kern_return_t;
+)

prometheus/process_collector_darwin.go

@@ -85,7 +85,13 @@ func (c *processCollector) processCollect(ch chan<- Metric) {
 		c.reportError(ch, c.cpuTotal, err)
 	}
 
-	// TODO: publish c.vsize and c.rss values
+	if info, err := getMemoryUsage(); err == nil {
+		ch <- MustNewConstMetric(c.rss, GaugeValue, float64(info.ResidentSize))
+		ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(info.VirtualSize))
+	} else {
+		c.reportError(ch, c.rss, err)
+		c.reportError(ch, c.vsize, err)
+	}
 
 	if fds, err := getOpenFileCount(); err == nil {
 		ch <- MustNewConstMetric(c.openFDs, GaugeValue, fds)

prometheus/process_collector_purego_darwin.go

@@ -0,0 +1,219 @@
+// Copyright 2024 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"github.com/ebitengine/purego"
+)
+
+func init() {
+	if lib, err := purego.Dlopen("/usr/lib/system/libsystem_kernel.dylib",
+		purego.RTLD_NOW|purego.RTLD_GLOBAL); err == nil {
+
+		// purego.RegisterLibFunc() panics if the symbol is missing.  Ignore any error,
+		// and the metric will simply be unavailable when it is queried, instead of
+		// bringing down the whole process.
+		defer func() {
+			if err := recover(); err != nil {
+				// TODO: Log this somehow
+			}
+		}()
+
+		purego.RegisterLibFunc(&machTaskSelf, lib, "mach_task_self")
+		purego.RegisterLibFunc(&taskInfo, lib, "task_info")
+		purego.RegisterLibFunc(&machVmRegion, lib, "mach_vm_region")
+	}
+}
+
+const (
+	// The task_info() flavor MACH_TASK_BASIC_INFO for retrieving machTaskBasicInfo.
+	mach_task_basic_info task_flavor_t = 20 /* always 64-bit basic info */
+
+	// The MACH_TASK_BASIC_INFO_COUNT value, which is passed to the Mach API as the size
+	// of the payload for MACH_TASK_BASIC_INFO commands.
+	machTaskBasicInfoCount mach_msg_type_number_t = machTaskBasicInfoSizeOf / 4
+)
+
+// Defined in xnu/osfmk/mach/policy.h, xnu/iokit/IOKit/IORPC.h, and xnu/osfmk/mach/message.h
+// respectively.  policy_t is explicitly 32-bit here to help decoding into a structure,
+// where 'int' types are not supported.
+type (
+	policy_t               = int32     // typedef int policy_t;
+	mach_port_t            = natural_t // typedef natural_t mach_port_t;
+	mach_msg_type_number_t = natural_t // typedef natural_t mach_msg_type_number_t;
+)
+
+// Defined in xnu/osfmk/mach/task_info.h.  Note that task_info_t is actually defined as
+// integer_t* and cast from the address of the structure in C.  Define it as []byte to
+// keep the type system happy.
+type (
+	task_flavor_t = natural_t // typedef natural_t task_flavor_t
+	task_info_t   = []byte    // typedef integer_t *task_info_t /* varying array of int */
+)
+
+// time_value_t is the type for kernel time values, defined in xnu/osfmk/mach/time_value.h
+type time_value_t struct {
+	Seconds      integer_t
+	MicroSeconds integer_t
+}
+
+var machTaskSelf func() mach_port_t
+
+var taskInfo func(
+	mach_port_t,
+	task_flavor_t,
+	task_info_t,
+	*mach_msg_type_number_t,
+) kern_return_t
+
+// machTaskBasicInfo is the representation of `struct mach_task_basic_info` defined in
+// xnu/osfmk/mach/task_info.h, which is the architecture independent payload for fetching
+// certain task values.
+type machTaskBasicInfo struct {
+	VirtualSize     mach_vm_size_t // virtual memory size (bytes)
+	ResidentSize    mach_vm_size_t // resident memory size (bytes)
+	ResidentSizeMax mach_vm_size_t // maximum resident memory size (bytes)
+	UserTime        time_value_t   // total user run time for terminated threads
+	SystemTime      time_value_t   // total system run time for terminated threads
+	Policy          policy_t       // default policy for new threads
+	SuspendCount    integer_t      // suspend count for task
+}
+
+func getBasicTaskInfo() (*machTaskBasicInfo, error) {
+	var info machTaskBasicInfo
+
+	if taskInfo == nil {
+		return nil, fmt.Errorf("task_info() is not supported")
+	}
+
+	var count = machTaskBasicInfoCount
+	buf := make([]byte, machTaskBasicInfoSizeOf)
+
+	if ret := taskInfo(machTaskSelf(), mach_task_basic_info, buf, &count); ret != 0 {
+		return nil, fmt.Errorf("task_info() returned %d", ret)
+	}
+
+	if err := loadStruct(buf, &info); err != nil {
+		return nil, err
+	}
+
+	return &info, nil
+}
+
+// Defined in xnu/osfmk/mach/memory_object_types.h, xnu/osfmk/mach/vm_behavior.h,
+// defined in xnu/osfmk/mach/vm_inherit.h, and defined in xnu/osfmk/mach/vm_prot.h
+// respectively.  These types are not identical to the native definitions, because the
+// struct decoding requires primitives with a specific width.  The widths here are the
+// same as the native types.
+type (
+	memory_object_offset_t = uint64 // typedef unsigned long long memory_object_offset_t;
+	vm_behavior_t          = int32  // typedef int vm_behavior_t
+	vm_inherit_t           = uint32 // typedef unsigned int vm_inherit_t;
+	vm_prot_t              = int32  // typedef int vm_prot_t;
+)
+
+// These are defined in xnu/osfmk/mach/vm_types.h.
+type (
+	vm_map_t = mach_port_t // typedef mach_port_t vm_map_t;
+)
+
+// Defined in xnu/osfmk/mach/vm_region.h.  vm_region_flavor_t is explicitly 32-bit here to
+// help decoding into a structure, where 'int' types are not supported, and vm_region_info_t
+// is []byte to keep the type system happy.
+type (
+	vm_region_flavor_t = int32  // typedef int vm_region_flavor_t;
+	vm_region_info_t   = []byte // typedef int *vm_region_info_t;
+)
+
+const (
+	// The mach_vm_region() flavor VM_REGION_BASIC_INFO_64 for retrieving
+	// vmRegionBasicInfo64.
+	vm_region_basic_info_64 vm_region_flavor_t = 9
+
+	// The VM_REGION_BASIC_INFO_COUNT_64 value, which is passed to the Mach API as the
+	// size of the payload for VM_REGION_BASIC_INFO_64 commands.
+	vmRegionBasicInfoCount64 mach_msg_type_number_t = vmRegionBasicInfo64SizeOf / 4
+)
+
+// vmRegionBasicInfo64 is the representation of `struct vm_region_basic_info_64` defined
+// in xnu/osfmk/mach/vm_region.h.  This is enclosed in `#pragma pack(push, 4)` in C, so
+// unsafe.SizeOf() won't match the actual sizeof(vm_region_basic_info_64).
+type vmRegionBasicInfo64 struct {
+	Protection     vm_prot_t
+	MaxProtection  vm_prot_t
+	Inheritance    vm_inherit_t
+	Shared         boolean_t
+	Reserved       boolean_t
+	Offset         memory_object_offset_t
+	Behavior       vm_behavior_t
+	UserWiredCount uint16
+}
+
+var machVmRegion func(
+	vm_map_t,
+	*mach_vm_offset_t, /* IN/OUT */
+	*mach_vm_size_t, /* OUT */
+	vm_region_flavor_t, /* IN */
+	vm_region_info_t, /* OUT */
+	*mach_msg_type_number_t, /* IN/OUT */
+	*mach_port_t, /* OUT */
+) kern_return_t
+
+func getMemoryUsage() (*machTaskBasicInfo, error) {
+	// The logic in here follows how the ps(1) utility determines the memory values.  The
+	// basic_task_info command used here is a more modern, cross-architecture one that is
+	// suggested in the kernel header files.
+	//
+	// https://github.com/apple-oss-distributions/adv_cmds/blob/8744084ea0ff41ca4bb96b0f9c22407d0e48e9b7/ps/tasks.c#L132
+
+	info, err := getBasicTaskInfo()
+
+	if err != nil {
+		return nil, err
+	} else if machVmRegion != nil {
+
+		var textInfo vmRegionBasicInfo64
+
+		buf := make([]byte, vmRegionBasicInfo64SizeOf)
+		address := globalSharedTextSegment
+		var size mach_vm_size_t
+		var objectName mach_port_t
+
+		cmd := vm_region_basic_info_64
+		count := vmRegionBasicInfoCount64
+
+		ret := machVmRegion(machTaskSelf(), &address, &size, cmd, buf, &count, &objectName)
+
+		if ret == 0 {
+			if err := loadStruct(buf, &textInfo); err == nil {
+				adjustment := sharedTextRegionSize + sharedDataRegionSize
+				if textInfo.Reserved != 0 && size == sharedTextRegionSize && info.VirtualSize > adjustment {
+					info.VirtualSize -= adjustment
+				}
+			}
+		}
+	}
+
+	return info, nil
+}
+
+func loadStruct(buffer []byte, data any) error {
+	r := bytes.NewReader(buffer)
+
+	// TODO: NativeEndian was added in go 1.21
+	return binary.Read(r, binary.LittleEndian, data)
+}