diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1931e5597b6b..87c243b3bbba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -875,6 +875,14 @@ /internal/third_party/kubernetes @DataDog/container-integrations /internal/third_party/golang/ @DataDog/container-integrations +# AIX +/packaging/aix/ @DataDog/agent-delivery @pgimalac +/packaging/aix/stages/ @DataDog/agent-build @pgimalac +/packaging/aix/build.sh @DataDog/agent-build @pgimalac +/packaging/aix/clean.sh @DataDog/agent-build @pgimalac +/**/*_aix.go @DataDog/agent-runtimes @pgimalac +/**/*_aix_test.go @DataDog/agent-runtimes @pgimalac + # Temporary during Bazel migration # This needs to go after all other rules that may include BUILD.bazel files, it otherwise gets overridden /**/BUILD.bazel @DataDog/agent-build diff --git a/cmd/agent/subcommands/launchgui/open_browser_unix.go b/cmd/agent/subcommands/launchgui/open_browser_unix.go index 7e186a614f9a..df861ac9ece0 100644 --- a/cmd/agent/subcommands/launchgui/open_browser_unix.go +++ b/cmd/agent/subcommands/launchgui/open_browser_unix.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build freebsd || netbsd || openbsd || solaris || dragonfly || linux +//go:build freebsd || netbsd || openbsd || solaris || dragonfly || linux || aix package launchgui diff --git a/comp/core/agenttelemetry/impl/sender.go b/comp/core/agenttelemetry/impl/sender.go index c68906f301a9..b9a79a37b67e 100644 --- a/comp/core/agenttelemetry/impl/sender.go +++ b/comp/core/agenttelemetry/impl/sender.go @@ -18,9 +18,8 @@ import ( "strings" "time" - dto "github.com/prometheus/client_model/go" - "github.com/DataDog/zstd" + dto "github.com/prometheus/client_model/go" "github.com/DataDog/datadog-agent/comp/core/config" log "github.com/DataDog/datadog-agent/comp/core/log/def" @@ -438,7 +437,7 @@ func (s *senderImpl) flushSession(ss *senderSession) error { compressed = true reqBody = reqBodyCompressed } else { - s.logComp.Errorf("Failed to compress agent telemetry payload: %v", errTemp) + s.logComp.Warnf("Failed to compress agent telemetry payload: %v", errTemp) } } diff --git a/comp/core/config/params_nix.go b/comp/core/config/params_nix.go index 8258b2ba6653..ae10c5834842 100644 --- a/comp/core/config/params_nix.go +++ b/comp/core/config/params_nix.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build netbsd || openbsd || solaris || dragonfly || linux +//go:build netbsd || openbsd || solaris || dragonfly || linux || aix package config diff --git a/comp/core/gui/guiimpl/platform_nix.go b/comp/core/gui/guiimpl/platform_nix.go index 1da224e5de7b..0f93fa02b077 100644 --- a/comp/core/gui/guiimpl/platform_nix.go +++ b/comp/core/gui/guiimpl/platform_nix.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build freebsd || netbsd || openbsd || solaris || dragonfly || linux +//go:build freebsd || netbsd || openbsd || solaris || dragonfly || linux || aix package guiimpl diff --git a/comp/dogstatsd/replay/impl/writer.go b/comp/dogstatsd/replay/impl/writer.go index 02aeb3b02232..924a706e4096 100644 --- a/comp/dogstatsd/replay/impl/writer.go +++ b/comp/dogstatsd/replay/impl/writer.go @@ -17,7 +17,6 @@ import ( "sync" "time" - // Refactor relevant bits "github.com/DataDog/zstd" "github.com/spf13/afero" diff --git a/comp/metadata/host/hostimpl/payload_aix.go b/comp/metadata/host/hostimpl/payload_aix.go new file mode 100644 index 000000000000..c6d22eae6884 --- /dev/null +++ b/comp/metadata/host/hostimpl/payload_aix.go @@ -0,0 +1,35 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package hostimpl + +import ( + "context" + "encoding/json" + + "github.com/DataDog/datadog-agent/comp/metadata/host/hostimpl/utils" +) + +// Payload handles the JSON unmarshalling of the metadata payload +type Payload struct { + utils.CommonPayload + utils.Payload +} + +// MarshalJSON serializes a Payload to JSON +func (p *Payload) MarshalJSON() ([]byte, error) { + type PayloadAlias Payload + return json.Marshal((*PayloadAlias)(p)) +} + +// getPayload returns the complete metadata payload as seen in Agent v5. Note: gohai can't be used on AIX. +func (h *host) getPayload(ctx context.Context) *Payload { + return &Payload{ + CommonPayload: *utils.GetCommonPayload(h.hostname, h.config), + Payload: *utils.GetPayload(ctx, h.config, h.hostnameComp), + } +} diff --git a/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go b/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go index fa5ed1f8c3bd..e4e5adf9ca02 100644 --- a/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go +++ b/comp/metadata/inventoryhost/inventoryhostimpl/inventoryhost.go @@ -216,22 +216,32 @@ func (ih *invHost) fillData() { networkInfo, err := networkGet() if err == nil { - _, warnings, err = networkInfo.AsJSON() - } - if err != nil { - ih.log.Errorf("failed to retrieve host network metadata from gohai: %s", err) //nolint:errcheck - } else { - logWarnings(warnings) - - ih.data.IPAddress = networkInfo.IPAddress - ih.data.IPv6Address = networkInfo.IPAddressV6.ValueOrDefault() - ih.data.MacAddress = networkInfo.MacAddress - jsonInterfaces, err := json.Marshal(networkInfo.Interfaces) + var netJSON interface{} + var warnings []string + netJSON, warnings, err = networkInfo.AsJSON() if err != nil { - ih.log.Errorf("failed to marshal network interfaces: %s", err) //nolint:errcheck + ih.log.Errorf("failed to retrieve host network metadata from gohai: %s", err) //nolint:errcheck } else { - ih.data.Interfaces = string(jsonInterfaces) + logWarnings(warnings) + + ih.data.IPAddress = networkInfo.IPAddress + ih.data.IPv6Address = networkInfo.IPAddressV6.ValueOrDefault() + ih.data.MacAddress = networkInfo.MacAddress + // Use the interfaces from AsJSON() which correctly handles utils.Value[string] + // fields (omitting error-valued fields rather than serializing them as {}). + if netData, ok := netJSON.(map[string]interface{}); ok { + if ifaces, ok := netData["interfaces"]; ok { + jsonInterfaces, jsonErr := json.Marshal(ifaces) + if jsonErr != nil { + ih.log.Errorf("failed to marshal network interfaces: %s", jsonErr) //nolint:errcheck + } else { + ih.data.Interfaces = string(jsonInterfaces) + } + } + } } + } else { + ih.log.Errorf("failed to retrieve host network metadata from gohai: %s", err) //nolint:errcheck } if ih.conf.GetBool("metadata_ip_resolution_from_hostname") { diff --git a/comp/metadata/resources/resourcesimpl/resources_non_supported.go b/comp/metadata/resources/resourcesimpl/resources_non_supported.go index f9ac147f9d9d..667c2db2cea9 100644 --- a/comp/metadata/resources/resourcesimpl/resources_non_supported.go +++ b/comp/metadata/resources/resourcesimpl/resources_non_supported.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build windows || freebsd || netbsd || openbsd || solaris || dragonfly +//go:build windows || freebsd || netbsd || openbsd || solaris || dragonfly || aix package resourcesimpl diff --git a/comp/process/agent/agent_aix.go b/comp/process/agent/agent_aix.go new file mode 100644 index 000000000000..9a869741854c --- /dev/null +++ b/comp/process/agent/agent_aix.go @@ -0,0 +1,70 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2024-present Datadog, Inc. + +//go:build aix + +package agent + +import ( + "slices" + "sync" + + "github.com/DataDog/datadog-agent/comp/core/config" + log "github.com/DataDog/datadog-agent/comp/core/log/def" + "github.com/DataDog/datadog-agent/comp/process/types" + "github.com/DataDog/datadog-agent/pkg/config/setup" + "github.com/DataDog/datadog-agent/pkg/process/checks" + "github.com/DataDog/datadog-agent/pkg/util/flavor" +) + +var ( + enabled bool + Once sync.Once + + processCheckNames = []string{ + checks.ProcessCheckName, + checks.ContainerCheckName, + checks.DiscoveryCheckName, + } +) + +func enabledHelper(config config.Component, checkComponents []types.CheckComponent, l log.Component) bool { + if setup.IsCLCRunner(config) { + return false + } + + runInCoreAgent := config.GetBool("process_config.run_in_core_agent.enabled") + + var processEnabled bool + for _, check := range checkComponents { + if slices.Contains(processCheckNames, check.Object().Name()) && check.Object().IsEnabled() { + processEnabled = true + } + } + + switch flavor.GetFlavor() { + case flavor.ProcessAgent: + if runInCoreAgent { + l.Info("The process checks will run in the core agent via the process-component") + } else if processEnabled { + l.Info("Process/Container Collection in the Process Agent will be deprecated in a future release " + + "and will instead be run in the Core Agent.") + } + return !runInCoreAgent + case flavor.DefaultAgent: + return runInCoreAgent + default: + return false + } +} + +// Enabled determines whether the process agent is enabled based on the configuration. +// On AIX, process checks run in the core agent when process_config.run_in_core_agent.enabled is true. +func Enabled(config config.Component, checkComponents []types.CheckComponent, l log.Component) bool { + Once.Do(func() { + enabled = enabledHelper(config, checkComponents, l) + }) + return enabled +} diff --git a/comp/process/agent/agent_fallback.go b/comp/process/agent/agent_fallback.go index 43daf32d7ae3..69d127f19812 100644 --- a/comp/process/agent/agent_fallback.go +++ b/comp/process/agent/agent_fallback.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2024-present Datadog, Inc. -//go:build !linux +//go:build !linux && !aix package agent diff --git a/go.mod b/go.mod index 724ee7e5f2dd..ce4595cd9586 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,12 @@ replace ( go.opentelemetry.io/ebpf-profiler => github.com/DataDog/opentelemetry-ebpf-profiler v0.0.202610 ) +// AIX: patched gopsutil with AIX process monitoring support +replace github.com/DataDog/gopsutil => github.com/DataDog/gopsutil v1.2.5-0.20260303142149-85b991b13594 + +// AIX: upstream gopsutil with unreleased AIX fixes +replace github.com/shirou/gopsutil/v4 => github.com/shirou/gopsutil/v4 v4.26.3-0.20260314122645-1ec4f5030b7e + require ( code.cloudfoundry.org/bbs v0.0.0-20200403215808-d7bc971db0db code.cloudfoundry.org/garden v0.0.0-20210208153517-580cadd489d2 diff --git a/go.sum b/go.sum index 5c3ff29bd6e4..e0b31680e687 100644 --- a/go.sum +++ b/go.sum @@ -220,8 +220,8 @@ github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee h1:tXibLZk3G6HncIFJK github.com/DataDog/gohai v0.0.0-20230524154621-4316413895ee/go.mod h1:nTot/Iy0kW16bXgXr6blEc8gFeAS7vTqYlhAxh+dbc0= github.com/DataDog/gopacket v0.0.0-20251104174046-ae42df68210e h1:kWyz88c8oMYcA/26/mG3lmsbHF9MTdmukzK4vWRbwMw= github.com/DataDog/gopacket v0.0.0-20251104174046-ae42df68210e/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA= -github.com/DataDog/gopsutil v1.2.4 h1:MlBW/7hvu1rK4w2OXny/JFlQdWBUA+eZMSFAIyC5yII= -github.com/DataDog/gopsutil v1.2.4/go.mod h1:F4ZaL3U3VONjc+dpyNmNq8U7hcx118KrJzLq6HlElCw= +github.com/DataDog/gopsutil v1.2.5-0.20260303142149-85b991b13594 h1:vep8M00kZeEPriow0cTfmJ+u+JJXHmEf2gdKcz+dK2s= +github.com/DataDog/gopsutil v1.2.5-0.20260303142149-85b991b13594/go.mod h1:F4ZaL3U3VONjc+dpyNmNq8U7hcx118KrJzLq6HlElCw= github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= github.com/DataDog/jsonapi v0.12.0 h1:N4e9RpmUflcV5hzceltSz8XUpM3PMtQr5C9Bhv0g87s= @@ -2315,8 +2315,8 @@ github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= -github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= -github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.3-0.20260314122645-1ec4f5030b7e h1:/yskJ+wd91E2eEgghqRxK/fsd3PaNEhe23lB6eOLNq0= +github.com/shirou/gopsutil/v4 v4.26.3-0.20260314122645-1ec4f5030b7e/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/shoenig/go-m1cpu v0.1.7 h1:C76Yd0ObKR82W4vhfjZiCp0HxcSZ8Nqd84v+HZ0qyI0= diff --git a/packaging/aix/build.sh b/packaging/aix/build.sh new file mode 100755 index 000000000000..d4de879d0ba2 --- /dev/null +++ b/packaging/aix/build.sh @@ -0,0 +1,179 @@ +#!/bin/sh +# build.sh — top-level orchestrator for the AIX Datadog Agent BFF package build +# +# Usage: +# AGENT_VERSION=7.78.0 AGENT_BUILD=1 AGENT_COMMIT=abc1234 ./build.sh +# +# Required environment variables: +# AGENT_VERSION — human-readable release version (e.g. 7.78.0) +# AGENT_BUILD — build iteration / 4th VRMF digit (e.g. 1) +# AGENT_COMMIT — short git SHA of the source tree (embedded in binary version string) +# +# The agent source must already be present at /opt/datadog-agent before running +# this script. Transfer it from the build machine with: +# tar czf /tmp/dd-agent-src.tar.gz --exclude='.git' --exclude='bin' . +# scp /tmp/dd-agent-src.tar.gz aix-host:/tmp/ +# ssh aix-host 'mkdir -p /opt/datadog-agent && gunzip -c /tmp/dd-agent-src.tar.gz | tar xf - -C /opt/datadog-agent' +# +# All intermediate artifacts go under /opt/dd-build/. +# The final .bff package is written to /opt/dd-build/ and reported at the end. + +set -eu + +# ── Validate required inputs ────────────────────────────────────────────────── + +if [ -z "${AGENT_VERSION:-}" ]; then + printf 'ERROR: AGENT_VERSION must be set (e.g. AGENT_VERSION=7.78.0)\n' >&2 + exit 1 +fi +if [ -z "${AGENT_BUILD:-}" ]; then + printf 'ERROR: AGENT_BUILD must be set (e.g. AGENT_BUILD=1)\n' >&2 + exit 1 +fi +if [ -z "${AGENT_COMMIT:-}" ]; then + # Try to resolve from local .git if present + if [ -d /opt/datadog-agent/.git ]; then + AGENT_COMMIT=$(git -C /opt/datadog-agent rev-parse --short HEAD) + printf 'INFO: AGENT_COMMIT resolved from .git: %s\n' "$AGENT_COMMIT" + else + printf 'ERROR: AGENT_COMMIT must be set to the short SHA of the source tree\n' >&2 + # shellcheck disable=SC2016 # literal $() is intentional in usage message + printf ' e.g.: AGENT_COMMIT=$(git rev-parse --short HEAD)\n' >&2 + exit 1 + fi +fi +export AGENT_COMMIT + +# ── Set PATH early so tool checks can find /opt/freeware and Go binaries ────── +# env.sh sets PATH too, but it is sourced later (after tool checks). +PATH=/opt/go/bin:/opt/freeware/bin:/usr/sbin:/usr/bin:/bin:$PATH +export PATH + +# ── Check required tools ────────────────────────────────────────────────────── +# Fail early with a clear message if a required build tool is missing. + +check_tool() { + _tool=$1; _pkg=${2:-$1} + if ! command -v "$_tool" >/dev/null 2>&1; then + printf 'ERROR: required tool not found: %s\n' "$_tool" >&2 + printf ' Install with: yum install %s\n' "$_pkg" >&2 + exit 1 + fi +} + +check_tool git git +check_tool curl curl +check_tool xz xz +check_tool make make +check_tool cmake cmake +check_tool gcc gcc +check_tool python3.12 python3.12 +check_tool go golang + +# Several libraries are taken from AIX Toolbox (source builds fail on AIX). +# Check that all required -devel packages are installed. +check_aix_devel() { + _hdr=$1; _pkg=$2 + if [ ! -f "$_hdr" ]; then + printf 'ERROR: %s not found (required for build)\n' "$_hdr" >&2 + printf ' Install with: yum install %s\n' "$_pkg" >&2 + exit 1 + fi +} +check_aix_devel /opt/freeware/include/ffi.h libffi-devel +check_aix_devel /opt/freeware/lib64/libffi.a libffi-devel +check_aix_devel /opt/freeware/include/ncurses.h ncurses-devel +check_aix_devel /opt/freeware/lib64/libncursesw.a ncurses-devel +check_aix_devel /opt/freeware/include/readline/readline.h readline-devel +check_aix_devel /opt/freeware/lib64/libreadline.a readline-devel +check_aix_devel /opt/freeware/include/sqlite3.h sqlite-devel +check_aix_devel /opt/freeware/lib64/libsqlite3.a sqlite-devel +check_aix_devel /opt/freeware/include/gdbm.h gdbm-devel +check_aix_devel /opt/freeware/lib/libgdbm.a gdbm-devel +check_aix_devel /opt/freeware/include/libxslt/xslt.h libxslt-devel +check_aix_devel /opt/freeware/lib/libxslt.a libxslt-devel + +# ── Source shared environment ───────────────────────────────────────────────── + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/lib/env.sh" + +# ── Bootstrap build directories ─────────────────────────────────────────────── + +mkdir -p "$BUILD_DIR/logs" +mkdir -p "$BUILD_DIR/.done" +mkdir -p "$STAGING" + +# ── Stage list ──────────────────────────────────────────────────────────────── + +STAGES=" +00-checkout +01-native-libs +02-python +03-rtloader +04-agent +05-python-extensions +06-pydantic +07-checks-base +08-integrations +09-strip-bytecode +10-assemble +" + +# ── Helper: run one stage script ────────────────────────────────────────────── + +run_stage() { + _stage="$1" + _script="$SCRIPT_DIR/stages/${_stage}.sh" + + if [ ! -f "$_script" ]; then + log "ERROR: stage script not found: $_script" + return 1 + fi + + log "--- Starting stage: $_stage ---" + if sh "$_script"; then + log "--- Stage complete: $_stage ---" + else + log "ERROR: Stage failed: $_stage (exit $?)" + log "Check log: $BUILD_DIR/logs/${_stage}.log" + return 1 + fi +} + +# ── Main ────────────────────────────────────────────────────────────────────── + +BUILD_START=$(date '+%Y-%m-%dT%H:%M:%S') +log "=== Datadog Agent AIX package build ===" +log " AGENT_VERSION = $AGENT_VERSION" +log " AGENT_BUILD = $AGENT_BUILD" +log " AGENT_COMMIT = $AGENT_COMMIT" +log " AGENT_VRMF = $AGENT_VRMF" +log " BUILD_DIR = $BUILD_DIR" +log " STAGING = $STAGING" +log " Started at = $BUILD_START" + +# Run all numbered stages in order +for stage in $STAGES; do + run_stage "$stage" || exit 1 +done + +# Run the final packaging step +log "--- Starting stage: package ---" +if sh "$SCRIPT_DIR/package.sh"; then + log "--- Stage complete: package ---" +else + log "ERROR: Stage failed: package" + log "Check log: $BUILD_DIR/logs/package.log" + exit 1 +fi + +# Report the output artifact +BFF_PATH="$BUILD_DIR/datadog-agent-${AGENT_VERSION}-${AGENT_BUILD}.aix.ppc64.bff" +BUILD_END=$(date '+%Y-%m-%dT%H:%M:%S') + +log "=== Build complete ===" +log " Started : $BUILD_START" +log " Finished: $BUILD_END" +log " Package : $BFF_PATH" diff --git a/packaging/aix/clean.sh b/packaging/aix/clean.sh new file mode 100755 index 000000000000..ce90d8bf6fc1 --- /dev/null +++ b/packaging/aix/clean.sh @@ -0,0 +1,106 @@ +#!/bin/sh +# clean.sh — reset build state for a clean rebuild while preserving expensive caches +# +# By default, removes: +# - All stage sentinel files ($BUILD_DIR/.done/) +# - The staging tree ($BUILD_DIR/staging/) +# - Build logs ($BUILD_DIR/logs/) +# - Temporary packaging files ($BUILD_DIR/.pkg_filelist.tmp, etc.) +# +# By default, PRESERVES: +# - Wheel cache ($BUILD_DIR/wheel-cache/) — pydantic-core (52-min Rust build), +# cryptography +# - Library cache ($BUILD_DIR/lib-cache/) — zlib, bzip2, OpenSSL, xz, libxml2 +# compiled artifacts; avoids ~9-min native-libs rebuild +# - integrations-core checkout ($BUILD_DIR/integrations-core/) — avoids re-clone +# +# Usage: +# ./clean.sh — clean build state, preserve all caches +# ./clean.sh --full — also remove wheel cache and lib cache (full rebuild) +# +# After running, re-run build.sh to perform a fresh build. + +set -eu + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/lib/env.sh" + +FULL_CLEAN=0 +for arg in "$@"; do + case "$arg" in + --full) + FULL_CLEAN=1 + ;; + -h|--help) + printf 'Usage: %s [--full]\n' "$0" + printf ' (no args) Clean build state; preserve wheel cache, lib cache, and integrations-core\n' + printf ' --full Also remove wheel cache and lib cache (forces full rebuild)\n' + exit 0 + ;; + *) + printf 'ERROR: unknown option: %s\n' "$arg" >&2 + exit 1 + ;; + esac +done + +printf '=== Datadog Agent AIX build clean ===\n' +printf ' BUILD_DIR = %s\n' "$BUILD_DIR" +printf ' STAGING = %s\n' "$STAGING" +printf ' WHEEL_CACHE = %s\n' "$WHEEL_CACHE" +printf ' LIB_CACHE = %s\n' "$LIB_CACHE" +printf '\n' + +# ── Remove stage sentinels ───────────────────────────────────────────────────── +if [ -d "$BUILD_DIR/.done" ]; then + _count=$(find "$BUILD_DIR/.done" -type f | wc -l | tr -d ' ') + printf 'Removing %s stage sentinel(s) from %s/.done/\n' "$_count" "$BUILD_DIR" + rm -rf "$BUILD_DIR/.done" +else + printf 'No sentinels directory found — already clean.\n' +fi + +# ── Remove staging tree ──────────────────────────────────────────────────────── +# This removes all compiled libraries, Python installation, and the agent binary. +# The next build will recompile everything from source (except wheel cache below). +if [ -d "$STAGING" ]; then + printf 'Removing staging tree: %s\n' "$STAGING" + rm -rf "$STAGING" +else + printf 'Staging tree not found — already clean.\n' +fi + +# ── Remove build logs ────────────────────────────────────────────────────────── +if [ -d "$BUILD_DIR/logs" ]; then + printf 'Removing build logs: %s/logs/\n' "$BUILD_DIR" + rm -rf "$BUILD_DIR/logs" +fi + +# ── Remove temporary packaging files ────────────────────────────────────────── +rm -f "$BUILD_DIR/.pkg_filelist.tmp" + +# ── Wheel cache ──────────────────────────────────────────────────────────────── +if [ "$FULL_CLEAN" -eq 1 ]; then + if [ -d "$WHEEL_CACHE" ]; then + printf 'FULL CLEAN: removing wheel cache: %s\n' "$WHEEL_CACHE" + rm -rf "$WHEEL_CACHE" + fi + if [ -d "$LIB_CACHE" ]; then + printf 'FULL CLEAN: removing lib cache: %s\n' "$LIB_CACHE" + rm -rf "$LIB_CACHE" + fi + printf 'FULL CLEAN: next build will recompile all native libs + pydantic-core from source\n' +else + if [ -d "$WHEEL_CACHE" ]; then + printf 'Wheel cache preserved: %s\n' "$WHEEL_CACHE" + fi + if [ -d "$LIB_CACHE" ]; then + printf 'Library cache preserved: %s\n' "$LIB_CACHE" + fi + if [ -d "$BUILD_DIR/integrations-core" ]; then + printf 'integrations-core checkout preserved: %s/integrations-core\n' "$BUILD_DIR" + fi +fi + +printf '\n=== Clean complete. Run build.sh to rebuild from scratch. ===\n' diff --git a/packaging/aix/gen.template.in b/packaging/aix/gen.template.in new file mode 100644 index 000000000000..9a351a12eb3b --- /dev/null +++ b/packaging/aix/gen.template.in @@ -0,0 +1,23 @@ +Package Name: datadog-agent +Package VRMF: AGENT_VRMF_PLACEHOLDER +Update: N +Fileset + Fileset Name: datadog-agent + Fileset VRMF: AGENT_VRMF_PLACEHOLDER + Fileset Description: Datadog Infrastructure Monitoring Agent + USRLIBLPPFiles + Pre-installation Script: /opt/datadog-agent/embedded/share/installp/preinst + Post-installation Script: /opt/datadog-agent/embedded/share/installp/postinst + Configuration Script: /opt/datadog-agent/embedded/share/installp/config + Unconfiguration Script: /opt/datadog-agent/embedded/share/installp/unconfig + Pre_rm Script: /opt/datadog-agent/embedded/share/installp/prerm + EOUSRLIBLPPFiles + Bosboot required: N + License agreement acceptance required: N + Include license files in this package: N + Requisites: + ROOT Part: Y + USRFiles +__FILE_LIST__ + EOUSRFiles +EOFileset diff --git a/packaging/aix/lib/env.sh b/packaging/aix/lib/env.sh new file mode 100644 index 000000000000..0108137adc4f --- /dev/null +++ b/packaging/aix/lib/env.sh @@ -0,0 +1,108 @@ +# shellcheck shell=sh +# lib/env.sh — shared environment sourced by every stage script +# +# Usage: . "$SCRIPT_DIR/../lib/env.sh" (from a stages/NN-name.sh script) +# . "$SCRIPT_DIR/lib/env.sh" (from build.sh) +# +# This file is sourced, never executed directly. Callers control set -e/set -u. +# No validation of required variables is done here; each script validates its +# own inputs after sourcing this file. + +# ── Python version ──────────────────────────────────────────────────────────── +PYTHON_VERSION="3.13.12" +PYTHON_MAJ_MIN="${PYTHON_VERSION%.*}" # e.g. 3.13 +export PYTHON_VERSION PYTHON_MAJ_MIN + +# ── Build tree layout ───────────────────────────────────────────────────────── + +BUILD_DIR=/opt/dd-build +STAGING=$BUILD_DIR/staging + +# DESTDIR approach (critical — read before modifying): +# EMBEDDED = final install path baked into all binaries at configure time +# (sys.prefix, _sysconfigdata, XCOFF loader sections) +# EMBEDDED_DESTDIR = where files actually land during the build (staging tree) +# +# All ./configure calls use --prefix=$EMBEDDED. +# All make install calls use DESTDIR=$STAGING. +# All compiler -I/-L flags point to $EMBEDDED_DESTDIR (where files are during build). +# Never pass $EMBEDDED_DESTDIR to --prefix; never pass $EMBEDDED to -L or -I. +EMBEDDED=/opt/datadog-agent/embedded +EMBEDDED_DESTDIR=$STAGING/opt/datadog-agent/embedded + +INTEGRATIONS_CORE=$BUILD_DIR/integrations-core +WHEEL_CACHE=$BUILD_DIR/wheel-cache +LIB_CACHE=$BUILD_DIR/lib-cache + +# Number of available CPUs — nproc does not exist on AIX; lsdev is in /usr/sbin +NPROC=$(/usr/sbin/lsdev -Cc processor | wc -l | tr -d ' ') + +export BUILD_DIR STAGING EMBEDDED EMBEDDED_DESTDIR INTEGRATIONS_CORE WHEEL_CACHE LIB_CACHE NPROC + +# ── Agent version variables ─────────────────────────────────────────────────── +# AGENT_BRANCH, AGENT_VERSION, and AGENT_BUILD are required inputs. +# They must be set in the caller's environment before sourcing this file. +# AGENT_VRMF is derived here; it is the four-component installp version string. + +# Use ${VAR:-} (no-fail) so env.sh can be sourced under set -u before the caller +# validates AGENT_VERSION/AGENT_BUILD. The individual stage scripts call +# : "${AGENT_VERSION:?AGENT_VERSION must be set}" +# after sourcing this file; that is where the empty-variable error is reported. +AGENT_VRMF=$(printf '%s' "${AGENT_VERSION:-}" | sed 's/\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/').${AGENT_BUILD:-} + +export AGENT_VERSION AGENT_BUILD AGENT_BRANCH AGENT_VRMF + +# ── Toolchain ───────────────────────────────────────────────────────────────── + +CC=/opt/freeware/bin/gcc +CXX=/opt/freeware/bin/g++ +NM="/usr/bin/nm -X64" +ARFLAGS="-X64 -cru" +OBJECT_MODE=64 + +export CC CXX NM ARFLAGS OBJECT_MODE + +# ── Compiler/linker flags ───────────────────────────────────────────────────── +# -I and -L always reference $EMBEDDED_DESTDIR (staging), not $EMBEDDED (final path). + +CFLAGS="-maix64" +CXXFLAGS="-maix64" +# -Wl,-bbigtoc: remove the 64KB TOC limit (required for large libs like OpenSSL and Python) +# -Wl,-brtl: enable runtime linking for dlopen support +LDFLAGS="-maix64 -Wl,-brtl -Wl,-bbigtoc -L$EMBEDDED_DESTDIR/lib" +CPPFLAGS="-I$EMBEDDED_DESTDIR/include" + +export CFLAGS CXXFLAGS LDFLAGS CPPFLAGS + +# ── PATH and Go toolchain ───────────────────────────────────────────────────── + +PATH=/opt/go/bin:/opt/freeware/bin:/usr/sbin:/usr/bin:/bin:$PATH +GOPATH=/home/gopath +GOROOT=/opt/go +CGO_ENABLED=1 +CGO_CFLAGS="-I/opt/freeware/include" +CGO_LDFLAGS="-L/opt/freeware/lib -L/opt/freeware/lib64" +GOPROXY=https://proxy.golang.org,direct + +export PATH GOPATH GOROOT CGO_ENABLED CGO_CFLAGS CGO_LDFLAGS GOPROXY + +# ── Utility functions ───────────────────────────────────────────────────────── + +# log MESSAGE ... +# Print a timestamped log line to stdout. +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%dT%H:%M:%S')" "$*" +} + +# sentinel_done STAGE_NAME +# Returns 0 (true) if the stage sentinel file exists, 1 (false) otherwise. +sentinel_done() { + [ -f "$BUILD_DIR/.done/$1" ] +} + +# sentinel_mark STAGE_NAME +# Create the sentinel file that marks STAGE_NAME as complete. +sentinel_mark() { + mkdir -p "$BUILD_DIR/.done" + touch "$BUILD_DIR/.done/$1" +} diff --git a/packaging/aix/package-scripts/config b/packaging/aix/package-scripts/config new file mode 100755 index 000000000000..1649b7136535 --- /dev/null +++ b/packaging/aix/package-scripts/config @@ -0,0 +1,16 @@ +#!/bin/sh +chown -R dd-agent:dd-agent /etc/datadog-agent \ + /var/log/datadog \ + /var/run/datadog \ + /opt/datadog-agent 2>/dev/null || true + +if [ -f /etc/datadog-agent/datadog.yaml ]; then + startsrc -s datadog-agent + startsrc -s datadog-trace-agent +else + echo "" + echo "Datadog Agent installed." + echo "Create /etc/datadog-agent/datadog.yaml with your api_key to start." + echo "Then run: startsrc -s datadog-agent && startsrc -s datadog-trace-agent" + echo "" +fi diff --git a/packaging/aix/package-scripts/postinst b/packaging/aix/package-scripts/postinst new file mode 100755 index 000000000000..11edd714f210 --- /dev/null +++ b/packaging/aix/package-scripts/postinst @@ -0,0 +1,79 @@ +#!/bin/sh +# Create dd-agent group +lsgroup dd-agent > /dev/null 2>&1 || mkgroup dd-agent + +# Create dd-agent user (no interactive login). +# Use login=false rlogin=false instead of shell=/usr/bin/false: the shell= attribute +# requires the shell to be listed in the usw.shells list in /etc/security/login.cfg, +# which /usr/bin/false is not on a standard AIX system. login=false/rlogin=false +# achieves the same "no login" result without that constraint. +if ! id dd-agent > /dev/null 2>&1; then + mkuser pgrp=dd-agent login=false rlogin=false home=/opt/datadog-agent dd-agent +fi + +# Add dd-agent to adm group (required for sar — used by the CPU check) +chuser groups=dd-agent,adm dd-agent 2>/dev/null || true + +# Set ownership of runtime directories so the agent can write to them. +# The BFF installs these directories as root:root (build-time default); fix +# them here after the dd-agent user is guaranteed to exist. +chown dd-agent:dd-agent /var/log/datadog 2>/dev/null || true +chown dd-agent:dd-agent /var/run/datadog 2>/dev/null || true +chown dd-agent:dd-agent /opt/datadog-agent/run 2>/dev/null || true +# Config dir: owned by dd-agent so the agent can create the auth_token file there. +# datadog.yaml is installed with mode 640 (root-readable only from outside the group), +# but the agent process itself must be able to write auth_token and run subdirs. +chown dd-agent:dd-agent /etc/datadog-agent 2>/dev/null || true +chmod 750 /etc/datadog-agent 2>/dev/null || true + +# Write SRC wrapper script. +# AIX's mkssys does not support setting environment variables (-e is stderr +# redirect, not env-var injection). This wrapper sets LIBPATH so all shared +# libraries and Python C extensions are found, then exec's the agent binary. +WRAPPER=/opt/datadog-agent/bin/agent-svc +LIBPATH_VAL=/opt/datadog-agent/rtloader +LIBPATH_VAL=${LIBPATH_VAL}:/opt/datadog-agent/embedded/lib +LIBPATH_VAL=${LIBPATH_VAL}:/opt/mqm/lib64:/opt/mqm/lib +LIBPATH_VAL=${LIBPATH_VAL}:/opt/ibm/db2/clidriver/lib +LIBPATH_VAL=${LIBPATH_VAL}:/opt/freeware/lib64:/opt/freeware/lib + +printf '#!/bin/sh\n' > "$WRAPPER" +printf 'export LIBPATH=%s\n' "$LIBPATH_VAL" >> "$WRAPPER" +printf 'export PATH=/opt/datadog-agent/embedded/bin:/opt/freeware/bin:/usr/bin:/bin\n' >> "$WRAPPER" +printf 'exec /opt/datadog-agent/bin/agent "$@"\n' >> "$WRAPPER" +chmod 755 "$WRAPPER" + +# Write SRC wrapper for trace-agent. +TRACE_WRAPPER=/opt/datadog-agent/bin/trace-agent-svc +printf '#!/bin/sh\n' > "$TRACE_WRAPPER" +printf 'export LIBPATH=%s\n' "$LIBPATH_VAL" >> "$TRACE_WRAPPER" +printf 'export PATH=/opt/datadog-agent/embedded/bin:/opt/freeware/bin:/usr/bin:/bin\n' >> "$TRACE_WRAPPER" +printf 'exec /opt/datadog-agent/bin/trace-agent run --config /etc/datadog-agent/datadog.yaml\n' >> "$TRACE_WRAPPER" +chmod 755 "$TRACE_WRAPPER" + +# Remove stale SRC entries then register fresh. +# The wrappers are registered as the subsystem paths so SRC inherits the correct +# LIBPATH when it spawns the binaries via startsrc. +odmdelete -o SRCsubsys -q "subsysname='datadog-agent'" > /dev/null 2>&1 || true +odmdelete -o SRCsubsys -q "subsysname='datadog-trace-agent'" > /dev/null 2>&1 || true + +mkssys -s datadog-agent \ + -p /opt/datadog-agent/bin/agent-svc \ + -a "run -c /etc/datadog-agent/datadog.yaml" \ + -u dd-agent \ + -R -S -n 15 -f 9 + +mkssys -s datadog-trace-agent \ + -p /opt/datadog-agent/bin/trace-agent-svc \ + -u dd-agent \ + -R -S -n 15 -f 9 + +# Compile Python bytecode. +# LIBPATH must include the embedded Python libs; the installp shell context does not +# set LIBPATH, so we set it explicitly here. +PYTHON=/opt/datadog-agent/embedded/bin/python3.13 +if [ -x "$PYTHON" ]; then + LIBPATH=/opt/datadog-agent/embedded/lib:/opt/datadog-agent/rtloader${LIBPATH:+:$LIBPATH} \ + "$PYTHON" -m compileall /opt/datadog-agent/embedded/lib/python3.13/site-packages \ + -x 'test/' -q 2>/dev/null || true +fi diff --git a/packaging/aix/package-scripts/postrm b/packaging/aix/package-scripts/postrm new file mode 100755 index 000000000000..3de9265d8c64 --- /dev/null +++ b/packaging/aix/package-scripts/postrm @@ -0,0 +1,10 @@ +#!/bin/sh +# prerm already stopped the agent and waited for processes to exit. +# This script cleans up after installp has removed the package files. +odmdelete -o SRCsubsys -q "subsysname='datadog-agent'" > /dev/null 2>&1 || true +odmdelete -o SRCsubsys -q "subsysname='datadog-trace-agent'" > /dev/null 2>&1 || true +rmssys -s datadog-agent 2>/dev/null || true +rmssys -s datadog-trace-agent 2>/dev/null || true +rmuser dd-agent 2>/dev/null || true +rmgroup dd-agent 2>/dev/null || true +rm -rf /var/log/datadog /var/run/datadog 2>/dev/null || true diff --git a/packaging/aix/package-scripts/preinst b/packaging/aix/package-scripts/preinst new file mode 100755 index 000000000000..9ecce057f902 --- /dev/null +++ b/packaging/aix/package-scripts/preinst @@ -0,0 +1,8 @@ +#!/bin/sh +stopsrc -s datadog-agent 2>/dev/null || true +stopsrc -s datadog-trace-agent 2>/dev/null || true + +COMPILED_LIST=/opt/datadog-agent/embedded/.pyc_compiled_files.txt +if [ -f "$COMPILED_LIST" ]; then + while IFS= read -r f; do rm -f "$f" 2>/dev/null; done < "$COMPILED_LIST" +fi diff --git a/packaging/aix/package-scripts/prerm b/packaging/aix/package-scripts/prerm new file mode 100755 index 000000000000..d6a581d6d6b1 --- /dev/null +++ b/packaging/aix/package-scripts/prerm @@ -0,0 +1,28 @@ +#!/bin/sh +# Pre_rm Script — runs ONLY during deinstallation (installp -u), before files +# are removed. This is the right place for user/directory cleanup because: +# 1. The script file is still present (not yet removed by installp). +# 2. The agent has already been stopped by the Unconfiguration Script. +# 3. Pre_rm does NOT run during upgrades (only -u deinstall). + +# Belt-and-suspenders: stop services in case unconfig did not run. +stopsrc -s datadog-agent 2>/dev/null || true +stopsrc -s datadog-trace-agent 2>/dev/null || true + +# Wait up to 30 s for dd-agent processes to fully exit before cleanup. +i=0 +while [ $i -lt 30 ]; do + ps -fu dd-agent 2>/dev/null | awk 'NR>1{f=1}END{exit !f}' || break + sleep 1; i=$((i+1)) +done + +# Remove compiled bytecode files before the package files are deleted. +COMPILED_LIST=/opt/datadog-agent/embedded/.pyc_compiled_files.txt +if [ -f "$COMPILED_LIST" ]; then + while IFS= read -r f; do rm -f "$f" 2>/dev/null; done < "$COMPILED_LIST" +fi + +# Remove user, home directory, group, and runtime directories. +rmuser dd-agent 2>/dev/null || true +rmgroup dd-agent 2>/dev/null || true +rm -rf /var/log/datadog /var/run/datadog 2>/dev/null || true diff --git a/packaging/aix/package-scripts/unconfig b/packaging/aix/package-scripts/unconfig new file mode 100755 index 000000000000..b3e8e7edea63 --- /dev/null +++ b/packaging/aix/package-scripts/unconfig @@ -0,0 +1,27 @@ +#!/bin/sh +# Unconfiguration Script — the only lifecycle script reliably stored in +# liblpp.a and run by installp during deinstallation (installp -u). +# It also runs during upgrades, but preinst recreates any removed state. +stopsrc -s datadog-agent 2>/dev/null || true +stopsrc -s datadog-trace-agent 2>/dev/null || true +# Wait up to 30 s for dd-agent processes to fully exit before cleanup. +i=0 +while [ $i -lt 30 ]; do + ps -fu dd-agent 2>/dev/null | awk 'NR>1{f=1}END{exit !f}' || break + sleep 1; i=$((i+1)) +done +odmdelete -o SRCsubsys -q "subsysname='datadog-agent'" > /dev/null 2>&1 || true +odmdelete -o SRCsubsys -q "subsysname='datadog-trace-agent'" > /dev/null 2>&1 || true +rmssys -s datadog-agent 2>/dev/null || true +rmssys -s datadog-trace-agent 2>/dev/null || true +rmuser dd-agent 2>/dev/null || true +rmgroup dd-agent 2>/dev/null || true +rm -rf /var/run/datadog 2>/dev/null || true +# Remove files written at runtime or by preinst that are not in the package +# file list and would otherwise prevent installp from removing the directories. +rm -f /opt/datadog-agent/bin/agent-svc \ + /opt/datadog-agent/bin/trace-agent-svc \ + /opt/datadog-agent/.profile 2>/dev/null || true +# Python stdlib __pycache__ .pyc files created at runtime (not in package). +find /opt/datadog-agent -name "*.pyc" -type f -exec rm -f {} \; 2>/dev/null || true +rm -rf /opt/datadog-agent/run 2>/dev/null || true diff --git a/packaging/aix/package.sh b/packaging/aix/package.sh new file mode 100755 index 000000000000..ff0417d7f665 --- /dev/null +++ b/packaging/aix/package.sh @@ -0,0 +1,151 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, BUILD_DIR, AGENT_VRMF, log(), etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/lib/env.sh" + +STAGE_NAME="package" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${AGENT_VERSION:?AGENT_VERSION must be set}" +: "${AGENT_BUILD:?AGENT_BUILD must be set}" +: "${AGENT_VRMF:?AGENT_VRMF must be set}" +: "${STAGING:?STAGING must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" + +# --- Output artifact path --- +BFF_OUT="$BUILD_DIR/datadog-agent-${AGENT_VERSION}-${AGENT_BUILD}.aix.ppc64.bff" + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed. Removing partial outputs." + rm -f "$BFF_OUT" + fi +} +trap cleanup EXIT + +# ─── Step 1: Pre-flight — verify staging tree is assembled ──────────────────── +# +# The agent binary and the postinst lifecycle script are the two most critical +# components. If either is absent the staging tree is incomplete and mkinstallp +# will produce a broken or empty BFF. + +AGENT_BIN="$STAGING/opt/datadog-agent/bin/agent" +if [ ! -f "$AGENT_BIN" ]; then + log "ERROR: agent binary not found at $AGENT_BIN" + log " Did Stage 04 (04-agent) complete successfully?" + exit 1 +fi +log "Pre-flight: agent binary found at $AGENT_BIN" + +POSTINST="$STAGING/opt/datadog-agent/embedded/share/installp/postinst" +if [ ! -f "$POSTINST" ]; then + log "ERROR: postinst script not found at $POSTINST" + log " Did Stage 10 (10-assemble) complete successfully?" + exit 1 +fi +log "Pre-flight: postinst script found at $POSTINST" + +# ─── Step 2: Generate the mkinstallp template with VRMF and full file list ──── +# +# gen.template.in contains two placeholder tokens: +# AGENT_VRMF_PLACEHOLDER — replaced with the four-component VRMF (e.g. 7.78.0.1) +# __FILE_LIST__ — replaced with a complete list of all files/dirs to package +# +# AIX mkinstallp USRFiles requires individual file/directory paths; listing a +# directory path alone only packages the directory entry, not its contents. +# We use find to enumerate every path under each package directory and strip the +# staging prefix so each line is the absolute installed path (e.g. /opt/datadog-agent/bin/agent). + +log "Generating gen.template with VRMF=${AGENT_VRMF}" + +FILELIST="$BUILD_DIR/.pkg_filelist.tmp" +: > "$FILELIST" +for pkg_dir in opt/datadog-agent etc/datadog-agent var/log/datadog var/run/datadog; do + if [ -d "$STAGING/$pkg_dir" ]; then + # Filter out paths with spaces or parentheses: AIX mkinstallp splits USRFiles lines + # on whitespace, so filenames containing spaces cause checksum errors and are skipped. + # Files like "Lorem ipsum.txt" and "launcher manifest.xml" (setuptools vendor docs) + # are documentation only and not needed at runtime. + find "$STAGING/$pkg_dir" -print | sed "s|^$STAGING||" | grep -v '[ ()]' >> "$FILELIST" + fi +done +FILE_COUNT=$(wc -l < "$FILELIST" | tr -d ' ') +log "File list generated: $FILE_COUNT paths" + +# Build gen.template: substitute VRMF and expand __FILE_LIST__ with the file list +{ + while IFS= read -r tmpl_line; do + case "$tmpl_line" in + '__FILE_LIST__') + cat "$FILELIST" + ;; + *'AGENT_VRMF_PLACEHOLDER'*) + printf '%s\n' "$tmpl_line" | sed "s/AGENT_VRMF_PLACEHOLDER/${AGENT_VRMF}/g" + ;; + *) + printf '%s\n' "$tmpl_line" + ;; + esac + done < "$SCRIPT_DIR/gen.template.in" +} > "$STAGING/gen.template" +log "Template written to $STAGING/gen.template ($FILE_COUNT file entries)" + +# ─── Step 3: Run mkinstallp to generate the BFF ─────────────────────────────── +# +# mkinstallp writes its output to $STAGING/tmp/datadog-agent..bff. +# The -d flag specifies the staging root (all paths in USRFiles are relative to +# this directory). The -T flag specifies the template file. +# +# mkinstallp requires the tmp/ subdirectory to exist; create it explicitly so +# the error message is clear if it fails rather than getting a cryptic mkdir +# failure from inside mkinstallp. +# +# Remove any .info directory left by a previous failed mkinstallp run; mkinstallp +# refuses to overwrite its own control files and exits with error 0503-844. + +log "Running mkinstallp (this may take a few minutes for large packages)" +rm -rf "$STAGING/.info" +mkdir -p "$STAGING/tmp" +/usr/sbin/mkinstallp -d "$STAGING" -T "$STAGING/gen.template" +log "mkinstallp completed" + +# ─── Step 4: Copy BFF to final artifact name ────────────────────────────────── +# +# mkinstallp names the output file after the package name and VRMF. +# Rename to the canonical artifact name used by CI and release tooling: +# datadog-agent--.aix.ppc64.bff +# e.g. datadog-agent-7.78.0-1.aix.ppc64.bff + +BFF_SRC="$STAGING/tmp/datadog-agent.${AGENT_VRMF}.bff" +if [ ! -f "$BFF_SRC" ]; then + log "ERROR: expected BFF not found at $BFF_SRC" + log " Check mkinstallp output above for errors." + exit 1 +fi + +cp "$BFF_SRC" "$BFF_OUT" +ls -l "$BFF_OUT" +log "Package ready: $BFF_OUT" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/sitecustomize.py b/packaging/aix/sitecustomize.py new file mode 100644 index 000000000000..3250c45f267e --- /dev/null +++ b/packaging/aix/sitecustomize.py @@ -0,0 +1,21 @@ +# AIX: preload libc++abi.a so that __xlcxx_personality_v0 is available +# when pydantic_core's libunwind.a(libunwind.so.1) dependency is resolved. +# +# Background: +# pydantic_core is a Rust extension module compiled against AIX's libunwind.a. +# libunwind.so.1 declares __xlcxx_personality_v0 as an undefined symbol, +# expecting the IBM XLC++ C++ ABI runtime (libc++abi.a) to provide it. +# Python is compiled with GCC, so libc++abi.a is not loaded at Python startup. +# Without this preload, "import pydantic" raises: +# ImportError: rtld: 0712-001 Symbol __xlcxx_personality_v0 was referenced +# from module /usr/lib/libunwind.a(libunwind.so.1), but a runtime definition +# of the symbol was not found. +# +# Fix: load libc++abi.a with RTLD_GLOBAL before any checks run so the symbol +# is available to all subsequently loaded shared modules. +import ctypes as _ctypes + +try: + _ctypes.CDLL('/usr/lib/libc++abi.a(libc++abi.so.1)', _ctypes.RTLD_GLOBAL) +except Exception: + pass # best-effort; Python still starts even if this fails diff --git a/packaging/aix/stages/00-checkout.sh b/packaging/aix/stages/00-checkout.sh new file mode 100755 index 000000000000..f8d96a0df9f7 --- /dev/null +++ b/packaging/aix/stages/00-checkout.sh @@ -0,0 +1,117 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="00-checkout" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${STAGING:?STAGING must be set}" +: "${INTEGRATIONS_CORE:?INTEGRATIONS_CORE must be set}" + +# --- Cleanup on failure --- +PARTIAL_INTEGRATIONS_CORE= +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed." + if [ -n "$PARTIAL_INTEGRATIONS_CORE" ] && [ -d "$PARTIAL_INTEGRATIONS_CORE" ]; then + log "Removing partial integrations-core checkout: $PARTIAL_INTEGRATIONS_CORE" + rm -rf "$PARTIAL_INTEGRATIONS_CORE" + fi + fi +} +trap cleanup EXIT + +# ─── Step 1: Validate agent source tree ─────────────────────────────────────── +# +# The agent source is NOT cloned here — it must already be present at +# /opt/datadog-agent, transferred from the build machine by the caller. +# (The local clone contains AIX-specific changes not yet merged to main.) +# +# Expected transfer method: +# tar czf /tmp/dd-agent-src.tar.gz --exclude='.git' --exclude='bin' ... . +# scp /tmp/dd-agent-src.tar.gz aix-host:/tmp/ +# ssh aix-host 'mkdir -p /opt/datadog-agent && gunzip -c /tmp/dd-agent-src.tar.gz | tar xf - -C /opt/datadog-agent' + +AGENT_SRC=/opt/datadog-agent + +if [ ! -f "$AGENT_SRC/go.mod" ]; then + log "ERROR: agent source not found at $AGENT_SRC/go.mod" + log " Transfer the source tree to the AIX host first:" + log " On build machine: tar czf /tmp/dd-agent-src.tar.gz --exclude='.git' --exclude='bin' ." + log " scp /tmp/dd-agent-src.tar.gz aix-host:/tmp/" + log " On AIX host: mkdir -p /opt/datadog-agent" + log " gunzip -c /tmp/dd-agent-src.tar.gz | tar xf - -C /opt/datadog-agent" + exit 1 +fi + +log "Agent source found at $AGENT_SRC" +log " go.mod: $(head -1 $AGENT_SRC/go.mod)" + +# ─── Step 2: Read INTEGRATIONS_CORE_VERSION from release.json ───────────────── + +RELEASE_JSON="$AGENT_SRC/release.json" +if [ ! -f "$RELEASE_JSON" ]; then + log "ERROR: $RELEASE_JSON not found — is the source tree complete?" + exit 1 +fi + +log "Reading INTEGRATIONS_CORE_VERSION from $RELEASE_JSON" +INTEGRATIONS_CORE_VERSION=$(python3.12 -c \ + "import json; print(json.load(open('$RELEASE_JSON'))['dependencies']['INTEGRATIONS_CORE_VERSION'])") + +if [ -z "$INTEGRATIONS_CORE_VERSION" ]; then + log "ERROR: Could not read INTEGRATIONS_CORE_VERSION from $RELEASE_JSON" + exit 1 +fi + +log "INTEGRATIONS_CORE_VERSION = $INTEGRATIONS_CORE_VERSION" + +# ─── Step 3: Clone or fetch integrations-core ───────────────────────────────── + +log "Checking out DataDog/integrations-core at $INTEGRATIONS_CORE_VERSION into $INTEGRATIONS_CORE" + +mkdir -p "$(dirname "$INTEGRATIONS_CORE")" + +if [ -d "$INTEGRATIONS_CORE/.git" ]; then + log "integrations-core repository already exists — fetching latest refs" + git -C "$INTEGRATIONS_CORE" fetch --quiet +else + log "Cloning https://github.com/DataDog/integrations-core.git (shallow --depth=1)" + PARTIAL_INTEGRATIONS_CORE="$INTEGRATIONS_CORE" + # --depth=1 cuts clone time from ~3 min to ~30 sec. If the pinned SHA is not + # the current HEAD of the default branch, fetch it specifically afterwards. + # GitHub supports fetching public commits by SHA (git protocol v2). + git clone --depth=1 --quiet https://github.com/DataDog/integrations-core.git "$INTEGRATIONS_CORE" + if ! git -C "$INTEGRATIONS_CORE" cat-file -e "${INTEGRATIONS_CORE_VERSION}^{commit}" 2>/dev/null; then + log " Pinned commit not at clone HEAD; fetching $INTEGRATIONS_CORE_VERSION" + git -C "$INTEGRATIONS_CORE" fetch --quiet --depth=1 origin "$INTEGRATIONS_CORE_VERSION" + fi + PARTIAL_INTEGRATIONS_CORE= +fi + +git -C "$INTEGRATIONS_CORE" checkout --quiet "$INTEGRATIONS_CORE_VERSION" +log "Checked out integrations-core at $(git -C "$INTEGRATIONS_CORE" rev-parse HEAD)" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/01-native-libs.sh b/packaging/aix/stages/01-native-libs.sh new file mode 100755 index 000000000000..52c40ba47946 --- /dev/null +++ b/packaging/aix/stages/01-native-libs.sh @@ -0,0 +1,478 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="01-native-libs" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${AGENT_VERSION:?AGENT_VERSION must be set}" +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED:?EMBEDDED must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${NPROC:?NPROC must be set}" +: "${CC:?CC must be set}" +: "${CFLAGS:?CFLAGS must be set}" + +# --- Cleanup on failure --- +CURRENT_LIB= +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed (building: ${CURRENT_LIB:-unknown})." + if [ -n "$CURRENT_LIB" ]; then + log "Removing partial build directory for $CURRENT_LIB" + rm -rf "$BUILD_DIR/build/$CURRENT_LIB" + fi + fi +} +trap cleanup EXIT + +# --- Prepare source and build directories --- +mkdir -p "$BUILD_DIR/sources" +mkdir -p "$BUILD_DIR/build" +mkdir -p "$EMBEDDED_DESTDIR/bin" +mkdir -p "$EMBEDDED_DESTDIR/lib" +mkdir -p "$EMBEDDED_DESTDIR/lib/pkgconfig" +mkdir -p "$EMBEDDED_DESTDIR/include" +mkdir -p "$EMBEDDED_DESTDIR/share" + +# ─── Version pins ────────────────────────────────────────────────────────────── +# +# Versions of built-from-source libraries are kept in sync with the Linux +# omnibus pipeline. Source of truth: deps/repos.MODULE.bazel. +# +# Libraries taken from AIX Toolbox (yum install -devel) use whatever +# version is provided by the toolbox. +# +ZLIB_VERSION="1.3.1" +BZIP2_VERSION="1.0.8" +OPENSSL_VERSION="3.5.5" +XZ_VERSION="5.8.1" +LIBXML2_VERSION="2.14.5" # built from source (AIX Toolbox also available but we build) +LIBXSLT_VERSION="1.1.45" # from AIX Toolbox (yum install libxslt-devel; source build fails on AIX) + +# These are sourced from AIX Toolbox (build from source fails on AIX) +LIBFFI_VERSION="3.4.4" # yum install libffi-devel +NCURSES_VERSION="6.5" # yum install ncurses-devel +READLINE_VERSION="8.2" # yum install readline-devel +SQLITE_VERSION="3.50.4" # yum install sqlite-devel +GDBM_VERSION="1.23" # yum install gdbm-devel + +# ─── Helpers ────────────────────────────────────────────────────────────────── + +# AIX system tar does not support decompression flags (-z, -j, -J). +# Always decompress explicitly and pipe into tar xf -. +extract_gz() { + gunzip -c "$1" | tar xf - -C "$2" +} + +extract_xz() { + xz -dc "$1" | tar xf - -C "$2" +} + +# lib_done NAME VERSION — true if library is already installed to staging +lib_done() { + [ -f "$BUILD_DIR/.done/01-lib-$1-$2" ] +} + +# lib_mark NAME VERSION — record successful installation +lib_mark() { + mkdir -p "$BUILD_DIR/.done" + touch "$BUILD_DIR/.done/01-lib-$1-$2" +} + +# ─── Per-library artifact cache ──────────────────────────────────────────────── +# +# Built-from-source libraries (zlib, bzip2, OpenSSL, xz, libxml2) are archived +# after their first successful build. On subsequent clean builds, if the pinned +# version matches, the archive is restored directly into $STAGING — no recompile. +# +# Cache key: $LIB_CACHE/-.tar.gz +# Format: cpio archive (relative to $STAGING), gzip-compressed. +# +# Toolbox libraries (libffi, ncurses, readline, sqlite, gdbm, libxslt) are not +# cached; they are simple file copies taking < 1 second. +# +mkdir -p "$LIB_CACHE" + +# lib_cache_restore NAME VERSION +# If a cache archive exists for NAME-VERSION, extract it into $STAGING, +# mark the library done, and return 0. Returns 1 on cache miss. +lib_cache_restore() { + _lcr_name=$1; _lcr_ver=$2 + _lcr_file="$LIB_CACHE/${_lcr_name}-${_lcr_ver}.tar.gz" + if [ -f "$_lcr_file" ]; then + log "${_lcr_name} ${_lcr_ver}: cache hit — restoring (skipping compile)" + (cd "$STAGING" && gunzip -c "$_lcr_file" | cpio -idm 2>/dev/null) + lib_mark "$_lcr_name" "$_lcr_ver" + log "${_lcr_name} ${_lcr_ver}: restored from cache" + return 0 + fi + return 1 +} + +# lib_cache_save NAME VERSION PRE_TIMESTAMP_FILE +# Pack all files added to $STAGING since PRE_TIMESTAMP_FILE into a cache +# archive. Call immediately after make install, before lib_mark. +lib_cache_save() { + _lcs_name=$1; _lcs_ver=$2; _lcs_pre=$3 + _lcs_file="$LIB_CACHE/${_lcs_name}-${_lcs_ver}.tar.gz" + (cd "$STAGING" && find . -newer "$_lcs_pre" | cpio -o 2>/dev/null) | gzip > "$_lcs_file" + log "${_lcs_name} ${_lcs_ver}: cached to $_lcs_file" +} + +# stage_toolbox_lib LIB_NAME VERSION LIB64 HEADERS... +# Copy a library and headers from AIX Toolbox into the staging tree. +# LIB64 = full path of the 64-bit .a file +# HEADERS = space-separated list of header source paths to copy +stage_toolbox_lib() { + _name=$1; _ver=$2; _lib64=$3 + shift 3 + if lib_done "$_name" "$_ver"; then + log "${_name} ${_ver} already staged — skipping" + return 0 + fi + log "Staging ${_name} ${_ver} (from AIX Toolbox)" + if [ ! -f "$_lib64" ]; then + log "ERROR: ${_name} library not found: $_lib64" + log " Install with: yum install ${_name}-devel" + exit 1 + fi + cp "$_lib64" "$EMBEDDED_DESTDIR/lib/" + for _hdr in "$@"; do + if [ -f "$_hdr" ]; then + # Preserve one level of subdirectory (e.g. readline/readline.h) + _subdir=$(dirname "$_hdr" | sed "s|/opt/freeware/include||") + if [ -n "$_subdir" ] && [ "$_subdir" != "/" ] && [ "$_subdir" != "." ]; then + mkdir -p "$EMBEDDED_DESTDIR/include/$_subdir" + cp "$_hdr" "$EMBEDDED_DESTDIR/include/$_subdir/" + else + cp "$_hdr" "$EMBEDDED_DESTDIR/include/" + fi + fi + done + lib_mark "$_name" "$_ver" + log "${_name} ${_ver} staged" +} + +# ─── Open-file limit ────────────────────────────────────────────────────────── +# shellcheck disable=SC3045 # ulimit -n is supported by AIX sh (not POSIX sh) +ulimit -n 65536 +# shellcheck disable=SC3045 +log "Open-file limit raised to $(ulimit -n)" + +# ─── Library build / stage functions ───────────────────────────────────────── +# Libraries are processed sequentially. +# Per-library sentinels allow re-runs to skip already-completed libraries. + +# ── zlib (build from source) ────────────────────────────────────────────────── +if lib_done zlib "$ZLIB_VERSION"; then + log "zlib ${ZLIB_VERSION} already installed — skipping" +elif lib_cache_restore zlib "$ZLIB_VERSION"; then + : +else + CURRENT_LIB="zlib-${ZLIB_VERSION}" + log "Building zlib ${ZLIB_VERSION}" + TARBALL="$BUILD_DIR/sources/zlib-${ZLIB_VERSION}.tar.gz" + # Primary: GitHub releases (canonical, stable across versions) + # Fallback: zlib.net (older releases moved to /fossils/) + [ -f "$TARBALL" ] || curl -fSL -o "$TARBALL" \ + "https://github.com/madler/zlib/releases/download/v${ZLIB_VERSION}/zlib-${ZLIB_VERSION}.tar.gz" || \ + curl -fSL -o "$TARBALL" "https://zlib.net/fossils/zlib-${ZLIB_VERSION}.tar.gz" || \ + curl -fSL -o "$TARBALL" "https://zlib.net/zlib-${ZLIB_VERSION}.tar.gz" + # Touch pre-timestamp before compile so install files are strictly newer (AIX has 1s mtime) + _pre="$BUILD_DIR/.lib-pre-zlib" + touch "$_pre" + rm -rf "$BUILD_DIR/build/zlib-${ZLIB_VERSION}" + extract_gz "$TARBALL" "$BUILD_DIR/build" + cd "$BUILD_DIR/build/zlib-${ZLIB_VERSION}" + ./configure --prefix="$EMBEDDED" + make -j"$NPROC" + make install DESTDIR="$STAGING" + lib_cache_save zlib "$ZLIB_VERSION" "$_pre" + rm -f "$_pre" + cd "$BUILD_DIR" + lib_mark zlib "$ZLIB_VERSION" + log "zlib ${ZLIB_VERSION} done" + CURRENT_LIB= +fi + +# ── bzip2 (build from source, static only) ──────────────────────────────────── +if lib_done bzip2 "$BZIP2_VERSION"; then + log "bzip2 ${BZIP2_VERSION} already installed — skipping" +elif lib_cache_restore bzip2 "$BZIP2_VERSION"; then + : +else + CURRENT_LIB="bzip2-${BZIP2_VERSION}" + log "Building bzip2 ${BZIP2_VERSION}" + TARBALL="$BUILD_DIR/sources/bzip2-${BZIP2_VERSION}.tar.gz" + [ -f "$TARBALL" ] || curl -fSL -o "$TARBALL" "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz" + # Touch pre-timestamp before compile so install files are strictly newer (AIX has 1s mtime) + _pre="$BUILD_DIR/.lib-pre-bzip2" + touch "$_pre" + rm -rf "$BUILD_DIR/build/bzip2-${BZIP2_VERSION}" + extract_gz "$TARBALL" "$BUILD_DIR/build" + cd "$BUILD_DIR/build/bzip2-${BZIP2_VERSION}" + # AIX does not support Linux-style .so shared libs for bzip2 (gcc -shared + -Wl,-soname fails). + # Build and install the static library only; Python links against libbz2.a at compile time. + make CC="$CC" CFLAGS="$CFLAGS" + make install PREFIX="$EMBEDDED" DESTDIR="$STAGING" + lib_cache_save bzip2 "$BZIP2_VERSION" "$_pre" + rm -f "$_pre" + cd "$BUILD_DIR" + lib_mark bzip2 "$BZIP2_VERSION" + log "bzip2 ${BZIP2_VERSION} done" + CURRENT_LIB= +fi + +# ── OpenSSL (build from source) ─────────────────────────────────────────────── +if lib_done openssl "$OPENSSL_VERSION"; then + log "OpenSSL ${OPENSSL_VERSION} already installed — skipping" +elif lib_cache_restore openssl "$OPENSSL_VERSION"; then + : +else + CURRENT_LIB="openssl-${OPENSSL_VERSION}" + log "Building OpenSSL ${OPENSSL_VERSION}" + TARBALL="$BUILD_DIR/sources/openssl-${OPENSSL_VERSION}.tar.gz" + # openssl.org/source redirects to GitHub for recent releases; use GitHub directly + [ -f "$TARBALL" ] || curl -fSL -o "$TARBALL" \ + "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" || \ + curl -fSL -o "$TARBALL" "https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz" + # Touch pre-timestamp before compile so install files are strictly newer (AIX has 1s mtime) + _pre="$BUILD_DIR/.lib-pre-openssl" + touch "$_pre" + rm -rf "$BUILD_DIR/build/openssl-${OPENSSL_VERSION}" + extract_gz "$TARBALL" "$BUILD_DIR/build" + cd "$BUILD_DIR/build/openssl-${OPENSSL_VERSION}" + ./Configure aix64-gcc \ + --prefix="$EMBEDDED" \ + --openssldir="$EMBEDDED/ssl" \ + -Wl,-brtl \ + shared + make -j"$NPROC" + make install_sw DESTDIR="$STAGING" + lib_cache_save openssl "$OPENSSL_VERSION" "$_pre" + rm -f "$_pre" + cd "$BUILD_DIR" + lib_mark openssl "$OPENSSL_VERSION" + log "OpenSSL ${OPENSSL_VERSION} done" + CURRENT_LIB= +fi + +# ── xz (build from source) ──────────────────────────────────────────────────── +if lib_done xz "$XZ_VERSION"; then + log "xz ${XZ_VERSION} already installed — skipping" +elif lib_cache_restore xz "$XZ_VERSION"; then + : +else + CURRENT_LIB="xz-${XZ_VERSION}" + log "Building xz ${XZ_VERSION}" + TARBALL="$BUILD_DIR/sources/xz-${XZ_VERSION}.tar.gz" + # tukaani.org redirects to GitHub; use GitHub directly + [ -f "$TARBALL" ] || curl -fSL -o "$TARBALL" \ + "https://github.com/tukaani-project/xz/releases/download/v${XZ_VERSION}/xz-${XZ_VERSION}.tar.gz" || \ + curl -fSL -o "$TARBALL" "https://tukaani.org/xz/xz-${XZ_VERSION}.tar.gz" + # Touch pre-timestamp before compile so install files are strictly newer (AIX has 1s mtime) + _pre="$BUILD_DIR/.lib-pre-xz" + touch "$_pre" + rm -rf "$BUILD_DIR/build/xz-${XZ_VERSION}" + extract_gz "$TARBALL" "$BUILD_DIR/build" + cd "$BUILD_DIR/build/xz-${XZ_VERSION}" + ./configure \ + --prefix="$EMBEDDED" \ + --disable-static + make -j"$NPROC" + make install DESTDIR="$STAGING" + lib_cache_save xz "$XZ_VERSION" "$_pre" + rm -f "$_pre" + cd "$BUILD_DIR" + lib_mark xz "$XZ_VERSION" + log "xz ${XZ_VERSION} done" + CURRENT_LIB= +fi + +# ── libffi (AIX Toolbox: yum install libffi-devel) ──────────────────────────── +# +# libffi source build fails on AIX: configure explicitly rejects powerpc64-ibm-aix +# and the autoconf bootstrap also fails. The AIX Toolbox provides a working +# 64-bit libffi. We stage it into the embedded tree so it ships with the package. +# +stage_toolbox_lib libffi "$LIBFFI_VERSION" \ + /opt/freeware/lib64/libffi.a \ + /opt/freeware/include/ffi.h \ + /opt/freeware/include/ffi_common.h \ + /opt/freeware/include/ffitarget.h +# Write pkg-config file +if [ ! -f "$EMBEDDED_DESTDIR/lib/pkgconfig/libffi.pc" ]; then + cat > "$EMBEDDED_DESTDIR/lib/pkgconfig/libffi.pc" < "$EMBEDDED_DESTDIR/lib/pkgconfig/${lib}.pc" < "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${AGENT_VERSION:?AGENT_VERSION must be set}" +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED:?EMBEDDED must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${NPROC:?NPROC must be set}" +: "${CC:?CC must be set}" +: "${CXX:?CXX must be set}" +: "${CFLAGS:?CFLAGS must be set}" +: "${CPPFLAGS:?CPPFLAGS must be set}" +: "${LDFLAGS:?LDFLAGS must be set}" +: "${ARFLAGS:?ARFLAGS must be set}" +: "${NM:?NM must be set}" + +# Confirm Stage 1 outputs are present before spending ~20 minutes compiling Python +if [ ! -f "$EMBEDDED_DESTDIR/lib/libssl.so" ] && [ ! -f "$EMBEDDED_DESTDIR/lib/libssl.a" ]; then + log "ERROR: OpenSSL not found at $EMBEDDED_DESTDIR/lib/libssl.so — did Stage 01 (01-native-libs) complete successfully?" + exit 1 +fi + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed. Removing partial outputs." + # Remove the partial Python build tree + rm -rf "$PYTHON_SRC" + # Remove partial install (python${PYTHON_MAJ_MIN} binary is the last step indicator) + if [ -f "$EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" ]; then + log "Removing partial $EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" + rm -f "$EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" + fi + # Do NOT remove $STAGING entirely — other stages may have already run. + fi +} +trap cleanup EXIT + +# --- Prepare directories --- +mkdir -p "$BUILD_DIR/sources" +mkdir -p "$BUILD_DIR/build" + +# ─── Step 1: Download tarball ───────────────────────────────────────────────── + +TARBALL="$BUILD_DIR/sources/$PYTHON_TARBALL" +if [ -f "$TARBALL" ]; then + log "Tarball already present: $TARBALL — skipping download." +else + log "Downloading Python ${PYTHON_VERSION} from $PYTHON_URL" + curl -fSL -o "$TARBALL" "$PYTHON_URL" + log "Download complete: $TARBALL" +fi + +# ─── Step 2: Extract source ─────────────────────────────────────────────────── + +if [ -d "$PYTHON_SRC" ]; then + log "Build directory already exists: $PYTHON_SRC — removing for clean build." + rm -rf "$PYTHON_SRC" +fi + +log "Extracting $TARBALL to $BUILD_DIR/build/" +# AIX system tar does not support decompression flags; decompress explicitly. +gunzip -c "$TARBALL" | tar xf - -C "$BUILD_DIR/build" +log "Extraction complete." + +# ─── Step 3: Apply AIX patches ─────────────────────────────────────────────── +# +# NOTE: These patches were identified from datadog-unix-agent's AIX patches for +# Python 3.8 (omnibus/config/patches/python3/). For Python 3.13, patch offsets +# will have shifted and some may no longer apply or may not be needed at all. +# We use sed-based substitutions rather than patch(1) files to avoid offset +# sensitivity. If a patch no longer applies (pattern not found), we log a +# warning rather than failing — it may mean the upstream fixed the issue. +# +# IMPORTANT: Validate all patches by doing a trial build of Python 3.13.12 on +# AIX before finalising this script. Add or remove sed substitutions based on +# actual configure/compile errors encountered. + +log "Applying AIX-specific patches to Python ${PYTHON_VERSION} source" + +cd "$PYTHON_SRC" + +# Patch: remove libintl dependency from Modules/getpath.c +# libintl (GNU message catalog library) is not present on stock AIX. +# The unix-agent carries this patch for Python 3.8; the include may or may +# not still be present in 3.13 — apply defensively. +if grep -q 'libintl\.h' Modules/getpath.c 2>/dev/null; then + sed 's/#include /\/* libintl not available on AIX *\//g' \ + Modules/getpath.c > Modules/getpath.c.tmp + mv Modules/getpath.c.tmp Modules/getpath.c + log "Applied: removed '#include ' from Modules/getpath.c" +else + log "INFO: libintl.h not found in Modules/getpath.c — patch not needed (OK for 3.13)." +fi + +# Patch: remove libintl dependency from Python/gettext.c (if present in 3.13) +# Some Python versions include libintl in additional translation-related files. +if grep -q 'libintl\.h' Python/gettext.c 2>/dev/null; then + sed 's/#include /\/* libintl not available on AIX *\//g' \ + Python/gettext.c > Python/gettext.c.tmp + mv Python/gettext.c.tmp Python/gettext.c + log "Applied: removed '#include ' from Python/gettext.c" +else + log "INFO: libintl.h not found in Python/gettext.c — patch not needed." +fi + +# Patch: Disable __thread TLS in libpython${PYTHON_MAJ_MIN}.so on AIX +# +# On AIX, GCC's __thread (and C11's _Thread_local) storage class uses the +# XCOFF "local-exec" TLS model. This model is ONLY valid for the main +# executable. When Python uses __thread in libpython${PYTHON_MAJ_MIN}.so (a startup-linked +# shared library), AIX's loader rejects the program with: +# 0509-187 The local-exec model was used for thread-local storage, +# but the module is not the main program. +# +# Fix: (1) Disable HAVE_THREAD_LOCAL on AIX in Include/pyport.h so Python +# falls back to its PyThread_tss_*() API. +# (2) Implement the PyThread_tss_t fallback in Python/pystate.c +# (the upstream code has #error stubs where this fallback should go). +# +# A Python 3.12 interpreter (from the AIX Toolbox) is used to apply these +# multi-line substitutions because sed on AIX doesn't support \n in replacements. + +if grep -q 'HAVE_THREAD_LOCAL' Include/pyport.h 2>/dev/null; then + +if python3.12 << 'PATCH_EOF' +import sys + +# ─── Patch 1: Include/pyport.h ───────────────────────────────────────────── +# On AIX, both _Thread_local (C11) and __thread (GCC extension) use the +# XCOFF "local-exec" TLS model, which AIX's loader rejects in shared libs. +# Insert an _AIX guard BEFORE all TLS model checks so HAVE_THREAD_LOCAL +# is undefined on AIX. Python then falls back to PyThread_tss_*(). +# +# Original block: Patched block: +# #define HAVE_THREAD_LOCAL 1 #define HAVE_THREAD_LOCAL 1 +# #ifdef thread_local #if defined(_AIX) +# ... #undef HAVE_THREAD_LOCAL +# #elif __STDC_VERSION__ >= 201112L #elif defined(thread_local) +# #define _Py_thread_local _Thread_ ... (rest unchanged) +# ... +with open('Include/pyport.h', 'r') as f: + src = f.read() + +OLD = ( + '# define HAVE_THREAD_LOCAL 1\n' + '# ifdef thread_local\n' + '# define _Py_thread_local thread_local\n' + '# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__)\n' + '# define _Py_thread_local _Thread_local\n' + '# elif defined(_MSC_VER) /* AKA NT_THREADS */\n' + '# define _Py_thread_local __declspec(thread)\n' + '# elif defined(__GNUC__) /* includes clang */' +) +NEW = ( + '# define HAVE_THREAD_LOCAL 1\n' + '# if defined(_AIX)\n' + ' /* AIX: _Thread_local and __thread use local-exec TLS model which\n' + ' is incompatible with shared library loading. Fall back to the\n' + ' PyThread_tss_*() API by undefining HAVE_THREAD_LOCAL. */\n' + '# undef HAVE_THREAD_LOCAL\n' + '# elif defined(thread_local)\n' + '# define _Py_thread_local thread_local\n' + '# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__)\n' + '# define _Py_thread_local _Thread_local\n' + '# elif defined(_MSC_VER) /* AKA NT_THREADS */\n' + '# define _Py_thread_local __declspec(thread)\n' + '# elif defined(__GNUC__) /* includes clang */' +) +if OLD in src: + src = src.replace(OLD, NEW, 1) + print('pyport.h: inserted _AIX guard before TLS model selection') +else: + print('pyport.h: HAVE_THREAD_LOCAL block not found — skipping', file=sys.stderr) + +with open('Include/pyport.h', 'w') as f: + f.write(src) + +# ─── Patch 2: Python/pystate.c ───────────────────────────────────────────── +# Replace the three #error stubs in current_fast_{get,set,clear} with a +# working Py_tss_t-based implementation. +with open('Python/pystate.c', 'r') as f: + src = f.read() + +# 2a: Add Py_tss_t declaration after the #ifdef HAVE_THREAD_LOCAL block +OLD_DECL = ( + '#ifdef HAVE_THREAD_LOCAL\n' + '_Py_thread_local PyThreadState *_Py_tss_tstate = NULL;\n' + '#endif' +) +NEW_DECL = ( + '#ifdef HAVE_THREAD_LOCAL\n' + '_Py_thread_local PyThreadState *_Py_tss_tstate = NULL;\n' + '#else\n' + '/* AIX: __thread uses local-exec TLS incompatible with shared libraries.\n' + ' Use a POSIX thread-specific key as fallback. */\n' + 'static Py_tss_t _Py_tss_current = Py_tss_NEEDS_INIT;\n' + '#endif' +) + +# 2b: current_fast_get fallback +OLD_GET = ( + ' // XXX Fall back to the PyThread_tss_*() API.\n' + '# error "no supported thread-local variable storage classifier"\n' + '#endif\n' + '}\n' + '\n' + 'static inline void\n' + 'current_fast_set' +) +NEW_GET = ( + ' /* AIX fallback: use Py_tss_t */\n' + ' if (!PyThread_tss_is_created(&_Py_tss_current)) {\n' + ' return NULL;\n' + ' }\n' + ' return (PyThreadState *)PyThread_tss_get(&_Py_tss_current);\n' + '#endif\n' + '}\n' + '\n' + 'static inline void\n' + 'current_fast_set' +) + +# 2c: current_fast_set fallback +OLD_SET = ( + ' // XXX Fall back to the PyThread_tss_*() API.\n' + '# error "no supported thread-local variable storage classifier"\n' + '#endif\n' + '}\n' + '\n' + 'static inline void\n' + 'current_fast_clear' +) +NEW_SET = ( + ' /* AIX fallback: use Py_tss_t */\n' + ' if (!PyThread_tss_is_created(&_Py_tss_current)) {\n' + ' (void)PyThread_tss_create(&_Py_tss_current);\n' + ' }\n' + ' (void)PyThread_tss_set(&_Py_tss_current, tstate);\n' + '#endif\n' + '}\n' + '\n' + 'static inline void\n' + 'current_fast_clear' +) + +# 2d: current_fast_clear fallback +OLD_CLR = ( + ' // XXX Fall back to the PyThread_tss_*() API.\n' + '# error "no supported thread-local variable storage classifier"\n' + '#endif\n' + '}\n' + '\n' + '#define tstate_verify_not_active' +) +NEW_CLR = ( + ' /* AIX fallback: use Py_tss_t */\n' + ' if (PyThread_tss_is_created(&_Py_tss_current)) {\n' + ' (void)PyThread_tss_set(&_Py_tss_current, NULL);\n' + ' }\n' + '#endif\n' + '}\n' + '\n' + '#define tstate_verify_not_active' +) + +patches = [ + ('pystate.c decl', OLD_DECL, NEW_DECL), + ('pystate.c current_fast_get', OLD_GET, NEW_GET), + ('pystate.c current_fast_set', OLD_SET, NEW_SET), + ('pystate.c current_fast_clear', OLD_CLR, NEW_CLR), +] +for name, old, new in patches: + if old in src: + src = src.replace(old, new, 1) + print(f'{name}: patched') + else: + print(f'{name}: pattern not found — skipping', file=sys.stderr) + +with open('Python/pystate.c', 'w') as f: + f.write(src) +PATCH_EOF +then + log "Applied: AIX TLS __thread patches to pyport.h and pystate.c" +else + log "ERROR: Python TLS patch script failed" + exit 1 +fi +else + log "INFO: pyport.h __GNUC__ pattern not found — TLS patch not needed (already patched?)." +fi + +log "AIX patching complete." + +# ─── Step 4: Configure ─────────────────────────────────────────────────────── +# +# Key AIX-specific flags (inherited from env.sh): +# -maix64 : 64-bit PowerPC code model +# -Wl,-brtl : enable runtime linking (required for dlopen on AIX) +# ARFLAGS=-X64 : 64-bit archive member handling +# NM="/usr/bin/nm -X64" : 64-bit symbol table +# +# --with-openssl=$EMBEDDED_DESTDIR : points to staging path (where OpenSSL +# headers and libs ARE during the build, not $EMBEDDED which is the final path) +# --with-dbmliborder=gdbm : use gdbm built in Stage 1 +# --without-ensurepip : we bootstrap pip manually below (step 7) + +log "Configuring Python ${PYTHON_VERSION} (--prefix=$EMBEDDED)" +log " (Note: configure can take several minutes on POWER8)" + +cd "$PYTHON_SRC" +./configure \ + --prefix="$EMBEDDED" \ + --enable-shared \ + --with-openssl="$EMBEDDED_DESTDIR" \ + --with-dbmliborder=gdbm \ + --without-ensurepip \ + --without-mimalloc \ + CC="$CC" \ + CXX="$CXX" \ + CFLAGS="$CFLAGS -I$EMBEDDED_DESTDIR/include" \ + CPPFLAGS="$CPPFLAGS" \ + LDFLAGS="$LDFLAGS" \ + ARFLAGS="$ARFLAGS" \ + NM="$NM" + +log "Configure complete." + +# ─── Step 5: Build ─────────────────────────────────────────────────────────── +# +# Building CPython from source takes approximately 20 minutes on POWER8. +# This is expected — do not interrupt the build. + +log "Building Python ${PYTHON_VERSION} with make -j$NPROC" +log " (This step takes approximately 20 minutes on POWER8 — please be patient.)" + +cd "$PYTHON_SRC" +make -j"$NPROC" + +log "Build complete." + +# ─── Step 6: Install to staging ────────────────────────────────────────────── +# +# DESTDIR=$STAGING causes files to land in $STAGING/, i.e. +# $STAGING/opt/datadog-agent/embedded/... = $EMBEDDED_DESTDIR/... +# The $EMBEDDED prefix is baked into all binaries; at runtime on the installed +# system the files are at $EMBEDDED and all paths are correct. + +log "Installing Python ${PYTHON_VERSION} to staging tree (DESTDIR=$STAGING)" + +cd "$PYTHON_SRC" +make install DESTDIR="$STAGING" + +log "Install to staging complete." +log "Python executable: $EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" + +# Verify the executable landed where expected +if [ ! -f "$EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" ]; then + log "ERROR: $EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN} not found after make install — install failed." + exit 1 +fi + +# ─── Step 7: Bootstrap pip ─────────────────────────────────────────────────── +# +# We configured with --without-ensurepip, so we must bootstrap pip manually. +# We invoke the STAGING executable ($EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}). +# Python discovers sys.prefix from its executable path at runtime, so it finds +# its stdlib under $EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}/ and installs packages into +# the staging tree — not into $EMBEDDED (which does not exist yet on this host). +# At runtime on the user's system the files are at $EMBEDDED, which is correct. + +log "Bootstrapping pip using staging Python executable" +"$EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" -m ensurepip +log "ensurepip complete." + +log "Upgrading pip to 24.0, setuptools to 75.1.0, and installing wheel" +"$EMBEDDED_DESTDIR/bin/pip${PYTHON_MAJ_MIN}" install --upgrade "pip==24.0" "setuptools==75.1.0" "wheel" +log "pip bootstrap complete." + +# ─── Step 7b: Create AIX .a archive wrapper for libpython${PYTHON_MAJ_MIN}.so ───────────── +# +# On AIX, Go's CGO requires shared libraries to be wrapped in .a archives. +# Without this, the Go cgo tool cannot generate correct //go:cgo_import_dynamic +# directives (which must reference "libpython${PYTHON_MAJ_MIN}.a/libpython${PYTHON_MAJ_MIN}.so"). +# This archive is created in-place next to the .so file. + +log "Creating libpython${PYTHON_MAJ_MIN}.a archive wrapper (required for AIX CGO)" +# On AIX, Go's compiler requires archive member names to end in ".o" or contain ".so." +# Convention: name the 64-bit shared module member "shr_64.o" inside the .a archive. +cd "$EMBEDDED_DESTDIR/lib" +cp "libpython${PYTHON_MAJ_MIN}.so" shr_64.o +ar -X64 -r "libpython${PYTHON_MAJ_MIN}.a" shr_64.o +rm -f shr_64.o +log "Created: $EMBEDDED_DESTDIR/lib/libpython${PYTHON_MAJ_MIN}.a (member: shr_64.o)" + +# Create a version-agnostic symlink so Go code can link with -lpython3 instead +# of hardcoding the minor version. +ln -sf "libpython${PYTHON_MAJ_MIN}.a" "$EMBEDDED_DESTDIR/lib/libpython3.a" +log "Created symlink: libpython3.a -> libpython${PYTHON_MAJ_MIN}.a" + +# ─── Step 7c: Create runtime-path symlink (needed to build C extensions) ───── +# +# Python's sys.prefix is baked-in as $EMBEDDED (/opt/datadog-agent/embedded). +# When building C extensions (cffi, psutil, etc.) in later stages, Python looks +# for ld_so_aix and config files at that prefix. We create a symlink from the +# runtime path to the staging path so Python can find these files during the build. +# The 10-assemble stage will remove this symlink and replace it with the real files. + +if [ -L "$EMBEDDED" ] && [ "$(readlink "$EMBEDDED")" = "$EMBEDDED_DESTDIR" ]; then + log "INFO: $EMBEDDED symlink already correct." +else + if [ -e "$EMBEDDED" ] && [ ! -L "$EMBEDDED" ]; then + log "INFO: $EMBEDDED is a real path — removing to create symlink" + rm -rf "$EMBEDDED" + fi + mkdir -p "$(dirname "$EMBEDDED")" + ln -sf "$EMBEDDED_DESTDIR" "$EMBEDDED" + log "Created runtime-path symlink: $EMBEDDED -> $EMBEDDED_DESTDIR" +fi + +# ─── Step 8: Convenience symlinks ──────────────────────────────────────────── + +log "Creating convenience symlinks python3 -> python${PYTHON_MAJ_MIN} and pip3 -> pip${PYTHON_MAJ_MIN}" +ln -sf "python${PYTHON_MAJ_MIN}" "$EMBEDDED_DESTDIR/bin/python3" +ln -sf "pip${PYTHON_MAJ_MIN}" "$EMBEDDED_DESTDIR/bin/pip3" +log "Symlinks created." + +# ─── Step 9: Remove test directories to save space ─────────────────────────── + +log "Removing Python test directories to reduce package size" +rm -rf "$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}/test" +rm -rf "$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}/unittest/test" +log "Test directories removed." + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/03-rtloader.sh b/packaging/aix/stages/03-rtloader.sh new file mode 100644 index 000000000000..897fc8953d80 --- /dev/null +++ b/packaging/aix/stages/03-rtloader.sh @@ -0,0 +1,223 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="03-rtloader" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED:?EMBEDDED must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${NPROC:?NPROC must be set}" +: "${CC:?CC must be set}" +: "${CXX:?CXX must be set}" +: "${CFLAGS:?CFLAGS must be set}" +: "${CXXFLAGS:?CXXFLAGS must be set}" +: "${LDFLAGS:?LDFLAGS must be set}" + +# --- Pre-flight: confirm Stage 02 completed --- +if [ ! -f "$EMBEDDED_DESTDIR/lib/libpython${PYTHON_MAJ_MIN}.so" ]; then + log "ERROR: libpython${PYTHON_MAJ_MIN}.so not found at $EMBEDDED_DESTDIR/lib — did Stage 02 (02-python) complete successfully?" + exit 1 +fi + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed. Removing partial outputs." + rm -rf /opt/datadog-agent/rtloader/build + rm -f "$STAGING/opt/datadog-agent/rtloader/libdatadog-agent-rtloader.so" + rm -f "$STAGING/opt/datadog-agent/rtloader/libdatadog-agent-three.so" + fi +} +trap cleanup EXIT + +# ─── Step 1: Clean and create rtloader build directory ──────────────────────── + +log "Cleaning rtloader build directory" +rm -rf /opt/datadog-agent/rtloader/build +mkdir -p /opt/datadog-agent/rtloader/build + +# ─── Step 2: CMake configure ────────────────────────────────────────────────── +# +# OBJECT_MODE=64 is required on AIX so that the IBM linker produces 64-bit +# XCOFF objects. cmake is invoked from within the build directory. +# +# -DBUILD_DEMO=OFF : skip the demo binary (not needed in the package) +# -DDISABLE_PYTHON2=ON : only build the Python 3 binding (libdatadog-agent-three.so) +# -DPython3_INCLUDE_DIR : staging path (where headers are during the build) +# -DPython3_LIBRARY : EMBEDDED (installed) path, not staging path +# +# Embedded path trick: +# On AIX, the linker bakes the Python library path into the XCOFF loader +# section of libdatadog-agent-three.so. If we pass the staging path +# ($EMBEDDED_DESTDIR), the installed .so will look for Python at the build +# host's staging tree — which does not exist on a fresh target system. +# Instead, we pass the EMBEDDED (installed) path ($EMBEDDED/lib) and create +# a symlink there pointing to the staging copy so cmake can find the file +# during the build. After installp installs the real libpython${PYTHON_MAJ_MIN}.so to +# $EMBEDDED/lib, the baked-in path resolves correctly. + +log "Creating embedded-path symlink so rtloader embeds the installed Python path" +mkdir -p "$EMBEDDED/lib" +# Symlink the staging Python .so to the EMBEDDED path for the cmake build. +# installp will overwrite the symlink with the real .so on the target system. +ln -sf "$EMBEDDED_DESTDIR/lib/libpython${PYTHON_MAJ_MIN}.so" "$EMBEDDED/lib/libpython${PYTHON_MAJ_MIN}.so" 2>/dev/null || true + +log "Running cmake for rtloader" +cd /opt/datadog-agent/rtloader/build + +OBJECT_MODE=64 cmake \ + -DCMAKE_C_COMPILER="$CC" \ + -DCMAKE_CXX_COMPILER="$CXX" \ + -DCMAKE_C_FLAGS="$CFLAGS" \ + -DCMAKE_CXX_FLAGS="$CXXFLAGS" \ + -DCMAKE_SHARED_LINKER_FLAGS="$LDFLAGS" \ + -DBUILD_DEMO=OFF \ + -DDISABLE_PYTHON2=ON \ + -DPython3_INCLUDE_DIR="$EMBEDDED_DESTDIR/include/python${PYTHON_MAJ_MIN}" \ + -DPython3_LIBRARY="$EMBEDDED/lib/libpython${PYTHON_MAJ_MIN}.so" \ + .. + +log "cmake configure complete." + +# ─── Step 3: Build ──────────────────────────────────────────────────────────── + +log "Building rtloader with make -j$NPROC" +OBJECT_MODE=64 make -j"$NPROC" +log "rtloader build complete." + +# ─── Step 3b: Relink libdatadog-agent-three.so to use libpython${PYTHON_MAJ_MIN}.a ───────── +# +# cmake/make built three.so against libpython${PYTHON_MAJ_MIN}.so (the shared object file). +# However, on AIX, libpython${PYTHON_MAJ_MIN}.a(shr_64.o) and libpython${PYTHON_MAJ_MIN}.so are the same +# code but identified as DIFFERENT modules by the XCOFF loader (different names). +# +# The agent binary startup-loads libpython${PYTHON_MAJ_MIN}.a(shr_64.o) via python_aix.go. +# If three.so depends on libpython${PYTHON_MAJ_MIN}.so, the loader treats it as a SECOND Python +# instance. With two Python instances, Python C extensions fail with: +# SystemError: initialization of _datetime did not return an extension module +# because PyModule_Type lives at different addresses in the two copies. +# +# Fix: relink three.so using the saved link command, substituting .so with .a. +# This makes three.so depend on libpython${PYTHON_MAJ_MIN}.a(shr_64.o), which matches the +# agent binary's startup-loaded module — the loader deduplicates to ONE instance. +# +# Note: libpython${PYTHON_MAJ_MIN}.a and libpython${PYTHON_MAJ_MIN}.so are byte-for-byte identical on AIX. +# The .a form (archive containing shr_64.o) is the canonical AIX shared library. + +log "Relinking libdatadog-agent-three.so to use libpython${PYTHON_MAJ_MIN}.a(shr_64.o)" +cd /opt/datadog-agent/rtloader/build/three + +# Step 1: Re-run the cmake ExportImportList (export symbols file generation) +EXPORT_CMD=$(head -1 CMakeFiles/datadog-agent-three.dir/link.txt) +eval "$EXPORT_CMD" + +# Step 2: Re-run the link command, substituting .so with .a for libpython${PYTHON_MAJ_MIN} +LINK_CMD=$(tail -1 CMakeFiles/datadog-agent-three.dir/link.txt) +LINK_CMD_FIXED=$(printf '%s' "$LINK_CMD" | sed "s|libpython${PYTHON_MAJ_MIN}\\.so|libpython${PYTHON_MAJ_MIN}.a|g") +eval "$LINK_CMD_FIXED" +log "Relink complete. three.so now depends on libpython${PYTHON_MAJ_MIN}.a(shr_64.o)" + +# Verify the dependency switched to .a +if dump -X64 -Hv libdatadog-agent-three.so 2>/dev/null | grep "libpython${PYTHON_MAJ_MIN}.so"; then + log "ERROR: libdatadog-agent-three.so still references libpython${PYTHON_MAJ_MIN}.so after relink!" + exit 1 +fi +log "Verified: libdatadog-agent-three.so depends on libpython${PYTHON_MAJ_MIN}.a(shr_64.o)" + +# ─── Step 4: Copy outputs to staging ────────────────────────────────────────── +# +# The two produced .so files must land in $STAGING/opt/datadog-agent/rtloader/ +# so the agent binary can find them at runtime via LIBPATH. + +log "Copying rtloader .so files to staging" +mkdir -p "$STAGING/opt/datadog-agent/rtloader" +cp rtloader/libdatadog-agent-rtloader.so \ + three/libdatadog-agent-three.so \ + "$STAGING/opt/datadog-agent/rtloader/" +log "Copy complete." + +# ─── Step 4b: Create AIX .a archive wrappers ────────────────────────────────── +# +# On AIX, Go's CGO requires shared libraries wrapped in .a archives. +# Without these, Go cannot generate correct //go:cgo_import_dynamic directives +# (which must reference "lib.a/lib.so" format). +# Archives are created in the build tree where CGO_LDFLAGS points. + +log "Creating .a archive wrappers for rtloader .so files (AIX CGO requirement)" +# On AIX, Go's compiler (lex.go) requires the archive member name to either end in +# ".o" or contain ".so." (a version number). The conventional AIX name for the +# 64-bit shared module inside an archive is "shr_64.o". +cd /opt/datadog-agent/rtloader/build/rtloader +cp libdatadog-agent-rtloader.so shr_64.o +ar -X64 -r libdatadog-agent-rtloader.a shr_64.o +rm -f shr_64.o +cd /opt/datadog-agent/rtloader/build/three +cp libdatadog-agent-three.so shr_64.o +ar -X64 -r libdatadog-agent-three.a shr_64.o +rm -f shr_64.o +log "Archive wrappers created (member: shr_64.o in each .a)." + +# ─── Step 4c: Copy .a archive wrappers to staging ───────────────────────────── +# +# The AIX dynamic linker resolves shared library dependencies by looking for +# lib.a(shr_64.o) archives in LIBPATH. If only the .so file is present +# the loader raises "Dependent module lib.a(shr_64.o) could not be loaded." +# Both the .so and the .a must exist in the same directory in the package. + +log "Copying rtloader .a archive wrappers to staging" +cp /opt/datadog-agent/rtloader/build/rtloader/libdatadog-agent-rtloader.a \ + /opt/datadog-agent/rtloader/build/three/libdatadog-agent-three.a \ + "$STAGING/opt/datadog-agent/rtloader/" +log "Archive wrappers copied to staging." + +# ─── Step 5: Verify XCOFF64 magic bytes ─────────────────────────────────────── +# +# XCOFF64 files begin with magic bytes 01 f7 (big-endian 0x01F7 = XCOFF64_MAGIC). +# We read the first line of od output (8 bytes) and check that the first two bytes +# match. If they do not, the build produced a wrong-format binary. + +log "Verifying libdatadog-agent-three.so is XCOFF64" +MAGIC=$(od -A x -t x1 "$STAGING/opt/datadog-agent/rtloader/libdatadog-agent-three.so" | head -1 | awk '{print $2 $3}') +if [ "$MAGIC" != "01f7" ]; then + log "ERROR: libdatadog-agent-three.so is not XCOFF64 (got: $MAGIC)" + log " Expected magic bytes: 01 f7" + log " Check that OBJECT_MODE=64 is set and that $CXX produces 64-bit XCOFF output." + exit 1 +fi +log "XCOFF64 magic verified for libdatadog-agent-three.so (magic: $MAGIC)" + +log "Verifying libdatadog-agent-rtloader.so is XCOFF64" +MAGIC=$(od -A x -t x1 "$STAGING/opt/datadog-agent/rtloader/libdatadog-agent-rtloader.so" | head -1 | awk '{print $2 $3}') +if [ "$MAGIC" != "01f7" ]; then + log "ERROR: libdatadog-agent-rtloader.so is not XCOFF64 (got: $MAGIC)" + log " Expected magic bytes: 01 f7" + log " Check that OBJECT_MODE=64 is set and that $CXX produces 64-bit XCOFF output." + exit 1 +fi +log "XCOFF64 magic verified for libdatadog-agent-rtloader.so (magic: $MAGIC)" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/04-agent.sh b/packaging/aix/stages/04-agent.sh new file mode 100644 index 000000000000..522200631118 --- /dev/null +++ b/packaging/aix/stages/04-agent.sh @@ -0,0 +1,186 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="04-agent" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${AGENT_VERSION:?AGENT_VERSION must be set}" +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED:?EMBEDDED must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${NPROC:?NPROC must be set}" +: "${CC:?CC must be set}" +: "${GOPATH:?GOPATH must be set}" +: "${GOROOT:?GOROOT must be set}" +: "${CGO_ENABLED:?CGO_ENABLED must be set}" + +# --- Pre-flight: confirm Stage 03 completed --- +if [ ! -f "$STAGING/opt/datadog-agent/rtloader/libdatadog-agent-rtloader.so" ]; then + log "ERROR: libdatadog-agent-rtloader.so not found at $STAGING/opt/datadog-agent/rtloader — did Stage 03 (03-rtloader) complete successfully?" + exit 1 +fi + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed. Removing partial outputs." + rm -f "$STAGING/opt/datadog-agent/bin/agent" + rm -f "$STAGING/opt/datadog-agent/bin/trace-agent" + fi +} +trap cleanup EXIT + +# ─── Step 1: Create output directory ────────────────────────────────────────── + +log "Creating staging bin directory" +mkdir -p "$STAGING/opt/datadog-agent/bin" + +# ─── Step 2: Set rtloader CGO flags ─────────────────────────────────────────── +# +# The global CGO_CFLAGS/CGO_LDFLAGS from env.sh point to /opt/freeware headers +# and libs. We extend them here with rtloader-specific paths so that the Go +# packages that import rtloader (pkg/collector/python/) can find its C headers +# and link against the .so files we built in Stage 03. +# +# Note: we point -L at the BUILD paths (rtloader/build/rtloader and +# rtloader/build/three), not the staging paths. The .so files are in the build +# tree; the staging copies are for the final package only. + +log "Setting rtloader CGO flags" +export CGO_CFLAGS="$CGO_CFLAGS -I/opt/datadog-agent/rtloader/include" +# +# -lpython3 (via libpython3.a symlink) causes libpython3.a(shr_64.o) to appear in the agent binary's +# XCOFF startup-load chain. This is necessary but not sufficient: the binary +# must also EXPORT Python API symbols so that extension modules with IMPid="." +# (which means "look in the main program's export table") can find them. +# +# -Wl,-bE:python.exp adds ~2762 Python API symbols to the agent binary's own +# EXP (export) table. Go's CGO security filter rejects -bE in #cgo LDFLAGS, +# so it must be passed here via CGO_LDFLAGS instead. The python_aix.go file +# handles the Py_IsInitialized() call that creates the live Go→CGO reference +# needed to trigger the //go:cgo_import_dynamic for the Python shared library. +# +# The libpython3.a -> libpython3.X.a symlink is created by Stage 02. +PYTHON_EXP="$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}/config-${PYTHON_MAJ_MIN}/python.exp" +if [ ! -f "$PYTHON_EXP" ]; then + log "ERROR: $PYTHON_EXP not found — did Stage 02 (02-python) complete?" + exit 1 +fi +log "Using Python export file: $PYTHON_EXP" +export CGO_LDFLAGS="$CGO_LDFLAGS \ + -L/opt/datadog-agent/rtloader/build/rtloader \ + -L/opt/datadog-agent/rtloader/build/three \ + -L$EMBEDDED_DESTDIR/lib \ + -lpython3 \ + -Wl,-bE:$PYTHON_EXP \ + -Wl,-blibpath:/opt/datadog-agent/rtloader:/opt/datadog-agent/embedded/lib:/opt/freeware/lib64:/opt/freeware/lib:/usr/lib:/lib" + +# ─── Step 3: Get commit hash ────────────────────────────────────────────────── +# +# AGENT_COMMIT may be pre-set by the caller (e.g. when the source was transferred +# without the .git directory). If unset, resolve it from the local git repo. + +if [ -n "${AGENT_COMMIT:-}" ]; then + COMMIT=$AGENT_COMMIT + log "Using pre-set commit hash: $COMMIT" +elif [ -d /opt/datadog-agent/.git ]; then + COMMIT=$(git -C /opt/datadog-agent rev-parse --short HEAD) + log "Resolved commit hash from .git: $COMMIT" +else + log "ERROR: AGENT_COMMIT env var not set and /opt/datadog-agent/.git not found." + log " Set AGENT_COMMIT to the short SHA of the source tree being built," + log " e.g.: AGENT_COMMIT=\$(git -C /path/to/repo rev-parse --short HEAD)" + exit 1 +fi +log "Building agent version $AGENT_VERSION at commit $COMMIT" + +# ─── Step 4: Build the agent binary ─────────────────────────────────────────── +# +# Build tags come from tasks/build_tags.py AIX_AGENT_TAGS + COMMON_TAGS: +# python, otlp, osusergo, datadog.no_waf, zstd +# + grpcnotrace, retrynotrace, no_dynamic_plugins, trivy_no_javadb (COMMON_TAGS) +# +# Note: pythonHome3 must be set explicitly here. +# The agent binary computes Python home as filepath.Join(executableFolder, "../../embedded"). +# On Linux (standard omnibus), the binary lives at bin/agent/agent so "../../embedded" +# correctly resolves to /opt/datadog-agent/embedded. +# On AIX we place the binary at bin/agent (one level shallower) so the relative path +# would resolve to /opt/embedded — which does not exist. +# Setting pythonHome3 via -ldflags overrides the relative calculation. + +log "Building agent binary via inv agent.build" +cd /opt/datadog-agent +rm -f "$STAGING/opt/datadog-agent/bin/agent" +python3.12 -m invoke agent.build \ + --rebuild \ + --skip-assets \ + --exclude-rtloader \ + --rtloader-root=/opt/datadog-agent/rtloader \ + --embedded-path="$EMBEDDED_DESTDIR" \ + --python-home-3="$EMBEDDED" \ + --agent-bin="$STAGING/opt/datadog-agent/bin/agent" + +strip -X64 "$STAGING/opt/datadog-agent/bin/agent" +log "agent binary build complete: $STAGING/opt/datadog-agent/bin/agent" + +# ─── Step 5: Build the trace-agent binary ───────────────────────────────────── +# +# Build tags come from tasks/build_tags.py AGENT_TAGS minus AIX_EXCLUDE_TAGS, plus COMMON_TAGS. + +log "Building trace-agent binary via inv trace-agent.build" +cd /opt/datadog-agent +python3.12 -m invoke trace-agent.build --rebuild +rm -f "$STAGING/opt/datadog-agent/bin/trace-agent" +cp /opt/datadog-agent/bin/trace-agent/trace-agent "$STAGING/opt/datadog-agent/bin/trace-agent" +strip -X64 "$STAGING/opt/datadog-agent/bin/trace-agent" +log "trace-agent binary build complete: $STAGING/opt/datadog-agent/bin/trace-agent" + +# ─── Step 6: Verify XCOFF64 magic bytes ─────────────────────────────────────── +# +# Both binaries must be XCOFF64 (magic bytes 01 f7). A non-XCOFF64 result +# would indicate a cross-compile or wrong-format build. + +log "Verifying agent binary is XCOFF64" +MAGIC=$(od -A x -t x1 "$STAGING/opt/datadog-agent/bin/agent" | head -1 | awk '{print $2 $3}') +if [ "$MAGIC" != "01f7" ]; then + log "ERROR: agent binary is not XCOFF64 (got: $MAGIC)" + log " Expected magic bytes: 01 f7" + log " Ensure CGO_ENABLED=1 and that GOROOT points to the AIX Go toolchain." + exit 1 +fi +log "XCOFF64 magic verified for agent binary (magic: $MAGIC)" + +log "Verifying trace-agent binary is XCOFF64" +MAGIC=$(od -A x -t x1 "$STAGING/opt/datadog-agent/bin/trace-agent" | head -1 | awk '{print $2 $3}') +if [ "$MAGIC" != "01f7" ]; then + log "ERROR: trace-agent binary is not XCOFF64 (got: $MAGIC)" + log " Expected magic bytes: 01 f7" + log " Ensure CGO_ENABLED=1 and that GOROOT points to the AIX Go toolchain." + exit 1 +fi +log "XCOFF64 magic verified for trace-agent binary (magic: $MAGIC)" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/05-python-extensions.sh b/packaging/aix/stages/05-python-extensions.sh new file mode 100755 index 000000000000..5022ed5eb21e --- /dev/null +++ b/packaging/aix/stages/05-python-extensions.sh @@ -0,0 +1,307 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="05-python-extensions" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${AGENT_VERSION:?AGENT_VERSION must be set}" +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${WHEEL_CACHE:?WHEEL_CACHE must be set}" +: "${INTEGRATIONS_CORE:?INTEGRATIONS_CORE must be set}" + +PIP=$EMBEDDED_DESTDIR/bin/pip${PYTHON_MAJ_MIN} +PYTHON_BIN=$EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN} + +# --- Pre-flight: confirm pip${PYTHON_MAJ_MIN} exists --- +if [ ! -x "$PIP" ]; then + log "ERROR: $PIP not found — did Stage 02 (02-python) complete successfully?" + exit 1 +fi + +# --- Pre-flight: confirm integrations-core is checked out --- +LOCKFILE="$INTEGRATIONS_CORE/.deps/resolved/linux-x86_64_${PYTHON_MAJ_MIN}.txt" +if [ ! -f "$LOCKFILE" ]; then + log "ERROR: $LOCKFILE not found — did Stage 00 (00-checkout) complete successfully?" + exit 1 +fi + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed." + log " Re-run after fixing the error by deleting the sentinel:" + log " rm $SENTINEL" + fi +} +trap cleanup EXIT + +# ─── Version pins ────────────────────────────────────────────────────────────── +# +# Versions are read from the integrations-core lockfile so they stay in sync +# with the Linux omnibus pipeline automatically when the integrations-core +# commit is updated in release.json. +# +# The lockfile uses PEP 508 URL format: +# package @ https://...registry.../package-VERSION-...-platform.whl#sha256=... +# We extract the version from the package name embedded in the wheel filename. + +extract_version() { + _pkg=$1 + # Match "pkgname @ https://...pkgname-VERSION-..." and extract VERSION + grep -i "^${_pkg} @ " "$LOCKFILE" | head -1 \ + | sed "s/.*${_pkg}-\([0-9][0-9.]*\)-.*/\1/" +} + +CFFI_VERSION=$(extract_version cffi) +PSUTIL_VERSION=$(extract_version psutil) +LXML_VERSION=$(extract_version lxml) +CRYPTOGRAPHY_VERSION=$(extract_version cryptography) + +if [ -z "$CFFI_VERSION" ] || [ -z "$PSUTIL_VERSION" ] || [ -z "$LXML_VERSION" ] || [ -z "$CRYPTOGRAPHY_VERSION" ]; then + log "ERROR: could not read one or more package versions from $LOCKFILE" + log " cffi=$CFFI_VERSION psutil=$PSUTIL_VERSION lxml=$LXML_VERSION cryptography=$CRYPTOGRAPHY_VERSION" + log " Check that $LOCKFILE contains cffi, psutil, lxml, cryptography entries." + exit 1 +fi + +log "Package versions (from integrations-core lockfile):" +log " cffi==$CFFI_VERSION" +log " psutil==$PSUTIL_VERSION" +log " lxml==$LXML_VERSION" +log " cryptography==$CRYPTOGRAPHY_VERSION" + +# ─── Step 1: cffi (C extension, bundled libffi) ──────────────────────────────── +# +# cffi is required by cryptography (and many other packages). It builds against +# the libffi we bundled in Stage 1 via the standard CFLAGS/CPPFLAGS/LDFLAGS that +# point to $EMBEDDED_DESTDIR. +# +# AIX/GCC TLS issue (0509-187 / USE__THREAD): +# cffi's setup.py calls config.try_compile('__thread int x;') which succeeds +# on AIX/GCC (the syntax is valid) and then adds -DUSE__THREAD to the build. +# -DUSE__THREAD makes _cffi_backend.c use __thread for TLS variables, which +# AIX's dynamic linker rejects in shared libraries (error 0509-187: local-exec +# model). -ftls-model=global-dynamic is not a fix because GCC on AIX generates +# calls to __tls_get_addr (ELF TLS) which does not exist on XCOFF/AIX. +# +# Fix: patch cffi's setup.py before install to skip USE__THREAD on AIX. +# Without USE__THREAD, cffi uses pthread_key_t for TLS which works correctly. +# +# AIX/GCC __lwsync note: cffi's call_python.c uses __lwsync() which is an IBM +# XLC intrinsic not available in GCC. We redefine it to GCC's +# __sync_synchronize() which provides an equivalent full memory barrier. + +log "Installing cffi==$CFFI_VERSION (C extension, bundled libffi)" +log " Downloading cffi source and patching for AIX TLS compatibility" + +# Download cffi source via pip (handles URL resolution, caching, etc.) +CFFI_SRCDIR="/tmp/cffi-${CFFI_VERSION}-aix-src" +rm -rf "$CFFI_SRCDIR" +mkdir -p "$CFFI_SRCDIR" +$PIP download --no-deps --no-binary cffi "cffi==$CFFI_VERSION" -d "$CFFI_SRCDIR" +CFFI_TARBALL=$(find "$CFFI_SRCDIR" -name 'cffi-*.tar.gz' 2>/dev/null | head -1) +if [ -z "$CFFI_TARBALL" ]; then + log "ERROR: could not download cffi source tarball" + exit 1 +fi +log " cffi source: $CFFI_TARBALL" +CFFI_BUILDDIR="/tmp/cffi-${CFFI_VERSION}-build" +rm -rf "$CFFI_BUILDDIR" +# Use Python tarfile module for extraction: AIX native tar rejects modern +# tar formats (pax headers) used by cffi's PyPI distribution. +$PYTHON_BIN -c "import tarfile, os; tarfile.open('$CFFI_TARBALL').extractall('/tmp')" +mv "/tmp/cffi-${CFFI_VERSION}" "$CFFI_BUILDDIR" + +# Patch setup.py: add sys.platform != 'aix' check to ask_supports_thread() +# so that __thread is not used in the shared library (causes 0509-187 on AIX). +sed "s/sys.platform != 'win32' and\$/sys.platform != 'win32' and sys.platform != 'aix' and/" \ + "$CFFI_BUILDDIR/setup.py" > "$CFFI_BUILDDIR/setup.py.new" +mv "$CFFI_BUILDDIR/setup.py.new" "$CFFI_BUILDDIR/setup.py" +log " setup.py patched: AIX check added to ask_supports_thread()" + +CFLAGS="-maix64 -D__lwsync=__sync_synchronize" \ + $PIP install --no-cache-dir "$CFFI_BUILDDIR" +log "cffi==$CFFI_VERSION installed successfully" + +# ─── Step 2: psutil (C extension) ───────────────────────────────────────────── +# +# psutil provides process and system information. --no-binary forces a source +# build so it is compiled against our bundled headers. + +log "Installing psutil==$PSUTIL_VERSION (C extension)" +$PIP install --no-binary psutil "psutil==$PSUTIL_VERSION" +log "psutil==$PSUTIL_VERSION installed successfully" + +# ─── Step 3: lxml (C extension, bundled libxml2/libxslt) ────────────────────── +# +# lxml is required by the ibm_was check. We pass explicit CFLAGS/LDFLAGS to +# ensure it links against our bundled libxml2 and libxslt from Stage 1. +# The libxml2 headers are under include/libxml2/ so we add that path explicitly. + +log "Installing lxml==$LXML_VERSION (C extension, bundled libxml2/libxslt)" +CFLAGS="-maix64 -I$EMBEDDED_DESTDIR/include/libxml2" \ +LDFLAGS="-maix64 -Wl,-brtl -L$EMBEDDED_DESTDIR/lib" \ + $PIP install --no-binary lxml "lxml==$LXML_VERSION" +log "lxml==$LXML_VERSION installed successfully" + +# ─── Step 4: cryptography (Rust/PyO3 extension) ─────────────────────────────── +# +# cryptography requires a Rust build. The wheel cache (keyed by version) avoids +# the ~15-minute Rust compilation on subsequent builds. +# +# AIX-specific Rust flags: +# CARGO_PROFILE_RELEASE_STRIP=none — IBM Rust 1.92 bug: stripping .info section +# from proc-macro artifacts breaks rustc +# CARGO_PROFILE_RELEASE_LTO=off — LLVM fat LTO uses .ipa bitcode sections +# that do not exist in AIX XCOFF format +# CC=/opt/freeware/bin/gcc — cc-rs defaults to IBM xlc which rejects GCC +# flags like -fPIC, -ffunction-sections, -maix64 +# OPENSSL_DIR — tells the openssl-sys crate where our +# bundled OpenSSL lives (staging path) +# ARFLAGS unset — env.sh sets ARFLAGS="-X64 -cru" which +# includes the ar operation code; cc-rs appends +# its own "cq" operation, producing "ar -cru cq" +# which AIX ar rejects (two operation codes). +# OBJECT_MODE=64 already ensures 64-bit archives. +# RUSTFLAGS="-C link-arg=-bbigtoc" — cryptography-rust has ~144KB TOC entries +# which exceeds AIX ld's default 64KB TOC limit. +# -bbigtoc removes this limit for AIX XCOFF. + +log "Installing cryptography==$CRYPTOGRAPHY_VERSION (Rust/PyO3 extension)" +log " Setting Rust environment: PATH=/opt/freeware/lib/RustSDK/1.92/bin:..." +export PATH=/opt/freeware/lib/RustSDK/1.92/bin:"$PATH" +export CARGO_HOME=/opt/cargo + +# Check wheel cache (keyed by version so a version bump triggers a fresh build) +CRYPTO_CACHE_DIR="$WHEEL_CACHE/cryptography-$CRYPTOGRAPHY_VERSION" +mkdir -p "$CRYPTO_CACHE_DIR" +# Match only full-AIX-tag wheels (e.g. aix_7302_2419_64) not legacy aix_ppc64 renames. +# aix_*_* requires at least one underscore within the AIX portion, which aix_ppc64 lacks. +CACHED_CRYPTO=$(find "$CRYPTO_CACHE_DIR" -name "cryptography-${CRYPTOGRAPHY_VERSION}-*-aix_*_*.whl" 2>/dev/null | head -1) + +if [ -n "$CACHED_CRYPTO" ]; then + log "Found cached cryptography wheel: $CACHED_CRYPTO" + # Install by direct path — bypasses PyPI lookup and works with any AIX tag + # that matches the current interpreter's supported tags. + $PIP install "$CACHED_CRYPTO" + log "cryptography==$CRYPTOGRAPHY_VERSION installed from cache" +else + log "No cached wheel found for cryptography==$CRYPTOGRAPHY_VERSION — building from source" + # Pre-install maturin (cryptography build dep) into staging. + # We use --no-build-isolation below so pip does not create an isolated venv, + # allowing cryptography's build script to import cffi from our staging env + # (which was compiled with AIX fixes) rather than from a fresh isolated build + # that would fail with AIX TLS error 0509-187. + # ARFLAGS="" prevents the cc-rs "ar -cru cq" double-operation conflict. + log "Pre-installing maturin (required for --no-build-isolation cryptography build)" + ARFLAGS="" \ + $PIP install "maturin>=1,<2" + log "maturin installed" + + OPENSSL_DIR=$EMBEDDED_DESTDIR \ + CARGO_PROFILE_RELEASE_STRIP=none \ + CARGO_PROFILE_RELEASE_LTO=off \ + RUSTFLAGS="-C link-arg=-bbigtoc" \ + ARFLAGS="" \ + $PIP install --no-build-isolation --no-binary cryptography "cryptography==$CRYPTOGRAPHY_VERSION" + log "cryptography==$CRYPTOGRAPHY_VERSION build complete" + + # Remove maturin (build-time tool; not needed at runtime) + $PIP uninstall -y maturin 2>/dev/null || true + + # Cache the built wheel for subsequent builds. + # Keep the original filename (with the full AIX platform tag from this system) + # so that pip can match it by tag on cache restore. + BUILT_WHEEL=$(find "${HOME}/.cache/pip" -name "cryptography-${CRYPTOGRAPHY_VERSION}-*.whl" 2>/dev/null | head -1) + if [ -n "$BUILT_WHEEL" ]; then + CACHE_NAME=$(basename "$BUILT_WHEEL") + cp "$BUILT_WHEEL" "$CRYPTO_CACHE_DIR/$CACHE_NAME" + log "Cached wheel to $CRYPTO_CACHE_DIR/$CACHE_NAME" + else + log "WARNING: could not locate built cryptography wheel — next build will rebuild from source" + fi +fi + +log "cryptography==$CRYPTOGRAPHY_VERSION installed successfully" + +# ─── Step 5: pymqi (conditional — IBM MQ Client required) ───────────────────── +# +# pymqi is a C extension wrapping the IBM MQ C Client API. It is required by +# the ibm_mq and ibm_ace checks. The MQ Client shared libraries (libmqm.so, +# libmqmcs.so) are NOT bundled — they are a user-installed prerequisite on the +# target system. We skip gracefully if the build host does not have MQ headers. + +if [ -d /opt/mqm/inc ]; then + log "IBM MQ Client found at /opt/mqm — building pymqi" + MQ_HOME=/opt/mqm + CFLAGS="$CFLAGS -I${MQ_HOME}/inc" \ + LDFLAGS="$LDFLAGS -L${MQ_HOME}/lib64 -L${MQ_HOME}/lib -Wl,-brtl -lmqm" \ + $PIP install --no-binary pymqi "pymqi==1.12.13" + log "pymqi installed successfully" +else + log "WARNING: IBM MQ Client not found at /opt/mqm — skipping pymqi (ibm_mq/ibm_ace checks will not work)" + log " Install IBM MQ Client 9.1 LTS from IBM Fix Central and re-run this stage to enable MQ checks." +fi + +# ─── Step 6: pyodbc (conditional — unixODBC headers required) ───────────────── +# +# pyodbc is a C++ extension wrapping unixODBC. It is required by the ibm_i +# check. The IBM i Access ODBC driver is a separate user-installed prerequisite +# on the target system. We skip gracefully if the build host lacks sql.h. + +if [ -f /opt/freeware/include/sql.h ] || [ -f /usr/include/sql.h ]; then + log "unixODBC headers found — building pyodbc" + CFLAGS="$CFLAGS -I/opt/freeware/include" \ + LDFLAGS="$LDFLAGS -L/opt/freeware/lib -lodbc" \ + $PIP install --no-binary pyodbc "pyodbc==5.3.0" + log "pyodbc installed successfully" +else + log "WARNING: unixODBC headers not found — skipping pyodbc (ibm_i check will not work)" + log " Install unixODBC development headers (yum install unixODBC unixODBC-devel) and re-run this stage." +fi + +# ─── Step 7: ibm_db (conditional — IBM DB2 CLI driver required) ─────────────── +# +# ibm_db is a C++ extension for IBM DB2. It is required by the ibm_db2 check. +# The CLI driver shared libraries are NOT bundled — user-installed prerequisite +# on the target system. We skip gracefully if no driver is found. + +if [ -n "${IBM_DB_HOME:-}" ] || [ -d /opt/ibm/db2/clidriver ]; then + DB2_HOME=${IBM_DB_HOME:-/opt/ibm/db2/clidriver} + log "IBM DB2 CLI driver found at $DB2_HOME — building ibm_db" + IBM_DB_HOME=$DB2_HOME \ + CFLAGS="$CFLAGS -I${DB2_HOME}/include" \ + LDFLAGS="$LDFLAGS -L${DB2_HOME}/lib -Wl,-brtl -ldb2" \ + $PIP install --no-binary ibm_db "ibm_db==3.2.6" + log "ibm_db installed successfully" +else + log "WARNING: IBM DB2 CLI driver not found — skipping ibm_db (ibm_db2 check will not work)" + log " Install the IBM DB2 CLI Driver (e.g. to /opt/ibm/db2/clidriver) or set IBM_DB_HOME" + log " and re-run this stage." +fi + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/06-pydantic.sh b/packaging/aix/stages/06-pydantic.sh new file mode 100755 index 000000000000..ef2db90b076a --- /dev/null +++ b/packaging/aix/stages/06-pydantic.sh @@ -0,0 +1,181 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="06-pydantic" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${WHEEL_CACHE:?WHEEL_CACHE must be set}" +: "${INTEGRATIONS_CORE:?INTEGRATIONS_CORE must be set}" + +PIP=$EMBEDDED_DESTDIR/bin/pip${PYTHON_MAJ_MIN} + +# --- Pre-flight: confirm pip${PYTHON_MAJ_MIN} exists --- +if [ ! -x "$PIP" ]; then + log "ERROR: $PIP not found — did Stage 02 (02-python) complete successfully?" + exit 1 +fi + +# --- Pre-flight: confirm integrations-core is checked out --- +PYPROJECT="$INTEGRATIONS_CORE/datadog_checks_base/pyproject.toml" +if [ ! -f "$PYPROJECT" ]; then + log "ERROR: $PYPROJECT not found — did Stage 00 (00-checkout) complete successfully?" + exit 1 +fi + +# --- Cleanup on failure --- +# pip installs are not easy to roll back; the sentinel not being written is +# sufficient to trigger a re-run. +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed." + log " Re-run after fixing the error by deleting the sentinel:" + log " rm $SENTINEL" + log " Common causes:" + log " - Rust SDK not installed: yum install rust1.92.ppc cargo1.92.ppc rust1.92-std-static.ppc" + log " - Insufficient disk space (need ~7 GB free in /tmp, ~4 GB in /)" + log " - CC not set to GCC (xlc rejects -fPIC, -maix64 flags)" + fi +} +trap cleanup EXIT + +# ─── Step 1: Read required pydantic version from integrations-core ───────────── +# +# datadog_checks_base/pyproject.toml pins an exact pydantic version, e.g.: +# "pydantic==2.11.7", +# pydantic itself pins exactly one pydantic-core version. By installing the +# exact pydantic version required by the checks, we guarantee the right +# pydantic-core version is pulled in. +# +# The wheel cache is keyed by this pydantic version so that upgrading pydantic +# (via a new integrations-core commit) automatically invalidates the cache and +# triggers a fresh Rust build. + +PYDANTIC_VERSION=$(grep '"pydantic==' "$PYPROJECT" | sed 's/.*pydantic==\([0-9][^"]*\)".*/\1/' | head -1) + +if [ -z "$PYDANTIC_VERSION" ]; then + log "ERROR: could not parse pydantic version from $PYPROJECT" + log " Expected a line like: \"pydantic==2.11.7\"," + exit 1 +fi + +log "Required pydantic version (from datadog_checks_base): $PYDANTIC_VERSION" + +# ─── Step 2: Set Rust environment ───────────────────────────────────────────── +# +# AIX-specific Rust build flags: +# CARGO_PROFILE_RELEASE_STRIP=none — IBM Rust 1.92 bug: stripping .info section +# from proc-macro artifacts breaks rustc +# CARGO_PROFILE_RELEASE_LTO=off — LLVM fat LTO uses .ipa bitcode sections +# that do not exist in AIX XCOFF format; +# fails after 50+ minutes of compilation +# CC=/opt/freeware/bin/gcc — cc-rs defaults to IBM xlc which rejects GCC +# flags like -fPIC, -ffunction-sections, -maix64 +# RUSTFLAGS="-C link-arg=-bbigtoc" — pydantic-core exceeds AIX ld's 64KB TOC limit; +# -bbigtoc removes this limit for AIX XCOFF. + +log "Setting Rust environment for pydantic-core build" +export CC=/opt/freeware/bin/gcc +export CXX=/opt/freeware/bin/g++ +export PATH=/opt/freeware/lib/RustSDK/1.92/bin:"$PATH" +export CARGO_HOME=/opt/cargo + +log " CC=$CC" +log " CXX=$CXX" +log " CARGO_HOME=$CARGO_HOME" +log " Rust toolchain: $(cargo --version 2>/dev/null || echo 'cargo not found — install rust1.92.ppc')" + +# ─── Step 3: Check wheel cache ──────────────────────────────────────────────── +# +# The cache is keyed by pydantic version (e.g. $WHEEL_CACHE/pydantic-2.11.7/). +# This ensures that when integrations-core bumps pydantic (and thereby +# pydantic-core), the old cached wheel is not used — a new subdirectory is +# created and a fresh Rust build is triggered automatically. +# +# pydantic-core takes ~52 minutes to build from source on POWER8. If a +# pre-built wheel is present in the versioned cache directory, install from +# it and skip the Rust build entirely. + +WHEEL_CACHE_DIR="$WHEEL_CACHE/pydantic-$PYDANTIC_VERSION" +mkdir -p "$WHEEL_CACHE_DIR" + +# Match only full-AIX-tag wheels (e.g. aix_7302_2419_64) not legacy aix_ppc64 renames. +# aix_*_* requires at least one underscore within the AIX portion, which aix_ppc64 lacks. +CACHED_WHEEL=$(find "$WHEEL_CACHE_DIR" -name 'pydantic_core-*-cp313-cp313-aix_*_*.whl' 2>/dev/null | head -1) + +if [ -n "$CACHED_WHEEL" ]; then + log "Found cached pydantic-core wheel: $CACHED_WHEEL" + log "Installing pydantic==$PYDANTIC_VERSION and pydantic-core from wheel cache (skipping Rust build)" + # --find-links lets pip use the local cached wheel for pydantic-core while + # downloading pydantic itself (pure Python) from PyPI. + # The wheel filename retains the original AIX platform tag so pip can match it. + $PIP install \ + --find-links "$WHEEL_CACHE_DIR" \ + "pydantic==$PYDANTIC_VERSION" + log "pydantic and pydantic-core installed from cache successfully" +else + log "No cached wheel found for pydantic==$PYDANTIC_VERSION — building pydantic-core from source" + log "WARNING: This step takes approximately 52 minutes on POWER8." + log " Disk space required: ~7 GB in /tmp, ~4 GB in /" + log " Cache directory: $WHEEL_CACHE_DIR" + + CARGO_PROFILE_RELEASE_STRIP=none \ + CARGO_PROFILE_RELEASE_LTO=off \ + RUSTFLAGS="-C link-arg=-bbigtoc" \ + ARFLAGS="" \ + $PIP install "pydantic==$PYDANTIC_VERSION" --no-binary pydantic-core + + log "pydantic-core build complete" + + # Cache the built wheel for next time. maturin produces a system-specific + # platform tag (aix_3_XXXXXXXX_XXXXXXXX); rename it to the portable aix_ppc64 + # tag so the same wheel can be used on any AIX 7.x POWER system. + BUILT_WHEEL=$(find "${HOME}/.cache/pip" -name "pydantic_core-*.whl" 2>/dev/null | head -1) + if [ -n "$BUILT_WHEEL" ]; then + # Keep the original filename (with the full AIX platform tag from this system) + # so that pip's --find-links can match it by tag on cache restore. + CACHE_NAME=$(basename "$BUILT_WHEEL") + cp "$BUILT_WHEEL" "$WHEEL_CACHE_DIR/$CACHE_NAME" + log "Cached wheel to $WHEEL_CACHE_DIR/$CACHE_NAME" + log " Preserved for all future builds using pydantic==$PYDANTIC_VERSION." + else + log "WARNING: could not locate built pydantic-core wheel in pip cache" + log " Next build will rebuild from source (~52 minutes)" + fi +fi + +# ─── Step 4: Install typing_extensions ──────────────────────────────────────── +# +# pydantic-core >= 2.41 requires typing_extensions >= 4.14.1. Install it +# explicitly after pydantic-core so the constraint is satisfied even if pydantic +# itself did not pull in a new enough version. + +log "Installing typing_extensions (required by pydantic-core >= 2.41)" +$PIP install "typing_extensions>=4.14.1" +log "typing_extensions installed successfully" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/07-checks-base.sh b/packaging/aix/stages/07-checks-base.sh new file mode 100755 index 000000000000..0b389efcf668 --- /dev/null +++ b/packaging/aix/stages/07-checks-base.sh @@ -0,0 +1,118 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="07-checks-base" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${INTEGRATIONS_CORE:?INTEGRATIONS_CORE must be set}" + +PIP=$EMBEDDED_DESTDIR/bin/pip${PYTHON_MAJ_MIN} + +# --- Pre-flight checks --- +if [ ! -x "$PIP" ]; then + log "ERROR: $PIP not found — did Stage 02 (02-python) complete successfully?" + exit 1 +fi + +if [ ! -f "$INTEGRATIONS_CORE/datadog_checks_base/pyproject.toml" ]; then + log "ERROR: $INTEGRATIONS_CORE/datadog_checks_base/pyproject.toml not found" + log " Did Stage 00 (00-checkout) clone integrations-core at the correct commit?" + exit 1 +fi + +# --- Cleanup on failure --- +# pip installs are not easy to roll back; the sentinel not being written is +# sufficient to trigger a re-run. +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed." + log " Re-run after fixing the error by deleting the sentinel:" + log " rm $SENTINEL" + log " Common causes:" + log " - Native deps (pydantic-core, cryptography) not installed: ensure Stage 05 and Stage 06 completed" + log " - Network access required for transitive pure-Python deps from PyPI" + log " - integrations-core not checked out: ensure INTEGRATIONS_CORE=$INTEGRATIONS_CORE is correct" + fi +} +trap cleanup EXIT + +# ─── Step 1: Install datadog-checks-base ────────────────────────────────────── +# +# Install datadog-checks-base from the pinned integrations-core checkout with +# full dependency resolution. pip resolves all transitive pure-Python deps +# (pyyaml, requests, prometheus_client, etc.) from PyPI. Native deps +# (pydantic-core, cryptography) were already installed from Stages 05-06 at +# satisfying versions; pip detects they satisfy the requirements and does not +# attempt to download or rebuild them. + +log "Installing datadog-checks-base from $INTEGRATIONS_CORE/datadog_checks_base" +$PIP install "$INTEGRATIONS_CORE/datadog_checks_base" +log "datadog-checks-base installed successfully" + +# ─── Step 1b: Install datadog-checks-base [deps] extra packages ─────────────── +# +# The [deps] extra lists all runtime-required packages, but pip will NOT install +# them unless you request [deps] explicitly. We cannot use +# pip install datadog-checks-base[deps] +# because two packages in the [deps] extra require a Rust toolchain to build +# from source and are not available as pre-built wheels for AIX/ppc64: +# - ddtrace (requires Rust, not available on AIX) +# - jellyfish (requires Rust, not available on AIX) +# We therefore install all other [deps] packages explicitly using the exact +# versions pinned in the datadog-checks-base METADATA. + +log "Installing datadog-checks-base deps (excluding ddtrace and jellyfish which require Rust)" +$PIP install \ + 'lazy-loader==0.4' \ + 'PyYAML==6.0.2' \ + 'cachetools==6.2.0' \ + 'requests==2.32.5' \ + 'wrapt==1.17.3' \ + 'simplejson==3.20.1' \ + 'requests-toolbelt==1.0.0' \ + 'requests-unixsocket2==1.0.0' \ + 'python-dateutil==2.9.0.post0' \ + 'urllib3==2.6.3' \ + 'prometheus-client==0.22.1' \ + 'protobuf==6.33.5' \ + 'binary==1.0.2' +log "datadog-checks-base deps installed successfully" + +# ─── Step 2: Freeze installed state to constraints file ─────────────────────── +# +# Freeze the complete installed state into a constraints file. Stage 08 passes +# this to every check install so pip pins all transitive deps to the exact same +# versions rather than resolving to whatever is latest on PyPI at build time. +# Any missing dep that has no AIX-compatible wheel will fail loudly here rather +# than silently at runtime. + +log "Freezing installed packages to $STAGING/constraints.txt" +$PIP freeze > "$STAGING/constraints.txt" +log "Constraints written to $STAGING/constraints.txt ($(wc -l < "$STAGING/constraints.txt") packages)" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/08-integrations.sh b/packaging/aix/stages/08-integrations.sh new file mode 100755 index 000000000000..54414ec98e6e --- /dev/null +++ b/packaging/aix/stages/08-integrations.sh @@ -0,0 +1,124 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="08-integrations" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" +: "${INTEGRATIONS_CORE:?INTEGRATIONS_CORE must be set}" +: "${WHEEL_CACHE:?WHEEL_CACHE must be set}" + +PIP=$EMBEDDED_DESTDIR/bin/pip${PYTHON_MAJ_MIN} + +# --- Pre-flight checks --- +if [ ! -x "$PIP" ]; then + log "ERROR: $PIP not found — did Stage 02 (02-python) complete successfully?" + exit 1 +fi + +if [ ! -f "$STAGING/constraints.txt" ]; then + log "ERROR: $STAGING/constraints.txt not found — Stage 07 (07-checks-base) must complete first" + exit 1 +fi + +# --- Cleanup on failure --- +# pip installs are not easy to roll back; the sentinel not being written is +# sufficient to trigger a re-run. +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed." + log " Re-run after fixing the error by deleting the sentinel:" + log " rm $SENTINEL" + log " Common causes:" + log " - Stage 07 constraints.txt missing: ensure Stage 07 completed" + log " - integrations-core check not found: verify INTEGRATIONS_CORE=$INTEGRATIONS_CORE" + log " - Network access required for any dep not yet in site-packages" + fi +} +trap cleanup EXIT + +# ─── Step 1: Copy built-in Go check example configs ─────────────────────────── +# +# Built-in Go checks (cpu, memory, disk, load, network) are implemented in the +# agent binary itself and do not require pip install. We bundle their example +# configuration files so the operator can see the available options. + +log "Copying built-in Go check default configs" +# Built-in check config files are named conf.yaml.default (not conf.yaml.example). +# The network check is intentionally excluded: datadog_checks.network is a Python +# check that is not bundled in the AIX package; including its config causes the +# agent to attempt loading the Python check and log an ImportError at startup. +for check in cpu memory disk load; do + mkdir -p "$STAGING/etc/datadog-agent/conf.d/${check}.d" + cp "/opt/datadog-agent/cmd/agent/dist/conf.d/${check}.d/conf.yaml.default" \ + "$STAGING/etc/datadog-agent/conf.d/${check}.d/" 2>/dev/null || \ + log "WARNING: no conf.yaml.default for built-in check: $check" +done +log "Built-in check configs copied" + +# ─── Step 2: Install Python checks from integrations-core ───────────────────── +# +# Install each check from the pinned integrations-core checkout. +# --constraint pins all transitive deps to the exact versions frozen by Stage 07, +# matching the Linux omnibus approach and failing loudly if a dep is unavailable. +# --find-links allows pip to locate native AIX wheels (pydantic-core, cryptography) +# from the local cache if needed rather than hitting PyPI. +# +# IBM checks (ibm_mq, ibm_ace, ibm_db2, ibm_i) are installed regardless of +# whether the corresponding C extension (pymqi, ibm_db, pyodbc) was built in +# Stage 05. The check code installs successfully; it will surface a clear +# ImportError at runtime if the missing native extension is not present on the +# target system. + +PYTHON_CHECKS="openmetrics ibm_mq ibm_ace ibm_db2 ibm_i ibm_was ibm_spectrum_lsf" + +log "Installing Python checks: $PYTHON_CHECKS" + +for check in $PYTHON_CHECKS; do + CHECK_DIR="$INTEGRATIONS_CORE/$check" + if [ -f "$CHECK_DIR/pyproject.toml" ]; then + log "Installing check: $check" + $PIP install \ + --constraint "$STAGING/constraints.txt" \ + --find-links "$WHEEL_CACHE" \ + "$CHECK_DIR" + mkdir -p "$STAGING/etc/datadog-agent/conf.d/${check}.d" + EXAMPLE="$CHECK_DIR/datadog_checks/$check/data/conf.yaml.example" + if [ -f "$EXAMPLE" ]; then + cp "$EXAMPLE" "$STAGING/etc/datadog-agent/conf.d/${check}.d/" + else + log "WARNING: no conf.yaml.example found for $check at $EXAMPLE" + fi + log "Check $check installed successfully" + else + log "WARNING: $check not found in integrations-core at $CHECK_DIR — skipping" + fi +done + +log "All Python checks processed" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/09-strip-bytecode.sh b/packaging/aix/stages/09-strip-bytecode.sh new file mode 100755 index 000000000000..7344cf847035 --- /dev/null +++ b/packaging/aix/stages/09-strip-bytecode.sh @@ -0,0 +1,99 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="09-strip-bytecode" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed. Removing partial outputs." + rm -f "$EMBEDDED_DESTDIR/.pyc_compiled_files.txt" + fi +} +trap cleanup EXIT + +# ─── Step 1: Strip debug info from shared libraries ─────────────────────────── +# +# /opt/freeware/bin/strip supports -X64 (required for XCOFF64 binaries on AIX). +# The system /usr/bin/strip is 32-bit only and will refuse or silently corrupt +# 64-bit XCOFF objects. Use while-read rather than for-f-in-$(find) to avoid +# command substitution size limits and to handle filenames with spaces safely. + +log "Stripping debug info from .so files under $EMBEDDED_DESTDIR/lib" +find "$EMBEDDED_DESTDIR/lib" -name "*.so*" | while IFS= read -r f; do + /opt/freeware/bin/strip -X64 "$f" 2>/dev/null || true +done +log "Strip pass complete" + +# ─── Step 2: Remove build artefacts not needed at runtime ───────────────────── +# +# Headers, pkg-config metadata, and man pages are only needed during compilation. +# Source files (.c/.h) inside the Python stdlib tree are not needed at runtime. +# __pycache__ directories may contain stale .pyc files from an earlier compileall +# run or a pip install; delete them before the fresh compileall in Step 4 so +# there are no stale bytecode files with incorrect magic numbers. + +log "Removing build-time artefacts (headers, pkgconfig, man pages, .c/.h files)" +rm -rf "$EMBEDDED_DESTDIR/include" +rm -rf "$EMBEDDED_DESTDIR/lib/pkgconfig" +rm -rf "$EMBEDDED_DESTDIR/share/man" +find "$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}" -name "*.c" -exec rm -f {} \; +find "$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}" -name "*.h" -exec rm -f {} \; +find "$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}" -name "*.pyc" -exec rm -f {} \; 2>/dev/null || true +log "Build artefacts removed" + +# ─── Step 3: Compile .py to .pyc for faster agent startup ───────────────────── +# +# -x 'test/' skips test directories which contain code that may not compile +# cleanly or that imports Linux-only modules. -q suppresses per-file output; +# errors (if any) are still printed. || true: a few files may legitimately +# fail to compile (e.g. Python 2-only syntax in vendored code); do not abort +# the stage for these. + +log "Compiling .py to .pyc under $EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}/site-packages" +"$EMBEDDED_DESTDIR/bin/python${PYTHON_MAJ_MIN}" -m compileall \ + "$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}/site-packages" \ + -x 'test/' -q 2>/dev/null || true +log "Bytecode compilation complete" + +# ─── Step 4: Record compiled bytecode files ─────────────────────────────────── +# +# The preinst and prerm package scripts read this manifest to delete .pyc files +# before file replacement (upgrade) or removal (uninstall). mkinstallp does not +# track .pyc files created post-install (they are generated at package-build time +# and again at postinst time), so the manifest is the only reliable way to clean +# them up without leaving orphaned bytecode behind. + +log "Recording .pyc/.pyo files to $EMBEDDED_DESTDIR/.pyc_compiled_files.txt" +find "$EMBEDDED_DESTDIR" \( -name "*.pyc" -o -name "*.pyo" \) -print \ + > "$EMBEDDED_DESTDIR/.pyc_compiled_files.txt" +log "Recorded $(wc -l < "$EMBEDDED_DESTDIR/.pyc_compiled_files.txt") .pyc files" + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/packaging/aix/stages/10-assemble.sh b/packaging/aix/stages/10-assemble.sh new file mode 100755 index 000000000000..75d846b80b2f --- /dev/null +++ b/packaging/aix/stages/10-assemble.sh @@ -0,0 +1,205 @@ +#!/bin/sh +set -eu + +# Source shared environment (defines STAGING, EMBEDDED, EMBEDDED_DESTDIR, etc.) +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +# shellcheck source=/dev/null +. "$SCRIPT_DIR/../lib/env.sh" + +STAGE_NAME="10-assemble" +SENTINEL="$BUILD_DIR/.done/$STAGE_NAME" +LOG="$BUILD_DIR/logs/$STAGE_NAME.log" + +# Redirect all output to log file (follow with: tail -f "$LOG") +mkdir -p "$BUILD_DIR/logs" +exec > "$LOG" 2>&1 + +log "=== Stage: $STAGE_NAME ===" + +# --- Idempotency check --- +if [ -f "$SENTINEL" ]; then + log "Already complete (sentinel: $SENTINEL) — skipping." + exit 0 +fi + +# --- Input validation --- +: "${AGENT_VERSION:?AGENT_VERSION must be set}" +: "${AGENT_VRMF:?AGENT_VRMF must be set}" +: "${STAGING:?STAGING must be set}" +: "${EMBEDDED_DESTDIR:?EMBEDDED_DESTDIR must be set}" +: "${BUILD_DIR:?BUILD_DIR must be set}" + +# --- Cleanup on failure --- +cleanup() { + if [ $? -ne 0 ]; then + log "ERROR: $STAGE_NAME failed. Removing partial outputs." + # Remove only the files written by this stage; leave earlier stage outputs intact. + rm -f "$STAGING/etc/datadog-agent/datadog.yaml.example" + rm -rf "$STAGING/etc/datadog-agent/conf.d" + rm -rf "$EMBEDDED_DESTDIR/share/installp" + fi +} +trap cleanup EXIT + +# ─── Step 1: Pre-flight — verify agent binary exists ────────────────────────── +# +# The agent binary is the primary deliverable. If it is absent the staging +# tree is incomplete and packaging will produce a broken BFF. + +AGENT_BIN="$STAGING/opt/datadog-agent/bin/agent" +if [ ! -f "$AGENT_BIN" ]; then + log "ERROR: agent binary not found at $AGENT_BIN" + log " Did Stage 04 (04-agent) complete successfully?" + exit 1 +fi +log "Pre-flight: agent binary found at $AGENT_BIN" + +# ─── Step 2: Copy main config example ───────────────────────────────────────── +# +# Install the upstream datadog.yaml as a .example file so the operator can copy +# and edit it. The agent will not start without a real datadog.yaml; the .example +# suffix makes it clear that manual configuration is required. + +log "Copying main config example" +mkdir -p "$STAGING/etc/datadog-agent" +cp /opt/datadog-agent/cmd/agent/dist/datadog.yaml \ + "$STAGING/etc/datadog-agent/datadog.yaml.example" +log "Config example written to $STAGING/etc/datadog-agent/datadog.yaml.example" + +# ─── Step 2b: Install default check configs ─────────────────────────────────── +# +# Copy conf.yaml.default files for the system checks that work on AIX. +# These enable the checks automatically on first install without any operator +# action — the same behaviour as Linux packages. +# network is excluded: datadog_checks.network is not bundled in the embedded +# Python, so including its default config would produce a permanent loading +# error in agent status. + +log "Installing default check configs" +DIST_CONFD=/opt/datadog-agent/cmd/agent/dist/conf.d +STAGING_CONFD="$STAGING/etc/datadog-agent/conf.d" +# Remove and recreate to avoid stale entries from prior builds. +rm -rf "$STAGING_CONFD" +mkdir -p "$STAGING_CONFD" +for check in cpu disk io load memory ntp uptime; do + src="$DIST_CONFD/${check}.d/conf.yaml.default" + if [ -f "$src" ]; then + mkdir -p "$STAGING_CONFD/${check}.d" + cp "$src" "$STAGING_CONFD/${check}.d/conf.yaml.default" + log " $check.d/conf.yaml.default" + else + log " WARNING: $src not found — skipping $check" + fi +done +log "Default check configs installed" + +# ─── Step 2c: Install sitecustomize.py ──────────────────────────────────────── +# +# On AIX, pydantic_core's bundled libunwind.a(libunwind.so.1) has an undefined +# reference to __xlcxx_personality_v0 (from the XLC++ ABI library libc++abi.a). +# GCC-compiled Python does not load libc++abi.a automatically, so importing +# pydantic_core fails with "Could not load dynamic library" at runtime. +# +# sitecustomize.py is executed by Python on every startup before any user code. +# We use it to pre-load /usr/lib/libc++abi.a with RTLD_GLOBAL so that +# __xlcxx_personality_v0 is available when pydantic_core (or any other C++ +# extension using libunwind) is imported. + +SITECUSTOMIZE_SRC="$(dirname "$0")/../sitecustomize.py" +PYTHON_LIB_DIR="$EMBEDDED_DESTDIR/lib/python${PYTHON_MAJ_MIN}" +if [ ! -f "$SITECUSTOMIZE_SRC" ]; then + log "ERROR: sitecustomize.py not found at $SITECUSTOMIZE_SRC" + exit 1 +fi +if [ ! -d "$PYTHON_LIB_DIR" ]; then + log "ERROR: Python lib dir not found at $PYTHON_LIB_DIR" + log " Did Stage 02 (02-python) complete successfully?" + exit 1 +fi +cp "$SITECUSTOMIZE_SRC" "$PYTHON_LIB_DIR/sitecustomize.py" +# Remove any stale .pyc that may have been compiled against an older version of +# this file in a previous build. The .pyc is not needed in the BFF — Python +# regenerates it on first import on the target host. +rm -f "$PYTHON_LIB_DIR/__pycache__/sitecustomize.cpython-313.pyc" +log "sitecustomize.py installed to $PYTHON_LIB_DIR/sitecustomize.py" + +# ─── Step 3: Create required empty directories ──────────────────────────────── +# +# mkinstallp will include these directories in the package so that installp +# creates them on the target system. The agent and its postinst script expect +# them to exist at runtime. + +log "Creating required runtime directories" +mkdir -p "$STAGING/var/log/datadog" +mkdir -p "$STAGING/var/run/datadog" +mkdir -p "$STAGING/opt/datadog-agent/run" +mkdir -p "$STAGING/opt/datadog-agent/checks.d" +mkdir -p "$STAGING/opt/datadog-agent/conf.d" +log "Runtime directories created" + +# ─── Step 4: Copy package lifecycle scripts into the staging tree ────────────── +# +# mkinstallp requires lifecycle scripts to be present at their final installed +# path inside the staging tree. The gen.template.in references them at +# /opt/datadog-agent/embedded/share/installp/. They must be executable. +# +# Fail clearly if any script is missing rather than producing a BFF that silently +# has no pre/post install hooks. + +log "Copying package lifecycle scripts" +SCRIPTS_DIR="$EMBEDDED_DESTDIR/share/installp" +mkdir -p "$SCRIPTS_DIR" +# mkinstallp checks lifecycle scripts at their absolute installed path, not +# relative to the staging tree. Copy them to BOTH the staging tree (so they +# are included in the BFF) AND the installed path (so mkinstallp can find +# them when building the BFF on the same host that will run the agent). +SCRIPTS_INSTALLED="$EMBEDDED/share/installp" +mkdir -p "$SCRIPTS_INSTALLED" +PKGSCRIPTS_SRC="$(dirname "$0")/../package-scripts" + +for script in preinst postinst config unconfig prerm postrm; do + SRC="$PKGSCRIPTS_SRC/$script" + if [ ! -f "$SRC" ]; then + log "ERROR: package script not found: $SRC" + log " All six lifecycle scripts (preinst postinst config unconfig prerm postrm)" + log " must exist under packaging/aix/package-scripts/ before running this stage." + exit 1 + fi + cp "$SRC" "$SCRIPTS_DIR/$script" + chmod 755 "$SCRIPTS_DIR/$script" + cp "$SRC" "$SCRIPTS_INSTALLED/$script" + chmod 755 "$SCRIPTS_INSTALLED/$script" + log " Staging: $SCRIPTS_DIR/$script" + log " Installed: $SCRIPTS_INSTALLED/$script" +done +log "All package lifecycle scripts installed" + +# ─── Step 5: Set correct ownership ──────────────────────────────────────────── +# +# mkinstallp records the owning uid:gid of every file and directory in the +# generated BFF. If files are owned by a build user rather than root, installp +# will install them with that non-root ownership on the target system, which +# causes permission errors at runtime. chown -h (portable spelling: -Rh) also +# fixes symbolic link ownership without following the link target. + +log "Setting root ownership on staging tree" +chown -Rh 0:0 "$STAGING/opt" "$STAGING/etc" "$STAGING/var" +log "Ownership set" + +# ─── Step 6: Print staging tree size summary ────────────────────────────────── +# +# Provides a quick sanity check that key components were built and are present. +# du -s prints single-directory totals in 512-byte blocks (AIX default). +# Use || true so a missing optional directory (e.g. rtloader if not yet built) +# does not abort the summary. + +log "Staging tree size summary:" +du -s \ + "$STAGING/opt/datadog-agent/bin" \ + "$STAGING/opt/datadog-agent/embedded/lib" \ + "$STAGING/opt/datadog-agent/rtloader" 2>/dev/null || true + +# --- Mark complete --- +mkdir -p "$(dirname "$SENTINEL")" +touch "$SENTINEL" +log "=== $STAGE_NAME complete ===" diff --git a/pkg/collector/corechecks/system/filehandles/file_handles.go b/pkg/collector/corechecks/system/filehandles/file_handles.go index 66b1ee125bbd..2b3957753851 100644 --- a/pkg/collector/corechecks/system/filehandles/file_handles.go +++ b/pkg/collector/corechecks/system/filehandles/file_handles.go @@ -2,7 +2,7 @@ // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows && !freebsd && !darwin +//go:build !windows && !freebsd && !darwin && !aix package filehandles diff --git a/pkg/collector/corechecks/system/filehandles/file_handles_aix.go b/pkg/collector/corechecks/system/filehandles/file_handles_aix.go new file mode 100644 index 000000000000..370771ff0d48 --- /dev/null +++ b/pkg/collector/corechecks/system/filehandles/file_handles_aix.go @@ -0,0 +1,20 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. +//go:build aix + +package filehandles + +import ( + "github.com/DataDog/datadog-agent/pkg/collector/check" + "github.com/DataDog/datadog-agent/pkg/util/option" +) + +// CheckName is the name of the check +const CheckName = "file_handle" + +// Factory returns a no-op option because file handle metrics are not available on AIX. +func Factory() option.Option[func() check.Check] { + return option.None[func() check.Check]() +} diff --git a/pkg/collector/corechecks/system/filehandles/file_handles_test.go b/pkg/collector/corechecks/system/filehandles/file_handles_test.go index 3bd258940999..32219930d649 100644 --- a/pkg/collector/corechecks/system/filehandles/file_handles_test.go +++ b/pkg/collector/corechecks/system/filehandles/file_handles_test.go @@ -2,7 +2,7 @@ // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows && !freebsd && !darwin +//go:build !windows && !freebsd && !darwin && !aix package filehandles diff --git a/pkg/collector/python/containers.go b/pkg/collector/python/containers.go index 68b097c45f05..acb518756b1c 100644 --- a/pkg/collector/python/containers.go +++ b/pkg/collector/python/containers.go @@ -14,7 +14,8 @@ import ( /* #include -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static */ import "C" diff --git a/pkg/collector/python/datadog_agent.go b/pkg/collector/python/datadog_agent.go index 95ea79bc00de..82a0b4aee756 100644 --- a/pkg/collector/python/datadog_agent.go +++ b/pkg/collector/python/datadog_agent.go @@ -31,7 +31,8 @@ import ( ) /* -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static #include "datadog_agent_rtloader.h" diff --git a/pkg/collector/python/init.go b/pkg/collector/python/init.go index 3742a54d9184..89c93f95e359 100644 --- a/pkg/collector/python/init.go +++ b/pkg/collector/python/init.go @@ -34,7 +34,11 @@ import ( ) /* -#cgo !windows LDFLAGS: -L${SRCDIR}/../../../rtloader/build/rtloader -ldatadog-agent-rtloader -ldl +// On AIX, Go's CGO requires shared libraries to be wrapped in .a archives. +// The stage 03 build script (03-rtloader.sh) creates libdatadog-agent-rtloader.a +// from the .so file using "ar -X64 -r". We then use -L/-l to reference it. +#cgo aix LDFLAGS: -L${SRCDIR}/../../../rtloader/build/rtloader -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -L${SRCDIR}/../../../rtloader/build/rtloader -ldatadog-agent-rtloader -ldl #cgo windows LDFLAGS: -L${SRCDIR}/../../../rtloader/build/rtloader -ldatadog-agent-rtloader -lstdc++ -static #cgo CFLAGS: -I "${SRCDIR}/../../../rtloader/include" -I "${SRCDIR}/../../../rtloader/common" diff --git a/pkg/collector/python/init_nix.go b/pkg/collector/python/init_nix.go index c9170fceb9ff..d3fda27b51ec 100644 --- a/pkg/collector/python/init_nix.go +++ b/pkg/collector/python/init_nix.go @@ -16,7 +16,8 @@ import ( ) /* -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #include #include diff --git a/pkg/collector/python/kubeutil.go b/pkg/collector/python/kubeutil.go index f3d81277ce53..6f47cb712140 100644 --- a/pkg/collector/python/kubeutil.go +++ b/pkg/collector/python/kubeutil.go @@ -20,7 +20,8 @@ import ( /* #include -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static */ import "C" diff --git a/pkg/collector/python/memory.go b/pkg/collector/python/memory.go index 8f95ad1d5f9f..2fe133c8c6bf 100644 --- a/pkg/collector/python/memory.go +++ b/pkg/collector/python/memory.go @@ -18,7 +18,8 @@ import ( ) /* -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static #if defined(__linux__) || defined(_WIN32) diff --git a/pkg/collector/python/no_kubeutil.go b/pkg/collector/python/no_kubeutil.go index 66432f8be25c..475c7ee9f68a 100644 --- a/pkg/collector/python/no_kubeutil.go +++ b/pkg/collector/python/no_kubeutil.go @@ -9,7 +9,8 @@ package python /* #include -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static */ import "C" diff --git a/pkg/collector/python/python_aix.go b/pkg/collector/python/python_aix.go new file mode 100644 index 000000000000..81f3374a5c82 --- /dev/null +++ b/pkg/collector/python/python_aix.go @@ -0,0 +1,76 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix && python + +package python + +// Force libpython3.13.so into the agent binary's XCOFF startup-load chain. +// +// Background +// ---------- +// Python C extension modules (lib-dynload/_decimal.so, etc.) carry Python API +// symbols (PyArg_ParseTuple, PyBool_FromLong, …) with no explicit library +// dependency in their XCOFF loader section (IMPid = "."). On AIX, such +// unresolved symbols are looked up in the process-global symbol table, which +// is populated only by libraries that appear in the XCOFF startup-load chain +// (i.e. in the XCOFF loader section of the main executable or its startup- +// linked dependencies). +// +// Without this file, libpython3.13.so is loaded only via dlopen, transitively +// through libdatadog-agent-three.so when rtloader initialises Python. A +// dlopen-loaded library does NOT add its symbols to the global symbol table. +// As a result every Python C extension fails with: +// ImportError: Symbol PyXxx is not exported from dependent module agent. +// +// Fix +// --- +// The init() function below calls C.Py_IsInitialized() once at process startup. +// Go's CGO layer sees this Go→C call and emits a //go:cgo_import_dynamic +// directive that places libpython3.13.a(shr_64.o) in the agent binary's XCOFF +// loader section. Consequently libpython3.13.so is loaded at process startup, +// before any Python C extension module is imported, and all ~1680 Python API +// symbols enter the global symbol table where the extension modules can find them. +// +// Note: a C-internal reference (static void* holding &Py_IsInitialized) is NOT +// sufficient — Go's linker only generates XCOFF import entries for symbols that +// are called from the Go side through CGO. A direct Go→C call is required. +// +// Py_IsInitialized() returns 0 when called here (Python not yet started) and +// has no side effects — it is a safe, idempotent read of an internal flag. +// +// Part 2 — Exporting Python API symbols from the agent binary +// The -bE:python.exp flag (passed via CGO_LDFLAGS env at build time, NOT via +// #cgo LDFLAGS which would be rejected by Go's CGO security filter) causes the +// linker to add all ~2762 Python API symbols to the agent binary's own EXP +// (export) table. Extension modules have Python API symbols with IMPid="." +// in their XCOFF, meaning "find in the main program's EXP table". Without +// -bE, those symbols are not exported from the agent and every extension fails: +// ImportError: Symbol PyXxx is not exported from dependent module agent. +// With -bE:python.exp, the agent exports the symbols and extensions load cleanly. +// See packaging/aix/stages/04-agent.sh for how -bE is passed at build time. +// +// The -L path resolves to /opt/datadog-agent/embedded/lib at build time +// (${SRCDIR}/../../../embedded is a symlink to the staging tree created by +// Stage 02). At runtime the same directory exists as a real directory +// installed by installp, and LIBPATH in the agent-svc wrapper includes it. + +/* +#cgo LDFLAGS: -L${SRCDIR}/../../../embedded/lib -lpython3 + +// Forward declaration — we do not include Python.h to avoid pulling in +// the entire CPython header tree; Py_IsInitialized has a stable ABI. +extern int Py_IsInitialized(void); +*/ +import "C" + +func init() { + // Call Py_IsInitialized() to create a live Go→C reference. This forces + // the CGO linker to emit a //go:cgo_import_dynamic for Py_IsInitialized, + // placing libpython3.13.a(shr_64.o) in the XCOFF startup-load chain. + // Python is not yet started here so the call always returns 0; we discard + // the result. There are no side effects. + _ = C.Py_IsInitialized() +} diff --git a/pkg/collector/python/tagger.go b/pkg/collector/python/tagger.go index 92354171fdac..9a074aed6fee 100644 --- a/pkg/collector/python/tagger.go +++ b/pkg/collector/python/tagger.go @@ -16,7 +16,8 @@ import ( ) /* -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static #include "datadog_agent_rtloader.h" diff --git a/pkg/collector/python/util.go b/pkg/collector/python/util.go index 14be3f6d065c..9a27c0f405c0 100644 --- a/pkg/collector/python/util.go +++ b/pkg/collector/python/util.go @@ -9,7 +9,8 @@ package python /* #include -#cgo !windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo !aix,!windows LDFLAGS: -ldatadog-agent-rtloader -ldl +#cgo aix LDFLAGS: -ldl #cgo windows LDFLAGS: -ldatadog-agent-rtloader -lstdc++ -static */ import "C" diff --git a/pkg/config/setup/process.go b/pkg/config/setup/process.go index 174e91429123..8457a25750d4 100644 --- a/pkg/config/setup/process.go +++ b/pkg/config/setup/process.go @@ -89,8 +89,8 @@ func setupProcesses(config pkgconfigmodel.Setup) { procBindEnvAndSetDefault(config, "process_config.container_collection.enabled", true) procBindEnvAndSetDefault(config, "process_config.process_collection.enabled", false) - // This allows for the process check to run in the core agent but is for linux only - procBindEnvAndSetDefault(config, "process_config.run_in_core_agent.enabled", runtime.GOOS == "linux") + // This allows for the process check to run in the core agent but is for linux and aix only + procBindEnvAndSetDefault(config, "process_config.run_in_core_agent.enabled", runtime.GOOS == "linux" || runtime.GOOS == "aix") config.BindEnv("process_config.process_dd_url", //nolint:forbidigo // TODO: replace by 'SetDefaultAndBindEnv' "DD_PROCESS_CONFIG_PROCESS_DD_URL", @@ -179,10 +179,10 @@ func setupProcesses(config pkgconfigmodel.Setup) { }) } -// overrideRunInCoreAgentConfig sets the process_config.run_in_core_agent.enabled to false in non-Linux environments. +// overrideRunInCoreAgentConfig sets the process_config.run_in_core_agent.enabled to false in non-Linux/AIX environments. // Otherwise, it is a no-op. func overrideRunInCoreAgentConfig(config pkgconfigmodel.Config) { - if runtime.GOOS != "linux" { + if runtime.GOOS != "linux" && runtime.GOOS != "aix" { config.Set("process_config.run_in_core_agent.enabled", false, pkgconfigmodel.SourceAgentRuntime) } } diff --git a/pkg/fleet/installer/telemetry/client.go b/pkg/fleet/installer/telemetry/client.go index 99182dfa2f25..36bdd69f0b4e 100644 --- a/pkg/fleet/installer/telemetry/client.go +++ b/pkg/fleet/installer/telemetry/client.go @@ -138,7 +138,7 @@ type httpClient interface { func newClient(httpClient httpClient, endpoints []*endpoint, service string, debug bool) *client { info, err := host.Info() if err != nil { - log.Errorf("failed to retrieve host info: %v", err) + log.Warnf("failed to retrieve host info: %v", err) info = &host.InfoStat{} } return &client{ diff --git a/pkg/gohai/cpu/cpu_aix.go b/pkg/gohai/cpu/cpu_aix.go new file mode 100644 index 000000000000..9e20b24e3e85 --- /dev/null +++ b/pkg/gohai/cpu/cpu_aix.go @@ -0,0 +1,80 @@ +// This file is licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build aix + +package cpu + +import ( + "os/exec" + "strconv" + "strings" + + "github.com/DataDog/datadog-agent/pkg/gohai/utils" +) + +func getCPUInfo() *Info { + info := &Info{ + VendorID: utils.NewValue("IBM"), + ModelName: utils.NewErrorValue[string](utils.ErrNotCollectable), + CPUCores: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + CPULogicalProcessors: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + Mhz: utils.NewErrorValue[float64](utils.ErrNotCollectable), + CacheSizeKB: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + Family: utils.NewErrorValue[string](utils.ErrNotCollectable), + Model: utils.NewErrorValue[string](utils.ErrNotCollectable), + Stepping: utils.NewErrorValue[string](utils.ErrNotCollectable), + CPUPkgs: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + CPUNumaNodes: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + CacheSizeL1Bytes: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + CacheSizeL2Bytes: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + CacheSizeL3Bytes: utils.NewErrorValue[uint64](utils.ErrNotCollectable), + } + + out, err := exec.Command("prtconf").Output() + if err != nil { + return info + } + + for _, line := range strings.Split(string(out), "\n") { + key, val, found := strings.Cut(line, ":") + if !found { + continue + } + key = strings.TrimSpace(key) + val = strings.TrimSpace(val) + + switch key { + case "Processor Type": + info.ModelName = utils.NewValue(val) + case "Number Of Processors": + if n, parseErr := strconv.ParseUint(val, 10, 64); parseErr == nil { + info.CPUCores = utils.NewValue(n) + // Default logical to physical; overridden below if bindprocessor succeeds. + info.CPULogicalProcessors = utils.NewValue(n) + } + case "Processor Clock Speed": + // "2000 MHz" - grab just the numeric part + fields := strings.Fields(val) + if len(fields) > 0 { + if mhz, parseErr := strconv.ParseFloat(fields[0], 64); parseErr == nil { + info.Mhz = utils.NewValue(mhz) + } + } + } + } + + // Use bindprocessor -q to count logical CPUs (includes SMT threads). + // Output: "The available processors are: 0 1 2 3 ... N" + if bpOut, bpErr := exec.Command("bindprocessor", "-q").Output(); bpErr == nil { + fields := strings.Fields(string(bpOut)) + // Skip the 4-word prefix "The available processors are:" + const prefixWords = 4 + if len(fields) > prefixWords { + info.CPULogicalProcessors = utils.NewValue(uint64(len(fields) - prefixWords)) + } + } + + return info +} diff --git a/pkg/gohai/filesystem/filesystem_aix.go b/pkg/gohai/filesystem/filesystem_aix.go new file mode 100644 index 000000000000..4438df869c62 --- /dev/null +++ b/pkg/gohai/filesystem/filesystem_aix.go @@ -0,0 +1,52 @@ +// This file is licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build aix + +package filesystem + +import ( + "os/exec" + "strconv" + "strings" +) + +func getFileSystemInfo() ([]MountInfo, error) { + // df -k reports 1024-byte block counts; output format: + // Filesystem 1024-blocks Free %Used Iused %Iused Mounted on + // /dev/hd4 131072 91312 31% 7011 5% / + out, err := exec.Command("df", "-k").Output() + if err != nil { + return nil, err + } + + var mounts []MountInfo + lines := strings.Split(string(out), "\n") + for _, line := range lines[1:] { // skip header + fields := strings.Fields(line) + if len(fields) < 7 { + continue + } + device := fields[0] + // Skip pseudo/remote filesystems + if device == "-" || device == "none" || strings.Contains(device, ":") { + continue + } + // Skip if size field is "-" (e.g. /proc on AIX) + if fields[1] == "-" { + continue + } + sizeKB, parseErr := strconv.ParseUint(fields[1], 10, 64) + if parseErr != nil || sizeKB == 0 { + continue + } + mountedOn := fields[6] + mounts = append(mounts, MountInfo{ + Name: device, + SizeKB: sizeKB, + MountedOn: mountedOn, + }) + } + return mounts, nil +} diff --git a/pkg/gohai/memory/memory_aix.go b/pkg/gohai/memory/memory_aix.go new file mode 100644 index 000000000000..e6713331ac79 --- /dev/null +++ b/pkg/gohai/memory/memory_aix.go @@ -0,0 +1,88 @@ +// This file is licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build aix + +package memory + +import ( + "os/exec" + "strconv" + "strings" + + "github.com/DataDog/datadog-agent/pkg/gohai/utils" +) + +func (info *Info) fillMemoryInfo() { + info.TotalBytes = utils.NewErrorValue[uint64](utils.ErrNotCollectable) + info.SwapTotalKb = utils.NewErrorValue[uint64](utils.ErrNotCollectable) + + out, err := exec.Command("prtconf").Output() + if err != nil { + return + } + + for _, line := range strings.Split(string(out), "\n") { + key, val, found := strings.Cut(line, ":") + if !found { + continue + } + key = strings.TrimSpace(key) + val = strings.TrimSpace(val) + + if key == "Memory Size" { + // e.g. "8192 Megabytes" or "8 Gigabytes" or "16384 MB" + fields := strings.Fields(val) + if len(fields) >= 2 { + n, parseErr := strconv.ParseUint(fields[0], 10, 64) + if parseErr == nil { + unit := strings.ToLower(fields[1]) + var bytes uint64 + switch { + case strings.HasPrefix(unit, "g"): + bytes = n * 1024 * 1024 * 1024 + case strings.HasPrefix(unit, "m"): + bytes = n * 1024 * 1024 + case strings.HasPrefix(unit, "k"): + bytes = n * 1024 + default: + bytes = n + } + info.TotalBytes = utils.NewValue(bytes) + } + } + } + } + + // Collect swap from lsps -s: "Total Paging Space Percent Used\n 2048MB 1%" + swapOut, swapErr := exec.Command("lsps", "-s").Output() + if swapErr == nil { + lines := strings.Split(strings.TrimSpace(string(swapOut)), "\n") + if len(lines) >= 2 { + fields := strings.Fields(lines[1]) + if len(fields) >= 1 { + sizeStr := strings.ToLower(fields[0]) + var swapKb uint64 + var parseErr error + switch { + case strings.HasSuffix(sizeStr, "gb"): + n, err := strconv.ParseUint(strings.TrimSuffix(sizeStr, "gb"), 10, 64) + parseErr = err + swapKb = n * 1024 * 1024 + case strings.HasSuffix(sizeStr, "mb"): + n, err := strconv.ParseUint(strings.TrimSuffix(sizeStr, "mb"), 10, 64) + parseErr = err + swapKb = n * 1024 + case strings.HasSuffix(sizeStr, "kb"): + n, err := strconv.ParseUint(strings.TrimSuffix(sizeStr, "kb"), 10, 64) + parseErr = err + swapKb = n + } + if parseErr == nil && swapKb > 0 { + info.SwapTotalKb = utils.NewValue(swapKb) + } + } + } + } +} diff --git a/pkg/gohai/platform/platform_aix.go b/pkg/gohai/platform/platform_aix.go new file mode 100644 index 000000000000..78d55cdf8bb2 --- /dev/null +++ b/pkg/gohai/platform/platform_aix.go @@ -0,0 +1,41 @@ +// This file is licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build aix + +package platform + +import ( + "runtime" + + "github.com/DataDog/datadog-agent/pkg/gohai/utils" + "golang.org/x/sys/unix" +) + +func (info *Info) fillPlatformInfo() { + info.Family = utils.NewErrorValue[string](utils.ErrNotCollectable) + info.OS = utils.NewValue("AIX") + + var uname unix.Utsname + if err := unix.Uname(&uname); err == nil { + info.KernelName = utils.NewValue(utils.StringFromBytes(uname.Sysname[:])) + info.Hostname = utils.NewValue(utils.StringFromBytes(uname.Nodename[:])) + info.KernelRelease = utils.NewValue(utils.StringFromBytes(uname.Release[:])) + machine := utils.StringFromBytes(uname.Machine[:]) + info.Machine = utils.NewValue(machine) + // On AIX, uname.Machine is the hardware serial (e.g. "00F9D80F4C00"), not the + // CPU architecture. Use runtime.GOARCH for the processor/hardware platform fields. + info.Processor = utils.NewValue(runtime.GOARCH) + info.HardwarePlatform = utils.NewValue(runtime.GOARCH) + info.KernelVersion = utils.NewValue(utils.StringFromBytes(uname.Version[:])) + } else { + info.KernelName = utils.NewErrorValue[string](err) + info.Hostname = utils.NewErrorValue[string](err) + info.KernelRelease = utils.NewErrorValue[string](err) + info.Machine = utils.NewErrorValue[string](err) + info.Processor = utils.NewErrorValue[string](err) + info.HardwarePlatform = utils.NewErrorValue[string](err) + info.KernelVersion = utils.NewErrorValue[string](err) + } +} diff --git a/pkg/gohai/processes/processes_aix.go b/pkg/gohai/processes/processes_aix.go new file mode 100644 index 000000000000..39ab1fa6e459 --- /dev/null +++ b/pkg/gohai/processes/processes_aix.go @@ -0,0 +1,13 @@ +// This file is licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build aix + +package processes + +import "errors" + +func getProcessGroups(_ int) ([]ProcessGroup, error) { + return nil, errors.New("process groups not supported on AIX") +} diff --git a/pkg/networkdevice/pinger/pinger_aix.go b/pkg/networkdevice/pinger/pinger_aix.go new file mode 100644 index 000000000000..228a887add1c --- /dev/null +++ b/pkg/networkdevice/pinger/pinger_aix.go @@ -0,0 +1,15 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package pinger + +import "errors" + +// New creates a new pinger; not supported on AIX. +func New(_ Config) (Pinger, error) { + return nil, errors.New("pinger not supported on AIX") +} diff --git a/pkg/process/checks/process.go b/pkg/process/checks/process.go index 90eab1ab3145..e28875750315 100644 --- a/pkg/process/checks/process.go +++ b/pkg/process/checks/process.go @@ -495,6 +495,11 @@ func fmtProcesses( // Hide disallow-listed args if the Scrubber is enabled fp.Cmdline = scrubber.ScrubProcessCommand(fp) + var voluntaryCtxSwitches, involuntaryCtxSwitches uint64 + if fp.Stats.CtxSwitches != nil { + voluntaryCtxSwitches = uint64(fp.Stats.CtxSwitches.Voluntary) + involuntaryCtxSwitches = uint64(fp.Stats.CtxSwitches.Involuntary) + } proc := &model.Process{ Pid: fp.Pid, NsPid: fp.NsPid, @@ -506,8 +511,8 @@ func fmtProcesses( OpenFdCount: fp.Stats.OpenFdCount, State: model.ProcessState(model.ProcessState_value[fp.Stats.Status]), IoStat: formatIO(fp.Stats, lastProcs[fp.Pid].Stats.IOStat, now, lastRun), - VoluntaryCtxSwitches: uint64(fp.Stats.CtxSwitches.Voluntary), - InvoluntaryCtxSwitches: uint64(fp.Stats.CtxSwitches.Involuntary), + VoluntaryCtxSwitches: voluntaryCtxSwitches, + InvoluntaryCtxSwitches: involuntaryCtxSwitches, ContainerId: ctrByProc[int(fp.Pid)], ProcessContext: serviceExtractor.GetServiceContext(fp.Pid), // SERVICE DISCOVERY FIELDS diff --git a/pkg/process/checks/process_rt.go b/pkg/process/checks/process_rt.go index 113300fa6597..d8969e2d0224 100644 --- a/pkg/process/checks/process_rt.go +++ b/pkg/process/checks/process_rt.go @@ -124,6 +124,11 @@ func fmtProcessStats( ioStat = formatIO(fp, lastProcs[pid].IOStat, now, lastRun) } + var voluntaryCtxSwitches, involuntaryCtxSwitches uint64 + if fp.CtxSwitches != nil { + voluntaryCtxSwitches = uint64(fp.CtxSwitches.Voluntary) + involuntaryCtxSwitches = uint64(fp.CtxSwitches.Involuntary) + } stat := &model.ProcessStat{ Pid: pid, CreateTime: fp.CreateTime, @@ -134,8 +139,8 @@ func fmtProcessStats( OpenFdCount: fp.OpenFdCount, ProcessState: model.ProcessState(model.ProcessState_value[fp.Status]), IoStat: ioStat, - VoluntaryCtxSwitches: uint64(fp.CtxSwitches.Voluntary), - InvoluntaryCtxSwitches: uint64(fp.CtxSwitches.Involuntary), + VoluntaryCtxSwitches: voluntaryCtxSwitches, + InvoluntaryCtxSwitches: involuntaryCtxSwitches, ContainerId: pidToCid[int(pid)], } diff --git a/pkg/process/checks/system_info.go b/pkg/process/checks/system_info.go index 1f101e6ff5b7..f3e24ae1f11f 100644 --- a/pkg/process/checks/system_info.go +++ b/pkg/process/checks/system_info.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows && !darwin +//go:build !windows && !darwin && !aix package checks diff --git a/pkg/process/checks/system_info_aix.go b/pkg/process/checks/system_info_aix.go new file mode 100644 index 000000000000..93ee22744d90 --- /dev/null +++ b/pkg/process/checks/system_info_aix.go @@ -0,0 +1,51 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package checks + +import ( + model "github.com/DataDog/agent-payload/v5/process" + "github.com/shirou/gopsutil/v4/cpu" + "github.com/shirou/gopsutil/v4/host" + "github.com/shirou/gopsutil/v4/mem" +) + +// CollectSystemInfo collects a set of system-level information that will not +// change until a restart. On AIX, fields that cannot be retrieved are left empty. +func CollectSystemInfo() (*model.SystemInfo, error) { + hi, err := host.Info() + if err != nil || hi == nil { + hi = &host.InfoStat{} + } + + cpuInfo, _ := cpu.Info() + + mi, err := mem.VirtualMemory() + if err != nil || mi == nil { + mi = &mem.VirtualMemoryStat{} + } + + cpus := make([]*model.CPUInfo, 0, len(cpuInfo)) + for _, c := range cpuInfo { + cpus = append(cpus, &model.CPUInfo{ + Cores: c.Cores, + }) + } + + return &model.SystemInfo{ + Uuid: hi.HostID, + Os: &model.OSInfo{ + Name: hi.OS, + Platform: hi.Platform, + Family: hi.PlatformFamily, + Version: hi.PlatformVersion, + KernelVersion: hi.KernelVersion, + }, + Cpus: cpus, + TotalMemory: int64(mi.Total), + }, nil +} diff --git a/pkg/process/procutil/process_model.go b/pkg/process/procutil/process_model.go index 325a21e8877e..7b6caa047ad3 100644 --- a/pkg/process/procutil/process_model.go +++ b/pkg/process/procutil/process_model.go @@ -346,18 +346,27 @@ func ConvertAllFilledProcessesToStats(processes map[int32]*process.FilledProcess // ConvertFilledProcessesToStats takes a group of FilledProcess objects and convert them into Stats func ConvertFilledProcessesToStats(p *process.FilledProcess) *Stats { - return &Stats{ + s := &Stats{ CreateTime: p.CreateTime, Status: p.Status, Nice: p.Nice, OpenFdCount: p.OpenFdCount, NumThreads: p.NumThreads, CPUTime: ConvertFromCPUStat(p.CpuTime), - MemInfo: ConvertFromMemInfo(p.MemInfo), - MemInfoEx: ConvertFromMemInfoEx(p.MemInfoEx), - IOStat: ConvertFromIOStats(p.IOStat), - CtxSwitches: ConvertFromCtxSwitches(p.CtxSwitches), } + if p.MemInfo != nil { + s.MemInfo = ConvertFromMemInfo(p.MemInfo) + } + if p.MemInfoEx != nil { + s.MemInfoEx = ConvertFromMemInfoEx(p.MemInfoEx) + } + if p.IOStat != nil { + s.IOStat = ConvertFromIOStats(p.IOStat) + } + if p.CtxSwitches != nil { + s.CtxSwitches = ConvertFromCtxSwitches(p.CtxSwitches) + } + return s } // ConvertFromFilledProcess takes a FilledProcess object and convert it into Process diff --git a/pkg/system-probe/config/config_aix.go b/pkg/system-probe/config/config_aix.go new file mode 100644 index 000000000000..62affd9df666 --- /dev/null +++ b/pkg/system-probe/config/config_aix.go @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package config + +import "github.com/DataDog/datadog-agent/pkg/config/model" + +// eBPFMapPreallocationSupported returns false on AIX (no eBPF support). +func eBPFMapPreallocationSupported() bool { + return false +} + +// ProcessEventDataStreamSupported returns false on AIX. +func ProcessEventDataStreamSupported() bool { + return false +} + +// RedisMonitoringSupported returns false on AIX. +func RedisMonitoringSupported() bool { + return false +} + +// HTTP2MonitoringSupported returns false on AIX. +func HTTP2MonitoringSupported() bool { + return false +} + +func allowPrebuiltEbpfFallback(_ model.Config) { +} diff --git a/pkg/util/compression/selector/no-zlib-no-zstd.go b/pkg/util/compression/selector/no-zlib-no-zstd.go index 282c3e369715..256932aee2ed 100644 --- a/pkg/util/compression/selector/no-zlib-no-zstd.go +++ b/pkg/util/compression/selector/no-zlib-no-zstd.go @@ -26,7 +26,7 @@ func NewCompressor(kind string, level int) common.Compressor { case common.NoneKind: return implnoop.New() default: - log.Error("invalid compression set") + log.Errorf("unknown compression kind %q, falling back to noop", kind) return implnoop.New() } } diff --git a/pkg/util/compression/selector/no-zlib-zstd.go b/pkg/util/compression/selector/no-zlib-zstd.go new file mode 100644 index 000000000000..0a4f96f56944 --- /dev/null +++ b/pkg/util/compression/selector/no-zlib-zstd.go @@ -0,0 +1,43 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build !zlib && zstd + +// Package selector provides correct compression impl to fx +package selector + +import ( + common "github.com/DataDog/datadog-agent/pkg/util/compression" + implgzip "github.com/DataDog/datadog-agent/pkg/util/compression/impl-gzip" + implnoop "github.com/DataDog/datadog-agent/pkg/util/compression/impl-noop" + implzstd "github.com/DataDog/datadog-agent/pkg/util/compression/impl-zstd" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// NewCompressor returns a new Compressor based on serializer_compressor_kind. +// This function is called when zstd is included but zlib is not. +func NewCompressor(kind string, level int) common.Compressor { + switch kind { + case common.ZstdKind: + return implzstd.New(implzstd.Requires{ + Level: common.ZstdCompressionLevel(level), + }) + case common.ZlibKind: + log.Warn("zlib build tag not included, falling back to gzip") + return implgzip.New(implgzip.Requires{Level: level}) + case common.GzipKind: + return implgzip.New(implgzip.Requires{Level: level}) + case common.NoneKind: + return implnoop.New() + default: + log.Errorf("unknown compression kind %q, falling back to noop", kind) + return implnoop.New() + } +} + +// NewNoopCompressor returns a new Noop Compressor. +func NewNoopCompressor() common.Compressor { + return implnoop.New() +} diff --git a/pkg/util/defaultpaths/path_nix.go b/pkg/util/defaultpaths/path_nix.go index c775691935c3..521333ee9709 100644 --- a/pkg/util/defaultpaths/path_nix.go +++ b/pkg/util/defaultpaths/path_nix.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build netbsd || openbsd || solaris || dragonfly || linux +//go:build netbsd || openbsd || solaris || dragonfly || linux || aix package defaultpaths diff --git a/pkg/util/hostinfo/hostinfo_aix.go b/pkg/util/hostinfo/hostinfo_aix.go new file mode 100644 index 000000000000..47cbdae06407 --- /dev/null +++ b/pkg/util/hostinfo/hostinfo_aix.go @@ -0,0 +1,36 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package hostinfo + +import ( + "github.com/shirou/gopsutil/v4/host" + + "github.com/DataDog/datadog-agent/pkg/util/cache" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// GetInformation returns an InfoStat object with host metadata. +// On AIX, fields that cannot be retrieved are left empty; the function +// never returns a nil pointer and never propagates errors so the result +// is always cached. +func GetInformation() *host.InfoStat { + info, _ := cache.Get[*host.InfoStat]( + hostInfoCacheKey, + func() (*host.InfoStat, error) { + info, err := host.Info() + if err != nil { + log.Warnf("failed to retrieve host info on AIX, using partial data: %s", err) + if info == nil { + info = &host.InfoStat{} + } + } + // Never return an error so the (possibly partial) result is cached. + return info, nil + }) + return info +} diff --git a/pkg/util/hostinfo/hostinfo_nix.go b/pkg/util/hostinfo/hostinfo_nix.go index af25947778d6..69e750b0d684 100644 --- a/pkg/util/hostinfo/hostinfo_nix.go +++ b/pkg/util/hostinfo/hostinfo_nix.go @@ -12,7 +12,7 @@ // repository, but is not designed as a general-purpose logging system. Its // API may change incompatibly. -//go:build !windows +//go:build !windows && !aix package hostinfo diff --git a/pkg/util/lsof/lsof_aix.go b/pkg/util/lsof/lsof_aix.go new file mode 100644 index 000000000000..f0003ce6a69e --- /dev/null +++ b/pkg/util/lsof/lsof_aix.go @@ -0,0 +1,12 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package lsof + +func openFiles(_ int) (Files, error) { + return nil, ErrNotImplemented +} diff --git a/pkg/util/port/portlist/poller_aix.go b/pkg/util/port/portlist/poller_aix.go new file mode 100644 index 000000000000..2bb01b3e0976 --- /dev/null +++ b/pkg/util/port/portlist/poller_aix.go @@ -0,0 +1,14 @@ +// Copyright (c) Tailscale Inc & AUTHORS +// SPDX-License-Identifier: BSD-3-Clause +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2014-present Datadog, Inc. + +//go:build aix + +package portlist + +import "errors" + +func (p *Poller) init() { + p.initErr = errors.New("portlist polling not supported on AIX") +} diff --git a/pkg/util/uuid/uuid_aix.go b/pkg/util/uuid/uuid_aix.go new file mode 100644 index 000000000000..d37219ec648c --- /dev/null +++ b/pkg/util/uuid/uuid_aix.go @@ -0,0 +1,34 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +//go:build aix + +package uuid + +import ( + gopsutilhost "github.com/shirou/gopsutil/v4/host" + + "github.com/DataDog/datadog-agent/pkg/util/cache" + "github.com/DataDog/datadog-agent/pkg/util/log" +) + +// getUUID returns the host ID. +// On AIX, host.Info() may fail (e.g. if 'bootinfo' is not in PATH), in which +// case an empty string is returned without propagating the error. +func getUUID() string { + guid, _ := cache.Get[string]( + guidCacheKey, + func() (string, error) { + info, err := gopsutilhost.Info() + if err != nil { + log.Warnf("failed to retrieve host info on AIX, UUID will be empty: %s", err) + // Return empty string without an error so the result is cached and + // we don't retry (and re-log) on every call. + return "", nil + } + return info.HostID, nil + }) + return guid +} diff --git a/pkg/util/uuid/uuid_nix.go b/pkg/util/uuid/uuid_nix.go index 9f8eb0ab8506..772264f951c8 100644 --- a/pkg/util/uuid/uuid_nix.go +++ b/pkg/util/uuid/uuid_nix.go @@ -3,7 +3,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2016-present Datadog, Inc. -//go:build !windows +//go:build !windows && !aix package uuid diff --git a/rtloader/common/stringutils.c b/rtloader/common/stringutils.c index 33a4f5de792c..5290e45a8381 100644 --- a/rtloader/common/stringutils.c +++ b/rtloader/common/stringutils.c @@ -2,11 +2,13 @@ // under the Apache License Version 2.0. // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-present Datadog, Inc. +// stringutils.h includes Python.h which must come before system headers (CPython requirement) +#include "stringutils.h" + #include #include "rtloader_mem.h" #include "rtloader_types.h" -#include "stringutils.h" PyObject * jloads = NULL; diff --git a/rtloader/rtloader/api.cpp b/rtloader/rtloader/api.cpp index 7d091dc6d91c..375f7039e2c4 100644 --- a/rtloader/rtloader/api.cpp +++ b/rtloader/rtloader/api.cpp @@ -42,6 +42,8 @@ # define DATADOG_AGENT_THREE "libdatadog-agent-three.dylib" #elif __FreeBSD__ # define DATADOG_AGENT_THREE "libdatadog-agent-three.so" +#elif defined(_AIX) +# define DATADOG_AGENT_THREE "libdatadog-agent-three.so" #elif _WIN32 # define DATADOG_AGENT_THREE "libdatadog-agent-three.dll" #else @@ -404,7 +406,11 @@ DATADOG_AGENT_RTLOADER_API int handle_crashes(const int enable_coredump, const i if (alt_stack == nullptr) { // Note: this memory is never freed, but it is necessary for the duration of the program alt_stack = _malloc(alt_stack_size); - stack_t new_stack{ .ss_sp = alt_stack, .ss_flags = 0, .ss_size = alt_stack_size }; + stack_t new_stack; + memset(&new_stack, 0, sizeof(new_stack)); + new_stack.ss_sp = (decltype(new_stack.ss_sp))alt_stack; + new_stack.ss_size = alt_stack_size; + new_stack.ss_flags = 0; int ret = sigaltstack(&new_stack, nullptr); if (ret != 0) { std::ostringstream err_msg; diff --git a/rtloader/three/three.h b/rtloader/three/three.h index e84634876274..c0387432d00f 100644 --- a/rtloader/three/three.h +++ b/rtloader/three/three.h @@ -6,13 +6,18 @@ #ifndef DATADOG_AGENT_RTLOADER_THREE_H #define DATADOG_AGENT_RTLOADER_THREE_H +// Python.h must be included before any system headers (CPython embedding requirement) +// On AIX (and other platforms), Python.h sets preprocessor definitions that affect +// how system headers like unistd.h define their types. Including Python.h after +// C++ stdlib headers causes conflicting type declarations in GCC's include-fixed headers. +#include + #include #include #include #include #include -#include #include class Three : public RtLoader diff --git a/tasks/agent.py b/tasks/agent.py index 771a4aeb8ad3..21a299062878 100644 --- a/tasks/agent.py +++ b/tasks/agent.py @@ -158,9 +158,10 @@ def build( """ flavor = AgentFlavor[flavor] - if not exclude_rtloader and not flavor.is_iot(): + if not exclude_rtloader and not flavor.is_iot() and sys.platform != "aix": # If embedded_path is set, we should give it to rtloader as it should install the headers/libs # in the embedded path folder because that's what is used in get_build_flags() + # On AIX, rtloader is built natively in advance (see packaging/aix/stages/03-rtloader.sh). with gitlab_section("Install embedded rtloader", collapsed=True): rtloader_make(ctx, install_prefix=embedded_path, cmake_options=cmake_options) rtloader_install(ctx) @@ -237,7 +238,8 @@ def build( if embedded_path is None: embedded_path = get_embedded_path(ctx) - assert embedded_path, "Failed to find embedded path" + if sys.platform != "aix": + assert embedded_path, "Failed to find embedded path" for build in bundled_agents: if build == "agent": @@ -660,6 +662,7 @@ def check_supports_python_version(check_dir, python): Check if a Python project states support for a given major Python version. """ import toml + from packaging.specifiers import SpecifierSet if python not in ['2', '3']: diff --git a/tasks/build_tags.py b/tasks/build_tags.py index 9cbbb485419f..73cf78954699 100644 --- a/tasks/build_tags.py +++ b/tasks/build_tags.py @@ -270,6 +270,31 @@ # List of tags to always remove when not building on Linux LINUX_ONLY_TAGS = {"netcgo", "systemd", "jetson", "linux_bpf", "nvml", "pcap", "podman", "trivy"} +# List of tags to always remove when building on AIX +AIX_EXCLUDE_TAGS = { + "cel", + "clusterchecks", + "containerd", + "cri", + "crio", + "docker", + "fargateprocess", + "jetson", + "jmx", + "kubeapiserver", + "kubelet", + "linux_bpf", + "netcgo", + "npm", + "nvml", + "orchestrator", + "pcap", + "podman", + "systemd", + "systemprobechecks", + "trivy", +} + # List of tags to always remove when building on Windows WINDOWS_EXCLUDE_TAGS = { "linux_bpf", @@ -366,6 +391,7 @@ _GOOS_TO_SYS_PLATFORM = { "windows": "win32", + "aix": "aix", } @@ -456,6 +482,9 @@ def filter_incompatible_tags(include, platform=sys.platform): if platform == "darwin": exclude = exclude.union(DARWIN_EXCLUDED_TAGS) + if platform == "aix": + exclude = exclude.union(AIX_EXCLUDE_TAGS) + return get_build_tags(include, exclude) diff --git a/tasks/libs/common/utils.py b/tasks/libs/common/utils.py index a1851757bd28..c794cb575812 100644 --- a/tasks/libs/common/utils.py +++ b/tasks/libs/common/utils.py @@ -256,7 +256,11 @@ def get_build_flags( if embedded_path is None: embedded_path = get_embedded_path(ctx) if embedded_path is None: - raise Exit("unable to locate embedded path please check your setup or set --embedded-path") + if sys.platform == "aix": + # On AIX, embedded_path may not exist; rtloader is resolved via --rtloader-root. + pass + else: + raise Exit("unable to locate embedded path please check your setup or set --embedded-path") rtloader_lib, rtloader_headers, rtloader_common_headers = get_rtloader_paths(embedded_path, rtloader_root) # setting the install path, allowing the agent to be installed in a custom location @@ -284,6 +288,8 @@ def get_build_flags( ) env['DYLD_LIBRARY_PATH'] = os.environ.get('DYLD_LIBRARY_PATH', '') + f":{':'.join(rtloader_lib)}" # OSX env['LD_LIBRARY_PATH'] = os.environ.get('LD_LIBRARY_PATH', '') + f":{':'.join(rtloader_lib)}" # linux + if sys.platform == "aix": + env['LIBPATH'] = os.environ.get('LIBPATH', '') + f":{':'.join(rtloader_lib)}" # AIX env['CGO_LDFLAGS'] = os.environ.get('CGO_LDFLAGS', '') + f" -L{' -L '.join(rtloader_lib)}" if sys.platform == 'win32': @@ -305,7 +311,8 @@ def get_build_flags( ldflags += "-s -w -linkmode=external " extldflags += "-static " elif rtloader_lib: - ldflags += f"-r {':'.join(rtloader_lib)} " + if sys.platform != "aix": # AIX uses full-path linking in pkg/collector/python/init.go + ldflags += f"-r {':'.join(rtloader_lib)} " if os.environ.get("DELVE"): gcflags = "all=-N -l" diff --git a/tasks/libs/types/arch.py b/tasks/libs/types/arch.py index 8a7ee03c7e42..1bc0dfef18b5 100644 --- a/tasks/libs/types/arch.py +++ b/tasks/libs/types/arch.py @@ -120,6 +120,10 @@ def from_str(arch: str | Literal["local"] | Arch) -> Arch: if arch == "local": arch = platform.machine().lower() + # On AIX, platform.machine() returns a machine model number (e.g. "00f9d80f4c00"), + # not the architecture name. Fall back to platform.processor() in that case. + if not any(arch in a.spellings for a in ALL_ARCHS): + arch = platform.processor().lower() # Not the most efficient way to do this, but the list is small # enough and this way we avoid having to maintain a dictionary @@ -155,4 +159,15 @@ def local() -> Arch: spellings={"amd64", "x86_64", "x64", "x86-64", "x86"}, ) -ALL_ARCHS = [ARCH_AMD64, ARCH_ARM64] +ARCH_PPC64 = Arch( + name="ppc64", + go_arch="ppc64", + gcc_arch="powerpc64", + kernel_arch="powerpc", + kmt_arch=None, + windows_arch="", + ci_arch="ppc64", + spellings={"ppc64", "powerpc64", "powerpc"}, +) + +ALL_ARCHS = [ARCH_AMD64, ARCH_ARM64, ARCH_PPC64]