Skip to content

Commit

Permalink
Merge branch 'main' into fix-memory-metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
@aphralG
aphralG committed Sep 28, 2023
2 parents 297d95d + df8cf6e commit 6b7d4dc
Show file tree
Hide file tree
Showing 50 changed files with 1,493 additions and 386 deletions.
16 changes: 8 additions & 8 deletions Makefile.packaging
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
# Create deb packages

@for arch in $(DEB_ARCHS); do \
GOWORK=off CGO_ENABLED=0 GOARCH=$${arch} GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${arch} GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
for distro in $(DEB_DISTROS); do \
deb_codename=`echo $$distro | cut -d- -f 2`; \
VERSION=$(shell echo ${VERSION} | tr -d 'v')~$${deb_codename} ARCH=$${arch} nfpm pkg --config .nfpm.yaml --packager deb --target ${PACKAGES_DIR}/deb/${PACKAGE_PREFIX}_$(shell echo ${VERSION} | tr -d 'v')~$${deb_codename}_$${arch}.deb; \
Expand All @@ -59,7 +59,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##

# Create rpm packages

@GOWORK=off CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent
@GOWORK=off CGO_ENABLED=0 GOARCH=amd64 GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent
@for distro in $(RPM_DISTROS); do \
rpm_distro=`echo $$distro | cut -d- -f 1`; \
rpm_major=`echo $$distro | cut -d- -f 2`; \
Expand All @@ -82,7 +82,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
@for arch in $(REDHAT_ARCHS); do \
goarch=amd64; \
if [ "$$arch" = "aarch64" ]; then goarch="arm64"; fi; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
for distro in $(REDHAT_VERSIONS); do \
rpm_distro=`echo $$distro | cut -d- -f 1`; \
rpm_major=`echo $$distro | cut -d- -f 2`; \
Expand All @@ -99,7 +99,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
@for arch in $(ALMA_ARCHS); do \
goarch=amd64; \
if [ "$$arch" = "aarch64" ]; then goarch="arm64"; fi; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
for distro in $(ALMA_VERSIONS); do \
rpm_distro=`echo $$distro | cut -d- -f 1`; \
rpm_major=`echo $$distro | cut -d- -f 2`; \
Expand All @@ -116,7 +116,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
@for arch in $(ROCKY_ARCHS); do \
goarch=amd64; \
if [ "$$arch" = "aarch64" ]; then goarch="arm64"; fi; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
for distro in $(ROCKY_VERSIONS); do \
rpm_distro=`echo $$distro | cut -d- -f 1`; \
rpm_major=`echo $$distro | cut -d- -f 2`; \
Expand All @@ -134,7 +134,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
@for arch in $(AMAZON_ARCHS); do \
goarch=amd64; \
if [ "$$arch" = "aarch64" ]; then goarch="arm64"; fi; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
for version in $(AMAZON_VERSIONS); do \
rpm_major=`echo $$version | cut -d- -f 2`; \
rpm_codename="amzn$$rpm_major";\
Expand All @@ -150,7 +150,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
@for arch in $(APK_ARCHS); do \
goarch=amd64; \
if [ "$$arch" = "aarch64" ]; then goarch="arm64"; fi; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
GOWORK=off CGO_ENABLED=0 GOARCH=$${goarch} GOOS=linux go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent; \
for version in $(APK_VERSIONS); do \
if [ ! -d "$(PACKAGES_DIR)/apk/v$${version}/$${arch}" ]; then mkdir -p $(PACKAGES_DIR)/apk/v$${version}/$${arch}; fi; \
VERSION=$(shell echo ${VERSION} | tr -d 'v') ARCH=$${arch} nfpm pkg --config .nfpm.yaml --packager apk --target $(PACKAGES_DIR)/apk/v$${version}/$${arch}/${PACKAGE_PREFIX}-$(shell echo ${VERSION} | tr -d 'v').apk; \
Expand All @@ -163,7 +163,7 @@ package: gpg-key $(PACKAGES_DIR) $(GITHUB_PACKAGES_DIR) $(AZURE_PACKAGES_DIR) ##
# Create txz packages

rm -rf ./build/nginx-agent
@GOWORK=off CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags=${LDFLAGS} -o ./build/nginx-agent
@GOWORK=off CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -pgo=auto -ldflags=${LDFLAGS} -o ./build/nginx-agent

docker run -v ${PWD}:/nginx-agent/ build-signed-packager:1.0.0

Expand Down
2 changes: 1 addition & 1 deletion nginx-agent.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,4 @@ config_dirs: "/etc/nginx:/usr/local/etc/nginx:/usr/share/nginx/modules:/etc/nms"
# host: 127.0.0.1
#
# Set this value to a secure port number to prevent information leaks.
# port: 8038
# port: 8038
Binary file modified profile.cgo
View file
Binary file not shown.
172 changes: 113 additions & 59 deletions src/core/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/shirou/gopsutil/v3/host"
"github.com/shirou/gopsutil/v3/process"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/singleflight"
"golang.org/x/sys/unix"

"github.com/nginx/agent/sdk/v2/files"
Expand All @@ -52,7 +53,9 @@ type Environment interface {
DeleteFile(backup ConfigApplyMarker, fileName string) error
Processes() (result []*Process)
FileStat(path string) (os.FileInfo, error)
Disks() ([]*proto.DiskPartition, error)
DiskDevices() ([]string, error)
DiskUsage(mountPoint string) (*DiskUsage, error)
GetContainerID() (string, error)
GetNetOverflow() (float64, error)
IsContainer() bool
Expand Down Expand Up @@ -80,6 +83,13 @@ type Process struct {
Command string
}

type DiskUsage struct {
Total float64
Used float64
Free float64
UsedPercentage float64
}

const (
lengthOfContainerId = 64
versionId = "VERSION_ID"
Expand All @@ -91,16 +101,19 @@ const (
KernProc = 14 // struct: process entries
KernProcPathname = 12 // path to executable
SYS_SYSCTL = 202
IsContainerKey = "isContainer"
GetContainerIDKey = "GetContainerID"
GetSystemUUIDKey = "GetSystemUUIDKey"
)

var (
virtualizationFunc = host.VirtualizationWithContext
_ Environment = &EnvironmentType{}
basePattern = regexp.MustCompile("/([a-f0-9]{64})$")
colonPattern = regexp.MustCompile(":([a-f0-9]{64})$")
scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`)
containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})")
containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})")
virtualizationFunc = host.VirtualizationWithContext
singleflightGroup = &singleflight.Group{}
basePattern = regexp.MustCompile("/([a-f0-9]{64})$")
colonPattern = regexp.MustCompile(":([a-f0-9]{64})$")
scopePattern = regexp.MustCompile(`/.+-(.+?).scope$`)
containersPattern = regexp.MustCompile("containers/([a-f0-9]{64})")
containerdPattern = regexp.MustCompile("sandboxes/([a-f0-9]{64})")
)

func (env *EnvironmentType) NewHostInfo(agentVersion string, tags *[]string, configDirs string, clearCache bool) *proto.HostInfo {
Expand All @@ -123,6 +136,12 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer
tags = &[]string{}
}

disks, err := env.Disks()
if err != nil {
log.Warnf("Unable to get disks information from the host: %v", err)
disks = nil
}

hostInfoFacacde := &proto.HostInfo{
Agent: agentVersion,
Boot: hostInformation.BootTime,
Expand All @@ -131,7 +150,7 @@ func (env *EnvironmentType) NewHostInfoWithContext(ctx context.Context, agentVer
OsType: hostInformation.OS,
Uuid: env.GetSystemUUID(),
Uname: getUnixName(),
Partitons: diskPartitions(),
Partitons: disks,
Network: env.networks(),
Processor: processors(hostInformation.KernelArch),
Release: releaseInfo("/etc/os-release"),
Expand Down Expand Up @@ -194,24 +213,32 @@ func (env *EnvironmentType) GetHostname() string {
}

func (env *EnvironmentType) GetSystemUUID() string {
ctx := context.Background()
defer ctx.Done()
res, err, _ := singleflightGroup.Do(GetSystemUUIDKey, func() (interface{}, error) {
var err error
ctx := context.Background()
defer ctx.Done()

if env.IsContainer() {
containerID, err := env.GetContainerID()
if err != nil {
log.Errorf("Unable to read docker container ID: %v", err)
return ""
if env.IsContainer() {
containerID, err := env.GetContainerID()
if err != nil {
return "", fmt.Errorf("unable to read docker container ID: %v", err)
}
return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String(), err
}
return uuid.NewMD5(uuid.NameSpaceDNS, []byte(containerID)).String()
}

hostInfo, err := host.InfoWithContext(ctx)
hostID, err := host.HostIDWithContext(ctx)
if err != nil {
log.Warnf("Unable to read host id from dataplane, defaulting value. Error: %v", err)
return "", err
}
return uuid.NewMD5(uuid.Nil, []byte(hostID)).String(), err
})
if err != nil {
log.Infof("Unable to read host id from dataplane, defaulting value. Error: %v", err)
log.Warnf("Unable to set hostname due to %v", err)
return ""
}
return uuid.NewMD5(uuid.Nil, []byte(hostInfo.HostID)).String()

return res.(string)
}

func (env *EnvironmentType) ReadDirectory(dir string, ext string) ([]string, error) {
Expand Down Expand Up @@ -307,18 +334,25 @@ func (env *EnvironmentType) IsContainer() bool {
k8sServiceAcct = "/var/run/secrets/kubernetes.io/serviceaccount"
)

for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} {
if _, err := os.Stat(filename); err == nil {
log.Debugf("is a container because (%s) exists", filename)
return true
res, err, _ := singleflightGroup.Do(IsContainerKey, func() (interface{}, error) {
for _, filename := range []string{dockerEnv, containerEnv, k8sServiceAcct} {
if _, err := os.Stat(filename); err == nil {
log.Debugf("Is a container because (%s) exists", filename)
return true, nil
}
}
}
// v1 check
if result, err := cGroupV1Check(selfCgroup); err == nil && result {
return result
// v1 check
if result, err := cGroupV1Check(selfCgroup); err == nil && result {
return result, err
}
return false, nil
})

if err != nil {
log.Warnf("Unable to retrieve values from cache (%v)", err)
}

return false
return res.(bool)
}

// cGroupV1Check returns if running cgroup v1
Expand All @@ -345,20 +379,24 @@ func cGroupV1Check(cgroupFile string) (bool, error) {

// GetContainerID returns the ID of the current environment if running in a container
func (env *EnvironmentType) GetContainerID() (string, error) {
const mountInfo = "/proc/self/mountinfo"
res, err, _ := singleflightGroup.Do(GetContainerIDKey, func() (interface{}, error) {
const mountInfo = "/proc/self/mountinfo"

if !env.IsContainer() {
return "", errors.New("not in docker")
}
if !env.IsContainer() {
return "", errors.New("not in docker")
}

containerID, err := getContainerID(mountInfo)
if err != nil {
return "", fmt.Errorf("could not get container ID: %v", err)
}
containerID, err := getContainerID(mountInfo)
if err != nil {
return "", fmt.Errorf("could not get container ID: %v", err)
}

log.Debugf("Container ID: %s", containerID)

log.Debugf("Container ID: %s", containerID)
return containerID, err
})

return containerID, err
return res.(string), err
}

// getContainerID returns the container ID of the current running environment.
Expand Down Expand Up @@ -428,6 +466,42 @@ func (env *EnvironmentType) DiskDevices() ([]string, error) {
}
}

func (env *EnvironmentType) Disks() (partitions []*proto.DiskPartition, err error) {
ctx := context.Background()
defer ctx.Done()
parts, err := disk.PartitionsWithContext(ctx, false)
if err != nil {
// return an array of 0
log.Errorf("Could not read disk partitions for host: %v", err)
return []*proto.DiskPartition{}, err
}
for _, part := range parts {
pm := proto.DiskPartition{
MountPoint: part.Mountpoint,
Device: part.Device,
FsType: part.Fstype,
}
partitions = append(partitions, &pm)
}
return partitions, nil
}

func (env *EnvironmentType) DiskUsage(mountPoint string) (*DiskUsage, error) {
ctx := context.Background()
defer ctx.Done()
usage, err := disk.UsageWithContext(ctx, mountPoint)
if err != nil {
return nil, errors.New("unable to obtain disk usage stats")
}

return &DiskUsage{
Total: float64(usage.Total),
Used: float64(usage.Used),
Free: float64(usage.Free),
UsedPercentage: float64(usage.UsedPercent),
}, nil
}

func (env *EnvironmentType) GetNetOverflow() (float64, error) {
return network.GetNetOverflow()
}
Expand Down Expand Up @@ -795,26 +869,6 @@ func virtualization() (string, string) {
return virtualizationSystem, virtualizationRole
}

func diskPartitions() (partitions []*proto.DiskPartition) {
ctx := context.Background()
defer ctx.Done()
parts, err := disk.PartitionsWithContext(ctx, false)
if err != nil {
// return an array of 0
log.Errorf("Could not read disk partitions for host: %v", err)
return []*proto.DiskPartition{}
}
for _, part := range parts {
pm := proto.DiskPartition{
MountPoint: part.Mountpoint,
Device: part.Device,
FsType: part.Fstype,
}
partitions = append(partitions, &pm)
}
return partitions
}

func releaseInfo(osReleaseFile string) (release *proto.ReleaseInfo) {
hostReleaseInfo := getHostReleaseInfo()
osRelease, err := getOsRelease(osReleaseFile)
Expand Down
Loading

0 comments on commit 6b7d4dc

Please sign in to comment.