runtime/cgo: cgo can't work with some old versions glibc

[lifubang]

Go version

go version go1.22.3 linux/amd64

Output of go env in your module/workspace:

GO111MODULE=''
GOARCH='amd64'
GOBIN=''
GOCACHE='/root/.cache/go-build'
GOENV='/root/.config/go/env'
GOEXE=''
GOEXPERIMENT=''
GOFLAGS=''
GOHOSTARCH='amd64'
GOHOSTOS='linux'
GOINSECURE=''
GOMODCACHE='/root/go/pkg/mod'
GONOPROXY=''
GONOSUMDB=''
GOOS='linux'
GOPATH='/root/go'
GOPRIVATE=''
GOPROXY='https://proxy.golang.org,direct'
GOROOT='/usr/local/go'
GOSUMDB='sum.golang.org'
GOTMPDIR=''
GOTOOLCHAIN='auto'
GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64'
GOVCS=''
GOVERSION='go1.22.3'
GCCGO='gccgo'
GOAMD64='v1'
AR='ar'
CC='gcc'
CXX='g++'
CGO_ENABLED='1'
GOMOD='/dev/null'
GOWORK=''
CGO_CFLAGS='-O2 -g'
CGO_CPPFLAGS=''
CGO_CXXFLAGS='-O2 -g'
CGO_FFLAGS='-O2 -g'
CGO_LDFLAGS='-O2 -g'
PKG_CONFIG='pkg-config'
GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=/tmp/go-build3943136906=/tmp/go-build -gno-record-gcc-switches'

What did you do?

root@iZj6cjf6t1l6rdg0tqzfrfZ:~# docker run -dit --name=test busybox
0a20c06c6413ee5b417f5292ccdd95650d6206494c6f180060172e69e4f908e9
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# docker top test
UID                 PID                 PPID                C                   STIME               TTY                 TIME                CMD
root                354966              354946              0                   22:44               ?                   00:00:00            sh
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# # We want to enter the process with pid 354966's mount namespace
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# vim setns.go
root@iZj6cjf6t1l6rdg0tqzfrfZ:~#
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# go build -o setns setns.go
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# ./setns
enter mnt ns(3): 0
Segmentation fault

cat setns.go

package main

/*
#cgo CFLAGS: -Wall
#define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <sched.h>
#include <unistd.h>

void test() {
        int fd = open("/proc/354966/ns/mnt", O_RDONLY);
        int r = setns(fd, CLONE_NEWNS);
        printf("enter mnt ns(%d): %d\n", fd, r);
}
void __attribute__((constructor)) init(void) {
        test();
}
*/
import "C"

import "fmt"

func main() {
        fmt.Println("hello world!")
}

What did you see happen?

The go program crashed: Segmentation fault

This is similar with the issue #65625, but not the same as it.
I wrote a c programe, and found that this is also a bug of pthread_getattr_np in some old glibc.

define _GNU_SOURCE
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <stdlib.h>
#include <stdio.h>
#include <sched.h>
#include <unistd.h>

int main(int argc, char **argv) {
        //printf("%d\n", argc);
        int pid;
        sscanf(argv[1], "%d", &pid);
        char path[256];
        sprintf(path, "/proc/%d/ns/mnt", pid);
        int fd = open(path, O_RDONLY);
        int r = setns(fd, CLONE_NEWNS);
        printf("setns pidns(%d): %d\n", fd, r);
        pthread_attr_t attr;
        pthread_attr_init(&attr);
        r = pthread_getattr_np(pthread_self(), &attr);
        printf("getattr: %d\n", r);
}

root@iZj6cjf6t1l6rdg0tqzfrfZ:~# gcc -o setnsc setnsc.c -lpthread
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# ./setnsc 354966
setns pidns(3): 0
getattr: 2
root@iZj6cjf6t1l6rdg0tqzfrfZ:~# strace -f ./setnsc 354966
execve("./setnsc", ["./setnsc", "354966"], 0x7ffc612b61e0 /* 22 vars */) = 0
brk(NULL)                               = 0x556d6270a000
arch_prctl(0x3001 /* ARCH_??? */, 0x7ffd12ad2890) = -1 EINVAL (Invalid argument)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=25966, ...}) = 0
mmap(NULL, 25966, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f817b1e9000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220q\0\0\0\0\0\0"..., 832) = 832
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\f\4K\246\21\256\356\256\273\203t\346`\6\0374"..., 68, 824) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=157224, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f817b1e7000
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\f\4K\246\21\256\356\256\273\203t\346`\6\0374"..., 68, 824) = 68
mmap(NULL, 140408, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f817b1c4000
mmap(0x7f817b1ca000, 69632, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f817b1ca000
mmap(0x7f817b1db000, 24576, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f817b1db000
mmap(0x7f817b1e1000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1c000) = 0x7f817b1e1000
mmap(0x7f817b1e3000, 13432, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f817b1e3000
close(3)                                = 0
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300A\2\0\0\0\0\0"..., 832) = 832
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\356\276]_K`\213\212S\354Dkc\230\33\272"..., 68, 880) = 68
fstat(3, {st_mode=S_IFREG|0755, st_size=2029592, ...}) = 0
pread64(3, "\6\0\0\0\4\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0@\0\0\0\0\0\0\0"..., 784, 64) = 784
pread64(3, "\4\0\0\0\20\0\0\0\5\0\0\0GNU\0\2\0\0\300\4\0\0\0\3\0\0\0\0\0\0\0", 32, 848) = 32
pread64(3, "\4\0\0\0\24\0\0\0\3\0\0\0GNU\0\356\276]_K`\213\212S\354Dkc\230\33\272"..., 68, 880) = 68
mmap(NULL, 2037344, PROT_READ, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f817afd2000
mmap(0x7f817aff4000, 1540096, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x22000) = 0x7f817aff4000
mmap(0x7f817b16c000, 319488, PROT_READ, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19a000) = 0x7f817b16c000
mmap(0x7f817b1ba000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7f817b1ba000
mmap(0x7f817b1c0000, 13920, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f817b1c0000
close(3)                                = 0
mmap(NULL, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f817afcf000
arch_prctl(ARCH_SET_FS, 0x7f817afcf740) = 0
mprotect(0x7f817b1ba000, 16384, PROT_READ) = 0
mprotect(0x7f817b1e1000, 4096, PROT_READ) = 0
mprotect(0x556d60a62000, 4096, PROT_READ) = 0
mprotect(0x7f817b21d000, 4096, PROT_READ) = 0
munmap(0x7f817b1e9000, 25966)           = 0
set_tid_address(0x7f817afcfa10)         = 355143
set_robust_list(0x7f817afcfa20, 24)     = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f817b1cabf0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f817b1d8420}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f817b1cac90, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f817b1d8420}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
prlimit64(0, RLIMIT_STACK, NULL, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
openat(AT_FDCWD, "/proc/354966/ns/mnt", O_RDONLY) = 3
setns(3, CLONE_NEWNS)                   = 0
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(0x88, 0x1), ...}) = 0
brk(NULL)                               = 0x556d6270a000
brk(0x556d6272b000)                     = 0x556d6272b000
write(1, "setns pidns(3): 0\n", 18setns pidns(3): 0
)     = 18
openat(AT_FDCWD, "/proc/self/maps", O_RDONLY|O_CLOEXEC) = -1 ENOENT (No such file or directory)
write(1, "getattr: 2\n", 11getattr: 2
)            = 11
exit_group(0)                           = ?
+++ exited with 0 +++

What did you expect to see?

Should see hello world! in the output, looks like in go 1.22.3 with glibc 2.35.

root@iZj6c2w08pe0hl9lwk8mdiZ:~# go version
go version go1.22.3 linux/amd64
root@iZj6c2w08pe0hl9lwk8mdiZ:~# ldd --version
ldd (Ubuntu GLIBC 2.35-0ubuntu3.7) 2.35
Copyright (C) 2022 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
Written by Roland McGrath and Ulrich Drepper.
root@iZj6c2w08pe0hl9lwk8mdiZ:~# go build -o setns setns.go
root@iZj6c2w08pe0hl9lwk8mdiZ:~# ./setns
enter mnt ns(-1): -1
hello world!

[lifubang]

@ianlancetaylor May I ask you some tech questions, if you can show me some datailed explanations, it will be appreciated.

1. I think the stack low address and the stack size are pthread's private data, why we need to get these data in cgo?
2. Why we choose using pthread_getattr_np to get the stack low address instead of __builtin_frame_address(0) + 4096 - size which has been used for many years before go 1.22.

Besides this issue and #65625 , we have found two issues about swith to pthread_getattr_np, and we can't sure that there is no other issues for pthread_getattr_np in old versions glibc. So I think we should consoder some ways to dertermine how to have a backward compatibility with previous versions. I have some ways to discuss:

1. switch back to __builtin_frame_address(0) + 4096 - size;
2. using __GLIBC_PREREQ(maj, min) macro to check glibc's version;
3. fail back to __builtin_frame_address(0) + 4096 - size. (Like the Patch Set 4 in runtime/cgo: provide backward compatibility with old glibc for cgo #67328 (comment) / https://go-review.googlesource.com/c/go/+/585019, obviously need rebase)

Looking forward to your reply.
And welcome some other people's opinions.
Thanks a lot.

[lifubang]

As we should know that if apply the patch https://go-review.googlesource.com/c/go/+/587919 , it can really make the program work, but the low address becomes 0, I don't know whether it will let the programme in a dangerous state or not.

[ianlancetaylor]

I think the stack low address and the stack size are pthread's private data, why we need to get these data in cgo?

1. They aren't strictly private data, as shown by the fact that there are functions that return them.

2. Go uses an automatically extensible stack. In order to implement that, the start of every function checks whether the stack is close to the bounds. That means that we need to know the stack bounds in order to run Go code. In particular, when a thread started by C code calls into Go code, we need to know the stack bounds in order to run the Go code. See callbackUpdateSystemStack in runtime/cgocall.go.

Why we choose using pthread_getattr_np to get the stack low address instead of __builtin_frame_address(0) + 4096 - size which has been used for many years before go 1.22.

See #59294 and https://go.dev/cl/479915.

I don't think we should make any changes for older versions of glibc until we find specific problems that we can test.

[lifubang]

Thanks your explanation, I agree your opinion. Because it has no impact for normal users, just only for container runtime written by golang.

Looking forward go 1.22.4, thanks a lot.

As we should know that if apply the patch https://go-review.googlesource.com/c/go/+/587919 , it can really make the program work, but the low address becomes 0, I don't know whether it will let the programme in a dangerous state or not.

[mknyszek]

@lifubang Given that the problems we know about will be fixed in Go 1.22.4, is there anything else to be done here? Thanks!

[lifubang]

Given that the problems we know about will be fixed in Go 1.22.4, is there anything else to be done here? Thanks!

After looked into the code, I think there is maybe no other thing needs to be worked here. Because in cgo we have aleady checked whether the low address returned by pthread libaray is zero or not:
https://github.com/golang/go/blob/master/src/runtime/cgocall.go#L302-L305

So, this one can be closed.