From 37a4fea12ad76439525c20dc1ecb9dcc3db02485 Mon Sep 17 00:00:00 2001 From: Jelle van den Hooff Date: Thu, 21 Nov 2024 20:44:52 -0800 Subject: [PATCH] initial public commit --- .ci/crossarch-tests/Dockerfile.test | 13 + .ci/crossarch-tests/docker-compose.yml | 21 + .ci/crossarch-tests/test-amd64.sh | 5 + .ci/crossarch-tests/test-arm64.sh | 5 + .ci/gofmt | 28 + .ci/gosim | 3 + .github/workflows/main.yml | 27 + .gitignore | 2 + .vscode/launch.json | 19 + .vscode/settings.json | 13 + LICENSE | 20 + README.md | 316 ++++ Taskfile.yml | 57 + cmd/gosim/doc.go | 60 + cmd/gosim/imports.go | 9 + cmd/gosim/main.go | 501 +++++++ cmd/gosim/main_test.go | 37 + cmd/gosim/selftest.go | 168 +++ doc.go | 65 + docs/design.md | 555 +++++++ go.mod | 68 + go.sum | 192 +++ gosimruntime/chan.go | 605 ++++++++ gosimruntime/doc.go | 8 + gosimruntime/fnv64.go | 57 + gosimruntime/globals.go | 204 +++ gosimruntime/log.go | 122 ++ gosimruntime/map.go | 492 ++++++ gosimruntime/map_test.go | 175 +++ gosimruntime/raceutil.go | 15 + gosimruntime/raceutil_norace.go | 39 + gosimruntime/raceutil_race.go | 72 + gosimruntime/rand.go | 52 + gosimruntime/rand_test.go | 13 + gosimruntime/runtime.go | 1043 +++++++++++++ gosimruntime/runtime_test.go | 80 + gosimruntime/sema.go | 292 ++++ gosimruntime/testmain.go | 147 ++ gosimruntime/time.go | 144 ++ gosimruntime/timer_heap.go | 114 ++ gosimruntime/timer_heap_test.go | 133 ++ gosimruntime/trace.go | 96 ++ gosimruntime/tracekey_string.go | 29 + imports.go | 27 + internal/coro/coro_linkname.go | 75 + internal/coro/coro_nolinkname.go | 39 + internal/coro/upcallcoro_norace.go | 28 + internal/coro/upcallcoro_race.go | 64 + internal/gosimtool/gosimtool.go | 279 ++++ internal/hooks/go123/doc.go | 6 + internal/hooks/go123/entrypoint.go | 28 + internal/hooks/go123/golangorg_x_sys_unix.go | 40 + .../golangorg_x_sys_unix_gensyscall_amd64.go | 113 ++ .../golangorg_x_sys_unix_gensyscall_arm64.go | 113 ++ internal/hooks/go123/hash_maphash.go | 7 + internal/hooks/go123/internal_abi.go | 9 + internal/hooks/go123/internal_bytealg.go | 55 + internal/hooks/go123/internal_chacha8.go | 12 + internal/hooks/go123/internal_cpu.go | 21 + internal/hooks/go123/internal_godebug.go | 16 + internal/hooks/go123/internal_poll.go | 63 + internal/hooks/go123/internal_syscall_unix.go | 13 + internal/hooks/go123/maps.go | 9 + internal/hooks/go123/math_rand.go | 7 + internal/hooks/go123/math_rand_v2.go | 7 + internal/hooks/go123/net.go | 7 + internal/hooks/go123/os.go | 32 + internal/hooks/go123/runtime_debug.go | 43 + internal/hooks/go123/runtime_trace.go | 17 + internal/hooks/go123/sync.go | 109 ++ internal/hooks/go123/sync_atomic.go | 394 +++++ internal/hooks/go123/syscall.go | 88 ++ .../hooks/go123/syscall_gensyscall_amd64.go | 113 ++ .../hooks/go123/syscall_gensyscall_arm64.go | 113 ++ internal/hooks/go123/time.go | 112 ++ internal/prettylog/prettylog.go | 408 +++++ internal/prettylog/prettylog_test.go | 93 ++ internal/prettylog/testdata/simple.txt | 5 + internal/prettylog/testdata/simple.txt.golden | 24 + internal/race/doc.go | 2 + internal/race/norace.go | 44 + internal/race/race.go | 54 + internal/reflect/deepequal.go | 251 ++++ internal/reflect/doc.go | 6 + internal/reflect/makefunc.go | 10 + internal/reflect/swapper.go | 7 + internal/reflect/type.go | 410 +++++ internal/reflect/value.go | 555 +++++++ internal/reflect/visiblefields.go | 7 + internal/simulation/fs/chunkedfile.go | 272 ++++ internal/simulation/fs/chunkedfile_test.go | 238 +++ internal/simulation/fs/depkind_string.go | 47 + internal/simulation/fs/filesystem.go | 851 +++++++++++ internal/simulation/fs/pendingops.go | 417 ++++++ internal/simulation/fs/pendingops_test.go | 169 +++ internal/simulation/generate.go | 5 + internal/simulation/gensyscall/main.go | 781 ++++++++++ internal/simulation/gosim.go | 206 +++ internal/simulation/gosim_gensyscall.go | 374 +++++ internal/simulation/linux_gensyscall_amd64.go | 837 +++++++++++ internal/simulation/linux_gensyscall_arm64.go | 837 +++++++++++ internal/simulation/network/delayqueue.go | 111 ++ internal/simulation/network/net.go | 228 +++ internal/simulation/network/stack.go | 720 +++++++++ internal/simulation/os_linux.go | 1180 +++++++++++++++ internal/simulation/os_other.go | 20 + internal/simulation/simulation.go | 180 +++ internal/simulation/syscallabi/errno.go | 44 + internal/simulation/syscallabi/poll.go | 259 ++++ internal/simulation/syscallabi/syscall.go | 200 +++ internal/simulation/userspace.go | 100 ++ internal/testing/match.go | 320 ++++ internal/testing/missing.go | 88 ++ internal/testing/testing.go | 1022 +++++++++++++ internal/tests/Makefile | 5 + internal/tests/behavior/chan_test.go | 336 +++++ internal/tests/behavior/context_test.go | 91 ++ internal/tests/behavior/crash_meta_test.go | 47 + internal/tests/behavior/crash_test.go | 760 ++++++++++ internal/tests/behavior/disk_crash_test.go | 615 ++++++++ internal/tests/behavior/disk_sim_test.go | 179 +++ internal/tests/behavior/disk_test.go | 911 ++++++++++++ internal/tests/behavior/globals.go | 10 + internal/tests/behavior/globals_for_test.go | 13 + internal/tests/behavior/globals_test.go | 85 ++ internal/tests/behavior/log_meta_test.go | 95 ++ internal/tests/behavior/log_test.go | 64 + internal/tests/behavior/machine_meta_test.go | 42 + internal/tests/behavior/machine_test.go | 209 +++ internal/tests/behavior/map_test.go | 779 ++++++++++ internal/tests/behavior/meta_test.go | 24 + internal/tests/behavior/nemesis_meta_test.go | 56 + internal/tests/behavior/nemesis_test.go | 46 + internal/tests/behavior/net_test.go | 1323 +++++++++++++++++ internal/tests/behavior/os_sim_test.go | 37 + internal/tests/behavior/os_test.go | 21 + internal/tests/behavior/rand_meta_test.go | 82 + internal/tests/behavior/rand_test.go | 35 + internal/tests/behavior/rand_v2_test.go | 16 + internal/tests/behavior/reflect_test.go | 212 +++ internal/tests/behavior/sync_test.go | 31 + internal/tests/behavior/testing_meta_test.go | 218 +++ internal/tests/behavior/testing_test.go | 296 ++++ internal/tests/behavior/time_test.go | 392 +++++ internal/tests/race/race_test.go | 223 +++ internal/tests/race/testdata/annotate_test.go | 151 ++ internal/tests/race/testdata/atomic_test.go | 330 ++++ internal/tests/race/testdata/chan_test.go | 789 ++++++++++ internal/tests/race/testdata/io_test.go | 75 + internal/tests/race/testdata/map_test.go | 336 +++++ internal/tests/race/testdata/mutex_test.go | 152 ++ internal/tests/race/testdata/os_test.go | 276 ++++ internal/tests/race/testdata/race_test.go | 91 ++ internal/tests/race/testdata/rwmutex_test.go | 156 ++ internal/tests/race/testdata/select_test.go | 272 ++++ internal/tests/race/testdata/sync_test.go | 207 +++ .../tests/race/testdata/time_simonly_test.go | 79 + internal/tests/race/testdata/time_test.go | 144 ++ .../tests/race/testdata/waitgroup_test.go | 368 +++++ internal/tests/script/script_test.go | 77 + .../tests/script/testdata/gosimpassfail.txtar | 22 + .../tests/script/testdata/simmetatest.txtar | 25 + .../tests/script/testdata/testbuilding.txtar | 75 + internal/tests/testpb.proto | 18 + internal/tests/testpb/testpb.pb.go | 213 +++ internal/tests/testpb/testpb_grpc.pb.go | 103 ++ internal/translate/cache.go | 162 ++ internal/translate/cache/cache.go | 115 ++ internal/translate/cache/cache_test.go | 79 + internal/translate/chan.go | 409 +++++ internal/translate/globals.go | 462 ++++++ internal/translate/globals_analysis.go | 244 +++ internal/translate/globals_analysis_ssa.go | 260 ++++ internal/translate/go.go | 234 +++ internal/translate/hooks_go123.go | 200 +++ .../translate/hooks_go123_amd64_gensyscall.go | 57 + .../translate/hooks_go123_arm64_gensyscall.go | 57 + internal/translate/main.go | 594 ++++++++ internal/translate/map.go | 461 ++++++ internal/translate/outputwriter.go | 273 ++++ internal/translate/rewrites.go | 268 ++++ .../translate/testdata/basic.translate.txt | 1300 ++++++++++++++++ .../translate/testdata/comment.translate.txt | 100 ++ .../testdata/directives.translate.txt | 42 + .../translate/testdata/packages.translate.txt | 157 ++ .../translate/testdata/select.translate.txt | 219 +++ .../translate/testdata/slog.translate.txt | 23 + .../translate/testdata/test.translate.txt | 59 + internal/translate/tests.go | 212 +++ internal/translate/translate.go | 637 ++++++++ internal/translate/translate_test.go | 221 +++ internal/translate/types.go | 462 ++++++ internal/translate/workqueue.go | 153 ++ machine.go | 202 +++ metatesting/doc.go | 46 + metatesting/metatest.go | 426 ++++++ metatesting/metatest_test.go | 28 + nemesis/meta_test.go | 24 + nemesis/nemesis.go | 158 ++ nemesis/nemesis_test.go | 122 ++ simulation.go | 84 ++ test.sh | 5 + 202 files changed, 39324 insertions(+) create mode 100644 .ci/crossarch-tests/Dockerfile.test create mode 100644 .ci/crossarch-tests/docker-compose.yml create mode 100755 .ci/crossarch-tests/test-amd64.sh create mode 100755 .ci/crossarch-tests/test-arm64.sh create mode 100755 .ci/gofmt create mode 100755 .ci/gosim create mode 100644 .github/workflows/main.yml create mode 100644 .gitignore create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json create mode 100644 LICENSE create mode 100644 README.md create mode 100644 Taskfile.yml create mode 100644 cmd/gosim/doc.go create mode 100644 cmd/gosim/imports.go create mode 100644 cmd/gosim/main.go create mode 100644 cmd/gosim/main_test.go create mode 100644 cmd/gosim/selftest.go create mode 100644 doc.go create mode 100644 docs/design.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 gosimruntime/chan.go create mode 100644 gosimruntime/doc.go create mode 100644 gosimruntime/fnv64.go create mode 100644 gosimruntime/globals.go create mode 100644 gosimruntime/log.go create mode 100644 gosimruntime/map.go create mode 100644 gosimruntime/map_test.go create mode 100644 gosimruntime/raceutil.go create mode 100644 gosimruntime/raceutil_norace.go create mode 100644 gosimruntime/raceutil_race.go create mode 100644 gosimruntime/rand.go create mode 100644 gosimruntime/rand_test.go create mode 100644 gosimruntime/runtime.go create mode 100644 gosimruntime/runtime_test.go create mode 100644 gosimruntime/sema.go create mode 100644 gosimruntime/testmain.go create mode 100644 gosimruntime/time.go create mode 100644 gosimruntime/timer_heap.go create mode 100644 gosimruntime/timer_heap_test.go create mode 100644 gosimruntime/trace.go create mode 100644 gosimruntime/tracekey_string.go create mode 100644 imports.go create mode 100644 internal/coro/coro_linkname.go create mode 100644 internal/coro/coro_nolinkname.go create mode 100644 internal/coro/upcallcoro_norace.go create mode 100644 internal/coro/upcallcoro_race.go create mode 100644 internal/gosimtool/gosimtool.go create mode 100644 internal/hooks/go123/doc.go create mode 100644 internal/hooks/go123/entrypoint.go create mode 100644 internal/hooks/go123/golangorg_x_sys_unix.go create mode 100644 internal/hooks/go123/golangorg_x_sys_unix_gensyscall_amd64.go create mode 100644 internal/hooks/go123/golangorg_x_sys_unix_gensyscall_arm64.go create mode 100644 internal/hooks/go123/hash_maphash.go create mode 100644 internal/hooks/go123/internal_abi.go create mode 100644 internal/hooks/go123/internal_bytealg.go create mode 100644 internal/hooks/go123/internal_chacha8.go create mode 100644 internal/hooks/go123/internal_cpu.go create mode 100644 internal/hooks/go123/internal_godebug.go create mode 100644 internal/hooks/go123/internal_poll.go create mode 100644 internal/hooks/go123/internal_syscall_unix.go create mode 100644 internal/hooks/go123/maps.go create mode 100644 internal/hooks/go123/math_rand.go create mode 100644 internal/hooks/go123/math_rand_v2.go create mode 100644 internal/hooks/go123/net.go create mode 100644 internal/hooks/go123/os.go create mode 100644 internal/hooks/go123/runtime_debug.go create mode 100644 internal/hooks/go123/runtime_trace.go create mode 100644 internal/hooks/go123/sync.go create mode 100644 internal/hooks/go123/sync_atomic.go create mode 100644 internal/hooks/go123/syscall.go create mode 100644 internal/hooks/go123/syscall_gensyscall_amd64.go create mode 100644 internal/hooks/go123/syscall_gensyscall_arm64.go create mode 100644 internal/hooks/go123/time.go create mode 100644 internal/prettylog/prettylog.go create mode 100644 internal/prettylog/prettylog_test.go create mode 100644 internal/prettylog/testdata/simple.txt create mode 100644 internal/prettylog/testdata/simple.txt.golden create mode 100644 internal/race/doc.go create mode 100644 internal/race/norace.go create mode 100644 internal/race/race.go create mode 100644 internal/reflect/deepequal.go create mode 100644 internal/reflect/doc.go create mode 100644 internal/reflect/makefunc.go create mode 100644 internal/reflect/swapper.go create mode 100644 internal/reflect/type.go create mode 100644 internal/reflect/value.go create mode 100644 internal/reflect/visiblefields.go create mode 100644 internal/simulation/fs/chunkedfile.go create mode 100644 internal/simulation/fs/chunkedfile_test.go create mode 100644 internal/simulation/fs/depkind_string.go create mode 100644 internal/simulation/fs/filesystem.go create mode 100644 internal/simulation/fs/pendingops.go create mode 100644 internal/simulation/fs/pendingops_test.go create mode 100644 internal/simulation/generate.go create mode 100644 internal/simulation/gensyscall/main.go create mode 100644 internal/simulation/gosim.go create mode 100644 internal/simulation/gosim_gensyscall.go create mode 100644 internal/simulation/linux_gensyscall_amd64.go create mode 100644 internal/simulation/linux_gensyscall_arm64.go create mode 100644 internal/simulation/network/delayqueue.go create mode 100644 internal/simulation/network/net.go create mode 100644 internal/simulation/network/stack.go create mode 100644 internal/simulation/os_linux.go create mode 100644 internal/simulation/os_other.go create mode 100644 internal/simulation/simulation.go create mode 100644 internal/simulation/syscallabi/errno.go create mode 100644 internal/simulation/syscallabi/poll.go create mode 100644 internal/simulation/syscallabi/syscall.go create mode 100644 internal/simulation/userspace.go create mode 100644 internal/testing/match.go create mode 100644 internal/testing/missing.go create mode 100644 internal/testing/testing.go create mode 100644 internal/tests/Makefile create mode 100644 internal/tests/behavior/chan_test.go create mode 100644 internal/tests/behavior/context_test.go create mode 100644 internal/tests/behavior/crash_meta_test.go create mode 100644 internal/tests/behavior/crash_test.go create mode 100644 internal/tests/behavior/disk_crash_test.go create mode 100644 internal/tests/behavior/disk_sim_test.go create mode 100644 internal/tests/behavior/disk_test.go create mode 100644 internal/tests/behavior/globals.go create mode 100644 internal/tests/behavior/globals_for_test.go create mode 100644 internal/tests/behavior/globals_test.go create mode 100644 internal/tests/behavior/log_meta_test.go create mode 100644 internal/tests/behavior/log_test.go create mode 100644 internal/tests/behavior/machine_meta_test.go create mode 100644 internal/tests/behavior/machine_test.go create mode 100644 internal/tests/behavior/map_test.go create mode 100644 internal/tests/behavior/meta_test.go create mode 100644 internal/tests/behavior/nemesis_meta_test.go create mode 100644 internal/tests/behavior/nemesis_test.go create mode 100644 internal/tests/behavior/net_test.go create mode 100644 internal/tests/behavior/os_sim_test.go create mode 100644 internal/tests/behavior/os_test.go create mode 100644 internal/tests/behavior/rand_meta_test.go create mode 100644 internal/tests/behavior/rand_test.go create mode 100644 internal/tests/behavior/rand_v2_test.go create mode 100644 internal/tests/behavior/reflect_test.go create mode 100644 internal/tests/behavior/sync_test.go create mode 100644 internal/tests/behavior/testing_meta_test.go create mode 100644 internal/tests/behavior/testing_test.go create mode 100644 internal/tests/behavior/time_test.go create mode 100644 internal/tests/race/race_test.go create mode 100644 internal/tests/race/testdata/annotate_test.go create mode 100644 internal/tests/race/testdata/atomic_test.go create mode 100644 internal/tests/race/testdata/chan_test.go create mode 100644 internal/tests/race/testdata/io_test.go create mode 100644 internal/tests/race/testdata/map_test.go create mode 100644 internal/tests/race/testdata/mutex_test.go create mode 100644 internal/tests/race/testdata/os_test.go create mode 100644 internal/tests/race/testdata/race_test.go create mode 100644 internal/tests/race/testdata/rwmutex_test.go create mode 100644 internal/tests/race/testdata/select_test.go create mode 100644 internal/tests/race/testdata/sync_test.go create mode 100644 internal/tests/race/testdata/time_simonly_test.go create mode 100644 internal/tests/race/testdata/time_test.go create mode 100644 internal/tests/race/testdata/waitgroup_test.go create mode 100644 internal/tests/script/script_test.go create mode 100644 internal/tests/script/testdata/gosimpassfail.txtar create mode 100644 internal/tests/script/testdata/simmetatest.txtar create mode 100644 internal/tests/script/testdata/testbuilding.txtar create mode 100644 internal/tests/testpb.proto create mode 100644 internal/tests/testpb/testpb.pb.go create mode 100644 internal/tests/testpb/testpb_grpc.pb.go create mode 100644 internal/translate/cache.go create mode 100644 internal/translate/cache/cache.go create mode 100644 internal/translate/cache/cache_test.go create mode 100644 internal/translate/chan.go create mode 100644 internal/translate/globals.go create mode 100644 internal/translate/globals_analysis.go create mode 100644 internal/translate/globals_analysis_ssa.go create mode 100644 internal/translate/go.go create mode 100644 internal/translate/hooks_go123.go create mode 100644 internal/translate/hooks_go123_amd64_gensyscall.go create mode 100644 internal/translate/hooks_go123_arm64_gensyscall.go create mode 100644 internal/translate/main.go create mode 100644 internal/translate/map.go create mode 100644 internal/translate/outputwriter.go create mode 100644 internal/translate/rewrites.go create mode 100644 internal/translate/testdata/basic.translate.txt create mode 100644 internal/translate/testdata/comment.translate.txt create mode 100644 internal/translate/testdata/directives.translate.txt create mode 100644 internal/translate/testdata/packages.translate.txt create mode 100644 internal/translate/testdata/select.translate.txt create mode 100644 internal/translate/testdata/slog.translate.txt create mode 100644 internal/translate/testdata/test.translate.txt create mode 100644 internal/translate/tests.go create mode 100644 internal/translate/translate.go create mode 100644 internal/translate/translate_test.go create mode 100644 internal/translate/types.go create mode 100644 internal/translate/workqueue.go create mode 100644 machine.go create mode 100644 metatesting/doc.go create mode 100644 metatesting/metatest.go create mode 100644 metatesting/metatest_test.go create mode 100644 nemesis/meta_test.go create mode 100644 nemesis/nemesis.go create mode 100644 nemesis/nemesis_test.go create mode 100644 simulation.go create mode 100755 test.sh diff --git a/.ci/crossarch-tests/Dockerfile.test b/.ci/crossarch-tests/Dockerfile.test new file mode 100644 index 0000000..44e7ab3 --- /dev/null +++ b/.ci/crossarch-tests/Dockerfile.test @@ -0,0 +1,13 @@ +# XXX: parametrize somehow? +FROM golang:1.23 + +ENV GOCACHE=/cache/GOCACHE GOMODCACHE=/cache/GOMODCACHE GOSIMCACHE=/cache/GOSIMCACHE + +WORKDIR /go/src/ +COPY . /go/src/ + +# clean up .gosim/ if copied +RUN rm -rf .gosim/ + +# run with limited concurrency to limit memory usage +CMD ./test.sh --concurrency 1 diff --git a/.ci/crossarch-tests/docker-compose.yml b/.ci/crossarch-tests/docker-compose.yml new file mode 100644 index 0000000..b33b01b --- /dev/null +++ b/.ci/crossarch-tests/docker-compose.yml @@ -0,0 +1,21 @@ +services: + test-linux-arm64: + platform: linux/amd64 + build: + context: ../.. + dockerfile: ./.ci/crossarch-tests/Dockerfile.test + volumes: + - cache:/cache + + test-linux-amd64: + platform: linux/amd64 + build: + context: ../.. + dockerfile: ./.ci/crossarch-tests/Dockerfile.test + volumes: + - cache:/cache + +volumes: + cache: + + diff --git a/.ci/crossarch-tests/test-amd64.sh b/.ci/crossarch-tests/test-amd64.sh new file mode 100755 index 0000000..40c2239 --- /dev/null +++ b/.ci/crossarch-tests/test-amd64.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd "${0%/*}" +set -e + +docker compose -f ./docker-compose.yml run --build --rm test-linux-amd64 diff --git a/.ci/crossarch-tests/test-arm64.sh b/.ci/crossarch-tests/test-arm64.sh new file mode 100755 index 0000000..9b0086d --- /dev/null +++ b/.ci/crossarch-tests/test-arm64.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd "${0%/*}" +set -e + +docker compose -f ./docker-compose.yml run --build --rm test-linux-arm64 diff --git a/.ci/gofmt b/.ci/gofmt new file mode 100755 index 0000000..b6da387 --- /dev/null +++ b/.ci/gofmt @@ -0,0 +1,28 @@ +#!/bin/bash +# gofmt checks that code is formatted with gofumpt and sorted inputs + +gofiles=$(find . -name "*.go" -and -not -ipath "./gosimout/*" -exec sh -c 'head -1 {} | grep -q "DO NOT EDIT." || echo {}' \;) +goimports="go run golang.org/x/tools/cmd/goimports -local=github.com/jellevandenhooff/gosim -format-only" +gofumpt="go run mvdan.cc/gofumpt" + +if [[ $1 == "check" ]]; then + echo goimports + output=$($goimports -l $gofiles) + if [[ -n "$output" ]]; then + echo $output + exit 1 + fi + echo gofumpt + output=$($gofumpt -l $gofiles) + if [[ -n "$output" ]]; then + echo $output + exit 1 + fi +elif [[ $1 == "fix" ]]; then + echo goimports + $goimports -w $gofiles + echo gofumpt + $gofumpt -w $gofiles +else + echo "usage $0 [check|fix]" +fi diff --git a/.ci/gosim b/.ci/gosim new file mode 100755 index 0000000..d5a17c8 --- /dev/null +++ b/.ci/gosim @@ -0,0 +1,3 @@ +#!/bin/sh +# gosim is shorthand for running cmd/gosim: +go run github.com/jellevandenhooff/gosim/cmd/gosim "$@" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..8f9f2eb --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + # run tests with -trimpath to hopefully share cache between gosim + # invocations in different folders + - name: Test + run: GOFLAGS=-trimpath ./test.sh --concurrency 1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e2aba58 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.gosim/ +next.txt diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..c38e65b --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Connect to dlv-dap server on localhost:2345", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "${workspaceFolder}", + "port": 2345, + "host": "127.0.0.1", + "debugAdapter": "dlv-dap", + "stopOnEntry": true, + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ce38db4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,13 @@ +{ + "gopls": { + "build.env": { + "GOOS": "linux", + }, + "formatting.local": "github.com/jellevandenhooff/gosim", + "formatting.gofumpt": true + }, + "go.buildFlags": [ + "-tags=sim,linkname", + "-ldflags=-checklinkname=0", + ], +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f44134f --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2024 Jelle van den Hooff + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..2acbf02 --- /dev/null +++ b/README.md @@ -0,0 +1,316 @@ +# Gosim: Simulation testing for Go + +Gosim is a simulation testing framework that aims to make it easier to write +reliable distributed systems code in go. Gosim takes standard go programs and +tests and runs them with a simulated network, filesystem, multiple machines, and +more. + +In Gosim's simulated environment you can run an entire distributed system in a +single go process, without having to set up any real servers. The simulation can +also introduce all kinds of problems, such as flaky networks, restarting +machines, or lost writes on a filesystem, to test that the program survive those +problems before they happen in real life. + +An interesting and entertaining introduction to simulation testing is the talk +[Testing Distributed Systems w/ Deterministic Simulation from Strangeloop +2014](https://www.youtube.com/watch?v=4fFDFbi3toc). + +Gosim is an experimental project. Feedback, suggestions, and ideas are all very +welcome. The underlying design feels quite solid, but the implemented and +simulated APIs are still limited: files work, but not directories; TCP works, +but not UDP; IP addresses work, but not hostnames. The API can and will change. +Now that those warnings are out of the way, please take a look at what gosim +can do. + +# Using Gosim + +## From go test to gosim test +Gosim tests standard go code and is used just like another go package. To get +started with `gosim`, you need to import it in your module: +``` +> mkdir example && cd example +> go mod init example +> go get github.com/jellevandenhooff/gosim +``` +To test code with Gosim, write a small a test in a file `simple_test.go` and then run +it using `go test`: +```go +package example_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim" +) + +func TestGosim(t *testing.T) { + t.Logf("Are we in the Matrix? %v", gosim.IsSim()) +} +``` +``` +> go test -v -run TestGosim +=== RUN TestGosim + simple_test.go:10: Are we in the Matrix? false +--- PASS: TestGosim (0.00s) +PASS +ok example 0.216s +``` +To run this test with `gosim` instead, replace `go test` with +`go run github.com/jellevandenhooff/gosim/cmd/gosim test`: +``` +> go run github.com/jellevandenhooff/gosim/cmd/gosim test -v -run TestGosim +=== RUN TestGosim + 1 main/4 14:10:03.000 INF example/simple_gosim_test.go:11 > Are we in the Matrix? true method=t.Logf +--- PASS: TestGosim (0.00s) +ok translated/example 0.204s +``` +The `gosim test` command has flags similar to `go test`. The test output is +more involved than a normal `go test`. Every log line includes the simulated +machine and the goroutine that invoked the log to help debug tests running on +multiple machines. + +If running gosim fails with errors about missing `go.sum` entries, run +`go mod tidy` to update your `go.sum` file. + +## Simulation +Tests running in Gosim run inside Gosim's simulation environment. In the +simulation tests can create simulated machines that can talk to eachother +over a simulated network, crash and restart machine, introduce latency, +and more. The following longer example shows some of those features: +```go +package example_test + +import ( + "fmt" + "io" + "log" + "net/http" + "net/netip" + "testing" + "time" + + "github.com/jellevandenhooff/gosim" +) + +var count = 0 + +func server() { + log.Printf("starting server") + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + count++ + log.Printf("got a request from %s", r.RemoteAddr) + fmt.Fprintf(w, "hello from the server! request: %d", count) + }) + http.ListenAndServe("10.0.0.1:80", nil) +} + +func request() { + log.Println("making a request") + resp, err := http.Get("http://10.0.0.1/") + if err != nil { + log.Println(err) + return + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + log.Println(err) + return + } + + log.Println(string(body)) +} + +func TestMachines(t *testing.T) { + // run the server + serverMachine := gosim.NewMachine(gosim.MachineOpts{ + Label: "server", + Addr: netip.MustParseAddr("10.0.0.1"), + MainFunc: server, + }) + + // let the server start + time.Sleep(time.Second) + + // make some requests to see them work + request() + request() + + // restart the server + log.Println("restarting the server") + serverMachine.Crash() + serverMachine.Restart() + time.Sleep(time.Second) + + // make a new request to see a reset count + request() + + // add some latency + log.Println("adding latency") + gosim.SetDelay("10.0.0.1", "11.0.0.1", time.Second) + + // make another request to see the latency + request() +} +``` +This example contains a HTTP server running on its own gosim machine, and a +client that makes several requests to the server. Then the client +messes with the server, restarting it and adding latency. Let's see +what happens this test is run: + +``` +> go run github.com/jellevandenhooff/gosim/cmd/gosim test -v -run TestMachines +=== RUN TestMachines + 1 server/5 14:10:03.000 INF example/machines_gosim_test.go:17 > starting server + 2 main/4 14:10:04.000 INF example/machines_gosim_test.go:27 > making a request + 3 server/8 14:10:04.000 INF example/machines_gosim_test.go:20 > got a request from 11.0.0.1:10000 + 4 main/4 14:10:04.000 INF example/machines_gosim_test.go:41 > hello from the server! request: 1 + 5 main/4 14:10:04.000 INF example/machines_gosim_test.go:27 > making a request + 6 server/8 14:10:04.000 INF example/machines_gosim_test.go:20 > got a request from 11.0.0.1:10000 + 7 main/4 14:10:04.000 INF example/machines_gosim_test.go:41 > hello from the server! request: 2 + 8 main/4 14:10:04.000 INF example/machines_gosim_test.go:60 > restarting the server + 9 server/13 14:10:04.000 INF example/machines_gosim_test.go:17 > starting server + 10 main/4 14:10:05.000 INF example/machines_gosim_test.go:27 > making a request + 11 server/16 14:10:05.000 INF example/machines_gosim_test.go:20 > got a request from 11.0.0.1:10001 + 12 main/4 14:10:05.000 INF example/machines_gosim_test.go:41 > hello from the server! request: 1 + 13 main/4 14:10:05.000 INF example/machines_gosim_test.go:69 > adding latency + 14 main/4 14:10:05.000 INF example/machines_gosim_test.go:27 > making a request + 15 server/16 14:10:06.000 INF example/machines_gosim_test.go:20 > got a request from 11.0.0.1:10001 + 16 main/4 14:10:07.000 INF example/machines_gosim_test.go:41 > hello from the server! request: 2 +--- PASS: TestMachines (4.00s) +ok translated/example 0.237s +``` +In the log, each line is annotated with the machine and goroutine that wrote +the log. When debugging code running on multiple machines, it is helpful to +know where logs are coming from. + +The first thing that happens is the server starting at step 1. Then the client +makes a request at step 2, the server receives it, responds, and the client +prints the response. When the client makes another request the server this +time responds with a higher count at steps 6 and 7. + +Afterwards, the client restarts the server at step 8 and the server prints that +is starting again at step 9. When the client makes a request after the restart +the server responds once again with a count of 1 at steps 11 and 12. How did +that happen? After the server is restarted, the counter resets to zero. Each +machine has its own copy of global variables which get re-initialized when a +machine crashes. + +Notice that the client address (after "got a request from:" in the logs) also +changes after the restart, and the goroutine handling the requests on the server +changes. The simulated TCP connection breaks when the machine crashes and the +`net/http` client makes a new connection afterwards. + +After the restart, the client calls a Gosim API to add latency to the simulated network +at step 13. The earlier requests all got sent and received at the same timestamp, +such as on steps 10, 11, and 12 all at 14:10:05. The request after adding +latency is sent at 14:10:05 on step 14 and received a second later at 14:10:06 +on step 15. This is the added latency in action. + +Even though the test takes over 4 simulated seconds, running the tests takes +less than a second. Simulated time can be sped up when Gosim knows that all +goroutines are waiting for time advance. With simulated time, tests can use +realistic latency and timeouts without development time becoming slow. + +## Debugging +Gosim's simulation is deterministic, which means that running a test twice +will result in the exact same behavior, from randomly generated numbers +to goroutine concurrency interleavings. That is cool because it means that +if we see an interesting log after running the program but do not fully +understand why that log printed a certain value, we can re-run the program +and see exactly what happened at the moment of printing. + +The numbers at the start of each log are Gosim's step numbers. Take step 11 from +the log above, from the line "got a request ...". We can debug the state of the +program at the time the log was printed with `gosim debug`: +``` +> go run github.com/jellevandenhooff/gosim/cmd/gosim debug -package=. -test=TestMachines -step=11 +... + 26: func (w gosimSlogHandler) Handle(ctx context.Context, r slog.Record) error { + 27: r.AddAttrs(slog.Int("goroutine", gosimruntime.GetGoroutine())) +=> 28: r.AddAttrs(slog.Int("step", gosimruntime.Step())) + 29: return w.inner.Handle(ctx, r) + 30: } +... +``` +The `gosim debug` command runs the requested test inside of the Delve go +debugger and stops the test at the requested step. The debugger here puts us in +Gosim's `log/slog` handler which calls `gosimruntime.Step()` and includes the +step number. + +Now we can debug the program as a standard Go program. To see what was happening +in the HTTP handler that called print, we run backtrace +``` +(dlv) bt + 0 0x000000010129e7ac in translated/github.com/jellevandenhooff/gosim/internal_/simulation.gosimSlogHandler.Handle + at ./github.com/jellevandenhooff/gosim/internal_/simulation/userspace_gosim.go:28 + 1 0x00000001012a5e24 in translated/github.com/jellevandenhooff/gosim/internal_/simulation.(*gosimSlogHandler).Handle + at :1 + 2 0x0000000101283d68 in translated/log/slog.(*handlerWriter).Write + at ./log/slog/logger_gosim.go:99 + 3 0x000000010124a600 in translated/log.(*Logger).output + at ./log/log_gosim.go:242 + 4 0x000000010124ae38 in translated/log.Printf + at ./log/log_gosim.go:394 + 5 0x000000010160e03c in translated/example_test.server.func1 + at ./example/machines_gosim_test.go:20 +... +``` +and then select the HTTP handler's stack frame that called `log.Printf` +``` +(dlv) frame 5 +> translated/github.com/jellevandenhooff/gosim/internal_/simulation.gosimSlogHandler.Handle() ./github.com/jellevandenhooff/gosim/internal_/simulation/userspace_gosim.go:28 (PC: 0x1029ce45c) +Frame 5: ./example/machines_gosim_test.go:19 (PC: 102d3decc) + 14: ) + 15: + 16: func server() { + 17: http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + 18: G().count++ +=> 19: log.Printf("got a request from %s", r.RemoteAddr) + 20: fmt.Fprintf(w, "hello from the server! request: %d", G().count) + 21: }) + 22: http.ListenAndServe("10.0.0.1:80", nil) + 23: } + 24: +``` +Now we can inspect the state of memory to see details that the log line might have missed +``` +(dlv) print r.RemoteAddr +"11.0.0.1:10001" +``` + +# API and documentation + +A description of Gosim's architecture and design decisions is in +[docs/design.md](docs/design.md). + +Gosim's public packages are: + +- [github.com/jellevandenhooff/gosim](./): the core API for creating machines + and manipulating the simulation environment + +- [github.com/jellevandenhooff/gosim/cmd/gosim](./cmd/gosim): the CLI for + running Gosim tests + +- [github.com/jellevandenhooff/gosim/metatesting](./metatesting/): a package + for running Gosim tests inside of normal go test. + +- [github.com/jellevandenhooff/gosim/nemesis](./nemesis/): a package, still + sparse, to introduce chaos into simulations + +# Development + +Gosim and its tools are tested using the `./test.sh` script, which invokes the +tests defined in `./Taskfile.yml`. These test the Gosim tooling as well as the +behavior of simulated code. Ideally tests verify that the simulation behaves +like reality: The tests for the filesystem in +[github.com/jellevandenhooff/gosim/internal/tests/behavior/disk_test.go](./internal/tests/behavior/disk_test.go) +are run in both simulated and non-simulated builds. + +Gosim simulates Linux, but should work on macOS, Linux, and Windows (not tested +on Windows) running on either arm64 or amd64. To test that code works on Linux +amd64 and arm64, there are scripts `.ci/crossarch-tests/test-amd64.sh` and +`.ci/crossarch-tests/test-arm64.sh` that run the tests using Docker. A Github +Action runs the tests on Linux amd64. + diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..0b55933 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,57 @@ +version: '3' + +output: prefixed + +vars: + # GOSIM: go run github.com/jellevandenhooff/gosim/cmd/gosim + GOSIM: $(pwd)/.gosim/gosimtool + SIMPACKAGES: ./internal/tests/behavior ./nemesis + +tasks: + fix-fmt: + cmds: + - .ci/gofmt fix + + check-fmt: + cmds: + - .ci/gofmt check + + go-generate: + cmds: + - go generate ./... + + gosim-tool: + run: once + cmds: + - mkdir -p ./.gosim && go build -o ./.gosim/gosimtool ./cmd/gosim + + gosim-prepare-selftest: + run: once + cmds: + - "{{.GOSIM}} prepare-selftest" + deps: + - gosim-tool + + go-test: + cmds: + - "{{.GOSIM}} build-tests {{.SIMPACKAGES}}" + - go test -ldflags=-checklinkname=0 -tags=linkname ./... + deps: + - gosim-tool + - gosim-prepare-selftest + + go-race-test: + cmds: + - "{{.GOSIM}} build-tests -race {{.SIMPACKAGES}} ./internal/tests/race/testdata" + # todo: get rid of this janky thing??? + - go test -ldflags=-checklinkname=0 -tags=linkname -race ./... + deps: + - gosim-tool + - gosim-prepare-selftest + + test: + deps: + - check-fmt + - go-test + - go-race-test + diff --git a/cmd/gosim/doc.go b/cmd/gosim/doc.go new file mode 100644 index 0000000..6b957e8 --- /dev/null +++ b/cmd/gosim/doc.go @@ -0,0 +1,60 @@ +/* +Gosim is a tool for working with gosim simulation testing framework. + +Usage: gosim [arguments] + +The commands are: + + test test packages + debug debug a test in a package + build-tests build packages for metatesting + translate translate packages + help print this help + +The 'translate' command: + +Usage: gosim translate [-race] [packages] + +The translate command translates packages without running the resulting code. +The output will be placed in /.gosim/translated. Translation +is cached by package, so re-running translate after modifying some files +in the current package should be fast. + +Packages should be listed as if they were arguments to 'go test' or 'go build' +command. All listed packages must be part of the current module. The current +module must have a dependency on gosim by importing +'github.com/jellevandenhooff/gosim' somewhere in the code. + +Translate translates code with specific build flags sets. The GOOS is fixed to +linux, and the GOARCH is the one used to compile translate. + +The -race flag translates all code with the race build tag set. + +The 'test' command: + +Usage: gosim test [-race] [-run=...] [-v] [packages] + +The test command translates and runs tests for the specified packages. It first +invokes translate, and then invokes 'go test' on the translated code, passing +through the -run and -v flags. + +The 'debug' command: + +Usage: gosim debug [-race] [-headless] -package=[package] -test=[test] -step=[step] + +The debug command translates and runs a specific test using the delve debugger. +It first invokes translate, and then runs 'dlv test' on the specific test. The +-step flag is the step to pause at as seen in the logs from running 'gosim test'. + +The -headless flag optionally runs delve in headless mode for use with an +external interface like an IDE. + +The 'build-tests' command: + +Usage: gosim build-tests [-race] [packages] + +The build-tests command translates and then builds tests for use with the +metatesting package. Metatesting in a cached go test run requires pre-building +tests; see the metatesting package documentation for details. +*/ +package main diff --git a/cmd/gosim/imports.go b/cmd/gosim/imports.go new file mode 100644 index 0000000..45fe621 --- /dev/null +++ b/cmd/gosim/imports.go @@ -0,0 +1,9 @@ +//go:build never + +package main + +import ( + // Janky plan to ensure that a tools import of the CLI also pulls in + // everything else + _ "github.com/jellevandenhooff/gosim" +) diff --git a/cmd/gosim/main.go b/cmd/gosim/main.go new file mode 100644 index 0000000..fde1d69 --- /dev/null +++ b/cmd/gosim/main.go @@ -0,0 +1,501 @@ +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/translate" +) + +const doc = `Gosim is a tool for working with gosim simulation testing framework. + +Usage: gosim [arguments] + +The commands are: + + test test packages + debug debug a test in a package + build-tests build packages for metatesting + translate translate packages + help print this help + +The 'translate' command: + +Usage: gosim translate [-race] [packages] + +The translate command translates packages without running the resulting code. +The output will be placed in /.gosim/translated. Translation +is cached by package, so re-running translate after modifying some files +in the current package should be fast. + +Packages should be listed as if they were arguments to 'go test' or 'go build' +command. All listed packages must be part of the current module. The current +module must have a dependency on gosim by importing +'github.com/jellevandenhooff/gosim' somewhere in the code. + +Translate translates code with specific build flags sets. The GOOS is fixed to +linux, and the GOARCH is the one used to compile translate. + +The -race flag translates all code with the race build tag set. + +The 'test' command: + +Usage: gosim test [-race] [-run=...] [-v] [packages] + +The test command translates and runs tests for the specified packages. It first +invokes translate, and then invokes 'go test' on the translated code, passing +through the -run and -v flags. + +The 'debug' command: + +Usage: gosim debug [-race] [-headless] -package=[package] -test=[test] -step=[step] + +The debug command translates and runs a specific test using the delve debugger. +It first invokes translate, and then runs 'dlv test' on the specific test. The +-step flag is the step to pause at as seen in the logs from running 'gosim test'. + +The -headless flag optionally runs delve in headless mode for use with an +external interface like an IDE. + +The 'build-tests' command: + +Usage: gosim build-tests [-race] [packages] + +The build-tests command translates and then builds tests for use with the +metatesting package. Metatesting in a cached go test run requires pre-building +tests; see the metatesting package documentation for details. +` + +func commandName(cmd string) string { + return fmt.Sprintf("%s %s", path.Base(os.Args[0]), cmd) +} + +func packageName(pkgPath string) string { + return pkgPath[strings.LastIndex(pkgPath, "/")+1:] +} + +func batchPackagesWithDifferentNames(packages []string) [][]string { + var groups [][]string + count := make(map[string]int) + + for _, pkg := range packages { + lastName := packageName(pkg) + group := count[lastName] + count[lastName]++ + if group >= len(groups) { + groups = append(groups, nil) + } + groups[group] = append(groups[group], pkg) + } + + return groups +} + +func hashFile(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +func copyFile(from, to string, perm fs.FileMode) error { + src, err := os.Open(from) + if err != nil { + return err + } + defer src.Close() + dst, err := os.OpenFile(to, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) + if err != nil { + return err + } + defer dst.Close() + if _, err := io.Copy(dst, src); err != nil { + return err + } + return nil +} + +func copyFileIfChanged(src, dst string, perm fs.FileMode) error { + // skip unchanged so we keep the old modified timestamp + dstHash, err := hashFile(dst) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if dstHash != nil { + srcHash, err := hashFile(src) + if err != nil { + return err + } + if bytes.Equal(srcHash, dstHash) { + // skip because unchanged, maybe log? + return nil + } + } + + if err := copyFile(src, dst, perm); err != nil { + return err + } + + return nil +} + +func writeFileIfDifferent(b []byte, dst string, perm fs.FileMode) error { + // skip unchanged so we keep the old modified timestamp + dstBytes, err := os.ReadFile(dst) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + if bytes.Equal(b, dstBytes) { + // skip because unchanged, maybe log? + return nil + } + + if err := os.WriteFile(dst, b, perm); err != nil { + return err + } + + return nil +} + +const toolsgoTemplate = `//go:build never + +package main + +import ( + _ "` + gosimtool.Module + `" +) +` + +func main() { + flag.Usage = func() { + fmt.Printf(doc) + } + flag.Parse() + + if len(flag.Args()) < 1 { + flag.Usage() + os.Exit(2) + } + cmd := flag.Args()[0] + cmdArgs := flag.Args()[1:] + + cfg := gosimtool.BuildConfig{ + GOOS: "linux", + GOARCH: runtime.GOARCH, + Race: false, + } + + switch cmd { + case "translate": + translateflags := flag.NewFlagSet(commandName("translate"), flag.ExitOnError) + race := translateflags.Bool("race", false, "build in -race mode") + translateflags.Parse(cmdArgs) + + cfg.Race = *race + + _, err := translate.Translate(&translate.TranslateInput{ + Packages: translateflags.Args(), + Cfg: cfg, + }) + if err != nil { + log.Fatal(err) + } + + case "test": + testflags := flag.NewFlagSet(commandName("test"), flag.ExitOnError) + verbose := testflags.Bool("v", false, "verbose output") + race := testflags.Bool("race", false, "build in -race mode") + run := testflags.String("run", "", "tests to run (as in go test -run)") + logformat := testflags.String("logformat", "pretty", "gosim log formatting: raw|indented|pretty") + testflags.Parse(cmdArgs) + + cfg.Race = *race + + packages := testflags.Args() + if len(packages) == 0 { + packages = []string{"."} + } + + output, err := translate.Translate(&translate.TranslateInput{ + Packages: packages, + Cfg: cfg, + }) + if err != nil { + log.Fatal(err) + } + + name := "go" + args := []string{"test"} + + // TODO: only for go1.23? + args = append(args, "-ldflags=-checklinkname=0", "-tags=linkname") + if *verbose { + args = append(args, "-v") + } + if *race { + args = append(args, "-race") + } + args = append(args, "-trimpath") + if *run != "" { + args = append(args, "-run", *run) + } + args = append(args, output.Packages...) + if *logformat != "pretty" { + // TODO: configure this default somewhere? + args = append(args, "-logformat", *logformat) + } + + cmd := exec.Command(name, args...) + cmd.Env = append(os.Environ(), "FORCE_COLOR=1") + cmd.Dir = output.RootOutputDir + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + log.Fatal(err) + } + + case "build-tests": + // do we supply packages or does it just work? + // for now, let's supply packages... + + modDir, err := gosimtool.FindGoModDir() + if err != nil { + log.Fatal(err) + } + + testflags := flag.NewFlagSet(commandName("build-tests"), flag.ExitOnError) + race := testflags.Bool("race", false, "build in -race mode") + testflags.Parse(cmdArgs) + + cfg.Race = *race + + output, err := translate.Translate(&translate.TranslateInput{ + Packages: testflags.Args(), + Cfg: cfg, + }) + if err != nil { + log.Fatal(err) + } + + // tests are named after the last part of the directory, even if they + // have a different internal name (nice.) + + groups := batchPackagesWithDifferentNames(output.Packages) + + // log.Println(groups) + + dstDir := filepath.Join(modDir, gosimtool.OutputDirectory, "metatest", cfg.AsDirname()) + if err := os.MkdirAll(dstDir, 0o755); err != nil { + log.Fatal(err) + } + + buildDir, err := os.MkdirTemp("", "gosimbuild") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(buildDir) + + for _, group := range groups { + // copy over existing binaries to make it faster + for _, pkg := range group { + binName := packageName(pkg) + ".test" + src := filepath.Join(dstDir, gosimtool.PreparedTestBinName(pkg)) + dst := filepath.Join(buildDir, binName) + if err := copyFile(src, dst, 0o755); err != nil { + if errors.Is(err, os.ErrNotExist) { + // ignore missing source files, this is just an optimization + continue + } + log.Fatal(err) + } + } + + // TODO: only for go1.23? + name := "go" + args := []string{"test"} + args = append(args, "-ldflags=-checklinkname=0", "-tags=linkname") + if *race { + args = append(args, "-race") + } + args = append(args, "-trimpath") + + args = append(args, "-c") + args = append(args, "-o", buildDir) + + args = append(args, group...) + + cmd := exec.Command(name, args...) + cmd.Dir = output.RootOutputDir + + // TODO: consider if we want to do this? + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + log.Fatal(err) + } + + for _, pkg := range group { + binName := packageName(pkg) + ".test" + src := filepath.Join(buildDir, binName) + dst := filepath.Join(dstDir, gosimtool.PreparedTestBinName(pkg)) + + // skip unchanged so we keep the old modified timestamp + if err := copyFileIfChanged(src, dst, 0o755); err != nil { + log.Fatal(err) + } + + depBytes, err := json.MarshalIndent(output.Deps[pkg], "", " ") // log.Println(binName, pkg, output.Deps[pkg]) + if err != nil { + log.Fatal(err) + } + + if err := writeFileIfDifferent(depBytes, filepath.Join(dstDir, gosimtool.PreparedTestInfoName(pkg)), 0o644); err != nil { + log.Fatal(err) + } + } + } + + case "debug": + // TODO: make -headless flag write launch configuration? + // TODO: for -headless, make ctrl-c work? + // TODO: for -headless, use delve api to send initial continue? + + debugflags := flag.NewFlagSet(commandName("debug"), flag.ExitOnError) + race := debugflags.Bool("race", false, "build in -race mode") + pkg := debugflags.String("package", "", "package path to debug") + test := debugflags.String("test", "", "full test name to debug") + step := debugflags.Int("step", 0, "step to break at") + headless := debugflags.Bool("headless", false, "run headless for IDE debugging") + debugflags.Parse(cmdArgs) + + modDir, err := gosimtool.FindGoModDir() + if err != nil { + log.Fatal(err) + } + + cfg.Race = *race + + output, err := translate.Translate(&translate.TranslateInput{ + Packages: []string{*pkg}, + Cfg: cfg, + }) + if err != nil { + log.Fatal(err) + } + + if len(output.Packages) != 1 { + log.Fatalf("expected 1 output packages, got %v", output.Packages) + } + translated := output.Packages[0] + + script := fmt.Sprintf(`continue +stepout`) + scriptPath := filepath.Join(modDir, gosimtool.OutputDirectory, "debug-script") + if err := os.WriteFile(scriptPath, []byte(script), 0o644); err != nil { + log.Fatal(err) + } + + name := "dlv" + flags := []string{ + "test", + translated, + "--build-flags=-ldflags=-checklinkname=0 -tags=linkname", + // TODO: does this actually set linkname? + // TODO: pass on -race + } + + if *headless { + flags = append(flags, + "--listen=:2345", + "--accept-multiclient", + "--headless", + ) + log.Printf("running delve in headless mode, connect with vscode using a launch configuration (.vscode/launch.json) like:\n" + + `{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Connect to dlv-dap server on localhost:2345", + "type": "go", + "request": "attach", + "mode": "remote", + "remotePath": "${workspaceFolder}", + "port": 2345, + "host": "127.0.0.1", + "debugAdapter": "dlv-dap", + "stopOnEntry": true, + } + ] +}`) + } else { + flags = append(flags, + "--init="+scriptPath, + ) + log.Println("running delve in the terminal...") + } + + flags = append(flags, + "--", + "-test.run", + "^"+*test+"$", + "-test.v", + "-step-breakpoint="+fmt.Sprint(*step), + ) + + cmd := exec.Command(name, flags...) + cmd.Dir = path.Join(modDir, gosimtool.OutputDirectory, "translated", cfg.AsDirname()) + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + log.Println(cmd.Args) + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + log.Fatal(err) + } + + case "prepare-selftest": + prepareSelftest() + + default: + flag.Usage() + os.Exit(2) + } +} + +// TODO: test with and without gosim dependency? (script test?) diff --git a/cmd/gosim/main_test.go b/cmd/gosim/main_test.go new file mode 100644 index 0000000..67e4a5b --- /dev/null +++ b/cmd/gosim/main_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestGroup(t *testing.T) { + grouped := batchPackagesWithDifferentNames([]string{ + "hello", + "github.com/bar", + "github.com/foo/baz", + "github.com/foo/bar", + "github.com/foo/hello", + "github.com/ok", + "goodbye/hello", + }) + + if diff := cmp.Diff(grouped, [][]string{ + { + "hello", + "github.com/bar", + "github.com/foo/baz", + "github.com/ok", + }, + { + "github.com/foo/bar", + "github.com/foo/hello", + }, + { + "goodbye/hello", + }, + }); diff != "" { + t.Error(diff) + } +} diff --git a/cmd/gosim/selftest.go b/cmd/gosim/selftest.go new file mode 100644 index 0000000..5e11947 --- /dev/null +++ b/cmd/gosim/selftest.go @@ -0,0 +1,168 @@ +package main + +import ( + "errors" + "io/fs" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + + "golang.org/x/tools/go/packages" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/translate" +) + +// prepareSelftest prepares for running tests in the gosim module. +func prepareSelftest() { + modDir, err := gosimtool.FindGoModDir() + if err != nil { + log.Fatal(err) + } + + buildDir, err := os.MkdirTemp("", "gosimbuild") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(buildDir) + + // Prepare the racetest binary + raceBuildPath := filepath.Join(buildDir, "testdata.test") + raceFinalPath := filepath.Join(modDir, gosimtool.OutputDirectory, "racetest", "testdata.test") + if err := os.MkdirAll(filepath.Dir(raceFinalPath), 0o755); err != nil { + log.Fatal(err) + } + // try to reuse the old binary + if err := copyFile(raceFinalPath, raceBuildPath, 0o755); err != nil { + if !errors.Is(err, os.ErrNotExist) { + log.Fatalf("copying source: %s", err) + } + } + // do the build + name := "go" + args := []string{"test"} + args = append(args, "-ldflags=-checklinkname=0", "-tags=linkname") + args = append(args, "-race") + args = append(args, "-c") + args = append(args, "-o", buildDir) + args = append(args, "./internal/tests/race/testdata") + cmd := exec.Command(name, args...) + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + log.Fatal(err) + } + // place the binary + if err := copyFileIfChanged(raceBuildPath, raceFinalPath, 0o755); err != nil { + log.Fatalf("copying final: %s", err) + } + + // Prepare the gosim binary + // TODO: should we have a -race version as well? + gosimBuildPath, err := os.Executable() + if err != nil { + log.Fatal(err) + } + gosimFinalPath := filepath.Join(modDir, gosimtool.OutputDirectory, "scripttest", "bin", "gosim") + if err := os.MkdirAll(filepath.Dir(gosimFinalPath), 0o755); err != nil { + log.Fatal(err) + } + if err := copyFileIfChanged(gosimBuildPath, gosimFinalPath, 0o755); err != nil { + log.Fatalf("copying final: %s", err) + } + + // Prepare the module + copyModuleForScriptTest(modDir) +} + +// copyModuleForScriptTest copies the source code of this module to a slimmed-down +// version for the script tests in ./internal/tests to make them interact nicely +// with the go test cache. +func copyModuleForScriptTest(modDir string) { + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedImports | packages.NeedFiles | packages.NeedDeps | packages.NeedName, + Tests: false, + }, gosimtool.Module+"/...") + if err != nil { + log.Fatal(err) + } + + keep := make(map[string]bool) + for _, pkg := range translate.TranslatedRuntimePackages { + keep[pkg] = true + } + + var mark func(pkg *packages.Package) + seen := make(map[*packages.Package]bool) + mark = func(pkg *packages.Package) { + if !strings.HasPrefix(pkg.PkgPath, gosimtool.Module) { + return + } + + if seen[pkg] { + return + } + seen[pkg] = true + + for _, dep := range pkg.Imports { + mark(dep) + } + } + + for _, pkg := range pkgs { + if (strings.Contains(pkg.PkgPath, "/internal/") || strings.HasSuffix(pkg.PkgPath, "/internal")) && !keep[pkg.PkgPath] { + // log.Println("skipping internal", pkg.PkgPath) + continue + } + if pkg.Name == "main" { + // log.Println("skipping main", pkg.PkgPath) + continue + } + // log.Println(pkg.PkgPath) + mark(pkg) + } + + var files []string + wanted := make(map[string]bool) + + for pkg := range seen { + // log.Println("marked", pkg.PkgPath) + // log.Println(pkg.GoFiles) + files = append(files, pkg.GoFiles...) + files = append(files, pkg.IgnoredFiles...) + } + + files = append(files, filepath.Join(modDir, "go.mod")) + files = append(files, filepath.Join(modDir, "go.sum")) + + extractedModDir := filepath.Join(modDir, gosimtool.OutputDirectory, "scripttest/mod") + + for _, file := range files { + newP := filepath.Join(extractedModDir, strings.TrimPrefix(file, modDir)) + if err := os.MkdirAll(filepath.Dir(newP), 0o755); err != nil { + log.Fatal(err) + } + copyFileIfChanged(file, newP, 0o644) + wanted[newP] = true + } + + err = filepath.WalkDir(extractedModDir, func(path string, d fs.DirEntry, err error) error { + if d.IsDir() { + return nil + } + if !wanted[path] { + if err := os.Remove(path); err != nil { + return err + } + } + return nil + }) + if err != nil { + log.Fatal(err) + } + + // TODO: delete unwanted directories? +} diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..136ab66 --- /dev/null +++ b/doc.go @@ -0,0 +1,65 @@ +/* +Package gosim contains the main API for interacting with Gosim simulations. +Gosim is a simulation testing framework that aims to make it easier to write +reliable distributed systems code in go. See the README.md file for an +introduction to Gosim. + +# Introduction + +Distributed systems can fail in many surprising ways. Exhaustively thinking of +all things that can go wrong and writing tests for them is infeasible. Gosim +simulates the external components that a program might interact with, such as +the disk, network, clocks, and more. Gosim has an API to introduce failures in +these systems (like chaos testing) to test that a program can handle otherwise +difficult-to-reproduce failures. + +# Writing and running gosim tests + +Gosim tests are normal Go tests that are run in a Gosim simulation. To run Gosim +tests, use the 'gosim test' command with similar arguments as the standard 'go +test' command: + + # run an normal test + go test -run TestName -v ./path/to/pkg + # run a gosim test + go run github.com/jellevandenhooff/gosim/cmd/gosim test -run TestName -v ./path/to/pkg + +For more information on the 'gosim' command see its documentation at +[github.com/jellevandenhooff/gosim/cmd/gosim]. To run Gosim tests from with a +Go test, use the +[github.com/jellevandenhooff/gosim/metatesting] package. + +# Gosim simulation + +When tests run inside Gosim they run in a simulation and they do not interact +with the normal operating system. Gosim simulates: + + - Real time. The time is simulated by Gosim (like on the Go playground) so that + it advances automatically when all goroutines are paused. This means tests run + quickly even if they sleep for a long time. + + - The filesystem. Gosim implements its own filesystem that can simulate data + loss when writes are not fsync-ed properly. + + - The network. Gosim implements its own network that can introduce extra latency + and partition machines. + + - Machines. Inside of Gosim a single Go test can create multiple machines which + are new instantiations of a Go program with their own global variables, their + own disk, and their own network address. + +Gosim implements its simulation at the Go runtime and system call level, +emulating Linux system calls. Gosim compiles all code with GOOS set to linux. To +interact with the simulation, programs can use the standard library with +functions like [time.Sleep], [os.OpenFile], [net.Dial], and all others working +as you would expect. + +To control the simulation, this package has functions like [NewMachine] and +[Machine.Crash] to create and manipulate machines and [SetConnected] to +manipulate the network. + +# Gosim internals + +For a description of how Gosim works, see the design in docs/design.MD. +*/ +package gosim diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..7c3c083 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,555 @@ +# Goals + +The goal of Gosim is to make writing correct systems code in Go easier and more +fun. To accomplish that goal Gosim is designed to: + +- Fit in with Go. Testing with Gosim should be like normal testing. Its APIs +follow the conventions from `go test` and the `testing` packages. + +- Give fast feedback. Gosim should work well with a frequent `gosim test` +workflow. + +- Work with the Go ecosystem. Gosim should not require applications to be +rewritten in a different style, and should support commonly used packages like +gRPC. + +- Be understandable. Gosim's simulation is involved with many moving pieces. +Those should be as debuggable as possible for when things go wrong. + +# How Gosim works + +Gosim runs Go code in a simulated environment. Gosim includes its own +lightweight simulated Go runtime and implementations of Linux system calls +backed by a network and disk simulation. Gosim's implementation consists of +three layers: + +The first layer is Gosim's simulated Go runtime which implements the basic +features of the Go language and runtime: goroutines, time, maps, and +synchronization with channels and semaphores. The simulated runtime provides +determinism and complete control over scheduling and time. This lets Gosim +accelerate time when goroutines are paused. Gosim translates standard Go code to +call into its runtime, and then compiles the translated code using the standard +Go compiler. This translation replaces eg. `go foo()` with +`gosimruntime.Go(foo)`. + +The second layer is a set of hooks in the Go standard library to call into +Gosim's runtime instead of the normal runtime. This makes the standard +`sync.Mutex` or `time.AfterFunc` call into `gosimruntime` instead of the normal +Go runtime. + +The final layer is Gosim's simulated operating system. Gosim simulates Linux +system calls with its fake operating system. This fake operating system is +written in standard Go and runs on Gosim's runtime. All system calls in the +standard library are replaced to call into Gosim's OS, so that +`os.Open` eventually calls `simulation.LinuxOS.SysOpenat`. + +Gosim exposes a high-level API in the `gosim` package that lets tests create +simulated machines, crash them, disconnect them, and more, to verify that +applications can handle those scenarios. + +To run the tests Gosim includes a CLI `gosim` with a `gosim test` command that +translates a package, runs its with Gosim's runtime and OS, and prints +the result as if it were a normal `go test`. + +## The deterministic runtime + +Gosim's deterministic runtime runs goroutines in a controlled way using +cooperative scheduling. Only one goroutine runs at a time and at every +synchronization point the goroutine calls into the scheduler to consider +switching to another goroutine. This scheduling decision is made using a +deterministic random number generator, and if a program is run again it will +make the exact same scheduling decisions. + +Behind the scenes, each simulated goroutine is implemented using a coroutine +(using the API underlying `iter.Pull`). This lets Gosim quickly switch between +goroutines. + +Besides the non-deterministic concurrency primitives, Go also exposes +non-determinism in the `rand` package and with `map` iteration order. Gosim +replaces the randomness in `rand` with its own seeded random number generator, +and provides a `map` that has varying yet deterministic iteration order. + +Not all sources of non-determinism are controlled by Gosim. For example, memory +allocation will give different pointers. Code that behaves non-deterministically +can be flagged by Gosim's tracing mechanism, which calculates a running hash +over all events like spawning goroutines, context switches, etc. + +The implementation of the runtime is in the +[github.com/jellevandenhooff/gosim/gosimruntime](../gosimruntime/) package. + +## Simulated time + +To simulate time Gosim tracks the state of every goroutine. If all goroutines +are waiting for time to advance, Gosim advances the clock to the next earlier +timestamp that any goroutine is waiting for. This simulated time lets Gosim run +programs that wait for a long simulated time in less real time. + +For example, consider the program + +```go +// main goroutine +ch := make(chan struct{}) +go func() { // goroutine A + time.Sleep(time.Hour) + close(ch) +}() +var wg sync.WaitGroup +wg.Add(1) +go func() { // goroutine B + defer wg.Done() + <-ch + time.Sleep(time.Minute) +}() +wg.Wait() +``` + +At the start of this program, the main goroutine and the two spawned goroutines +will all be waiting. The main goroutine is waiting on `wg.Wait()`, B is waiting +on `<-ch`, and A is waiting on `time.Sleep(time.Hour)`. When running this code, +the Gosim runtime will see all goroutines paused and advance time by one hour. +Then A will close the channel, B will awaken and call `time.Sleep(time.Minute)`, +time will be advanced again, etc. + +## The translator + +To make existing Go programs use the deterministic runtime, Gosim translates +existing code to call into its own runtime. The translator works at the AST +level and replaces all interactions with channels, maps, and goroutines with +calls to its own runtime. For example, it rewrites + +```go +go func() { + log.Println("hello!") +}() +``` + +to + +```go +gosimruntime.Go(func() { + log.Println("hello!") +}) +``` + +and + +```go +m = map[int]string{ + 1: "ok", + 2: "bar", +} +log.Println(m[1]) +``` + +to + +```go +m = gosimruntime.MapLiteral[int, string]{ + {K: 1, V: "ok"}, + {K: 2, V: "bar"}, +}.Build() +log.Println(m.Get(1)) +``` + +After translating, Gosim compiles the translated code using the +standard Go compiler, and it can be debugged or instrumented with +standard Go tools like `go tool pprof` or `delve`. + +The translator runs on almost all code, including the standard library. Only +the `gosimruntime` packages remains untranslated. Translated code can access +non-translated code by annotating an import with `//gosim:notranslate`. This +lets, for example, Gosim's wrapper `reflect` package access the underlying real +`reflect` package. + +The github.com/jellevandenhooff/gosim/internal/translator package implements the +translator. It is exposed through the Gosim CLI. + +An alternative design could have modified the Go runtime or compiler to instead +compile standard Go programs into a deterministic version, but the runtime and +compiler are modified much more often than the Go language specification. +Hopefully maintaining this translator will be less work than maintaining a +custom compiler. + +## Standard library hooks + +Gosim translates not only application code to be tested, but also +the entire Go standard library. The interface between the Go standard +library and the runtime library is a (relatively) small set of functions, +such as + +```go +func newTimer(when, period int64, f func(arg any, seq uintptr, delay int64), arg any, c *hchan) *timeTimer +``` + +which lives in `runtime/time.go`. For all such functions Gosim has hooks in the +package github.com/jellevandenhooff/gosim/internals/hooks/go123. The hooks are +Go version specific because there are no API stability guarantees for these +low-level unexported runtime functions. The hooks in Gosim are implemented as +calls to the gosimruntime package. + +By translating the entire go standard library, applications should notice +as few differences between reality and simulation. + +An alternative design (and an earlier version of Gosim) instead replaced +higher-level functions like `os.OpenFile` with simulated or mocked versions. +However, implementing all functions used by programs in a faithful way is very +difficult. Although the internal API that Gosim now replaces does not have +stability guarantees, it is much smaller than the entire standard library. + +Gosim wraps the `reflect` package with its own version to hide the differences +between the built-in `map` and Gosim's `gosimruntime.Map`. + +## Globals and machines + +Gosims runs multiple simulated machines in own go process, each with their own +copy of global variables. This duplication is necessary because there are quite +a few important global variables, such as the `net/http` connection pool, that +make no sense to share between machines. + +The translator replaces all accesses to such variables to call into a special +gosimruntime call + +```go +var global string + +func Get() string { + return global +} +``` + +becomes + +```go +type Globals struct { + global string +} + +func G() *Globals { + // call to gosimruntime to get an appropriate pointer +} + +func Get() string { + return G().global +} +``` + +This lets gosim faithfully simulate multiple go processes in a single go process. + +Whenever gosim starts a new simulated machine, it allocates and initializes all +global variables by calling each package's `init()` functions. Some globals are +large, take a long time to initialize, and are never modified afterwards; for +example, the unicode tables. As an optimization, Gosim allocates and +initializes them only once. + +Communication between machines is tricky. Because each machine has its own +copy of global variables, sharing objects between machines can cause all kinds +of issues: For example, `io.EOF` can be a different object on each machine, +and so checking if an error is a sentinel value no longer works. + +To prevent problems, machines communicate only between eachother using the +simulated operating system. The operating system and each machine communicate +over a channel (which does work between machines) and work on raw values that +are safe to share. + +An alternative design might run each simulated go process in its own physical +process. However, the overhead for communicating and starting such processes +seems high, although I have not benchmarked this. + +## The system call interface + +Gosim simulates the operating system as an independent simulated go process. +The operating system has a model of a network, filesystem, etc., all stored as +Go structs, etc. The programs that Gosim tests call into this using its syscall +interface defined in +github.com/jellevandenhooff/gosim/internal/simulation/syscallabi.Syscall. +This is a RPC-like mechanism that works safely between Gosim's simulated machines. + +At a code level, Gosim's translator replaces the wrapper functions in the +`syscall` package that call `syscall6` with its own wrapper functions that +call into its `syscallabi` package. The `gensyscall` tool generates these +wrapper functions, as well as a dispatcher and interface to be implemented +by the simulated operating system. + +The calling code for eg. `SYS_PREAD64` becomes + +```go +func SyscallSysPread64(fd int, p []byte, offset int64) (n int, err error) { + // generated wrapper +} +``` + +and the implementation becomes + +```go +func (l *LinuxOS) SysPread64(fd int, data syscallabi.ByteSliceView, offset int64) (int, error) { + // figure out what to write + n := data.Write([]byte{...}) + return n, nil +} +``` + +The `syscallabi` runs all operating system code on its own simulated Go machine. +The boundary is illustrated by the `syscallabi.ByteSliceView` type which lets +the operating system read and write from userspace without triggering the race +detector and makes sure that there are no variables accidentally shared between +machines. + +To glue this call to the standard library, the calling code is wrapped in a +standard library hook + +```go +func Syscall_pread(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPread64(fd, p, offset) +} +``` + +and finally linked into the rewritten standard library with a linkname + +```go +//go:linkname pread translated/github.com/jellevandenhooff/gosim/internal_/hooks/go123.Syscall_pread +func pread(fd int, p []byte, offset int64) (n int, err error) +``` + +The translator uses a linkname to prevent cycles in the import graph. + +## The simulated operating system + +Gosim's simulated network implements a TCP API by sending TCP-like packets over +a virtual network. Each link on the network can have configurable latency, or +be temporarily disabled, to simulate challenging network conditions. The +simulated network implements this with normal Go timers, arranging for each +packet to be delivered at the correct time. + +Gosim's simulated filesystem implements a Posix-style filesystem API, +with read, write, and fsync calls. The simulated filesystem tracks +in-flight writes so that it can simulate machine crashes with all +the tricky behavior that Linux can portray. + +## The gosim API for tested programs + +Gosim's implementation is spread among a number of internal packages. To +provide a consistent API to handle machines, the simulated network, etc. the +[github.com/jellevandenhooff/gosim package](..) provides a high-level API to +create new machines, manipulate the network, etc. Behind the scenes these are +implemented as custom system calls to the simulated operating system. + +## The gosim CLI and integration with Go tests + +Gosim aspires to be as easy to use as the standard Go tooling. To run a test +using Gosim, ideally all that you need to do is replace `> go test` with +`> gosim test` on the command line. In practice, `gosim` becomes +`go run github.com/jellevandenhooff/gosim/cmd/gosim` +but otherwise the tool exposes flags like `go test`. + +When running such a test, `gosim` first translates the code, caching +packages to not retranslate eg. the standard library, and then invokes +`go test` on the translated code. + +To integrate with the `go test` tool, the translated code provides its own +`TestMain` function that takes over control of the test execution. Each test is +run on its own simulated machine, so that each test gets its own globals and +will run the same (given the same seed). + +## Logging and formatting + +To make logs from multiple machines and goroutines readable, Gosim annotates all +logs with their source machine, goroutine, and timestamp. To implement this, +Gosim uses a JSON-formatting `log/slog` handler in each machine that includes +the current machine and goroutine as extra fields. The default `log` handler +writes to the same `log/slog` handler, and all logging code in the `testing` +package is modified as well. The `gosim` CLI by default parses the JSON logs and +pretty-prints them. Behind the scenes all logs are JSON so they can be parsed by +metatesting code. + +A simplified log line in JSON might look like + +``` +{"time":"2020-01-15T14:10:03.000001234Z","level":"INFO","msg":"hello info","machine":"main","foo":"bar","goroutine":4,"step":1} +``` + +and gets formatted as + +``` + 1 main/4 14:10:03.000 INF behavior/log_gosim_test.go:53 > hello info foo=bar +``` + +## Metatesting + +A larger module might want to run Gosim tests as parts of its normal `go test` +suite. With the [github.com/jellevandenhooff/gosim/metatesting](../metatesting) +package, normal tests can invoke Gosim tests with varying seeds and inspecting +their logs. Such tests can check that the Gosim tests are deterministic, or that +programs behave as expected by reading their logs. + +A future goal of metatesting is to fuzz Gosim tests by controlling seeds and +randomness and seeing how they behave. + +## Race detection + +To help find concurrency bugs Gosim works with the built-in Go race detector. +Since Gosim translates Go code to call into own runtime and then runs it with +the standard Go compiler, it can use the built-in race detector with only minor +special handling. + +The Go compiler will insert all standard race detector read and write calls, +notifying when a goroutine accesses potentially shared memory. This does not +require any extra work. Gosim's runtime is instrumented to call into the race +detector whenever it creates a happens-before dependency, for example in its +channel and semapahore code. + +For gosim's simulation to correctly explore all behavior, it is important that +goroutines do not influence eachother between the synchronization points where +Gosim explicitly controls the scheduler. This is exactly what the race detector +verifies. + +Gosim's internal runtime state (the structs in the `gosimruntime` package) are +always accessed from a single system goroutine so that accesses do not race. +Whenever a userspace goroutine needs to modify this state (eg. to spawn a new +goroutine), it performs an upcall that temporarily switches to the system +goroutine to modify the runtime state before switching back. + +Functions implementing concurrency primitives, eg. channel reads, do no always +switch back to the system goroutine. Those functions are all marked +`//go:norace` to not trigger the race detector. + +To test the race detector, gosim includes a copy of (most of) the race detector +tests from the go toolchain. + +## Controlled non-determinism + +Gosim has an escape hatch to allow local non-determinism that has no externally +visible side-effects. An example of this is the type cache used in the +`encoding/json` package. Gosim allows this type cache to be shared between +machines (and even between Gosim tests) to speed up `encoding/json` code. +Rebuilding type descriptors for every test would be wasteful and does not +explore interesting behavior, so caching the type descriptors is appealing. +However, since this caching logic writes to a shared map and synchronizes access +using a mutex, it does trigger Gosim's non-determinism detector. + +A solution to this is the `gosimruntime.BeginControlledNondeterminism` call +which enters a code region that cannot yield nor consume randomness. In essence, +it promises that the non-determinism in that section will not impact other code +and pauses trace recording. If later code does get impacted hopefully the trace +will capture that non-determinism there, but debugging will be difficult. The +translator inserts calls as a patch to some packages. + +# Background + +## Bug finding in distributed systems + +Perhaps most famous in the distributed systems bug finding world is [Jepsen's +blog](https://jepsen.io/blog). Using their tools +[Jepsen](https://github.com/jepsen-io/jepsen), to run and mess with existing +distributed systems, and [Elle](https://github.com/jepsen-io/elle), to then +validate the behavior of those systems, Jepsen has found many bugs. + +Jepsen works with existing systems and runs them on actual cloud servers, +messing with the actual network. Gosim aims to have less overhead and complexity +by simulating the servers and network instead. + +## Simulation testing + +Simulation testing has a long history. Some current approaches and inspirations +follow. This is not an exhaustive list or deep evaluation. + +The 2014 talk +["Testing Distributed Systems w/ Deterministic Simulation" by Will Wilson](https://www.youtube.com/watch?v=4fFDFbi3toc) +introduces simulation testing as used by FoundationDB, later described in +the +[FoundationDB paper](https://www.foundationdb.org/files/fdb-paper.pdf). +The +current code for FoundationDB is once again open-source and contains +examples of the scenarios they run against like +[swizzle clogging](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/workloads/RandomClogging.actor.cpp) +mentioned in the talk. +FoundationDB's simulation testing intercepts interactions between application +code and its own custom Flow actor and RPC system. + +The start-up [Antithesis](https://antithesis.com), founded by some of the people +behind FoundationDB, offers simulation testing-as-a-service on the virtual +machine level. Their [blog](https://antithesis.com/blog/) describes the work +they do. Antithesis implements determinisim at the virtual machine level, and +can run arbitrary Docker images. + +The [Shadow](https://github.com/shadow/shadow) simulation testing tool focuses +on simulating networks and is used primarily by the Tor project. Shadow runs +arbitrary applications and intercepts the system calls they make. + +The C# framework [Coyote](https://microsoft.github.io/coyote) by Microsoft is a +simulation testing framework focused on testing actor systems. Coyote simulates +by instrumenting C# programs that use its RPC and actor mechanism. + +A framework for Rust [Madsim](https://github.com/madsim-rs/madsim/) is used to +test the RisingWave database https://risingwave.com. Madsim runs Rust programs +and intercepts calls at the Tokio library level. Programs must use Tokio to test +with Madsim. + +Another framework for Rust is [Tokio's Turmoil](https://github.com/tokio-rs/turmoil). Turmoil works similar to Madsim +and intercepts at the Tokio library level. Programs must use Tokio to test with +Turmoil. + +Gosim's approach for simulating a distributed system at the system call level is +remeniscent of the papers +["Efficient System-Enforced Deterministic Parallelism"](https://dedis.cs.yale.edu/2010/det/papers/osdi10.pdf) +or +["Deterministic Process Groups in dOS"](https://homes.cs.washington.edu/~luisceze/publications/osdi10-dos.pdf). + +The [TigerBeetle database](https://github.com/tigerbeetle/tigerbeetle/), written +in Zig, uses a homegrown simulation testing system, VOPR, described at +https://github.com/tigerbeetle/tigerbeetle/blob/main/docs/HACKING.md#simulation-tests. +VOPR simulates at a state-machine level and is tightly coupled to TigerBeetle. + +## Bug finding in concurrent code + +Gosim's scheduler could be used to test low-level concurrent or lock-free code. +The package does not include support, but the approach of carefully scheduling +goroutines at every synchronization point is supported by Gosim's runtime. + +The [Loom model checker](https://github.com/tokio-rs/loom) for Rust, now part of +Tokio, was used by AWS in validating a Rust service as described in their paper +[Using Lightweight Formal Methods to Validate a Key-Value Storage Node in Amazon S3](https://www.amazon.science/publications/using-lightweight-formal-methods-to-validate-a-key-value-storage-node-in-amazon-s3). + +Anoter simple but effective approach for finding bugs in concurrent code is the +PCT algorithm originally described in +[A Randomized Scheduler with Probabilistic Guarantees of Finding Bugs](https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/asplos277-pct.pdf). + +## Testing filesystem code + +Gosim's filesystem simulation with fine-grained fsync support is reminsicent +of +[Alice (Application-Level Intelligent Crash Explorer)](https://www.usenix.org/system/files/conference/osdi14/osdi14-paper-pillai.pdf). +Like Alice, Gosim's filesystem splits file system system calls into smaller +operations that can individually be persisted or lost on crash, and can test +that an application can handle those scenarios. + +## Testing strategies for existing Go code + +Many existing distributed systems are written in Go and they use a variety +of testing strategies for their critical components. + +The Raft library [github.com/etcd-io/raft](https://github.com/etcd-io/raft) is +used by both [Etcd](https://github.com/etcd-io/etcd) and [CockroachDB](https://github.com/cockroachdb/cockroach). The critical [Raft state machine in +Etcd](https://github.com/etcd-io/raft/blob/main/raft.go) is implemented with side-effect free +transition functions that take in events like time advancing +and receiving messages. Outgoing RPCs are buffered in the implementation. This design lends itself to +[thorough tests](https://github.com/etcd-io/raft/tree/main/testdata) that cover in detail +how the Raft state machine should behave among different inputs. + +The key-value library +[github.com/cockroachdb/pebble](https://github.com/cockroachdb/pebble) backing +CockroachDB tests, among others, with an error-injecting virtual filesystem. + +The Raft library [github.com/hashicorp/raft](https://github.com/hashicorp/raft) +used by [Consul](https://github.com/hashicorp/consul) has +[fuzzy tests](https://github.com/hashicorp/raft/tree/main/fuzzy) that check for +correctness under tricky network conditions by instrumenting their RPC stack + +Many tests for Go try to mock Go's built-in time package to test +time-outs without tests becoming slow. One initial package +was [github.com/benbjohnson/clock](https://github.com/benbjohnson/clock). +A big difficulty with mocking the clock is that is not clear when +time should advance. + +The [testing/synctest proposal](https://github.com/golang/go/issues/67434) +adds a new API for mocking the clock in a Go program scoped to +some goroutines under test. \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cf56a2e --- /dev/null +++ b/go.mod @@ -0,0 +1,68 @@ +module github.com/jellevandenhooff/gosim + +go 1.23.2 + +require ( + github.com/dave/dst v0.27.3 + github.com/go-task/task/v3 v3.40.0 + github.com/google/go-cmp v0.6.0 + github.com/mattn/go-isatty v0.0.20 + github.com/mattn/go-sqlite3 v1.14.24 + github.com/rogpeppe/go-internal v1.13.1 + golang.org/x/mod v0.22.0 + golang.org/x/sync v0.9.0 + golang.org/x/sys v0.27.0 + golang.org/x/tools v0.27.0 + google.golang.org/grpc v1.62.1 + google.golang.org/protobuf v1.32.0 + mvdan.cc/gofumpt v0.7.0 + pgregory.net/rapid v1.1.1-0.20240401182707-34cb5b24e44b +) + +require ( + dario.cat/mergo v1.0.1 // indirect + github.com/Ladicle/tabwriter v1.0.0 // indirect + github.com/Masterminds/semver/v3 v3.3.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.2 // indirect + github.com/alecthomas/chroma/v2 v2.14.0 // indirect + github.com/cloudflare/circl v1.5.0 // indirect + github.com/cyphar/filepath-securejoin v0.3.4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/dominikbraun/graph v0.23.0 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/fatih/color v1.18.0 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.6.0 // indirect + github.com/go-git/go-git/v5 v5.12.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/go-task/template v0.1.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-zglob v0.0.6 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/pjbgf/sha1cd v0.3.0 // indirect + github.com/radovskyb/watcher v1.0.7 // indirect + github.com/sajari/fuzzy v1.0.0 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/skeema/knownhosts v1.3.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/whilp/git-urls v1.0.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + golang.org/x/crypto v0.29.0 // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/term v0.26.0 // indirect + golang.org/x/text v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + mvdan.cc/sh/v3 v3.10.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..3f8eb30 --- /dev/null +++ b/go.sum @@ -0,0 +1,192 @@ +dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= +dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Ladicle/tabwriter v1.0.0 h1:DZQqPvMumBDwVNElso13afjYLNp0Z7pHqHnu0r4t9Dg= +github.com/Ladicle/tabwriter v1.0.0/go.mod h1:c4MdCjxQyTbGuQO/gvqJ+IA/89UEwrsD6hUCW98dyp4= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.2 h1:A7JbD57ThNqh7XjmHE+PXpQ3Dqt3BrSAC0AL0Go3KS0= +github.com/ProtonMail/go-crypto v1.1.2/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= +github.com/alecthomas/assert/v2 v2.7.0 h1:QtqSACNS3tF7oasA8CU6A6sXZSBDqnm7RfpLl9bZqbE= +github.com/alecthomas/assert/v2 v2.7.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/chroma/v2 v2.14.0 h1:R3+wzpnUArGcQz7fCETQBzO5n9IMNi13iIs46aU4V9E= +github.com/alecthomas/chroma/v2 v2.14.0/go.mod h1:QolEbTfmUHIMVpBqxeDnNBj2uoeI4EbYP4i6n68SG4I= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= +github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0= +github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= +github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM= +github.com/dave/dst v0.27.3 h1:P1HPoMza3cMEquVf9kKy8yXsFirry4zEnWOdYPOoIzY= +github.com/dave/dst v0.27.3/go.mod h1:jHh6EOibnHgcUW3WjKHisiooEkYwqpHLBSX1iOBhEyc= +github.com/dave/jennifer v1.5.0 h1:HmgPN93bVDpkQyYbqhCHj5QlgvUkvEOzMyEvKLgCRrg= +github.com/dave/jennifer v1.5.0/go.mod h1:4MnyiFIlZS3l5tSDn8VnzE6ffAhYBMB2SZntBsZGUok= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo= +github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= +github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE= +github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8= +github.com/go-git/go-billy/v5 v5.6.0/go.mod h1:sFDq7xD3fn3E0GOwUSZqHo9lrkmx8xJhA0ZrfvjBRGM= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= +github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/go-task/task/v3 v3.40.0 h1:1gKx+2UDz06Jtm0MBiN+EqVN87wWEyspuEze4LRGusk= +github.com/go-task/task/v3 v3.40.0/go.mod h1:Eb9p9TYX2LpNrd8rBL+Ceht7LzSqA+WniSFeHAJlsnI= +github.com/go-task/template v0.1.0 h1:ym/r2G937RZA1bsgiWedNnY9e5kxDT+3YcoAnuIetTE= +github.com/go-task/template v0.1.0/go.mod h1:RgwRaZK+kni/hJJ7/AaOE2lPQFPbAdji/DyhC6pxo4k= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-zglob v0.0.6 h1:mP8RnmCgho4oaUYDIDn6GNxYk+qJGUs8fJLn+twYj2A= +github.com/mattn/go-zglob v0.0.6/go.mod h1:MxxjyoXXnMxfIpxTK2GAkw1w8glPsQILx3N5wrKakiY= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= +github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE= +github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sajari/fuzzy v1.0.0 h1:+FmwVvJErsd0d0hAPlj4CxqxUtQY/fOoY0DwX4ykpRY= +github.com/sajari/fuzzy v1.0.0/go.mod h1:OjYR6KxoWOe9+dOlXeiCJd4dIbED4Oo8wpS89o0pwOo= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY= +github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/whilp/git-urls v1.0.0 h1:95f6UMWN5FKW71ECsXRUd3FVYiXdrE7aX4NZKcPmIjU= +github.com/whilp/git-urls v1.0.0/go.mod h1:J16SAmobsqc3Qcy98brfl5f5+e0clUvg1krgwk/qCfE= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ= +golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= +golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= +golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.26.0 h1:WEQa6V3Gja/BhNxg540hBip/kkaYtRg3cxg4oXSw4AU= +golang.org/x/term v0.26.0/go.mod h1:Si5m1o57C5nBNQo5z1iq+XDijt21BDBDp2bK0QI8e3E= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +mvdan.cc/gofumpt v0.7.0 h1:bg91ttqXmi9y2xawvkuMXyvAA/1ZGJqYAEGjXuP0JXU= +mvdan.cc/gofumpt v0.7.0/go.mod h1:txVFJy/Sc/mvaycET54pV8SW8gWxTlUuGHVEcncmNUo= +mvdan.cc/sh/v3 v3.10.0 h1:v9z7N1DLZ7owyLM/SXZQkBSXcwr2IGMm2LY2pmhVXj4= +mvdan.cc/sh/v3 v3.10.0/go.mod h1:z/mSSVyLFGZzqb3ZIKojjyqIx/xbmz/UHdCSv9HmqXY= +pgregory.net/rapid v1.1.1-0.20240401182707-34cb5b24e44b h1:s68/o7gMF5b0E/pW1p81Jd/5QBDirafROoc8itbeXLA= +pgregory.net/rapid v1.1.1-0.20240401182707-34cb5b24e44b/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/gosimruntime/chan.go b/gosimruntime/chan.go new file mode 100644 index 0000000..d6d287b --- /dev/null +++ b/gosimruntime/chan.go @@ -0,0 +1,605 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package gosimruntime + +import ( + "iter" + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/race" +) + +type waitq struct { + first, last *sudog +} + +//go:norace +func (sq *waitq) empty() bool { + return sq.first == nil +} + +//go:norace +func (sq *waitq) enqueue(elem *sudog) { + elem.queue = sq + elem.prev = sq.last + elem.next = nil + if sq.last != nil { + sq.last.next = elem + } + sq.last = elem + if sq.first == nil { + sq.first = elem + } +} + +//go:norace +func (sq *waitq) enqueuelifo(elem *sudog) { + elem.queue = sq + elem.prev = nil + elem.next = sq.first + if sq.first != nil { + sq.first.prev = elem + } + sq.first = elem + if sq.last == nil { + sq.last = elem + } +} + +//go:norace +func (sq *waitq) remove(elem *sudog) { + if elem.prev != nil { + elem.prev.next = elem.next + } + if elem.next != nil { + elem.next.prev = elem.prev + } + if sq.first == elem { + sq.first = elem.next + } + if sq.last == elem { + sq.last = elem.prev + } + elem.queue = nil +} + +//go:norace +func (sq *waitq) dequeue() *sudog { + if sq.first != nil { + elem := sq.first + + g := elem.goroutine + // remove all waiters of this goroutine from their respective queues (so we only trigger + // one branch in a select) + for sg := g.waiters; sg != nil; sg = sg.waitersNext { + sg.queue.remove(sg) + // TODO: optimize? remove does more work than necessary for the + // current waiter, since we know sq.first == elem and elem.prev == + // nil. + } + + if g.selected != nil { + panic("help") + } + g.selected = elem + getg().upcall(func() { + elem.goroutine.setWaiting(false) + }) + return elem + } + return nil +} + +type sudog struct { + queue *waitq + + prev, next *sudog // linked list in queue + + waitersNext *sudog // linked list in g.waiters + + goroutine *goroutine + selectIdx int + + success bool + + // TODO: make this a pointer to a selector instead? and then we can maybe skip a copy? + // TODO: set to nil when done + item interface{} + + // used instead of item in NotifyList because storing ints in interfaces allocates + index int +} + +var sudogPool []*sudog + +//go:norace +func allocSudog() *sudog { + if n := len(sudogPool); n > 0 { + sg := sudogPool[n-1] + sudogPool = sudogPool[:n-1] + return sg + } + return &sudog{} +} + +//go:norace +func freeSudog(sg *sudog) { + sudogPool = append(sudogPool, sg) +} + +type chanImpl[V any] struct { + buf []V + race []racesynchelper + start, end, len int + closed bool + sendQ waitq + recvQ waitq +} + +type Chan[V any] struct { + impl *chanImpl[V] +} + +//go:norace +func NewChan[V any](capacity int) Chan[V] { + var buf []V + if capacity != 0 { + buf = make([]V, capacity) + } + var r []racesynchelper + if race.Enabled { + r = make([]racesynchelper, max(1, capacity)) + } + return Chan[V]{ + impl: &chanImpl[V]{ + buf: buf, + race: r, + start: 0, + end: 0, + len: 0, + }, + } +} + +func NilChan[V any]() Chan[V] { + return Chan[V]{} +} + +func ExtractChan[M ~struct{ impl *chanImpl[V] }, V any](m M) Chan[V] { + return Chan[V](m) +} + +func (c Chan[V]) canSend() bool { + return c.impl.len < len(c.impl.buf) || !c.impl.recvQ.empty() || c.impl.closed +} + +//go:norace +func (c Chan[V]) send(v V) { + if c.impl.closed { + panic("closed") + } + + if other := c.impl.recvQ.dequeue(); other != nil { + if c.impl.buf != nil { + if race.Enabled { + racesyncdouble(other.goroutine, &c.impl.race[c.impl.end]) + } + c.impl.end = (c.impl.end + 1) % len(c.impl.buf) + c.impl.start = c.impl.end + } else { + if race.Enabled { + racesynccross(other.goroutine, &c.impl.race[0]) + } + } + other.item = v + other.success = true + } else { + if race.Enabled { + racesyncsingle(&c.impl.race[c.impl.end]) + } + c.impl.buf[c.impl.end] = v + c.impl.len++ + c.impl.end = (c.impl.end + 1) % len(c.impl.buf) + } +} + +//go:norace +func (c Chan[V]) Send(v V) { + g := getg() + g.yield() + + if race.Enabled { + race.Read(unsafe.Pointer(c.impl)) + } + + if c.canSend() { + c.send(v) + return + } + sudog := g.allocWaiter() + sudog.item = v + c.impl.sendQ.enqueue(sudog) + g.wait() + if !sudog.success { + race.Acquire(unsafe.Pointer(c.impl)) + g.releaseWaiters() + panic("closed") + } + g.releaseWaiters() +} + +func (c Chan[V]) completeSend(ok bool) { + if !ok { + race.Acquire(unsafe.Pointer(c.impl)) + panic("closed") + } +} + +func ChanCast[V any](v interface{}) V { + return chanCast[V](v) +} + +//go:norace +func (c Chan[V]) close() { + if c.impl.closed { + panic("closed") + } + c.impl.closed = true + + for !c.impl.sendQ.empty() { + next := c.impl.sendQ.dequeue() + next.success = false + } + for !c.impl.recvQ.empty() { + next := c.impl.recvQ.dequeue() + var v V + next.item = v + next.success = false + } +} + +//go:norace +func (c Chan[V]) Close() { + getg().yield() + + // XXX: race.Write + race.Write(unsafe.Pointer(c.impl)) + race.Release(unsafe.Pointer(c.impl)) + + c.close() +} + +//go:norace +func (c Chan[V]) canRecv() bool { + return c.impl.len > 0 || !c.impl.sendQ.empty() || c.impl.closed +} + +func chanCast[V any](v any) V { + // if we send nil to a iface channel (eg chan error), then casting v.(error) will fail. + // handle that gracefully + res, _ := v.(V) + return res +} + +//go:norace +func (c Chan[V]) recvOk() (V, bool) { + if c.impl.closed && c.impl.len == 0 { + race.Acquire(unsafe.Pointer(c.impl)) + var v V + return v, false + } + if other := c.impl.sendQ.dequeue(); other != nil { + if c.impl.buf != nil { + if race.Enabled { + racesyncdouble(other.goroutine, &c.impl.race[c.impl.start]) + } + other.success = true + v := c.impl.buf[c.impl.start] + c.impl.buf[c.impl.start] = chanCast[V](other.item) + c.impl.start = (c.impl.start + 1) % len(c.impl.buf) + c.impl.end = c.impl.start + return v, true + } else { + if race.Enabled { + racesynccross(other.goroutine, &c.impl.race[0]) + } + v := chanCast[V](other.item) + other.success = true + return v, true + } + } else { + if race.Enabled { + racesyncsingle(&c.impl.race[c.impl.start]) + } + v := c.impl.buf[c.impl.start] + c.impl.len-- + c.impl.start = (c.impl.start + 1) % len(c.impl.buf) + return v, true + } +} + +//go:norace +func (c Chan[V]) RecvOk() (V, bool) { + g := getg() + g.yield() + if !c.canRecv() { + sudog := g.allocWaiter() + c.impl.recvQ.enqueue(sudog) + g.wait() + v := chanCast[V](sudog.item) + ok := sudog.success + g.releaseWaiters() + if !sudog.success { + // closed, acquire + race.Acquire(unsafe.Pointer(c.impl)) + } + return v, ok + } + return c.recvOk() +} + +func (c Chan[V]) Range() iter.Seq[V] { + return func(yield func(V) bool) { + for { + v, ok := c.RecvOk() + if !ok { + break + } + if !yield(v) { + break + } + } + } +} + +func (c Chan[V]) completeRecv(ok bool) { + if !ok { + // closed, acquire + race.Acquire(unsafe.Pointer(c.impl)) + } +} + +func (c Chan[V]) Recv() V { + v, _ := c.RecvOk() + return v +} + +//go:norace +func (c Chan[V]) Len() int { + getg().yield() + + return c.impl.len +} + +//go:norace +func (c Chan[V]) Cap() int { + return len(c.impl.buf) +} + +type sendSelector[V any] struct { + c Chan[V] + v V // XXX: box already here? +} + +//go:norace +func (c sendSelector[V]) enqueue(g *sudog) { + g.item = c.v + c.c.impl.sendQ.enqueue(g) +} + +func (c sendSelector[V]) exec() (any, bool) { + c.c.send(c.v) + return nil, true +} + +func (c Chan[V]) SelectSendOrDefault(v V) (int, any, bool) { + getg().yield() + if !c.canSend() { + return 1, nil, false + } + c.send(v) + return 0, nil, true +} + +func (c sendSelector[V]) ready() bool { + return c.c.canSend() +} + +func (c sendSelector[V]) complete(ok bool) { + c.c.completeSend(ok) +} + +type recvSelector[V any] struct { + c Chan[V] +} + +func (c recvSelector[V]) enqueue(g *sudog) { + c.c.impl.recvQ.enqueue(g) +} + +func (c recvSelector[V]) exec() (any, bool) { + v, ok := c.c.recvOk() + return v, ok +} + +func (c Chan[V]) SelectRecvOrDefault() (int, any, bool) { + if c.impl == nil { + return 1, nil, false + } + getg().yield() + if !c.canRecv() { + return 1, nil, false + } + v, ok := c.recvOk() + return 0, v, ok +} + +func (c recvSelector[V]) ready() bool { + return c.c.canRecv() +} + +func (c recvSelector[V]) complete(ok bool) { + c.c.completeRecv(ok) +} + +func (c Chan[V]) SendSelector(item V) ChanSelector { + return sendSelector[V]{c: c, v: item} +} + +func (c Chan[V]) RecvSelector() ChanSelector { + if c.impl == nil { + return nilChanSelector{} + } + return recvSelector[V]{c: c} +} + +type nilChanSelector struct{} + +func (nilChanSelector) enqueue(g *sudog) {} +func (nilChanSelector) ready() bool { return false } +func (nilChanSelector) exec() (interface{}, bool) { panic("help") } +func (nilChanSelector) complete(bool) { panic("help") } + +type ChanSelector interface { + enqueue(g *sudog) + ready() bool + exec() (interface{}, bool) + complete(bool) +} + +func DefaultSelector() ChanSelector { + return nil +} + +//go:norace +func Select(selectors ...ChanSelector) (int, any, bool) { + // TODO: add test for not taking default + g := getg() + g.yield() + + if len(selectors) >= 128 { + panic("bad") + } + var readybuf [8]byte + ready := readybuf[:0] + + defaultIdx := -1 + count := 0 + for i, selector := range selectors { + if selector == nil { + defaultIdx = i + } else if selector != (nilChanSelector{}) { + count++ + if selector.ready() { + ready = append(ready, byte(i)) + } + } + } + if len(ready) == 0 && defaultIdx != -1 { + return defaultIdx, nil, false + } + if len(ready) > 0 { + var idx byte + if len(ready) == 1 { + idx = ready[0] + } else { + idx = ready[fastrandn(uint32(len(ready)))] + } + v, ok := selectors[idx].exec() + return int(idx), v, ok + } + + // add self to queue for every selector + for i, selector := range selectors { + if selector == (nilChanSelector{}) { + continue + } + sudog := g.allocWaiter() + sudog.selectIdx = i + selector.enqueue(sudog) + } + + // wait to be selected + g.wait() + idx, v, ok := g.selected.selectIdx, g.selected.item, g.selected.success + g.releaseWaiters() + selectors[idx].complete(ok) + + // return result + return idx, v, ok +} + +// racesynchelper tracks goroutines that still have to call race.Release and +// race.Acquire on a given channel element. +// +// When a blocked goroutine's channel operation is completed by another +// goroutine, the completing goroutine will call racesync[single/double/cross] +// to race.Acquire and race.Release a channel element for itself, and then +// (ideally) for the blocked goroutine. In the standard go runtime there are +// functions to do that but those are not exposed. We simulate the end result +// by tracking that the blocked goroutine should have called race.Release and +// race.Acquire. This requires some care because another goroutine might come +// along want to call race.Acquire and race.Release on the same variable: +// +// 1. A blocked goroutine releases blockedg.waitsyncvar before pausing. +// 2. The unblocking goroutine releases to blockedg.shouldacquire. +// 3. The unblocking goroutine stores the address of the channel element to +// blockedg.shouldrelease +// Scenario A: +// 4. The blocked goroutine acquires blockedg.shouldacquire +// 5. The blocked goroutine releases blockedg.shouldrelease +// Scenario B: +// 4. Another goroutine comes wants to race.Acquire and race.Release. It notices the +// pending blockedg, acquires blockedg.waitsyncvar, and sets +// blockedg.shouldrelease to false. +// 5. The blocked goroutine acquires blockedg.shouldacquire. +// +// In both scenario A and scenario B the same happens-before relationships are +// established. +type racesynchelper struct { + // pending.shouldrelease = &thisracesynchelper + pending *goroutine +} + +//go:norace +func racesyncsingle(addr *racesynchelper) { + if g := addr.pending; g != nil { + g.parksynctoken.Acquire() + addr.pending = nil + g.shouldrelease = nil + } + + race.Acquire(unsafe.Pointer(addr)) + race.Release(unsafe.Pointer(addr)) +} + +//go:norace +func racesyncdouble(g *goroutine, addr *racesynchelper) { + if g := addr.pending; g != nil { + g.parksynctoken.Acquire() + addr.pending = nil + g.shouldrelease = nil + } + + race.Acquire(unsafe.Pointer(addr)) + race.Release(unsafe.Pointer(addr)) + + g.shouldacquire = true + race.Release(unsafe.Pointer(&g.shouldacquire)) + addr.pending = g + g.shouldrelease = addr +} + +//go:norace +func racesynccross(g *goroutine, addr *racesynchelper) { + g.parksynctoken.Acquire() + + race.Acquire(unsafe.Pointer(addr)) + race.Release(unsafe.Pointer(addr)) + + g.shouldacquire = true + race.Release(unsafe.Pointer(&g.shouldacquire)) +} diff --git a/gosimruntime/doc.go b/gosimruntime/doc.go new file mode 100644 index 0000000..f8d586f --- /dev/null +++ b/gosimruntime/doc.go @@ -0,0 +1,8 @@ +/* +Package gosimruntime simulates the go runtime for programs tested using gosim. + +The gosimruntime API is internal to gosim and programs should not directly use +it. This package is exported only because translated code must be able to +import it. Instead, use the public API in the gosim package. +*/ +package gosimruntime diff --git a/gosimruntime/fnv64.go b/gosimruntime/fnv64.go new file mode 100644 index 0000000..d358336 --- /dev/null +++ b/gosimruntime/fnv64.go @@ -0,0 +1,57 @@ +// Copyright 2011 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package gosimruntime + +import ( + "encoding/binary" + "math" +) + +type ( + fnv64 uint64 +) + +const ( + fnv64init = 14695981039346656037 + prime64 = 1099511628211 +) + +// newFnv64 returns a new 64-bit FNV-1 hash.Hash. +// Its Sum method will lay the value out in big-endian byte order. +func newFnv64() fnv64 { + var s fnv64 = fnv64init + return s +} + +func (s *fnv64) Sum() uint64 { return uint64(*s) } + +func (s *fnv64) Hash(data []byte) { + hash := *s + for _, c := range data { + hash *= prime64 + hash ^= fnv64(c) + } + *s = hash +} + +func (s *fnv64) HashInt(data uint64) { + switch { + case data == 0: + case data < math.MaxUint8: + s.Hash([]byte{uint8(data)}) + case data < math.MaxUint16: + var n [2]byte + binary.LittleEndian.PutUint16(n[:], uint16(data)) + s.Hash(n[:]) + case data < math.MaxUint32: + var n [4]byte + binary.LittleEndian.PutUint32(n[:], uint32(data)) + s.Hash(n[:]) + default: + var n [8]byte + binary.LittleEndian.PutUint64(n[:], data) + s.Hash(n[:]) + } +} diff --git a/gosimruntime/globals.go b/gosimruntime/globals.go new file mode 100644 index 0000000..f01a14a --- /dev/null +++ b/gosimruntime/globals.go @@ -0,0 +1,204 @@ +package gosimruntime + +import ( + "cmp" + "fmt" + "log/slog" + "reflect" + "runtime" + "slices" + "time" + "unsafe" +) + +// An atomicValue is a value that can be read or written without triggering the +// race detector. +type atomicValue[T any] struct { + value T +} + +// get returns the current value. It is not marked go:norace to allow inlining. +func (v *atomicValue[T]) get() T { + return v.value +} + +// set writes the current value. It is marked go:norace to allow reads without +// triggering the race detector. +// +//go:norace +func (v *atomicValue[T]) set(t T) { + v.value = t +} + +var globalPtr atomicValue[unsafe.Pointer] + +func allocGlobals() unsafe.Pointer { + return reflect.New(mergedGlobalTyp).UnsafePointer() +} + +type Global[T any] struct { + offset uintptr +} + +func (g *Global[T]) typ() reflect.Type { + var t T + return reflect.TypeOf(t) +} + +//go:norace +func (g *Global[T]) setOffset(x uintptr) { + g.offset = x +} + +type Globaler interface { + setOffset(uintptr) + typ() reflect.Type +} + +func (g Global[T]) Get() *T { + return (*T)(unsafe.Pointer((uintptr(globalPtr.get()) + g.offset))) +} + +var mergedGlobalTyp reflect.Type + +var ( + packages []*PackageInfo + packagesByPath = make(map[string]*PackageInfo) +) + +//go:norace +func InitGlobals(benchmark, initializeShared bool) { + // now := time.Now() + // defer func() { + // totalInitTime += time.Since(now) + // }() + if benchmark { // XXX: optional debug package init time + type stat struct { + path string + duration time.Duration + allocs int64 + size int64 + } + + var stats []stat + + var memStats runtime.MemStats + runtime.ReadMemStats(&memStats) + prev := memStats.TotalAlloc + + outerStart := time.Now() + outerPrev := memStats.TotalAlloc + + for _, pkg := range packages { + if pkg.Globals != nil { + start := time.Now() + + pkg.Globals.Initializers(initializeShared) + + duration := time.Since(start) + runtime.ReadMemStats(&memStats) + stats = append(stats, stat{ + path: pkg.Path, + duration: duration, + allocs: int64(memStats.TotalAlloc - prev), + size: int64(pkg.Globals.Globals.typ().Size()), + }) + prev = memStats.TotalAlloc + } + if pkg.ForTest != nil { + start := time.Now() + pkg.ForTest.Initializers(initializeShared) + duration := time.Since(start) + runtime.ReadMemStats(&memStats) + stats = append(stats, stat{ + path: pkg.Path + " (for test)", + duration: duration, + allocs: int64(memStats.TotalAlloc - prev), + size: int64(pkg.Globals.Globals.typ().Size()), + }) + prev = memStats.TotalAlloc + } + } + + slices.SortFunc(stats, func(a, b stat) int { + return -cmp.Compare(a.duration, b.duration) + }) + + slog.Info("total initialization", "duration", time.Since(outerStart), "allocs", memStats.TotalAlloc-outerPrev, "size", mergedGlobalTyp.Size()) + + for _, pkg := range stats { + slog.Info("initialization", "pkg", pkg.path, "duration", pkg.duration, "allocs", pkg.allocs, "size", pkg.size) + } + } else { + for _, pkg := range packages { + if pkg.Globals != nil { + pkg.Globals.Initializers(initializeShared) + } + if pkg.ForTest != nil { + pkg.ForTest.Initializers(initializeShared) + } + } + } + // log.Println(totalInitTime) // XXX find a place to log this, from TestMain? + // XXX: and make it race safe +} + +//go:norace +func prepareGlobals() { + var fields []reflect.StructField + offset := 0 + for _, pkg := range packages { + if pkg.Globals != nil { + fields = append(fields, reflect.StructField{ + Name: "Globals" + fmt.Sprint(offset), + Type: pkg.Globals.Globals.typ(), + }) + offset++ + } + if pkg.ForTest != nil { + fields = append(fields, reflect.StructField{ + Name: "Globals" + fmt.Sprint(offset), + Type: pkg.ForTest.Globals.typ(), + }) + offset++ + } + } + mergedGlobalTyp = reflect.StructOf(fields) + offset = 0 + for _, pkg := range packages { + if pkg.Globals != nil { + pkg.Globals.Globals.setOffset(mergedGlobalTyp.Field(offset).Offset) + offset++ + } + if pkg.ForTest != nil { + pkg.ForTest.Globals.setOffset(mergedGlobalTyp.Field(offset).Offset) + offset++ + } + } +} + +type Globals struct { + Globals Globaler + Initializers func(initializeShared bool) +} + +type PackageInfo struct { + Path string + Globals *Globals + ForTest *Globals +} + +func RegisterPackage(path string) *PackageInfo { + if runtimeInitialized { + panic("calling register package after already initialized") + } + if p, ok := packagesByPath[path]; ok { + return p + } + p := &PackageInfo{ + Path: path, + } + packages = append(packages, p) + packagesByPath[path] = p + return p +} diff --git a/gosimruntime/log.go b/gosimruntime/log.go new file mode 100644 index 0000000..2bf9b8c --- /dev/null +++ b/gosimruntime/log.go @@ -0,0 +1,122 @@ +package gosimruntime + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "log/slog" + "time" + + "github.com/jellevandenhooff/gosim/internal/prettylog" +) + +var ( + logLevel = flag.String("log-level", "ERROR", "gosim slog log level") + forceTrace = flag.Bool("force-trace", false, "gosim force trace logging") +) + +type logger struct { + out io.Writer + level slog.Level + slog *slog.Logger +} + +func makeLogger(out io.Writer, level slog.Level) logger { + ho := slog.HandlerOptions{ + Level: level, + AddSource: true, + } + handler := slog.NewJSONHandler(out, &ho) + slog := slog.New(wrapHandler{inner: handler}) + + return logger{ + out: out, + level: level, + slog: slog, + } +} + +type wrapHandler struct { + inner slog.Handler +} + +func (w wrapHandler) Enabled(ctx context.Context, level slog.Level) bool { + return w.inner.Enabled(ctx, level) +} + +func (w wrapHandler) Handle(ctx context.Context, r slog.Record) error { + r.Time = time.Unix(0, Nanotime()) + if g := getg(); g != nil { + r.AddAttrs(slog.String("machine", g.machine.label), slog.Int("goroutine", g.ID)) + } + return w.inner.Handle(ctx, r) +} + +func (w wrapHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return wrapHandler{ + inner: w.inner.WithAttrs(attrs), + } +} + +func (w wrapHandler) WithGroup(name string) slog.Handler { + return wrapHandler{ + inner: w.inner.WithGroup(name), + } +} + +type logformatKind string + +const ( + logformatRaw logformatKind = "raw" + logformatIndented logformatKind = "indented" + logformatPretty logformatKind = "pretty" +) + +var logformatFlag logformatKind + +func init() { + logformatFlag = logformatPretty + flag.Func("logformat", "raw|indented|pretty", func(s string) error { + k := logformatKind(s) + if k != logformatRaw && k != logformatIndented && k != logformatPretty { + return fmt.Errorf("bad log kind %q", s) + } + logformatFlag = k + return nil + }) +} + +type indentedWriter struct { + out io.Writer +} + +func (w *indentedWriter) Write(p []byte) (n int, err error) { + if len(p) > 0 && p[len(p)-1] == '\n' { + var x any + if err := json.Unmarshal(p, &x); err == nil { + o := json.NewEncoder(w.out) + o.SetIndent("", " ") + o.Encode(x) + return len(p), nil + } + } + w.out.Write(p) + return len(p), nil +} + +func makeConsoleLogger(out io.Writer) io.Writer { + switch logformatFlag { + case logformatRaw: + return out + case logformatIndented: + return &indentedWriter{ + out: out, + } + case logformatPretty: + return prettylog.NewWriter(out) + default: + panic(logformatFlag) + } +} diff --git a/gosimruntime/map.go b/gosimruntime/map.go new file mode 100644 index 0000000..a963be8 --- /dev/null +++ b/gosimruntime/map.go @@ -0,0 +1,492 @@ +package gosimruntime + +import ( + "iter" + "reflect" + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/race" +) + +const checkMap = false + +// Map implements go's builtin map API with deterministic yet varying iteration +// order. +// +// The Map maintains an ordering of values based on their insertion time. +// Iterators start at a random index and then visit all items. +// +// Map is a value-type that wraps a pointer to an implementation like go's +// builtin map. +type Map[K comparable, V any] struct { + Impl *mapImpl[K, V] +} + +type mapElem[K comparable, V any] struct { + key K + value V + present bool +} + +type mapImpl[K comparable, V any] struct { + // underlying maps keys to a position in small concatenated with values + underlying map[K]int32 + + // small and values together form the backing array for the map. + // values might be nil for small maps. + small [4]mapElem[K, V] + values []mapElem[K, V] + + // free holds free indices in the backing array of the map. Only + // allocated and meaningful when values is non-nil. + free []int32 + + // TODO: use underlying equal etc from map? + // TODO: use raw bytes like map? + // TODO: somehow support reflect.MapOf? +} + +func NewMap[K comparable, V any]() Map[K, V] { + return Map[K, V]{ + Impl: &mapImpl[K, V]{}, + } +} + +func NilMap[K comparable, V any]() Map[K, V] { + return Map[K, V]{} +} + +// ExtractMap extracts a Map from named Map types. +func ExtractMap[M ~struct{ Impl *mapImpl[K, V] }, K comparable, V any](m M) Map[K, V] { + return Map[K, V](m) +} + +// A MapLiteral is a slice of key-value pairs that can be converted into a Map. +type MapLiteral[K comparable, V any] []KV[K, V] + +type KV[K comparable, V any] struct { + K K + V V +} + +func (pairs MapLiteral[K, V]) Build() Map[K, V] { + m := NewMap[K, V]() + for _, pair := range pairs { + m.Set(pair.K, pair.V) + } + return m +} + +func (m Map[K, V]) Clear() { + clear(m.Impl.underlying) + for i := range m.Impl.small { + m.Impl.small[i] = mapElem[K, V]{} + } + for i := range m.Impl.values { + m.Impl.values[i] = mapElem[K, V]{} + } + if m.Impl.free != nil { + n := len(m.Impl.values) + len(m.Impl.small) + m.Impl.free = m.Impl.free[:n] + for i := range n { + m.Impl.free[i] = int32(i) + } + } +} + +func (m Map[K, V]) elem(idx int) *mapElem[K, V] { + if idx < len(m.Impl.small) { + return &m.Impl.small[idx] + } + return &m.Impl.values[idx-len(m.Impl.small)] +} + +func (m Map[K, V]) GetOk(k K) (V, bool) { + if m.Impl == nil { + var v V + return v, false + } + + if m.Impl.underlying == nil { + for i := range m.Impl.small { + elem := &m.Impl.small[i] + if elem.present && elem.key == k { + return elem.value, true + } + } + var v V + return v, false + } + + idx, ok := m.Impl.underlying[k] + if !ok { + var v V + return v, false + } + + elem := m.elem(int(idx)) + if checkMap { + if elem.key != k { + panic("bad elem.key") + } + } + + return elem.value, true +} + +func (m Map[K, V]) Get(k K) V { + v, _ := m.GetOk(k) + return v +} + +func (m Map[K, V]) resizeInitial() { + oldN := len(m.Impl.small) + newN := oldN * 2 + + newValues := make([]mapElem[K, V], newN-len(m.Impl.small)) + newFree := make([]int32, oldN, newN) + + m.Impl.underlying = make(map[K]int32, newN) + for i := range m.Impl.small { + elem := &m.Impl.small[i] + if checkMap { + if !elem.present { + panic("bad elem.present") + } + } + m.Impl.underlying[elem.key] = int32(i * 2) + } + + for i := len(m.Impl.small) / 2; i < len(m.Impl.small); i++ { + newValues[2*i-len(m.Impl.small)] = m.Impl.small[i] + } + for i := len(m.Impl.small)/2 - 1; i >= 0; i-- { + m.Impl.small[i*2+1] = mapElem[K, V]{} + m.Impl.small[i*2] = m.Impl.small[i] + } + + for i := 0; i < oldN; i++ { + newFree[i] = int32(i*2 + 1) + } + + m.Impl.values = newValues + m.Impl.free = newFree +} + +func (m Map[K, V]) resizeLater() { + oldN := len(m.Impl.values) + len(m.Impl.small) + newN := oldN * 2 + + newValues := make([]mapElem[K, V], newN-len(m.Impl.small)) + newFree := make([]int32, oldN, newN) + + for k, idx := range m.Impl.underlying { + m.Impl.underlying[k] = idx * 2 + } + + for i := len(m.Impl.small) / 2; i < len(m.Impl.small); i++ { + newValues[2*i-len(m.Impl.small)] = m.Impl.small[i] + } + for i := len(m.Impl.small)/2 - 1; i >= 0; i-- { + m.Impl.small[i*2+1] = mapElem[K, V]{} + m.Impl.small[i*2] = m.Impl.small[i] + } + for i := len(m.Impl.small); i < oldN; i++ { + newValues[2*i-len(m.Impl.small)] = m.Impl.values[i-len(m.Impl.small)] + } + + for i := 0; i < oldN; i++ { + newFree[i] = int32(2*i + 1) + } + + m.Impl.values = newValues + m.Impl.free = newFree +} + +func (m Map[K, V]) Set(k K, v V) { + if m.Impl.underlying == nil { + freeIdx := -1 + for i := range m.Impl.small { + elem := &m.Impl.small[i] + if elem.present && elem.key == k { + elem.value = v + return + } + if !elem.present { + freeIdx = i + } + } + + if freeIdx != -1 { + m.Impl.small[freeIdx] = mapElem[K, V]{ + present: true, + key: k, + value: v, + } + return + } + + m.resizeInitial() + } else { + idx, ok := m.Impl.underlying[k] + if ok { + elem := m.elem(int(idx)) + if checkMap { + if elem.key != k { + panic("bad elem.key") + } + } + elem.value = v + return + } + } + + if len(m.Impl.free) == 0 { + m.resizeLater() + } + + freeIdx := int(fastrandn(uint32(len(m.Impl.free)))) + idx := m.Impl.free[freeIdx] + m.Impl.free[freeIdx] = m.Impl.free[len(m.Impl.free)-1] + m.Impl.free = m.Impl.free[:len(m.Impl.free)-1] + + m.Impl.underlying[k] = idx + *m.elem(int(idx)) = mapElem[K, V]{key: k, value: v, present: true} +} + +func (m Map[K, V]) Len() int { + if m.Impl == nil { + return 0 + } + + if race.Enabled { + // for TestRaceMapLenDelete + race.Read(unsafe.Pointer(m.Impl)) + } + + if m.Impl.underlying == nil { + count := 0 + for i := range m.Impl.small { + elem := &m.Impl.small[i] + if elem.present { + count++ + } + } + return count + } + + return len(m.Impl.underlying) +} + +func (m Map[K, V]) Delete(k K) { + if m.Impl == nil { + // XXX: wtf + return + } + + if race.Enabled { + // for TestRaceMapLenDelete + race.Write(unsafe.Pointer(m.Impl)) + } + + if m.Impl.underlying == nil { + for i := range m.Impl.small { + elem := &m.Impl.small[i] + if elem.present && elem.key == k { + elem.present = false + return + } + } + } else { + idx, ok := m.Impl.underlying[k] + if !ok { + return + } + + delete(m.Impl.underlying, k) + *m.elem(int(idx)) = mapElem[K, V]{} + m.Impl.free = append(m.Impl.free, idx) + } +} + +type MapIter[K comparable, V any] struct { + impl *mapImpl[K, V] + size int + start int + offset int +} + +func (i *MapIter[K, V]) Next() (K, V, bool) { + if i.impl == nil { + var k K + var v V + return k, v, false + } + + for i.size != len(i.impl.values)+len(i.impl.small) { + i.size *= 2 + i.offset *= 2 + i.start *= 2 + } + + for i.offset < i.size { + idx := i.start + i.offset + if idx >= i.size { + idx -= i.size + } + elem := i.impl.outer().elem(idx) + i.offset++ + + if elem.present { + return elem.key, elem.value, true + } + } + + var k K + var v V + return k, v, false +} + +func (m Map[K, V]) Iter() MapIter[K, V] { + if m.Impl == nil { + return MapIter[K, V]{} + } + size := len(m.Impl.values) + len(m.Impl.small) + start := int(fastrandn(uint32(size))) + return MapIter[K, V]{ + impl: m.Impl, + size: size, + start: start, + offset: 0, + } +} + +func (m Map[K, V]) Range() iter.Seq2[K, V] { + return func(yield func(K, V) bool) { + iter := m.Iter() + for { + k, v, ok := iter.Next() + if !ok { + break + } + if !yield(k, v) { + break + } + } + } +} + +// CloneMap clones a map, exposed by the runtime for maps.Clone. +func CloneMap(m any) any { + return m.(mapsCloner).mapsClone() +} + +type mapsCloner interface { + mapsClone() any +} + +func (m Map[K, V]) mapsClone() any { + result := NewMap[K, V]() + iter := m.Iter() + for k, v, ok := iter.Next(); ok; k, v, ok = iter.Next() { + result.Set(k, v) + } + return result +} + +// ReflectMap provides a reflection API to access a Map, used by gosim's +// internal/reflect package. +type ReflectMap interface { + GetIndex(reflect.Value) (reflect.Value, bool) + SetIndex(reflect.Value, reflect.Value) + Iter() ReflectMapIter + Len() int + UnsafePointer() unsafe.Pointer + Type() ReflectMapType +} + +type ReflectMapIter interface { + Key() reflect.Value + Value() reflect.Value + Next() bool +} + +type ReflectMapType interface { + Make() any + KeyType() reflect.Type + ElemType() reflect.Type +} + +var _ ReflectMap = &mapImpl[any, any]{} + +func (m *mapImpl[K, V]) Type() ReflectMapType { + return reflectMapType[K, V]{} +} + +func (m *mapImpl[K, V]) Len() int { + return m.outer().Len() +} + +func (m *mapImpl[K, V]) UnsafePointer() unsafe.Pointer { + return unsafe.Pointer(m) +} + +func (m *mapImpl[K, V]) SetIndex(key, value reflect.Value) { + m.outer().Set(key.Interface().(K), value.Interface().(V)) +} + +func (m *mapImpl[K, V]) outer() Map[K, V] { + return Map[K, V]{Impl: m} +} + +func (m *mapImpl[K, V]) Iter() ReflectMapIter { + return &reflectMapIter[K, V]{inner: m.outer().Iter()} +} + +func (m *mapImpl[K, V]) GetIndex(key reflect.Value) (reflect.Value, bool) { + v, ok := m.outer().GetOk(key.Interface().(K)) + return reflect.ValueOf(&v).Elem(), ok +} + +type reflectMapIter[K comparable, V any] struct { + key K + value V + inner MapIter[K, V] +} + +var _ ReflectMapIter = &reflectMapIter[any, any]{} + +func (i *reflectMapIter[K, V]) Next() bool { + k, v, ok := i.inner.Next() + i.key = k + i.value = v + return ok +} + +func (i *reflectMapIter[K, V]) Key() reflect.Value { + k := i.key + return reflect.ValueOf(&k).Elem() +} + +func (i *reflectMapIter[K, V]) Value() reflect.Value { + v := i.value + return reflect.ValueOf(&v).Elem() +} + +type reflectMapType[K comparable, V any] struct{} + +var _ ReflectMapType = reflectMapType[any, any]{} + +func (reflectMapType[K, V]) Make() any { + return NewMap[K, V]() +} + +func (reflectMapType[K, V]) KeyType() reflect.Type { + var k [0]K + return reflect.TypeOf(k).Elem() +} + +func (reflectMapType[K, V]) ElemType() reflect.Type { + var v [0]V + return reflect.TypeOf(v).Elem() +} diff --git a/gosimruntime/map_test.go b/gosimruntime/map_test.go new file mode 100644 index 0000000..43b20c4 --- /dev/null +++ b/gosimruntime/map_test.go @@ -0,0 +1,175 @@ +package gosimruntime_test + +import ( + "testing" + + "pgregory.net/rapid" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func TestMapIterDeleteDuplicatesHappen(t *testing.T) { + success := false + for !success { + m := make(map[int]struct{}) + m[0] = struct{}{} + m[1] = struct{}{} + count := 0 + for k := range m { + if k == 1 { + count++ + } + delete(m, 0) + delete(m, 1) + m[1] = struct{}{} + } + if count > 0 { + success = true + break + } + } +} + +func TestMapIterClearDuplicatesHappen(t *testing.T) { + success := false + for !success { + m := make(map[int]struct{}) + m[0] = struct{}{} + m[1] = struct{}{} + count := 0 + for k := range m { + if k == 1 { + count++ + } + clear(m) + m[1] = struct{}{} + } + if count > 0 { + success = true + break + } + } +} + +func TestCheckMap(t *testing.T) { + t.Cleanup(gosimruntime.CleanupMockFastrand) + rapid.Check(t, checkMap) +} + +func checkMap(t *rapid.T) { + gosimruntime.SetupMockFastrand() + + m := gosimruntime.NewMap[string, string]() + model := make(map[string]string) + + actions := make(map[string]func(t *rapid.T)) + + var iter gosimruntime.MapIter[string, string] + var iterRequired, iterSeen map[string]struct{} + haveIter := false + + actions["get"] = func(t *rapid.T) { + k := rapid.String().Draw(t, "k") + v, ok := m.GetOk(k) + expectedV, expectedOk := model[k] + if ok != expectedOk { + t.Errorf("got ok %v, expected %v", ok, expectedOk) + } + if v != expectedV { + t.Errorf("got v %q, expected %q", v, expectedV) + } + } + + actions["clear"] = func(t *rapid.T) { + m.Clear() + clear(model) + if haveIter { + clear(iterRequired) + // after clearing a duplicate is legal, see TestMapIterClearDuplicatesHappen. + clear(iterSeen) + } + } + + actions["set"] = func(t *rapid.T) { + k := rapid.String().Draw(t, "k") + v := rapid.String().Draw(t, "v") + m.Set(k, v) + model[k] = v + } + + actions["delete"] = func(t *rapid.T) { + k := rapid.String().Draw(t, "k") + m.Delete(k) + delete(model, k) + if haveIter { + delete(iterRequired, k) + // after deleting a duplicate is legal, see TestMapIterDeleteDuplicatesHappen. + delete(iterSeen, k) + } + } + + actions["start-iter"] = func(t *rapid.T) { + if haveIter { + t.Skip() + } + iter = m.Iter() + iterRequired = make(map[string]struct{}) + for k := range model { + iterRequired[k] = struct{}{} + } + iterSeen = make(map[string]struct{}) + haveIter = true + } + + actions["step-iter"] = func(t *rapid.T) { + if !haveIter { + t.Skip() + } + // might be too high but not a problem + steps := rapid.IntRange(1, 1+len(model)).Draw(t, "steps") + for i := 0; i < steps; i++ { + k, v, ok := iter.Next() + if !ok { + t.Log("got iter finished") + if len(iterRequired) > 0 { + t.Errorf("unexpected iter stop, still missing %v", iterRequired) + } + haveIter = false + break + } + t.Logf("got iter key %q value %q", k, v) + if _, ok := model[k]; !ok { + t.Errorf("unexpected iter key %q", k) + } + if expectedV := model[k]; expectedV != v { + t.Errorf("iter value mismatch for key %q: got %q, expected %q", k, v, expectedV) + } + if _, ok := iterSeen[k]; ok { + t.Errorf("duplicate iter key %q", k) + } + iterSeen[k] = struct{}{} + delete(iterRequired, k) + } + } + + actions[""] = func(t *rapid.T) { + if got, expected := m.Len(), len(model); got != expected { + t.Errorf("length mismatch: got %d, expected %d", got, expected) + } + iter := m.Iter() + for { + k, v, ok := iter.Next() + if !ok { + break + } + expectedV, expectedOk := model[k] + if !expectedOk { + t.Errorf("contents mismatch: got unexpected key %q with value %q", k, v) + } else if v != expectedV { + t.Errorf("contents mismatch: key %q got value %q, expected value %q", k, v, expectedV) + } + } + } + + t.Repeat(actions) +} diff --git a/gosimruntime/raceutil.go b/gosimruntime/raceutil.go new file mode 100644 index 0000000..0a3d6ca --- /dev/null +++ b/gosimruntime/raceutil.go @@ -0,0 +1,15 @@ +package gosimruntime + +// TestRaceToken is a publicly accessible RaceToken to allow synchronization +// between different machines that otherwise have no common happens-before +// edges. +// +// Machines do not have any happens-before edges between each other. This is +// fine in most cases because machines do not share any memory and have their +// own globals. +// +// In low-level gosim tests, though, it can be convenient to share some memory +// (eg. atomic.Bool indicating success). To do that without triggering the race +// detector, call TestRaceToken.Release before creating the machine and then +// TestRaceToken.Acquire inside the machine. +var TestRaceToken = MakeRaceToken() diff --git a/gosimruntime/raceutil_norace.go b/gosimruntime/raceutil_norace.go new file mode 100644 index 0000000..5596729 --- /dev/null +++ b/gosimruntime/raceutil_norace.go @@ -0,0 +1,39 @@ +//go:build !race + +package gosimruntime + +// RaceToken is a zero overhead version of RaceToken for non-race builds. It +// does not do anything. +type RaceToken struct{} + +func MakeRaceToken() RaceToken { + return RaceToken{} +} + +func EmptyRaceToken() RaceToken { + return RaceToken{} +} + +func (t RaceToken) Empty() bool { + return true +} + +func (t RaceToken) Release() { +} + +func (r RaceToken) Acquire() { +} + +// NoraceAppend is a zero overhead version of NoraceAppend. It wraps append. +func NoraceAppend[A any](to []A, from ...A) []A { + return append(to, from...) +} + +// NoraceCopy is a zero overhead version of NoraceCopy. It shallowly copies a +// slice. +func NoraceCopy[A any](a []A) []A { + if a == nil { + return nil + } + return append([]A(nil), a...) +} diff --git a/gosimruntime/raceutil_race.go b/gosimruntime/raceutil_race.go new file mode 100644 index 0000000..4419662 --- /dev/null +++ b/gosimruntime/raceutil_race.go @@ -0,0 +1,72 @@ +//go:build race + +package gosimruntime + +import ( + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/race" +) + +const debugRaceToken = false + +// A RaceToken is a token that can be used to establish happens-before +// dependencies for go's race detector. +type RaceToken struct { + elem *int +} + +//go:norace +func MakeRaceToken() RaceToken { + return RaceToken{ + // TODO: we could have a pool of... 1024 of these and randomly pick them? + // "should" catch most problems? + elem: new(int), + } +} + +//go:norace +func EmptyRaceToken() RaceToken { + return RaceToken{ + elem: nil, + } +} + +//go:norace +func (t RaceToken) Empty() bool { + return t.elem == nil +} + +//go:norace +func (t RaceToken) Release() { + if debugRaceToken && t.elem == nil { + panic("nil token") + } + race.Release(unsafe.Pointer(t.elem)) +} + +//go:norace +func (t RaceToken) Acquire() { + if debugRaceToken && t.elem == nil { + panic("nil token") + } + race.Acquire(unsafe.Pointer(t.elem)) +} + +// NoraceAppend appends to a slice without triggering the race detector. +// +//go:norace +func NoraceAppend[A any](to []A, from ...A) []A { + for _, x := range from { + to = append(to, x) + } + return to +} + +// NoraceCopy copies a slice without triggering the race detector. +func NoraceCopy[A any](a []A) []A { + if a == nil { + return nil + } + return NoraceAppend([]A(nil), a...) +} diff --git a/gosimruntime/rand.go b/gosimruntime/rand.go new file mode 100644 index 0000000..c9ec794 --- /dev/null +++ b/gosimruntime/rand.go @@ -0,0 +1,52 @@ +// Copyright 2023 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package gosimruntime + +import ( + "math/bits" +) + +func Fastrand64() uint64 { + return fastrand64() +} + +func Fastrandn(n uint32) uint32 { + return fastrandn(n) +} + +type fastrander struct { + state uint64 +} + +func (f *fastrander) fastrandn(n uint32) uint32 { + // This is similar to fastrand() % n, but faster. + // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ + return uint32(uint64(f.fastrand32()) * uint64(n) >> 32) +} + +func (f *fastrander) fastrand32() uint32 { + return uint32(f.fastrand64()) +} + +// norace to allow modifying state +// +//go:norace +func (f *fastrander) fastrand64() uint64 { + if inControlledNondeterminism() { + // TODO: return an actually random value? + return 0 + } + f.state += 0xa0761d6478bd642f + hi, lo := bits.Mul64(f.state, f.state^0xe7037ed1a0b428db) + return hi ^ lo +} + +func fastrandn(n uint32) uint32 { + return gs.get().fastrander.fastrandn(n) +} + +func fastrand64() uint64 { + return gs.get().fastrander.fastrand64() +} diff --git a/gosimruntime/rand_test.go b/gosimruntime/rand_test.go new file mode 100644 index 0000000..06a162d --- /dev/null +++ b/gosimruntime/rand_test.go @@ -0,0 +1,13 @@ +package gosimruntime + +// SetupMockFastrand mocks gs and fastrander to test the map implementation. +func SetupMockFastrand() { + gs.set(&scheduler{ + fastrander: fastrander{state: 0}, + }) +} + +// CleanupMockFastrand cleans up the mock gs. +func CleanupMockFastrand() { + gs.set(nil) +} diff --git a/gosimruntime/runtime.go b/gosimruntime/runtime.go new file mode 100644 index 0000000..afc434d --- /dev/null +++ b/gosimruntime/runtime.go @@ -0,0 +1,1043 @@ +package gosimruntime + +import ( + "bytes" + "context" + "errors" + "flag" + "fmt" + "io" + "log/slog" + mathrand "math/rand" + "os" + "reflect" + "runtime" + "strings" + "sync/atomic" + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/coro" + "github.com/jellevandenhooff/gosim/internal/race" +) + +type constErr struct { + error +} + +func makeConstErr(err error) error { + return constErr{error: err} +} + +var ( + ErrPaniced = makeConstErr(errors.New("paniced")) + ErrAborted = makeConstErr(errors.New("aborted")) + ErrMainReturned = makeConstErr(errors.New("main returned")) + ErrTimeout = makeConstErr(errors.New("timeout")) + ErrTestFailed = makeConstErr(errors.New("test failed")) +) + +func getEntryFunc(fun func()) string { + return runtime.FuncForPC(reflect.ValueOf(fun).Pointer()).Name() +} + +var aslrBaseAddr = uintptr(reflect.ValueOf(reflect.ValueOf).UnsafePointer()) + +// relativeFuncAddr returns the address of a function ignoring address space +// layout randomization. This makes function pointers comparable across +// processes to check determinism. +func relativeFuncAddr(fun func()) uintptr { + return uintptr(reflect.ValueOf(fun).UnsafePointer()) - aslrBaseAddr +} + +var ( + insideGosimRun atomic.Bool + gs atomicValue[*scheduler] +) + +func setgs(newgs *scheduler) func() { + if !insideGosimRun.CompareAndSwap(false, true) { + panic("gosimruntime already running; only support one per process") + } + gs.set(newgs) + if newgs.current.get() != nil { + panic("when setting gs expecting nil current") + } + globalPtr.set(nil) + + return func() { + gs.set(nil) + globalPtr.set(nil) + if !insideGosimRun.CompareAndSwap(true, false) { + panic("gosimruntime already stopped!!!???") + } + } +} + +func getg() *goroutine { + return gs.get().current.get() +} + +type Machine struct { + label string + globals unsafe.Pointer +} + +func newMachine(label string) *Machine { + return &Machine{ + label: label, + globals: allocGlobals(), + } +} + +type scheduler struct { + goroutines intrusiveList[*goroutine] + runnable intrusiveList[*goroutine] + nextGoroutineID int + + goroutinesById map[int]*goroutine + + envs []string + + current atomicValue[*goroutine] + + tracer *tracer + + fastrander fastrander + + clock *clock + + nextEventID int + + logger logger + + abortErr error + + // race token given to background goroutines so they can read gs fields + globaltoken RaceToken + + inupcall bool +} + +func newScheduler(seed int64, logger logger, tracer *tracer, extraenv []string) *scheduler { + clock := newClock() + + rand := mathrand.New(mathrand.NewSource(seed)) + + s := &scheduler{ + goroutines: make([]*goroutine, 0, 1024), + runnable: make([]*goroutine, 0, 1024), + nextGoroutineID: 1, + fastrander: fastrander{state: rand.Uint64()}, + goroutinesById: make(map[int]*goroutine), + clock: clock, + nextEventID: 1, + logger: logger, + tracer: tracer, + globaltoken: MakeRaceToken(), + envs: append(extraenv, + "GOSIM_LOG_LEVEL="+logger.level.String(), + ), + } + s.globaltoken.Release() + + return s +} + +type intrusiveList[A any] []A + +type intrusiveIndex struct{ idx int } + +func (l *intrusiveList[A]) add(elem A, idxFunc func(elem A) *intrusiveIndex) { + oldIdx := idxFunc(elem).idx + if oldIdx != -1 { + panic(elem) + } + idxFunc(elem).idx = len(*l) + *l = append(*l, elem) +} + +func (l *intrusiveList[A]) remove(elem A, idxFunc func(elem A) *intrusiveIndex) { + oldIdx := idxFunc(elem).idx + if oldIdx == -1 { + panic(elem) + } + n := len(*l) + replacement := (*l)[n-1] + (*l)[oldIdx] = replacement + idxFunc(replacement).idx = oldIdx + *l = (*l)[:n-1] + idxFunc(elem).idx = -1 +} + +func (s *scheduler) addGoroutine(g *goroutine) { + s.goroutines.add(g, (*goroutine).allIdxPtr) + s.goroutinesById[g.ID] = g + s.addRunnableGoroutine(g) +} + +func (s *scheduler) removeGoroutine(g *goroutine) { + s.goroutines.remove(g, (*goroutine).allIdxPtr) + delete(s.goroutinesById, g.ID) + if g.runnableIdx.idx != -1 { + s.removeRunnableGoroutine(g) + } +} + +func (s *scheduler) addRunnableGoroutine(g *goroutine) { + s.runnable.add(g, (*goroutine).runnableIdxPtr) +} + +func (s *scheduler) removeRunnableGoroutine(g *goroutine) { + s.runnable.remove(g, (*goroutine).runnableIdxPtr) +} + +func (s *scheduler) pickRandomOtherGoroutine(g *goroutine) *goroutine { + if len(s.goroutines) == 1 { + return nil + } + idx := fastrandn(uint32(len(s.goroutines) - 1)) + if s.goroutines[idx] == g { + idx++ + } + return s.goroutines[idx] +} + +func (s *scheduler) getNextGoroutineID() int { + id := s.nextGoroutineID + s.nextGoroutineID++ + return id +} + +func (s *scheduler) Go(fun func(), machine *Machine, token RaceToken) *goroutine { + id := s.getNextGoroutineID() + + eventID := NextEventID() + + if !token.Empty() { + // for getEntryFunc + token.Acquire() + } + if s.logger.slog.Enabled(context.TODO(), slog.LevelDebug) { + s.logger.slog.Log(context.TODO(), slog.LevelDebug, "spawning goroutine", "eventid", eventID, "childgoroutine", id, "entry", getEntryFunc(fun)) + } + + // XXX: log/trace happens-before spawn, happens-after start? + newg := newGoroutine(id, getg(), eventID, fun, machine, token) + s.addGoroutine(newg) + + return newg +} + +func (s *scheduler) Run() error { + // TODO: add a wall-clock timeout for running any given step. If the step + // hangs for too long, it probably is doing something weird (using + // uninstrumented sync or some spin.) Call out the stuck goroutine as the + // problem. + + // TODO: move scheduler deterministic seed/state initialization here? + + for len(s.goroutines) > 0 && s.abortErr == nil { + // XXX: can we make the scheduler itself run in the goroutines? so we + // don't have to ping-pong another goroutine switch + + for { + handler, ok := s.clock.maybefire() + if !ok { + break + } + if handler != nil { + handler.handler(handler) + // s.firetimer(handler) + } + } + + if len(s.runnable) == 0 && s.clock.anyWaiting() { + s.clock.doadvance() + continue + } + + if len(s.runnable) == 0 { + buf := make([]byte, 256*1024) + buf = buf[:runtime.Stack(buf, true)] + return fmt.Errorf("all goroutines blocked\n\n%s", string(buf)) + } + + pick := s.runnable[s.fastrander.fastrandn(uint32(len(s.runnable)))] + + if s.tracer != nil { + s.tracer.recordIntInt(traceKeyRunPick, uint64(pick.ID), uint64(s.fastrander.state)) + } + s.current.set(pick) + pick.step() + s.current.set(nil) + if s.tracer != nil { + var flags uint64 + if pick.finished { + flags |= 1 + } + if pick.waiting { + flags |= 1 << 1 + } + s.tracer.recordIntInt(tracekeyRunResult, flags, s.fastrander.state) + } + if pick.finished { + s.removeGoroutine(pick) + freeGoroutine(pick) + } + } + + // TODO: log exit reason? log that we're killing a bunch of goroutines? + + for _, pick := range s.goroutines { + pick.abort() + freeGoroutine(pick) + } + s.goroutines = nil + + // TODO: Test abortErr? + return s.abortErr +} + +var ( + globalRuntime func(func()) + runtimeInitialized bool +) + +var benchmarkGlobals = flag.Bool("benchmark-globals", false, "benchmark global initialization cost") + +func initializeRuntime(rt func(func())) { + if runtimeInitialized { + panic("already initialized") + } + runtimeInitialized = true + + globalRuntime = rt + prepareGlobals() + if err := initSharedGlobals(); err != nil { + panic(err) + } + + if *benchmarkGlobals { + doBenchmarkGlobals() + } +} + +type internalRunResult struct { + Seed int64 + Trace []byte + Failed bool + LogOutput []byte + Err error +} + +func run(f func(), seed int64, enableTracer bool, captureLog bool, logLevelOverride string, simLogger io.Writer, extraenv []string) internalRunResult { + if !runtimeInitialized { + panic("not yet initialized") + } + + enableTracer = enableTracer || *forceTrace + + logLevelStr := *logLevel + if logLevelOverride != "" { + logLevelStr = logLevelOverride + } + var logLevel slog.Level + if err := logLevel.UnmarshalText([]byte(logLevelStr)); err != nil { + panic("bad log-level " + logLevelStr) + } + + var logOut io.Writer = simLogger + var logBuffer *bytes.Buffer + if captureLog { + logBuffer = &bytes.Buffer{} + logOut = io.MultiWriter(logBuffer, logOut) + } + + logger := makeLogger(logOut, logLevel) + + var tracer *tracer + if enableTracer { + tracer = newTracer(logger.slog) + logOut = io.MultiWriter(&traceWriter{trace: tracer}, logger.out) + logger = makeLogger(logOut, logLevel) + } + + scheduler := newScheduler(seed, logger, tracer, extraenv) + defer setgs(scheduler)() + + InitSteps() + + defaultMachine := newMachine("") + + wrapper := func() { + InitGlobals(false, false) + globalRuntime(f) + } + + scheduler.Go(wrapper, defaultMachine, scheduler.globaltoken) + if enableTracer { + // XXX: add a top-level force-trace flag + // XXX: this flag is kinda sad... reorg? + tracer.recordIntInt(traceKeyRunStarted, 0, 0) + } + runErr := scheduler.Run() + if runErr == ErrMainReturned { + runErr = nil + } + var trace []byte + if enableTracer { + tracer.recordIntInt(traceKeyRunFinished, 0, 0) + trace = tracer.finalize() + } + var capturedLog []byte + if captureLog { + capturedLog = logBuffer.Bytes() + } + return internalRunResult{ + Seed: seed, + Trace: trace, + Failed: runErr != nil, // TODO: get rid of Failed or Err? + LogOutput: capturedLog, + Err: runErr, + } +} + +var sharedGlobalsInitialized bool + +func initSharedGlobals() error { + if sharedGlobalsInitialized { + panic("help already initialized") + } + sharedGlobalsInitialized = true + + logger := makeLogger(makeConsoleLogger(os.Stderr), slog.LevelInfo) + scheduler := newScheduler(0, logger, nil, nil) + + defer setgs(scheduler)() + + defaultMachine := newMachine("") + + scheduler.Go(func() { + InitGlobals(false, true) + SetAbortError(ErrMainReturned) + }, defaultMachine, scheduler.globaltoken) + if err := scheduler.Run(); err != nil && err != ErrMainReturned { + return err + } + + return nil +} + +func doBenchmarkGlobals() error { + logger := makeLogger(makeConsoleLogger(os.Stderr), slog.LevelInfo) + scheduler := newScheduler(0, logger, nil, nil) + + defer setgs(scheduler)() + + defaultMachine := newMachine("") + + scheduler.Go(func() { + InitGlobals(true, false) + SetAbortError(ErrMainReturned) + }, defaultMachine, scheduler.globaltoken) + if err := scheduler.Run(); err != nil && err != ErrMainReturned { + return err + } + + return nil +} + +type corotask int + +const ( + corotaskNone corotask = iota + corotaskAbort + corotaskStep +) + +type goroutine struct { + ID int + allIdx intrusiveIndex // idx in scheduler.goroutines, or -1 + runnableIdx intrusiveIndex // idx in scheduler.runnable, or -1 + + spawnEventID int + machine *Machine + parent *goroutine + + finished bool + + coro coro.Upcallcoro + + corotask corotask + + parksynctoken RaceToken + shouldacquire bool + shouldrelease *racesynchelper + + selected *sudog + waiters *sudog // linked-list + + // true if called yield(true) and not yet continued + waiting bool + // true if explicitly paused + paused bool + + parkWait bool + parkPaniced bool + parkPanicTraceback []byte + parkPanicValue string + + entryFun func() + entryToken RaceToken + + // per-goroutine syscall struct so we don't do allocs + syscallCache unsafe.Pointer // *Syscall +} + +func newGoroutine(id int, parent *goroutine, spawnEventID int, fun func(), machine *Machine, token RaceToken) *goroutine { + gs := gs.get() + + g := allocGoroutine() + g.syscallCache = syscallAllocator() // &Syscall{} // XXX: doing this rn because of reused sema + g.ID = id + g.parent = parent + g.spawnEventID = spawnEventID + g.allIdx.idx = -1 + g.runnableIdx.idx = -1 + g.machine = machine + g.parksynctoken = MakeRaceToken() + + if gs.tracer != nil { + gs.tracer.recordIntInt(traceKeyCreating, uint64(g.ID), uint64(relativeFuncAddr(fun))) // XXX: test this pointer + } + + prev := getg() + gs.current.set(g) + g.entryFun = fun + g.entryToken = token + race.Disable() // make sure no happens-before to the scheduler + g.coro.Start(goroutineEntrypoint) + race.Enable() + gs.current.set(prev) + + return g +} + +var goroutinePool []*goroutine + +func allocGoroutine() *goroutine { + if n := len(goroutinePool); n > 0 { + g := goroutinePool[n-1] + goroutinePool = goroutinePool[:n-1] + return g + } + return &goroutine{} +} + +// XXX: if ANYONE still refers to this goroutine we will be in a world of +// pain... eg. select or timer of an aborted goroutine??????????? +func freeGoroutine(g *goroutine) { + g.ID = 0 + g.allIdx.idx = -1 + g.runnableIdx.idx = -1 + g.spawnEventID = 0 + g.machine = nil + g.parent = nil + g.finished = false + g.coro = coro.Upcallcoro{} + g.corotask = corotaskNone + g.parksynctoken = RaceToken{} + g.shouldacquire = false + g.shouldrelease = nil + g.selected = nil + g.waiters = nil + g.waiting = false + g.paused = false + g.parkWait = false + g.parkPaniced = false + g.parkPanicTraceback = nil + g.parkPanicValue = "" + g.entryFun = nil + g.entryToken = RaceToken{} + g.syscallCache = nil + goroutinePool = append(goroutinePool, g) +} + +func (g *goroutine) allIdxPtr() *intrusiveIndex { + return &g.allIdx +} + +func (g *goroutine) runnableIdxPtr() *intrusiveIndex { + return &g.runnableIdx +} + +func (g *goroutine) setWaiting(waiting bool) { + gs := gs.get() + g.waiting = waiting + if g.runnableIdx.idx == -1 && !(g.waiting || g.paused) { + gs.addRunnableGoroutine(g) + } else if g.runnableIdx.idx != -1 && (g.waiting || g.paused) { + gs.removeRunnableGoroutine(g) + } +} + +func (g *goroutine) setPaused(paused bool) { + gs := gs.get() + g.paused = paused + if g.runnableIdx.idx == -1 && !(g.waiting || g.paused) { + gs.addRunnableGoroutine(g) + } else if g.runnableIdx.idx != -1 && (g.waiting || g.paused) { + gs.removeRunnableGoroutine(g) + } +} + +// TODO: should this be norace? think about it +// +//go:norace +func (g *goroutine) allocWaiter() *sudog { + sg := allocSudog() + sg.goroutine = g + sg.waitersNext = g.waiters + g.waiters = sg + return sg +} + +// TODO: should this be norace? think about it +// +//go:norace +func (g *goroutine) releaseWaiters() { + g.selected = nil + next := g.waiters + g.waiters = nil + for next != nil { + cur := next + cur.goroutine = nil + cur.item = nil + next = cur.waitersNext + cur.waitersNext = nil + freeSudog(cur) + } +} + +const debugUpcall = true + +//go:norace +func (g *goroutine) upcall(f func()) { + race.Disable() + if debugUpcall { + if gs.get().inupcall { + panic("already in upcall") + } + gs.get().inupcall = true + } + g.coro.Upcall(f) + if debugUpcall { + gs.get().inupcall = false + } + race.Enable() +} + +func (g *goroutine) step() { + globalPtr.set(g.machine.globals) + + if inControlledNondeterminism() { + panic("help") + } + + g.corotask = corotaskStep + g.coro.Next() + if g.parkWait { + g.setWaiting(true) + } + if g.parkPaniced { + if gs.get().abortErr == nil { + gs.get().abortErr = ErrPaniced + } + gs.get().logger.slog.Error("uncaught panic", + "traceback", strings.Split(strings.ReplaceAll(string(g.parkPanicTraceback), "\t", " "), "\n"), + "panic", g.parkPanicValue) + } +} + +// abort stops the goroutine. +// this is a dangerous call, make sure that noone will refer to the goroutine later. +func (g *goroutine) abort() { + gs := gs.get() + + // XXX: trace? + if getg() == g { + panic("aborting self") + } + + // if currently waiting, unhook from queues. for crash support. queue code + // better be resilient... + if g.waiters != nil && g.selected == nil { + for sg := g.waiters; sg != nil; sg = sg.waitersNext { + sg.queue.remove(sg) + } + } + g.releaseWaiters() + + // XXX: if g is in any kind of select operation, carefully unwind that??? so + // far, not necessary because we abort entire machines at a time. (though + // what about cross-machine selects??? help.) + + prev := getg() + gs.current.set(g) + g.corotask = corotaskAbort + g.coro.Next() + gs.current.set(prev) + + if !g.finished { + panic("expected finished") + } +} + +// Marked norace to read entryToken and entryFun. +// +//go:norace +func goroutineEntrypoint() { + g := getg() + // no need to allocate this entrypoint + if !g.entryToken.Empty() { + g.entryToken.Acquire() + } + + g.park(false) + defer g.exitpoint() + g.entryFun() +} + +// exitpoint is a helper for entrypoint that stores any exit panic value +// and marks the coroutine as finished. +// +// It cannot be inlined because go:norace does not apply internally. +// +// The g.finished = true must be in a recover call because the goroutine +// might exit by calling runtime.Goexit() +// +//go:norace +func (g *goroutine) exitpoint() { + if recovered := recover(); recovered != nil { + g.parkPaniced = true + traceback := make([]byte, 32*1024) + traceback = traceback[:runtime.Stack(traceback, false)] + // XXX: necessary? + g.parkPanicTraceback = NoraceCopy(traceback) + g.parkPanicValue = fmt.Sprint(recovered) + } + g.finished = true + g.coro.Finish() +} + +//go:norace +func (g *goroutine) park(wait bool) { + if inControlledNondeterminism() { + if wait { + panic("help") + } + return + } + + g.parkWait = wait + + // inform scheduler we're paused + // wait for us to resume + for { + if race.Enabled { + if wait { + g.parksynctoken.Release() + } + race.Disable() + } + + g.coro.Yield() + + if race.Enabled { + race.Enable() + if wait { + if g.shouldacquire { + race.Acquire(unsafe.Pointer(&g.shouldacquire)) + g.shouldacquire = false + } + if g.shouldrelease != nil { + addr := g.shouldrelease + race.Release(unsafe.Pointer(addr)) + addr.pending = nil + g.shouldrelease = nil + } + } + } + + switch g.corotask { + case corotaskAbort: + g.finished = true + g.coro.Finish() + case corotaskStep: + return + default: + panic(g.corotask) + } + g.corotask = corotaskNone + } +} + +func (g *goroutine) yield() { + // TODO: optimize by fast-pathing here to stay on the same goroutine before + // hitting the expensive coroutine switching in park? + g.park(false) +} + +func (g *goroutine) wait() { + g.park(true) +} + +var syscallAllocator func() unsafe.Pointer + +func SetSyscallAllocator(f func() unsafe.Pointer) { + syscallAllocator = f +} + +func GetGoroutineLocalSyscall() unsafe.Pointer { + g := getg() + return g.syscallCache +} + +// Go starts a new goroutine. +// +// Marked go:norace to let upcall access g. +// +//go:norace +func Go(fun func()) { + g := getg() + g.yield() + + token := MakeRaceToken() + token.Release() + g.upcall(func() { + gs.get().Go(fun, g.machine, token) + }) +} + +//go:norace +func GoWithMachine(fun func(), machine *Machine) { + g := getg() + g.yield() + + token := MakeRaceToken() + token.Release() + g.upcall(func() { + gs.get().Go(fun, machine, token) + }) +} + +//go:norace +func GoFromTimerHandler(fun func(), machine *Machine) { + if getg() != nil { + panic("must be called without g") + } + + gs.get().Go(fun, machine, gs.get().globaltoken) +} + +// Marked go:norace to return result. +// +//go:norace +func PickRandomOtherGoroutine() (goroutineID int) { + var result int + g := getg() + g.upcall(func() { + result = gs.get().pickRandomOtherGoroutine(g).ID + }) + return result +} + +func PauseGoroutine(goroutineID int) { + getg().upcall(func() { + gs.get().goroutinesById[goroutineID].setPaused(true) + }) +} + +func ResumeGoroutine(goroutineID int) { + getg().upcall(func() { + gs.get().goroutinesById[goroutineID].setPaused(false) + }) +} + +func StopMachine(machine *Machine) { + // TODO: janky method, expose different API? + getg().upcall(func() { + gs := gs.get() + + // XXX: why are we doing this list copying game? trying to not lose goroutines + // if they get shuffled...? + var toAbort []*goroutine + // XXX: inefficient? + for _, g := range gs.goroutines { + if g.machine == machine { + toAbort = append(toAbort, g) + } + } + for _, g := range toAbort { + g.abort() + gs.removeGoroutine(g) + freeGoroutine(g) + } + // unhook timer events + // XXX: inefficient: + gs.clock.removemachine(machine) + + // XXX: poison machine for future use? set globals to nil? + // XXX: could later also reuse globals struct (after zeroing...) (but maybe not in super paranoid mode) + machine.globals = nil + + // XXX: mark machine as bad for higher-level errors? + }) +} + +func IsSim() bool { + // TODO: test this doesn't race? + return gs.get() != nil +} + +func Yield() { + getg().yield() +} + +func SetFinalizer(obj any, finalizer any) { + // never do anything, hopefully no problems... + + // TODO: maybe could have deterministic GC if we disable automatic GC and + // invoke it ourselves? +} + +func SetAbortError(err error) { + if _, ok := err.(constErr); !ok { + panic("SetAbortErr must be called with one of the predefined errors in this package") + } + + getg().upcall(func() { + gs := gs.get() + if gs.abortErr == nil { + gs.abortErr = err + } + }) + + // call park so this goroutine can check abortErr and exit + getg().park(false) +} + +const GOOS = "linux" + +// Marked go:norace to read ID +// +//go:norace +func GetGoroutine() int { + return getg().ID +} + +// Marked go:norace to read machine +// +//go:norace +func CurrentMachine() *Machine { + return getg().machine +} + +// Marked go:norace to read and write nextEventID +// +//go:norace +func NextEventID() int { + gs := gs.get() + id := gs.nextEventID + gs.nextEventID++ + return id +} + +var logBuf []byte + +//go:norace +func WriteLog(b []byte) { + logBuf = NoraceAppend(logBuf[:0], b...) + getg().upcall(func() { + gs.get().logger.out.Write(logBuf) + }) +} + +//go:norace +func Envs() []string { + return NoraceCopy(gs.get().envs) +} + +//go:norace +func NewMachine(label string) *Machine { + var result *Machine + getg().upcall(func() { + result = newMachine(label) + }) + return result +} + +var controlledNondeterminismDepth int + +func inControlledNondeterminism() bool { + return controlledNondeterminismDepth > 0 +} + +// BeginControlledNondeterminism starts a section of code that is expected to +// behave non-deterministically inside the section but that will not show that +// outside the section. +// +// Calls to random in the section always return 0. Any attempt to pause the +// current thread (eg. mutex.Lock) will panic. +// +// Controlled nondeterminism is useful to cache work that can be shared between +// different machines that is expensive to recompute and complicated to +// precompute once before machines start. One use case is the JSON package's +// type cache, see the translate code that inserts calls. +// +//go:norace +func BeginControlledNondeterminism() { + controlledNondeterminismDepth++ +} + +//go:norace +func EndControlledNondeterminism() { + controlledNondeterminismDepth-- +} + +var globalCounter int + +//go:norace +func Nondeterministic() int { + globalCounter++ + return globalCounter +} + +// TODO: expose something like func AssertAllDone() which checks there are no +// other goroutines, no registered timers, ...? + +var ( + stepCounter int + stepBreakpoint int +) + +var stepBreakpointFlag = flag.Int("step-breakpoint", 0, "") + +//go:norace +func InitSteps() { + stepCounter = 0 + stepBreakpoint = *stepBreakpointFlag +} + +//go:norace +func Step() int { + stepCounter++ + if stepCounter == stepBreakpoint { + runtime.Breakpoint() + } + return stepCounter +} + +func GetStep() int { + return stepCounter +} diff --git a/gosimruntime/runtime_test.go b/gosimruntime/runtime_test.go new file mode 100644 index 0000000..7e0577a --- /dev/null +++ b/gosimruntime/runtime_test.go @@ -0,0 +1,80 @@ +package gosimruntime + +import ( + "os" + "runtime" + "sync/atomic" + "testing" + "time" + "unsafe" +) + +func TestIsSim(t *testing.T) { + if IsSim() { + t.Error("should not be sim") + } +} + +func dump() string { + var buf [32 * 1024]byte + return string(buf[:runtime.Stack(buf[:], true)]) +} + +const baseNumGoroutines = 2 + +func expectNumGoroutines(t *testing.T, situation string) { + t.Helper() + + // XXX: give any pre-existing goroutines time to fully exit + start := time.Now() + for runtime.NumGoroutine() != baseNumGoroutines { + if time.Since(start) >= 1*time.Second { + t.Errorf("too many goroutines at %s, expected %d", situation, baseNumGoroutines) + t.Log(dump()) + } + time.Sleep(1 * time.Millisecond) + } +} + +func init() { + SetSyscallAllocator(func() unsafe.Pointer { + return nil + }) + initializeRuntime(func(f func()) { + f() + SetAbortError(ErrMainReturned) + }) +} + +// TODO: Rewrite to use another API? Could add a call to dump goroutines to metatesting...? +func TestCleanupGoroutines(t *testing.T) { + expectNumGoroutines(t, "start") + + var hit atomic.Int32 + var deferred atomic.Int32 + f := func() { + waiting := NewChan[struct{}](0) + pause := NewChan[struct{}](0) + for i := 0; i < 5; i++ { + Go(func() { + defer func() { + deferred.Add(1) + }() + + waiting.Recv() // will not block + hit.Add(1) + pause.Recv() // should block + }) + waiting.Send(struct{}{}) + } + } + run(f, 1, false, false, "INFO", makeConsoleLogger(os.Stderr), nil) + + expectNumGoroutines(t, "end") + if hit.Load() != 5 { + t.Error("recv not hit") + } + if deferred.Load() != 0 { + t.Error("deferred hit") + } +} diff --git a/gosimruntime/sema.go b/gosimruntime/sema.go new file mode 100644 index 0000000..efc548e --- /dev/null +++ b/gosimruntime/sema.go @@ -0,0 +1,292 @@ +// Copyright 2009 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Based on https://go.googlesource.com/go/+/refs/heads/master/src/runtime/sema.go. + +package gosimruntime + +import ( + "unsafe" +) + +// TODO: use the go runtime treap plan instead? +type sema struct { + ptr *uint32 + q waitq + + next *sema +} + +const semasSize = 251 // prime just like in runtime + +var semas [semasSize]*sema + +// TODO: store these structs on g instead or something? +var semaPool []*sema + +// TODO: clean up freeing code (no need to do two lookups) + +// cleanup/handle sema with empty sudog queue (can happen when crashing machines) +// +//go:norace +func maybefree(semaPtr **sema) { + if *semaPtr == nil || (*semaPtr).q.first != nil { + return + } + + cur := *semaPtr + for cur != nil && cur.q.first == nil { + next := cur.next + cur.next = nil + cur.ptr = nil + semaPool = append(semaPool, cur) + cur = next + } + *semaPtr = cur +} + +//go:norace +func getsema(addr *uint32, alloc bool) *sema { + idx := (uintptr(unsafe.Pointer(addr)) >> 3) % semasSize + + maybefree(&semas[idx]) + cur := semas[idx] + for cur != nil { + if cur.ptr == addr { + return cur + } + maybefree(&cur.next) + cur = cur.next + } + + if !alloc { + return nil + } + + if n := len(semaPool); n > 0 { + cur = semaPool[n-1] + semaPool = semaPool[:n-1] + } else { + cur = &sema{} + } + cur.ptr = addr + cur.next = semas[idx] + semas[idx] = cur + + return cur +} + +//go:norace +func freesema(addr *uint32) { + idx := (uintptr(unsafe.Pointer(addr)) >> 3) % semasSize + + if semas[idx].ptr == addr { + freeing := semas[idx] + semas[idx] = semas[idx].next + freeing.ptr = nil + freeing.next = nil + semaPool = append(semaPool, freeing) + return + } + + cur := semas[idx] + for { + if cur.next.ptr == addr { + freeing := cur.next + cur.next = cur.next.next + freeing.ptr = nil + freeing.next = nil + semaPool = append(semaPool, freeing) + return + } + cur = cur.next + } +} + +// TODO: this needs tests + +//go:norace +func Semacquire(addr *uint32, lifo bool) { + g := getg() + g.yield() + + if *addr > 0 { + *addr-- + return + } + + sema := getsema(addr, true) + + sudog := g.allocWaiter() + if lifo { + sema.q.enqueuelifo(sudog) + } else { + sema.q.enqueue(sudog) + } + g.wait() + g.releaseWaiters() +} + +//go:norace +func Semrelease(addr *uint32) { + g := getg() + g.yield() + + if *addr > 0 { + *addr++ + return + } + + sema := getsema(addr, false) + if sema == nil { + *addr++ + return + } + + sema.q.dequeue() + if sema.q.first == nil { + freesema(addr) + } +} + +const AtomicYield = false + +type NotifyList struct { + wait, notify uint32 + q waitq + // TODO: nocopy? +} + +//go:norace +func (l *NotifyList) Add() uint32 { + // XXX: is the reordering important here? + g := getg() + g.yield() + + val := l.wait + l.wait++ + return val +} + +// less checks if a < b, considering a & b running counts that may overflow the +// 32-bit range, and that their "unwrapped" difference is always less than 2^31. +func less(a, b uint32) bool { + return int32(a-b) < 0 +} + +//go:norace +func (l *NotifyList) Wait(t uint32) { + g := getg() + g.yield() + + if less(t, l.notify) { + return + } + + sudog := g.allocWaiter() + sudog.index = int(t) + l.q.enqueue(sudog) + g.wait() + g.releaseWaiters() +} + +//go:norace +func (l *NotifyList) NotifyAll() { + g := getg() + g.yield() + + l.notify = l.wait + for l.q.dequeue() != nil { + } +} + +//go:norace +func (l *NotifyList) NotifyOne() { + g := getg() + g.yield() + + if l.notify == l.wait { + return + } + + t := l.notify + l.notify++ + + for c := l.q.first; c != nil; c = c.next { + if c.index == int(t) { + l.q.remove(c) + + if c.goroutine.waiters.waitersNext != nil || c.goroutine.selected != nil { + // other waiters should not exist, only happens in select {} + panic(c) + } + c.goroutine.selected = c + + g.upcall(func() { + c.goroutine.setWaiting(false) + }) + break + } + } +} + +type Uint32Futex struct { + val uint32 + q waitq + // TODO: nocopy? +} + +//go:norace +func (w *Uint32Futex) Get() uint32 { + g := getg() + g.yield() + + return w.val +} + +//go:norace +func (w *Uint32Futex) Set(val uint32) { + g := getg() + g.yield() + + w.val = val + if val != 0 { + for sudog := w.q.dequeue(); sudog != nil; sudog = w.q.dequeue() { + sudog.index = int(w.val) + } + } +} + +//go:norace +func (w *Uint32Futex) SetBit(index int, value bool) { + g := getg() + g.yield() + + if value { + w.val |= 1 << index + } else { + w.val &= ^(1 << index) + } + if w.val != 0 { + for sudog := w.q.dequeue(); sudog != nil; sudog = w.q.dequeue() { + sudog.index = int(w.val) + } + } +} + +//go:norace +func (w *Uint32Futex) WaitNonzero() uint32 { + g := getg() + g.yield() + + if w.val != 0 { + return w.val + } + sudog := g.allocWaiter() + w.q.enqueue(sudog) + g.wait() + ret := uint32(sudog.index) + g.releaseWaiters() + return ret +} diff --git a/gosimruntime/testmain.go b/gosimruntime/testmain.go new file mode 100644 index 0000000..382c8dd --- /dev/null +++ b/gosimruntime/testmain.go @@ -0,0 +1,147 @@ +package gosimruntime + +import ( + "encoding/json" + "flag" + "log" + "maps" + "os" + "slices" +) + +type Test struct { + Name string + Test any // func() +} + +var ( + allTests map[string]any + allTestsSlice []Test +) + +func SetAllTests(tests []Test) { + allTests = make(map[string]any) + allTestsSlice = tests + for _, test := range tests { + allTests[test.Name] = test.Test + } +} + +var metatest = flag.Bool("metatest", false, "") + +// Copied in metatesting.RunConfig. Keep in sync. +type runConfig struct { + Test string + Seed int64 + ExtraEnv []string +} + +// Copied in metatesting.RunResult. Keep in sync. +type runResult struct { + Seed int64 + Trace []byte + Failed bool + LogOutput []byte + Err string // TODO: reconsider this type? +} + +type Runtime interface { + Run(fn func()) + Setup() + TestEntrypoint(match string, skip string, tests []Test) bool +} + +func TestMain(rt Runtime) { + flag.Parse() + + rt.Setup() + initializeRuntime(rt.Run) + + if !*metatest { + // TODO: complain about unsupported test flags + // parallel := flag.Lookup("test.parallel").Value.(flag.Getter).Get().(int) + match := flag.Lookup("test.run").Value.(flag.Getter).Get().(string) + skip := flag.Lookup("test.skip").Value.(flag.Getter).Get().(string) + + seed := int64(1) + enableTracer := true + captureLog := true + logLevelOverride := "INFO" + + // TODO: allow -count, -seeds + + outerOk := true + + for _, test := range allTestsSlice { + // log.Println("running", test.Name) + result := run(func() { + ok := rt.TestEntrypoint(match, skip, []Test{ + test, + }) + if !ok { + SetAbortError(ErrTestFailed) + } + }, seed, enableTracer, captureLog, logLevelOverride, makeConsoleLogger(os.Stderr), []string{}) + + if result.Failed { + outerOk = false + } + } + + if !outerOk { + os.Exit(1) + } + os.Exit(0) + } + + in := json.NewDecoder(os.Stdin) + out := json.NewEncoder(os.Stdout) + + for { + var req runConfig + if err := in.Decode(&req); err != nil { + log.Fatal(err) + } + + // TODO: add "listtests" as a separate message type? + if req.Test == "listtests" { + out.Encode(slices.Sorted(maps.Keys(allTests))) + continue + } + + seed := req.Seed + enableTracer := true + captureLog := true + logLevelOverride := "INFO" + + func() { + result := run(func() { + // TODO: add an entrypoint that takes a single test? + // TODO: fail gracefully with non-existent tests? + ok := rt.TestEntrypoint(req.Test, "", []Test{ + { + Name: req.Test, + Test: allTests[req.Test], + }, + }) + if !ok { + SetAbortError(ErrTestFailed) + } + }, seed, enableTracer, captureLog, logLevelOverride, makeConsoleLogger(os.Stderr), req.ExtraEnv) + + metaResult := runResult{ + Seed: result.Seed, + Trace: result.Trace, + Failed: result.Failed, + LogOutput: result.LogOutput, + } + if result.Err != nil { + metaResult.Err = result.Err.Error() + } + + if err := out.Encode(metaResult); err != nil { + log.Fatal(err) + } + }() + } +} diff --git a/gosimruntime/time.go b/gosimruntime/time.go new file mode 100644 index 0000000..3d55d94 --- /dev/null +++ b/gosimruntime/time.go @@ -0,0 +1,144 @@ +package gosimruntime + +import ( + "time" +) + +// TODO: move clock to machine so we can simulate broken clocks and offsets + +// race detector: +// there is an edge from timer start/reset to func firing, or channel being +// written + +// TODO: +// - come up with some rules as to what thread code runs on (make a special time goroutine?) +// - come up with some rules as to when/if timer should ever yield +// - figure out happens-before rules +// - write some tests that assert all this + +//go:norace +func Nanotime() int64 { + return gs.get().clock.now +} + +type clock struct { + now int64 // unix nsec + heap *timerHeap +} + +func newClock() *clock { + return &clock{ + // TODO: make start timestamp random also + now: time.Date(2020, 1, 15, 14, 10, 3, 1234, time.UTC).UnixNano(), + heap: newTimerHeap(), + } +} + +//go:norace +func (c *clock) anyWaiting() bool { + return c.heap.len() > 0 +} + +//go:norace +func (c *clock) maybefire() (*Timer, bool) { + if c.heap.len() == 0 { + return nil, false + } + if c.heap.peek().when <= c.now { + e := c.heap.pop() + return e, true + } + return nil, false +} + +//go:norace +func (c *clock) doadvance() { + e := c.heap.peek() + t := e.when + if t <= c.now { + panic("bad") + } + // XXX: we allow adding at now/earlier to handle instant timers + c.now = t + if tracer := gs.get().tracer; tracer != nil { + tracer.recordIntInt(traceKeyTimeNow, uint64(c.now), 0) + } +} + +func (c *clock) removemachine(m *Machine) { + c.heap.removemachine(m) +} + +func unpause(t *Timer) { + if getg() != nil { + panic("expect to run without a goroutine") + } + t.Arg.(*goroutine).setWaiting(false) +} + +//go:norace +func Sleep(duration int64) { + start := gs.get().clock.now + if duration <= 0 { + return + } + end := start + duration + + // no sudog??? cool. + // XXX: use a sudog so we can have every wait paired with a sudog and some + // sensible checking for that? + g := getg() + + // XXX: reuse this timer? + NewTimer(unpause, g, g.machine, end) + + g.wait() +} + +type Timer struct { + when int64 + handler func(t *Timer) + Arg any + Machine *Machine + + pos int +} + +// TODO: reconsider if the timer needs to be go:norace. Maybe do a pivot to +// runtime stack here also? + +//go:norace +func NewTimer(handler func(arg *Timer), arg any, machine *Machine, when int64) *Timer { + t := &Timer{ + when: when, + handler: handler, + Arg: arg, + Machine: machine, + pos: -1, + } + + gs.get().clock.heap.add(t) + + return t +} + +//go:norace +func (t *Timer) Reset(when int64) bool { + stopped := t.pos != -1 + if stopped { + gs.get().clock.heap.adjust(t, when) + } else { + t.when = when + gs.get().clock.heap.add(t) + } + return stopped +} + +//go:norace +func (t *Timer) Stop() bool { + stopped := t.pos != -1 + if stopped { + gs.get().clock.heap.remove(t) + } + return stopped +} diff --git a/gosimruntime/timer_heap.go b/gosimruntime/timer_heap.go new file mode 100644 index 0000000..8d40d3a --- /dev/null +++ b/gosimruntime/timer_heap.go @@ -0,0 +1,114 @@ +package gosimruntime + +import ( + "container/heap" +) + +// timers implements heap.Interface +type timers []*Timer + +//go:norace +func (h timers) Len() int { return len(h) } + +//go:norace +func (h timers) Less(i, j int) bool { return h[i].when < h[j].when } + +//go:norace +func (h timers) Swap(i, j int) { + h[i], h[j] = h[j], h[i] + h[i].pos = i + h[j].pos = j +} + +//go:norace +func (h *timers) Push(x any) { + t := x.(*Timer) + if t.pos != -1 { + panic(t.pos) + } + t.pos = len(*h) + *h = append(*h, x.(*Timer)) +} + +//go:norace +func (h *timers) Pop() any { + old := *h + n := len(old) + x := old[n-1] + *h = old[0 : n-1] + x.pos = -1 + return x +} + +type timerHeap struct { + timers timers +} + +//go:norace +func newTimerHeap() *timerHeap { + return &timerHeap{} +} + +//go:norace +func (h *timerHeap) add(t *Timer) { + heap.Push(&h.timers, t) +} + +//go:norace +func (h *timerHeap) adjust(t *Timer, when int64) { + if t.pos == -1 || h.timers[t.pos] != t { + panic(t) + } + t.when = when + heap.Fix(&h.timers, t.pos) +} + +//go:norace +func (h *timerHeap) remove(t *Timer) { + if t.pos == -1 || h.timers[t.pos] != t { + panic(t) + } + heap.Remove(&h.timers, t.pos) +} + +//go:norace +func (h *timerHeap) len() int { + return len(h.timers) +} + +//go:norace +func (h *timerHeap) pop() *Timer { + return heap.Pop(&h.timers).(*Timer) +} + +//go:norace +func (h *timerHeap) peek() *Timer { + return h.timers[0] +} + +//go:norace +func (h *timerHeap) removemachine(m *Machine) { + // XXX: don't use a helper because it upsets the race detector + i, j := 0, 0 + changed := false + for i < len(h.timers) { + if h.timers[i].Machine == m { + changed = true + i++ + continue + } + if changed { + h.timers[j] = h.timers[i] + h.timers[j].pos = j + } + i++ + j++ + } + i = j + for i < len(h.timers) { + h.timers[i] = nil + i++ + } + h.timers = h.timers[:j] + heap.Init(&h.timers) +} diff --git a/gosimruntime/timer_heap_test.go b/gosimruntime/timer_heap_test.go new file mode 100644 index 0000000..37fdb29 --- /dev/null +++ b/gosimruntime/timer_heap_test.go @@ -0,0 +1,133 @@ +package gosimruntime + +import ( + "fmt" + "slices" + "testing" + "time" + + "pgregory.net/rapid" +) + +func TestTimerHeap(t *testing.T) { + baseTime := time.Date(2010, 5, 1, 10, 3, 1, 100, time.UTC).UnixNano() + + heap := newTimerHeap() + first := &Timer{when: baseTime + int64(0*time.Second), pos: -1} + heap.add(first) + if first.pos != 0 { + t.Errorf("expected pos 0, got %d", first.pos) + } + a := &Timer{when: baseTime + int64(1*time.Second), pos: -1} + heap.add(a) + heap.add(&Timer{when: baseTime + int64(2*time.Second), pos: -1}) + b := &Timer{when: baseTime + int64(3*time.Second), pos: -1} + heap.add(b) + if l := heap.len(); l != 4 { + t.Errorf("expected heap.len() = 4, got %d", l) + } + + if e := heap.pop(); e.when != baseTime+int64(0*time.Second) { + t.Errorf("expected t+0s, got %v", e.when-baseTime) + } + if e := heap.peek(); e.when != baseTime+int64(1*time.Second) { + t.Errorf("expected t+1s, got %v", e.when-baseTime) + } + + heap.adjust(a, baseTime+int64(4*time.Second)) + + if e := heap.pop(); e.when != baseTime+int64(2*time.Second) { + t.Errorf("expected t+2s, got %v", e.when-baseTime) + } + + heap.remove(b) + heap.add(&Timer{when: baseTime + int64(5*time.Second), pos: -1}) + + if e := heap.pop(); e.when != baseTime+int64(4*time.Second) { + t.Errorf("expected t+4s, got %v", e.when-baseTime) + } + + if e := heap.pop(); e.when != baseTime+int64(5*time.Second) { + t.Errorf("expected t+5s, got %v", e.when-baseTime) + } +} + +func TestCheckTimerHeap(t *testing.T) { + rapid.Check(t, checkTimerHeap) +} + +func checkTimerHeap(t *rapid.T) { + heap := newTimerHeap() + var model []*Timer + + var machines []*Machine + for i := 0; i < 5; i++ { + machines = append(machines, &Machine{label: fmt.Sprint(i)}) + } + + actions := make(map[string]func(t *rapid.T)) + + actions["add"] = func(t *rapid.T) { + when := rapid.Int64().Draw(t, "when") + machine := rapid.SampledFrom(machines).Draw(t, "machine") + timer := &Timer{when: when, Machine: machine, pos: -1} + model = append(model, timer) + heap.add(timer) + } + + actions["peek"] = func(t *rapid.T) { + if heap.len() == 0 { + t.Skip() + } + got := heap.peek() + for _, other := range model { + if other.when < got.when { + t.Errorf("found earlier when %d than returned %d", other.when, got.when) + } + } + } + + actions["pop"] = func(t *rapid.T) { + if heap.len() == 0 { + t.Skip() + } + got := heap.pop() + for _, other := range model { + if other.when < got.when { + t.Errorf("found earlier when %d than returned %d", other.when, got.when) + } + } + if got.pos != -1 { + t.Error("expected pos -1 after pop") + } + model = slices.DeleteFunc(model, func(t *Timer) bool { return t == got }) + } + + actions["adjust"] = func(t *rapid.T) { + if heap.len() == 0 { + t.Skip() + } + timer := rapid.SampledFrom(model).Draw(t, "timer") + when := rapid.Int64().Draw(t, "when") + heap.adjust(timer, when) + } + + actions["delete-machine"] = func(t *rapid.T) { + machine := rapid.SampledFrom(machines).Draw(t, "machine") + heap.removemachine(machine) + model = slices.DeleteFunc(model, func(t *Timer) bool { return t.Machine == machine }) + } + + actions[""] = func(t *rapid.T) { + if expected, actual := len(model), heap.len(); expected != actual { + t.Errorf("length mismatch: expected %d, got %d", expected, actual) + } + for _, timer := range model { + if timer.pos < 0 || timer.pos >= len(heap.timers) || heap.timers[timer.pos] != timer { + t.Errorf("wrong pos for timer") // XXX: helpful information somehow? + } + } + } + + t.Repeat(actions) +} diff --git a/gosimruntime/trace.go b/gosimruntime/trace.go new file mode 100644 index 0000000..5570326 --- /dev/null +++ b/gosimruntime/trace.go @@ -0,0 +1,96 @@ +package gosimruntime + +import ( + "context" + "encoding/binary" + "log/slog" +) + +type tracer struct { + step int + hash fnv64 + logger *slog.Logger +} + +func newTracer(logger *slog.Logger) *tracer { + return &tracer{ + step: 0, + hash: newFnv64(), + logger: logger, + } +} + +//go:generate go run golang.org/x/tools/cmd/stringer -type=traceKey + +type traceKey byte + +const ( + traceKeyRunPick traceKey = iota + tracekeyRunResult + tracekeyLogWrite + traceKeyCreating + traceKeyRunStarted + traceKeyRunFinished + traceKeyTimeNow +) + +func (t *tracer) recordIntInt(key traceKey, a, b uint64) { + if inControlledNondeterminism() { + panic("help") + } + + t.hash.Hash([]byte{byte(key)}) + t.hash.HashInt(a) + t.hash.HashInt(b) + + level := slog.LevelDebug + if *forceTrace { + level = slog.LevelError + } + if t.logger.Enabled(context.TODO(), level) { + t.logger.LogAttrs(context.TODO(), level, "dettrace", + slog.Int("step", t.step), + slog.String("key", key.String()), + slog.Int("a", int(a)), + slog.Int("b", int(b)), + slog.Int64("sum", int64(t.hash))) + } + t.step++ +} + +func (t *tracer) recordBytes(key traceKey, a []byte) { + if inControlledNondeterminism() { + panic("help") + } + + t.hash.Hash([]byte{byte(key)}) + t.hash.Hash(a) + + level := slog.LevelDebug + if *forceTrace { + level = slog.LevelError + } + if t.logger.Enabled(context.TODO(), level) { + t.logger.LogAttrs(context.TODO(), level, "dettrace", + slog.Int("step", t.step), + slog.String("key", key.String()), + slog.String("a", string(a)), + slog.Int64("sum", int64(t.hash))) + } + t.step++ +} + +func (t *tracer) finalize() []byte { + var n [8]byte + binary.LittleEndian.PutUint64(n[:], uint64(t.hash)) + return n[:] +} + +type traceWriter struct { + trace *tracer +} + +func (t traceWriter) Write(p []byte) (n int, err error) { + t.trace.recordBytes(tracekeyLogWrite, p) + return len(p), nil +} diff --git a/gosimruntime/tracekey_string.go b/gosimruntime/tracekey_string.go new file mode 100644 index 0000000..0d9ac21 --- /dev/null +++ b/gosimruntime/tracekey_string.go @@ -0,0 +1,29 @@ +// Code generated by "stringer -type=traceKey"; DO NOT EDIT. + +package gosimruntime + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[traceKeyRunPick-0] + _ = x[tracekeyRunResult-1] + _ = x[tracekeyLogWrite-2] + _ = x[traceKeyCreating-3] + _ = x[traceKeyRunStarted-4] + _ = x[traceKeyRunFinished-5] + _ = x[traceKeyTimeNow-6] +} + +const _traceKey_name = "traceKeyRunPicktracekeyRunResulttracekeyLogWritetraceKeyCreatingtraceKeyRunStartedtraceKeyRunFinishedtraceKeyTimeNow" + +var _traceKey_index = [...]uint8{0, 15, 32, 48, 64, 82, 101, 116} + +func (i traceKey) String() string { + if i >= traceKey(len(_traceKey_index)-1) { + return "traceKey(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _traceKey_name[_traceKey_index[i]:_traceKey_index[i+1]] +} diff --git a/imports.go b/imports.go new file mode 100644 index 0000000..e9c2870 --- /dev/null +++ b/imports.go @@ -0,0 +1,27 @@ +//go:build never + +package gosim + +// This file exists as a dependency for users of gosim. The CLI, through +// internal/translate, needs to be able to import all packages below when run +// _inside_ the user's module. By importing the gosim package their go.mod will +// get all the required dependencies. + +// TODO: allow fixing these at a different, independent version? what if there +// are multiple go.mod because we run on multiple versions? + +import ( + // Allow running the CLI. + _ "github.com/jellevandenhooff/gosim/cmd/gosim" + + // Packages listed in internal/translate.TranslatedRuntimePackages. + _ "github.com/jellevandenhooff/gosim/internal/hooks/go123" + _ "github.com/jellevandenhooff/gosim/internal/reflect" + _ "github.com/jellevandenhooff/gosim/internal/simulation" + _ "github.com/jellevandenhooff/gosim/internal/testing" + + // Tools used by gosim. For this repository. + _ "github.com/go-task/task/v3/cmd/task" + _ "golang.org/x/tools/cmd/goimports" + _ "mvdan.cc/gofumpt" +) diff --git a/internal/coro/coro_linkname.go b/internal/coro/coro_linkname.go new file mode 100644 index 0000000..464b612 --- /dev/null +++ b/internal/coro/coro_linkname.go @@ -0,0 +1,75 @@ +//go:build linkname + +package coro + +import ( + "unsafe" + _ "unsafe" + + "github.com/jellevandenhooff/gosim/internal/race" +) + +type coro struct{} + +//go:linkname newcoro runtime.newcoro +func newcoro(func(*coro)) *coro + +//go:linkname coroswitch runtime.coroswitch +func coroswitch(*coro) + +//go:linkname coroexit runtime.coroexit +func coroexit(*coro) + +// A Coro is a coroutine: A goroutine with explicit and cheap context switching. +// Coro uses go's runtime coroutine implementation (which otherwise powers +// iter.Pull) using linkname to get low overhead and direct access to coroexit. +type Coro struct { + coro *coro +} + +// Start starts the coroutine, running f in a new coroutine. It should be called only +// once on a Coro. It lets the coroutine run until the first call to Yield or the final +// call to Finish. +// +//go:norace +func (c *Coro) Start(f func()) { + c.coro = newcoro(func(*coro) { + race.Acquire(unsafe.Pointer(c)) + f() + panic("wtf") + }) + race.Release(unsafe.Pointer(c)) + coroswitch(c.coro) + race.Acquire(unsafe.Pointer(c)) +} + +// Next must be called from outside the coroutine. It lets the coroutine run +// until the next call to Yield or the final call to Finish. +// +//go:norace +func (c *Coro) Next() { + race.Release(unsafe.Pointer(c)) + coroswitch(c.coro) + race.Acquire(unsafe.Pointer(c)) +} + +// Yield be called form inside the coroutine. It pauses the coroutine, yielding to +// the caller of Next (or Start). +// +//go:norace +func (c *Coro) Yield() { + race.Release(unsafe.Pointer(c)) + coroswitch(c.coro) + race.Acquire(unsafe.Pointer(c)) +} + +// Finish must be called from inside the coroutine. It stops the coroutine, +// yielding to the caller of Next (or Start). Finish terminates the coroutine, +// any defers in the callstack will _not_ be run. +// +//go:norace +func (c *Coro) Finish() { + race.Release(unsafe.Pointer(c)) + coroexit(c.coro) + panic("wtf") +} diff --git a/internal/coro/coro_nolinkname.go b/internal/coro/coro_nolinkname.go new file mode 100644 index 0000000..e51e0b8 --- /dev/null +++ b/internal/coro/coro_nolinkname.go @@ -0,0 +1,39 @@ +//go:build !linkname + +package coro + +import ( + _ "unsafe" +) + +// Coro is an implementation of Coro in coro_linkname.go without using linkname +// to access the go runtime's internals. +type Coro struct { + runWaitCh chan struct{} +} + +//go:norace +func (c *Coro) Start(f func()) { + c.runWaitCh = make(chan struct{}) + go f() + <-c.runWaitCh +} + +//go:norace +func (c *Coro) Next() { + c.runWaitCh <- struct{}{} + <-c.runWaitCh +} + +//go:norace +func (c *Coro) Yield() { + c.runWaitCh <- struct{}{} + <-c.runWaitCh +} + +//go:norace +func (c *Coro) Finish() { + c.runWaitCh <- struct{}{} + select {} + panic("wtf") +} diff --git a/internal/coro/upcallcoro_norace.go b/internal/coro/upcallcoro_norace.go new file mode 100644 index 0000000..c4fbb86 --- /dev/null +++ b/internal/coro/upcallcoro_norace.go @@ -0,0 +1,28 @@ +//go:build !race + +package coro + +// Upcallcoro is a non-race instrumented version of Upcallcoro in upcallcoro_race.go. +type Upcallcoro struct { + coro Coro +} + +func (u *Upcallcoro) Start(f func()) { + u.coro.Start(f) +} + +func (u *Upcallcoro) Next() { + u.coro.Next() +} + +func (u *Upcallcoro) Upcall(f func()) { + f() +} + +func (u *Upcallcoro) Yield() { + u.coro.Yield() +} + +func (u *Upcallcoro) Finish() { + u.coro.Finish() +} diff --git a/internal/coro/upcallcoro_race.go b/internal/coro/upcallcoro_race.go new file mode 100644 index 0000000..3431262 --- /dev/null +++ b/internal/coro/upcallcoro_race.go @@ -0,0 +1,64 @@ +//go:build race + +package coro + +// Upcallcoro is a wrapper around a Coro that allows executing "upcalls": +// functions calls from inside the coroutine that execute on the stack outside +// the coroutine. +// +// Upcalls are used by the gosim runtime to have a single outside goroutine +// modify scheduler (and other) state without triggering the race detector. +// Whenever user code, for example, spawns a new goroutine which modifies +// shared scheduler state, it does that with Upcall. +// +// In non-race instrumented builds Upcall can invoke the function on the +// stack inside the coroutine. +type Upcallcoro struct { + coro Coro + + // upcall is set to non-nil whenever an upcall is made + upcall func() +} + +//go:norace +func (u *Upcallcoro) Start(f func()) { + u.coro.Start(f) + for u.upcall != nil { + u.upcall() + u.upcall = nil + u.coro.Next() + } +} + +func (u *Upcallcoro) Next() { + for { + u.coro.Next() + if u.upcall == nil { + break + } + u.upcall() + u.upcall = nil + } +} + +// Upcall must be called from inside the coroutine, and invokes f on the stack +// that called Start or Next. +// +//go:norace +func (u *Upcallcoro) Upcall(f func()) { + // norace to allow writing upcall + + // TODO: It might be more efficient to hook into the go runtime's race + // detector integration and manually switch race detector contexts instead + // of switching stacks entirely. + u.upcall = f + u.coro.Yield() +} + +func (u *Upcallcoro) Yield() { + u.coro.Yield() +} + +func (u *Upcallcoro) Finish() { + u.coro.Finish() +} diff --git a/internal/gosimtool/gosimtool.go b/internal/gosimtool/gosimtool.go new file mode 100644 index 0000000..11d3aa2 --- /dev/null +++ b/internal/gosimtool/gosimtool.go @@ -0,0 +1,279 @@ +package gosimtool + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/mod/modfile" + + "github.com/jellevandenhooff/gosim/internal/race" +) + +const ( + Module = "github.com/jellevandenhooff/gosim" + OutputDirectory = ".gosim" +) + +type BuildConfig struct { + GOOS string + GOARCH string + Race bool +} + +func (b BuildConfig) AsDirname() string { + name := fmt.Sprintf("%s_%s", b.GOOS, b.GOARCH) + if b.Race { + name += "_race" + } + return name +} + +type TranslateOutput struct { + RootOutputDir string + Packages []string // TODO: combine all this in a struct? also include input name? + Deps map[string]map[string]time.Time // pkg -> path -> time +} + +func FindGoMod() (string, *modfile.File, error) { + baseDir, err := os.Getwd() + if err != nil { + return "", nil, err + } + + for i := 0; i < 20; i++ { + p := path.Join(baseDir, "go.mod") + bytes, err := os.ReadFile(p) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return "", nil, err + } + + baseDir = path.Dir(baseDir) + if baseDir != "." { + continue + } + return "", nil, errors.New("could not find go.mod file, hit root") + } + + file, err := modfile.Parse(p, bytes, nil) + if err != nil { + return "", nil, err + } + + return p, file, nil + } + + return "", nil, errors.New("could not find go.mod file, tried too many dirs") +} + +func FindGoModDir() (string, error) { + modFile, _, err := FindGoMod() + if err != nil { + return "", err + } + + modDir := filepath.Dir(modFile) + return modDir, nil +} + +// MakeGoModForTest makes a go.mod and go.sum for tests that use the `gosim` CLI +// outside of the gosim module. It takes the go.mod from the gosim module strips +// it, and adds a `replace` directive pointing to the gosim module. +// +// Only dependencies that are listed in the original go.mod work. +func MakeGoModForTest(goSimModDir, testModDir string, requires []string) error { + origModPath := filepath.Join(goSimModDir, "go.mod") + origModBytes, err := os.ReadFile(origModPath) + if err != nil { + return err + } + + origFile, err := modfile.Parse(origModPath, origModBytes, nil) + if err != nil { + return err + } + + origSumBytes, err := os.ReadFile(filepath.Join(goSimModDir, "go.sum")) + if err != nil { + return err + } + + origRequires := make(map[string]*modfile.Require) + for _, mod := range origFile.Require { + origRequires[mod.Mod.Path] = mod + } + + newFile := &modfile.File{} + newFile.AddModuleStmt("test") + newFile.AddRequire(origFile.Module.Mod.Path, "v1.0.0") + newFile.AddGoStmt(origFile.Go.Version) + for _, pkg := range requires { + require, ok := origRequires[pkg] + if !ok { + return fmt.Errorf("could not find pkg %s", pkg) + } + newFile.AddRequire(require.Mod.Path, require.Mod.Version) + } + newFile.AddReplace(origFile.Module.Mod.Path, "", goSimModDir, "") + newModBytes, err := newFile.Format() + if err != nil { + return err + } + + if err := os.WriteFile(filepath.Join(testModDir, "go.mod"), newModBytes, 0o644); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(testModDir, "go.sum"), origSumBytes, 0o644); err != nil { + return err + } + return nil +} + +func isUncachedTestRun() bool { + // TODO: fallback only allowed if -count=1 (which bans caching) + flags := []string{"-test.count="} + + for _, arg := range os.Args { + for _, flag := range flags { + // if we have this flag and it looks non-empty, assume non-cached test run + if strings.HasPrefix(arg, flag) && len(arg) > len(flag) { + return true + } + } + } + + return false +} + +func ReplaceSpecialPackages(s string) string { + changed := false + parts := strings.Split(s, "/") + for i, part := range parts { + if part == "vendor" || part == "internal" { + parts[i] = part + "_" + changed = true + } + } + if !changed { + return s + } + return strings.Join(parts, "/") +} + +func PreparedTestBinName(pkg string) string { + return strings.ReplaceAll(pkg, "/", "-") + ".test" +} + +func PreparedTestInfoName(pkg string) string { + return strings.ReplaceAll(pkg, "/", "-") + ".info" +} + +func GetNewPackageName(pkg string) string { + return "translated/" + ReplaceSpecialPackages(pkg) +} + +var ( + buildOnceMu sync.Mutex + buildOnce = make(map[string]*sync.Once) +) + +func GetPathForPrecompiledTestBinary(t *testing.T, pkg string) string { + didBuild := false + + if isUncachedTestRun() { + // TODO: figure out if we should pass -race or not? + buildOnceMu.Lock() + once, ok := buildOnce[pkg] + if !ok { + once = new(sync.Once) + buildOnce[pkg] = once + } + buildOnceMu.Unlock() + + didBuild = true + once.Do(func() { + var name string + var flags []string + if prebuilt, ok := os.LookupEnv("GOSIMTOOL"); ok { + name = prebuilt + } else { + name = "go" + flags = []string{"run", Module + "/cmd/gosim"} + } + flags = append(flags, "build-tests") + if race.Enabled { + flags = append(flags, "-race") + } + flags = append(flags, pkg) + cmd := exec.Command(name, flags...) + t.Logf("looks like an uncached test run... running `%s`", strings.Join(cmd.Args, " ")) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("gosim build-tests failed: %s\n\n%s", err, string(out)) + } + }) + } + + goModDir, err := FindGoModDir() + if err != nil { + t.Fatalf("Could not find containing go module: %s", err) + } + + // TODO: test this race stuff somehow. make test output if race is enabled? + cfg := BuildConfig{ + GOOS: "linux", + GOARCH: runtime.GOARCH, + Race: race.Enabled, + } + + outDir := filepath.Join(goModDir, OutputDirectory, "metatest", cfg.AsDirname()) + path := filepath.Join(outDir, PreparedTestBinName(GetNewPackageName(pkg))) + infoPath := filepath.Join(outDir, PreparedTestInfoName(GetNewPackageName(pkg))) + + if _, err := os.Stat(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + t.Fatalf("Could not find pre-built gosim test binary for package %q. Pre-building is necessary to support the go test cache. "+ + "Pre-build the test binary using `go run %s/cmd/gosim build-tests %s` or "+ + "run `go test` with `-count=1` to disable the go test cache and build during the test.", pkg, Module, pkg) + } + } + + if !didBuild { + t.Logf("Using pre-built gosim test binary %q", path) + + // TODO: only do this once per binary? + // TODO: maybe this is less than optimal... now if files get touched but the binary doesn't change + // we still re-run tests... worthwhile trade-off to catch misuse? + + // check if the precompiled test binary is up-to-date "enough" + bytes, err := os.ReadFile(infoPath) + if err != nil { + t.Fatalf("failed to read info file: %s", err) + } + var deps map[string]time.Time + if err := json.Unmarshal(bytes, &deps); err != nil { + t.Fatalf("failed to parse info file: %s", err) + } + for dep, ts := range deps { + info, err := os.Stat(dep) + if err != nil { + t.Fatalf("failed to stat dependency %q: %s", dep, err) + } + if !ts.Equal(info.ModTime()) { + t.Fatalf("Pre-built gosim test binary is out of date. Input %q changed: %s vs %s.", dep, ts, info.ModTime()) + } + } + } + + return path +} diff --git a/internal/hooks/go123/doc.go b/internal/hooks/go123/doc.go new file mode 100644 index 0000000..71efb14 --- /dev/null +++ b/internal/hooks/go123/doc.go @@ -0,0 +1,6 @@ +/* +Package go123 is the wrapper layer between go1.23's standard library and the +gosim runtime. The gosim translator replaces all runtime linknames in the +standard library to use go123's functions instead. +*/ +package go123 diff --git a/internal/hooks/go123/entrypoint.go b/internal/hooks/go123/entrypoint.go new file mode 100644 index 0000000..2ffb62f --- /dev/null +++ b/internal/hooks/go123/entrypoint.go @@ -0,0 +1,28 @@ +package go123 + +import ( + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/simulation" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" + "github.com/jellevandenhooff/gosim/internal/testing" +) + +func Runtime() gosimruntime.Runtime { + return runtimeImpl{} +} + +type runtimeImpl struct{} + +var _ gosimruntime.Runtime = runtimeImpl{} + +func (r runtimeImpl) Run(fn func()) { + simulation.Runtime(fn) +} + +func (r runtimeImpl) Setup() { + syscallabi.Setup() +} + +func (r runtimeImpl) TestEntrypoint(match string, skip string, tests []gosimruntime.Test) bool { + return testing.Entrypoint(match, skip, tests) +} diff --git a/internal/hooks/go123/golangorg_x_sys_unix.go b/internal/hooks/go123/golangorg_x_sys_unix.go new file mode 100644 index 0000000..9f10164 --- /dev/null +++ b/internal/hooks/go123/golangorg_x_sys_unix.go @@ -0,0 +1,40 @@ +//go:build linux + +package go123 + +import ( + "syscall" + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +//go:nocheckptr +//go:uintptrescapes +func GolangOrgXSysUnix_RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { + return simulation.RawSyscall6(trap, a1, a2, a3, a4, a5, a6) +} + +func GolangOrgXSysUnix_SyscallNoError(trap, a1, a2, a3 uintptr) (r1, r2 uintptr) { + panic("gosim not implemented") +} + +func GolangOrgXSysUnix_RawSyscallNoError(trap, a1, a2, a3 uintptr) (r1, r2 uintptr) { + panic("gosim not implemented") +} + +func GolangOrgXSysUnix_Syscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) { + panic("gosim not implemented") +} + +func GolangOrgXSysUnix_Syscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { + return 0, 0, syscall.ENOSYS +} + +func GolangOrgXSysUnix_RawSyscall(trap, a1, a2, a3 uintptr) (r1, r2 uintptr, err syscall.Errno) { + panic("gosim not implemented") +} + +func GolangOrgXSysUnix_gettimeofday(timeval unsafe.Pointer) syscall.Errno { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/golangorg_x_sys_unix_gensyscall_amd64.go b/internal/hooks/go123/golangorg_x_sys_unix_gensyscall_amd64.go new file mode 100644 index 0000000..f5bee14 --- /dev/null +++ b/internal/hooks/go123/golangorg_x_sys_unix_gensyscall_amd64.go @@ -0,0 +1,113 @@ +//go:build linux + +// Code generated by gensyscall. DO NOT EDIT. +package go123 + +import ( + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +// prevent unused imports +var _ unsafe.Pointer + +func GolangOrgXSysUnix_accept4(s int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen, flags int) (fd int, err error) { + return simulation.SyscallSysAccept4(s, rsa, addrlen, flags) +} + +func GolangOrgXSysUnix_bind(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysBind(s, addr, addrlen) +} + +func GolangOrgXSysUnix_Close(fd int) (err error) { + return simulation.SyscallSysClose(fd) +} + +func GolangOrgXSysUnix_connect(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysConnect(s, addr, addrlen) +} + +func GolangOrgXSysUnix_Fstat(fd int, stat *simulation.Stat_t) (err error) { + return simulation.SyscallSysFstat(fd, stat) +} + +func GolangOrgXSysUnix_Fstatat(dirfd int, path string, stat *simulation.Stat_t, flags int) (err error) { + return simulation.SyscallSysNewfstatat(dirfd, path, stat, flags) +} + +func GolangOrgXSysUnix_Fsync(fd int) (err error) { + return simulation.SyscallSysFsync(fd) +} + +func GolangOrgXSysUnix_Ftruncate(fd int, length int64) (err error) { + return simulation.SyscallSysFtruncate(fd, length) +} + +func GolangOrgXSysUnix_Getdents(fd int, buf []byte) (n int, err error) { + return simulation.SyscallSysGetdents64(fd, buf) +} + +func GolangOrgXSysUnix_getpeername(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetpeername(fd, rsa, addrlen) +} + +func GolangOrgXSysUnix_Getpid() (pid int) { + return simulation.SyscallSysGetpid() +} + +func GolangOrgXSysUnix_Getrandom(buf []byte, flags int) (n int, err error) { + return simulation.SyscallSysGetrandom(buf, flags) +} + +func GolangOrgXSysUnix_getsockname(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockname(fd, rsa, addrlen) +} + +func GolangOrgXSysUnix_getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockopt(s, level, name, val, vallen) +} + +func GolangOrgXSysUnix_Listen(s int, n int) (err error) { + return simulation.SyscallSysListen(s, n) +} + +func GolangOrgXSysUnix_openat(dirfd int, path string, flags int, mode uint32) (fd int, err error) { + return simulation.SyscallSysOpenat(dirfd, path, flags, mode) +} + +func GolangOrgXSysUnix_pread(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPread64(fd, p, offset) +} + +func GolangOrgXSysUnix_pwrite(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPwrite64(fd, p, offset) +} + +func GolangOrgXSysUnix_read(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysRead(fd, p) +} + +func GolangOrgXSysUnix_Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) { + return simulation.SyscallSysRenameat(olddirfd, oldpath, newdirfd, newpath) +} + +func GolangOrgXSysUnix_setsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + return simulation.SyscallSysSetsockopt(s, level, name, val, vallen) +} + +func GolangOrgXSysUnix_socket(domain int, typ int, proto int) (fd int, err error) { + return simulation.SyscallSysSocket(domain, typ, proto) +} + +func GolangOrgXSysUnix_Uname(buf *simulation.Utsname) (err error) { + return simulation.SyscallSysUname(buf) +} + +func GolangOrgXSysUnix_Unlinkat(dirfd int, path string, flags int) (err error) { + return simulation.SyscallSysUnlinkat(dirfd, path, flags) +} + +func GolangOrgXSysUnix_write(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysWrite(fd, p) +} diff --git a/internal/hooks/go123/golangorg_x_sys_unix_gensyscall_arm64.go b/internal/hooks/go123/golangorg_x_sys_unix_gensyscall_arm64.go new file mode 100644 index 0000000..7d5bf38 --- /dev/null +++ b/internal/hooks/go123/golangorg_x_sys_unix_gensyscall_arm64.go @@ -0,0 +1,113 @@ +//go:build linux + +// Code generated by gensyscall. DO NOT EDIT. +package go123 + +import ( + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +// prevent unused imports +var _ unsafe.Pointer + +func GolangOrgXSysUnix_accept4(s int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen, flags int) (fd int, err error) { + return simulation.SyscallSysAccept4(s, rsa, addrlen, flags) +} + +func GolangOrgXSysUnix_bind(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysBind(s, addr, addrlen) +} + +func GolangOrgXSysUnix_Close(fd int) (err error) { + return simulation.SyscallSysClose(fd) +} + +func GolangOrgXSysUnix_connect(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysConnect(s, addr, addrlen) +} + +func GolangOrgXSysUnix_Fstat(fd int, stat *simulation.Stat_t) (err error) { + return simulation.SyscallSysFstat(fd, stat) +} + +func GolangOrgXSysUnix_Fstatat(fd int, path string, stat *simulation.Stat_t, flags int) (err error) { + return simulation.SyscallSysFstatat(fd, path, stat, flags) +} + +func GolangOrgXSysUnix_Fsync(fd int) (err error) { + return simulation.SyscallSysFsync(fd) +} + +func GolangOrgXSysUnix_Ftruncate(fd int, length int64) (err error) { + return simulation.SyscallSysFtruncate(fd, length) +} + +func GolangOrgXSysUnix_Getdents(fd int, buf []byte) (n int, err error) { + return simulation.SyscallSysGetdents64(fd, buf) +} + +func GolangOrgXSysUnix_getpeername(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetpeername(fd, rsa, addrlen) +} + +func GolangOrgXSysUnix_Getpid() (pid int) { + return simulation.SyscallSysGetpid() +} + +func GolangOrgXSysUnix_Getrandom(buf []byte, flags int) (n int, err error) { + return simulation.SyscallSysGetrandom(buf, flags) +} + +func GolangOrgXSysUnix_getsockname(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockname(fd, rsa, addrlen) +} + +func GolangOrgXSysUnix_getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockopt(s, level, name, val, vallen) +} + +func GolangOrgXSysUnix_Listen(s int, n int) (err error) { + return simulation.SyscallSysListen(s, n) +} + +func GolangOrgXSysUnix_openat(dirfd int, path string, flags int, mode uint32) (fd int, err error) { + return simulation.SyscallSysOpenat(dirfd, path, flags, mode) +} + +func GolangOrgXSysUnix_pread(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPread64(fd, p, offset) +} + +func GolangOrgXSysUnix_pwrite(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPwrite64(fd, p, offset) +} + +func GolangOrgXSysUnix_read(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysRead(fd, p) +} + +func GolangOrgXSysUnix_Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) { + return simulation.SyscallSysRenameat(olddirfd, oldpath, newdirfd, newpath) +} + +func GolangOrgXSysUnix_setsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + return simulation.SyscallSysSetsockopt(s, level, name, val, vallen) +} + +func GolangOrgXSysUnix_socket(domain int, typ int, proto int) (fd int, err error) { + return simulation.SyscallSysSocket(domain, typ, proto) +} + +func GolangOrgXSysUnix_Uname(buf *simulation.Utsname) (err error) { + return simulation.SyscallSysUname(buf) +} + +func GolangOrgXSysUnix_Unlinkat(dirfd int, path string, flags int) (err error) { + return simulation.SyscallSysUnlinkat(dirfd, path, flags) +} + +func GolangOrgXSysUnix_write(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysWrite(fd, p) +} diff --git a/internal/hooks/go123/hash_maphash.go b/internal/hooks/go123/hash_maphash.go new file mode 100644 index 0000000..3ac501a --- /dev/null +++ b/internal/hooks/go123/hash_maphash.go @@ -0,0 +1,7 @@ +package go123 + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func HashMaphash_runtime_rand() uint64 { + return gosimruntime.Fastrand64() +} diff --git a/internal/hooks/go123/internal_abi.go b/internal/hooks/go123/internal_abi.go new file mode 100644 index 0000000..91f4ef4 --- /dev/null +++ b/internal/hooks/go123/internal_abi.go @@ -0,0 +1,9 @@ +package go123 + +func InternalAbi_FuncPCABI0(f interface{}) uintptr { + panic("gosim not implemented") +} + +func InternalAbi_FuncPCABIInternal(f interface{}) uintptr { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/internal_bytealg.go b/internal/hooks/go123/internal_bytealg.go new file mode 100644 index 0000000..55f3375 --- /dev/null +++ b/internal/hooks/go123/internal_bytealg.go @@ -0,0 +1,55 @@ +package go123 + +import ( + "bytes" //gosim:notranslate + "strings" //gosim:notranslate + "unsafe" +) + +func InternalBytealg_IndexByteString(s string, c byte) int { + return strings.IndexByte(s, c) +} + +func InternalBytealg_Equal(a, b []byte) bool { + return bytes.Equal(a, b) +} + +func InternalBytealg_IndexByte(b []byte, c byte) int { + return bytes.IndexByte(b, c) +} + +func InternalBytealg_MakeNoZero(n int) []byte { + return make([]byte, n) +} + +func InternalBytealg_Compare(a, b []byte) int { + return bytes.Compare(a, b) +} + +func InternalBytealg_abigen_runtime_cmpstring(a, b string) int { + return strings.Compare(a, b) +} + +func InternalBytealg_Count(b []byte, c byte) int { + return bytes.Count(b, []byte{c}) +} + +func InternalBytealg_CountString(s string, c byte) int { + return strings.Count(s, string([]byte{c})) +} + +func InternalBytealg_abigen_runtime_memequal(a, b unsafe.Pointer, size uintptr) bool { + panic("gosim not implemented") +} + +func InternalBytealg_abigen_runtime_memequal_varlen(a, b unsafe.Pointer) bool { + panic("gosim not implemented") +} + +func InternalBytealg_Index(a, b []byte) int { + return bytes.Index(a, b) +} + +func InternalBytealg_IndexString(a, b string) int { + return strings.Index(a, b) +} diff --git a/internal/hooks/go123/internal_chacha8.go b/internal/hooks/go123/internal_chacha8.go new file mode 100644 index 0000000..df8073e --- /dev/null +++ b/internal/hooks/go123/internal_chacha8.go @@ -0,0 +1,12 @@ +package go123 + +import ( + _ "unsafe" +) + +//go:linkname block gosimnotranslate/internal/chacha8rand.block +func block(seed *[4]uint64, blocks *[32]uint64, counter uint32) + +func InternalChacha8rand_block(seed *[4]uint64, blocks *[32]uint64, counter uint32) { + block(seed, blocks, counter) +} diff --git a/internal/hooks/go123/internal_cpu.go b/internal/hooks/go123/internal_cpu.go new file mode 100644 index 0000000..f62b69a --- /dev/null +++ b/internal/hooks/go123/internal_cpu.go @@ -0,0 +1,21 @@ +package go123 + +func InternalCpu_getisar0() uint64 { + panic("gosim not implemented") +} + +func InternalCpu_getMIDR() uint64 { + panic("gosim not implemented") +} + +func InternalCpu_cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) { + panic("gosim not implemented") +} + +func InternalCpu_xgetbv() (eax, edx uint32) { + panic("gosim not implemented") +} + +func InternalCpu_getGOAMD64level() int32 { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/internal_godebug.go b/internal/hooks/go123/internal_godebug.go new file mode 100644 index 0000000..99199ac --- /dev/null +++ b/internal/hooks/go123/internal_godebug.go @@ -0,0 +1,16 @@ +package go123 + +import "unsafe" + +func InternalGodebug_setUpdate(update func(string, string)) { +} + +func InternalGodebug_registerMetric(name string, read func() uint64) { +} + +func InternalGodebug_setNewIncNonDefault(newIncNonDefault func(string) func()) { +} + +func InternalGodebug_write(fd uintptr, p unsafe.Pointer, n int32) int32 { + return n +} diff --git a/internal/hooks/go123/internal_poll.go b/internal/hooks/go123/internal_poll.go new file mode 100644 index 0000000..750fb67 --- /dev/null +++ b/internal/hooks/go123/internal_poll.go @@ -0,0 +1,63 @@ +//go:build linux + +package go123 + +import ( + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/simulation" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +func InternalPoll_runtimeNano() int64 { + panic("gosim not implemented") +} + +func InternalPoll_runtime_Semacquire(addr *uint32) { + gosimruntime.Semacquire(addr, false) +} + +func InternalPoll_runtime_Semrelease(addr *uint32) { + gosimruntime.Semrelease(addr) +} + +func InternalPoll_runtime_pollServerInit() { + // nop +} + +func InternalPoll_runtime_isPollServerDescriptor(fd uintptr) bool { + panic("gosim not implemented") +} + +// FIXME: returning a pointer here is very suspect, these are uintptrs on the +// other side and GC is ruthless + +func InternalPoll_runtime_pollOpen(fd uintptr) (*syscallabi.PollDesc, int) { + desc := syscallabi.AllocPollDesc(int(fd)) + code := simulation.SyscallPollOpen(desc.FD(), desc) + return desc, code +} + +func InternalPoll_runtime_pollClose(ctx *syscallabi.PollDesc) { + simulation.SyscallPollClose(ctx.FD(), ctx) + ctx.Close() +} + +func InternalPoll_runtime_pollWaitCanceled(ctx *syscallabi.PollDesc, mode int) { + panic("not implemented") // only used on windows +} + +func InternalPoll_runtime_pollReset(ctx *syscallabi.PollDesc, mode int) int { + return ctx.Reset(mode) +} + +func InternalPoll_runtime_pollWait(ctx *syscallabi.PollDesc, mode int) int { + return ctx.Wait(mode) +} + +func InternalPoll_runtime_pollSetDeadline(ctx *syscallabi.PollDesc, d int64, mode int) { + ctx.SetDeadline(d, mode) +} + +func InternalPoll_runtime_pollUnblock(ctx *syscallabi.PollDesc) { + ctx.Unblock() +} diff --git a/internal/hooks/go123/internal_syscall_unix.go b/internal/hooks/go123/internal_syscall_unix.go new file mode 100644 index 0000000..670c0d2 --- /dev/null +++ b/internal/hooks/go123/internal_syscall_unix.go @@ -0,0 +1,13 @@ +//go:build linux + +package go123 + +import "syscall" + +func InternalSyscallUnix_fcntl(fd int32, cmd int32, args int32) (int32, int32) { + return 0, int32(syscall.ENOSYS) +} + +func InternalSyscallUnix_GetRandom(p []byte, flags uintptr) (n int, err error) { + return GolangOrgXSysUnix_Getrandom(p, int(flags)) +} diff --git a/internal/hooks/go123/maps.go b/internal/hooks/go123/maps.go new file mode 100644 index 0000000..f0068b8 --- /dev/null +++ b/internal/hooks/go123/maps.go @@ -0,0 +1,9 @@ +package go123 + +import ( + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Maps_clone(m any) any { + return gosimruntime.CloneMap(m) +} diff --git a/internal/hooks/go123/math_rand.go b/internal/hooks/go123/math_rand.go new file mode 100644 index 0000000..410b35f --- /dev/null +++ b/internal/hooks/go123/math_rand.go @@ -0,0 +1,7 @@ +package go123 + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func MathRand_runtime_rand() uint64 { + return gosimruntime.Fastrand64() +} diff --git a/internal/hooks/go123/math_rand_v2.go b/internal/hooks/go123/math_rand_v2.go new file mode 100644 index 0000000..016b028 --- /dev/null +++ b/internal/hooks/go123/math_rand_v2.go @@ -0,0 +1,7 @@ +package go123 + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func MathRandV2_runtime_rand() uint64 { + return gosimruntime.Fastrand64() +} diff --git a/internal/hooks/go123/net.go b/internal/hooks/go123/net.go new file mode 100644 index 0000000..0bc8341 --- /dev/null +++ b/internal/hooks/go123/net.go @@ -0,0 +1,7 @@ +package go123 + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func Net_runtime_rand() uint64 { + return gosimruntime.Fastrand64() +} diff --git a/internal/hooks/go123/os.go b/internal/hooks/go123/os.go new file mode 100644 index 0000000..0e38916 --- /dev/null +++ b/internal/hooks/go123/os.go @@ -0,0 +1,32 @@ +package go123 + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func Os_runtime_args() []string { + // TODO: make this configurable / fetch this from the machine API? + return []string{"gosimapp"} +} + +func Os_sigpipe() { + panic("gosim not implemented") +} + +func Os_runtime_beforeExit(exitCode int) { + panic("gosim not implemented") +} + +func Os_runtime_rand() uint64 { + return gosimruntime.Fastrand64() +} + +func Os_checkClonePidfd() { + panic("gosim not implemented") +} + +func Os_ignoreSIGSYS() { + panic("gosim not implemented") +} + +func Os_restoreSIGSYS() { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/runtime_debug.go b/internal/hooks/go123/runtime_debug.go new file mode 100644 index 0000000..8272f97 --- /dev/null +++ b/internal/hooks/go123/runtime_debug.go @@ -0,0 +1,43 @@ +package go123 + +import "time" + +func RuntimeDebug_WriteHeapDump(fd uintptr) { + panic("gosim not implemented") +} + +func RuntimeDebug_SetTraceback(level string) { + panic("gosim not implemented") +} + +func RuntimeDebug_modinfo() string { + panic("gosim not implemented") +} + +func RuntimeDebug_readGCStats(*[]time.Duration) { + panic("gosim not implemented") +} + +func RuntimeDebug_freeOSMemory() { + panic("gosim not implemented") +} + +func RuntimeDebug_setMaxStack(int) int { + panic("gosim not implemented") +} + +func RuntimeDebug_setGCPercent(int32) int32 { + panic("gosim not implemented") +} + +func RuntimeDebug_setPanicOnFault(bool) bool { + panic("gosim not implemented") +} + +func RuntimeDebug_setMaxThreads(int) int { + panic("gosim not implemented") +} + +func RuntimeDebug_setMemoryLimit(int64) int64 { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/runtime_trace.go b/internal/hooks/go123/runtime_trace.go new file mode 100644 index 0000000..0d29dce --- /dev/null +++ b/internal/hooks/go123/runtime_trace.go @@ -0,0 +1,17 @@ +package go123 + +func RuntimeTrace_userTaskCreate(id, parentID uint64, taskType string) { + panic("gosim not implemented") +} + +func RuntimeTrace_userTaskEnd(id uint64) { + panic("gosim not implemented") +} + +func RuntimeTrace_userRegion(id, mode uint64, regionType string) { + panic("gosim not implemented") +} + +func RuntimeTrace_userLog(id uint64, category, message string) { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/sync.go b/internal/hooks/go123/sync.go new file mode 100644 index 0000000..90d1a00 --- /dev/null +++ b/internal/hooks/go123/sync.go @@ -0,0 +1,109 @@ +package go123 + +import ( + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Sync_runtime_Semacquire(addr *uint32) { + gosimruntime.Semacquire(addr, false) +} + +func Sync_runtime_Semrelease(addr *uint32, handoff bool, skipframes int) { + gosimruntime.Semrelease(addr) +} + +func Sync_runtime_SemacquireMutex(addr *uint32, lifo bool, skipframes int) { + gosimruntime.Semacquire(addr, lifo) +} + +func Sync_runtime_SemacquireRWMutexR(addr *uint32, lifo bool, skipframes int) { + gosimruntime.Semacquire(addr, lifo) +} + +func Sync_runtime_SemacquireRWMutex(addr *uint32, lifo bool, skipframes int) { + gosimruntime.Semacquire(addr, lifo) +} + +func Sync_runtime_registerPoolCleanup(cleanup func()) { + // noop +} + +func Sync_runtime_procPin() int { + // noop + // TODO: randomize? + return 0 +} + +func Sync_runtime_procUnpin() { + // noop +} + +func Sync_runtime_nanotime() int64 { + return gosimruntime.Nanotime() +} + +// Active spinning runtime support. +// runtime_canSpin reports whether spinning makes sense at the moment. +func Sync_runtime_canSpin(i int) bool { + return false +} + +// runtime_doSpin does active spinning. +func Sync_runtime_doSpin() { + panic("no") +} + +//go:norace +func Sync_runtime_LoadAcquintptr(ptr *uintptr) uintptr { + // XXX + if gosimruntime.AtomicYield { + gosimruntime.Yield() + } + return *ptr +} + +//go:norace +func Sync_runtime_StoreReluintptr(ptr *uintptr, val uintptr) uintptr { + if gosimruntime.AtomicYield { + gosimruntime.Yield() + } + *ptr = val + return 0 // XXX? +} + +type NotifyList struct { + inner gosimruntime.NotifyList +} + +func Sync_runtime_notifyListAdd(l *NotifyList) uint32 { + return l.inner.Add() +} + +func Sync_runtime_notifyListWait(l *NotifyList, t uint32) { + l.inner.Wait(t) +} + +func Sync_runtime_notifyListNotifyAll(l *NotifyList) { + l.inner.NotifyAll() +} + +func Sync_runtime_notifyListNotifyOne(l *NotifyList) { + l.inner.NotifyOne() +} + +// Ensure that sync and runtime agree on size of notifyList. +func Sync_runtime_notifyListCheck(size uintptr) { + // noop +} + +func Sync_runtime_randn(n uint32) uint32 { + return gosimruntime.Fastrandn(n) +} + +func Sync_throw(str string) { + panic(str) +} + +func Sync_fatal(str string) { + panic(str) +} diff --git a/internal/hooks/go123/sync_atomic.go b/internal/hooks/go123/sync_atomic.go new file mode 100644 index 0000000..7d8cb6b --- /dev/null +++ b/internal/hooks/go123/sync_atomic.go @@ -0,0 +1,394 @@ +package go123 + +import ( + "sync/atomic" //gosim:notranslate + "unsafe" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/race" +) + +func maybeAtomicYield() { + if gosimruntime.AtomicYield { + gosimruntime.Yield() + } +} + +func SyncAtomic_SwapInt32(addr *int32, new int32) (old int32) { + maybeAtomicYield() + if race.Enabled { + return atomic.SwapInt32(addr, new) + } + old = *addr + *addr = new + return +} + +func SyncAtomic_SwapInt64(addr *int64, new int64) (old int64) { + maybeAtomicYield() + if race.Enabled { + return atomic.SwapInt64(addr, new) + } + old = *addr + *addr = new + return +} + +func SyncAtomic_SwapUint32(addr *uint32, new uint32) (old uint32) { + maybeAtomicYield() + if race.Enabled { + return atomic.SwapUint32(addr, new) + } + old = *addr + *addr = new + return +} + +func SyncAtomic_SwapUint64(addr *uint64, new uint64) (old uint64) { + maybeAtomicYield() + if race.Enabled { + return atomic.SwapUint64(addr, new) + } + old = *addr + *addr = new + return +} + +func SyncAtomic_SwapUintptr(addr *uintptr, new uintptr) (old uintptr) { + maybeAtomicYield() + if race.Enabled { + return atomic.SwapUintptr(addr, new) + } + old = *addr + *addr = new + return +} + +func SyncAtomic_SwapPointer(addr *unsafe.Pointer, new unsafe.Pointer) (old unsafe.Pointer) { + maybeAtomicYield() + if race.Enabled { + return atomic.SwapPointer(addr, new) + } + old = *addr + *addr = new + return +} + +func SyncAtomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool) { + maybeAtomicYield() + if race.Enabled { + return atomic.CompareAndSwapInt32(addr, old, new) + } + if *addr != old { + return false + } + *addr = new + return true +} + +func SyncAtomic_CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool) { + maybeAtomicYield() + if race.Enabled { + return atomic.CompareAndSwapInt64(addr, old, new) + } + if *addr != old { + return false + } + *addr = new + return true +} + +func SyncAtomic_CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool) { + maybeAtomicYield() + if race.Enabled { + return atomic.CompareAndSwapUint32(addr, old, new) + } + if *addr != old { + return false + } + *addr = new + return true +} + +func SyncAtomic_CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool) { + maybeAtomicYield() + if race.Enabled { + return atomic.CompareAndSwapUint64(addr, old, new) + } + if *addr != old { + return false + } + *addr = new + return true +} + +func SyncAtomic_CompareAndSwapUintptr(addr *uintptr, old, new uintptr) (swapped bool) { + maybeAtomicYield() + if race.Enabled { + return atomic.CompareAndSwapUintptr(addr, old, new) + } + if *addr != old { + return false + } + *addr = new + return true +} + +func SyncAtomic_CompareAndSwapPointer(addr *unsafe.Pointer, old, new unsafe.Pointer) (swapped bool) { + maybeAtomicYield() + if race.Enabled { + return atomic.CompareAndSwapPointer(addr, old, new) + } + if *addr != old { + return false + } + *addr = new + return true +} + +func SyncAtomic_AddInt32(addr *int32, delta int32) (new int32) { + maybeAtomicYield() + if race.Enabled { + return atomic.AddInt32(addr, delta) + } + *addr += delta + return *addr +} + +func SyncAtomic_AddUint32(addr *uint32, delta uint32) (new uint32) { + maybeAtomicYield() + if race.Enabled { + return atomic.AddUint32(addr, delta) + } + *addr += delta + return *addr +} + +func SyncAtomic_AddInt64(addr *int64, delta int64) (new int64) { + maybeAtomicYield() + if race.Enabled { + return atomic.AddInt64(addr, delta) + } + *addr += delta + return *addr +} + +func SyncAtomic_AddUint64(addr *uint64, delta uint64) (new uint64) { + maybeAtomicYield() + if race.Enabled { + return atomic.AddUint64(addr, delta) + } + *addr += delta + return *addr +} + +func SyncAtomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr) { + maybeAtomicYield() + if race.Enabled { + return atomic.AddUintptr(addr, delta) + } + *addr += delta + return *addr +} + +func SyncAtomic_AndInt32(addr *int32, delta int32) (new int32) { + maybeAtomicYield() + if race.Enabled { + return atomic.AndInt32(addr, delta) + } + *addr &= delta + return *addr +} + +func SyncAtomic_AndUint32(addr *uint32, delta uint32) (new uint32) { + maybeAtomicYield() + if race.Enabled { + return atomic.AndUint32(addr, delta) + } + *addr &= delta + return *addr +} + +func SyncAtomic_AndInt64(addr *int64, delta int64) (new int64) { + maybeAtomicYield() + if race.Enabled { + return atomic.AndInt64(addr, delta) + } + *addr &= delta + return *addr +} + +func SyncAtomic_AndUint64(addr *uint64, delta uint64) (new uint64) { + maybeAtomicYield() + if race.Enabled { + return atomic.AndUint64(addr, delta) + } + *addr &= delta + return *addr +} + +func SyncAtomic_AndUintptr(addr *uintptr, delta uintptr) (new uintptr) { + maybeAtomicYield() + if race.Enabled { + return atomic.AndUintptr(addr, delta) + } + *addr &= delta + return *addr +} + +func SyncAtomic_OrInt32(addr *int32, delta int32) (new int32) { + maybeAtomicYield() + if race.Enabled { + return atomic.OrInt32(addr, delta) + } + *addr |= delta + return *addr +} + +func SyncAtomic_OrUint32(addr *uint32, delta uint32) (new uint32) { + maybeAtomicYield() + if race.Enabled { + return atomic.OrUint32(addr, delta) + } + *addr |= delta + return *addr +} + +func SyncAtomic_OrInt64(addr *int64, delta int64) (new int64) { + maybeAtomicYield() + if race.Enabled { + return atomic.OrInt64(addr, delta) + } + *addr |= delta + return *addr +} + +func SyncAtomic_OrUint64(addr *uint64, delta uint64) (new uint64) { + maybeAtomicYield() + if race.Enabled { + return atomic.OrUint64(addr, delta) + } + *addr |= delta + return *addr +} + +func SyncAtomic_OrUintptr(addr *uintptr, delta uintptr) (new uintptr) { + maybeAtomicYield() + if race.Enabled { + return atomic.OrUintptr(addr, delta) + } + *addr |= delta + return *addr +} + +func SyncAtomic_LoadInt32(addr *int32) (val int32) { + maybeAtomicYield() + if race.Enabled { + return atomic.LoadInt32(addr) + } + return *addr +} + +func SyncAtomic_LoadInt64(addr *int64) (val int64) { + maybeAtomicYield() + if race.Enabled { + return atomic.LoadInt64(addr) + } + return *addr +} + +func SyncAtomic_LoadUint32(addr *uint32) (val uint32) { + maybeAtomicYield() + if race.Enabled { + return atomic.LoadUint32(addr) + } + return *addr +} + +func SyncAtomic_LoadUint64(addr *uint64) (val uint64) { + maybeAtomicYield() + if race.Enabled { + return atomic.LoadUint64(addr) + } + return *addr +} + +func SyncAtomic_LoadUintptr(addr *uintptr) (val uintptr) { + maybeAtomicYield() + if race.Enabled { + return atomic.LoadUintptr(addr) + } + return *addr +} + +func SyncAtomic_LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer) { + maybeAtomicYield() + if race.Enabled { + return atomic.LoadPointer(addr) + } + return *addr +} + +func SyncAtomic_StoreInt32(addr *int32, val int32) { + maybeAtomicYield() + if race.Enabled { + atomic.StoreInt32(addr, val) + return + } + *addr = val +} + +func SyncAtomic_StoreInt64(addr *int64, val int64) { + maybeAtomicYield() + if race.Enabled { + atomic.StoreInt64(addr, val) + return + } + *addr = val +} + +func SyncAtomic_StoreUint32(addr *uint32, val uint32) { + maybeAtomicYield() + if race.Enabled { + atomic.StoreUint32(addr, val) + return + } + *addr = val +} + +func SyncAtomic_StoreUint64(addr *uint64, val uint64) { + maybeAtomicYield() + if race.Enabled { + atomic.StoreUint64(addr, val) + return + } + *addr = val +} + +func SyncAtomic_StoreUintptr(addr *uintptr, val uintptr) { + maybeAtomicYield() + if race.Enabled { + atomic.StoreUintptr(addr, val) + return + } + *addr = val +} + +func SyncAtomic_StorePointer(addr *unsafe.Pointer, val unsafe.Pointer) { + maybeAtomicYield() + if race.Enabled { + atomic.StorePointer(addr, val) + return + } + *addr = val +} + +func SyncAtomic_runtime_procPin() int { + // noop + // TODO: randomize? + return 0 +} + +func SyncAtomic_runtime_procUnpin() { + // noop +} diff --git a/internal/hooks/go123/syscall.go b/internal/hooks/go123/syscall.go new file mode 100644 index 0000000..a9b7caf --- /dev/null +++ b/internal/hooks/go123/syscall.go @@ -0,0 +1,88 @@ +//go:build linux + +package go123 + +import ( + "sync" + "syscall" + "unsafe" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +func Syscall_runtime_BeforeFork() { + panic("gosim not implemented") +} + +func Syscall_runtime_AfterFork() { + panic("gosim not implemented") +} + +func Syscall_runtime_AfterForkInChild() { + panic("gosim not implemented") +} + +func Syscall_runtime_BeforeExec() { + panic("gosim not implemented") +} + +func Syscall_runtime_AfterExec() { + panic("gosim not implemented") +} + +func Syscall_hasWaitingReaders(rw *sync.RWMutex) bool { + panic("gosim not implemented") +} + +func Syscall_Getpagesize() int { + panic("gosim not implemented") +} + +func Syscall_Exit(code int) { + panic("gosim not implemented") +} + +func Syscall_runtimeSetenv(k, v string) { + panic("gosim not implemented") +} + +func Syscall_runtimeUnsetenv(k string) { + panic("gosim not implemented") +} + +func Syscall_rawSyscallNoError(trap, a1, a2, a3 uintptr) (r1, r2 uintptr) { + panic("gosim not implemented") +} + +func Syscall_rawVforkSyscall(trap, a1, a2 uintptr) (r1 uintptr, err syscall.Errno) { + panic("gosim not implemented") +} + +func Syscall_runtime_doAllThreadsSyscall(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { + panic("gosim not implemented") +} + +func Syscall_cgocaller(unsafe.Pointer, ...uintptr) uintptr { + panic("gosim not implemented") +} + +func Syscall_runtime_envs() []string { + return gosimruntime.Envs() +} + +func Syscall_runtime_entersyscall() { +} + +func Syscall_runtime_exitsyscall() { +} + +//go:nocheckptr +//go:uintptrescapes +func Syscall_RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { + return simulation.RawSyscall6(trap, a1, a2, a3, a4, a5, a6) +} + +func Syscall_gettimeofday(timeval unsafe.Pointer) syscall.Errno { + panic("gosim not implemented") +} diff --git a/internal/hooks/go123/syscall_gensyscall_amd64.go b/internal/hooks/go123/syscall_gensyscall_amd64.go new file mode 100644 index 0000000..a31522e --- /dev/null +++ b/internal/hooks/go123/syscall_gensyscall_amd64.go @@ -0,0 +1,113 @@ +//go:build linux + +// Code generated by gensyscall. DO NOT EDIT. +package go123 + +import ( + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +// prevent unused imports +var _ unsafe.Pointer + +func Syscall_accept4(s int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen, flags int) (fd int, err error) { + return simulation.SyscallSysAccept4(s, rsa, addrlen, flags) +} + +func Syscall_bind(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysBind(s, addr, addrlen) +} + +func Syscall_Close(fd int) (err error) { + return simulation.SyscallSysClose(fd) +} + +func Syscall_connect(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysConnect(s, addr, addrlen) +} + +func Syscall_fcntl(fd int, cmd int, arg int) (val int, err error) { + return simulation.SyscallSysFcntl(fd, cmd, arg) +} + +func Syscall_Fstat(fd int, stat *simulation.Stat_t) (err error) { + return simulation.SyscallSysFstat(fd, stat) +} + +func Syscall_fstatat(fd int, path string, stat *simulation.Stat_t, flags int) (err error) { + return simulation.SyscallSysNewfstatat(fd, path, stat, flags) +} + +func Syscall_Fsync(fd int) (err error) { + return simulation.SyscallSysFsync(fd) +} + +func Syscall_Ftruncate(fd int, length int64) (err error) { + return simulation.SyscallSysFtruncate(fd, length) +} + +func Syscall_Getdents(fd int, buf []byte) (n int, err error) { + return simulation.SyscallSysGetdents64(fd, buf) +} + +func Syscall_getpeername(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetpeername(fd, rsa, addrlen) +} + +func Syscall_Getpid() (pid int) { + return simulation.SyscallSysGetpid() +} + +func Syscall_getsockname(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockname(fd, rsa, addrlen) +} + +func Syscall_getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockopt(s, level, name, val, vallen) +} + +func Syscall_Listen(s int, n int) (err error) { + return simulation.SyscallSysListen(s, n) +} + +func Syscall_openat(dirfd int, path string, flags int, mode uint32) (fd int, err error) { + return simulation.SyscallSysOpenat(dirfd, path, flags, mode) +} + +func Syscall_pread(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPread64(fd, p, offset) +} + +func Syscall_pwrite(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPwrite64(fd, p, offset) +} + +func Syscall_read(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysRead(fd, p) +} + +func Syscall_Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) { + return simulation.SyscallSysRenameat(olddirfd, oldpath, newdirfd, newpath) +} + +func Syscall_setsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + return simulation.SyscallSysSetsockopt(s, level, name, val, vallen) +} + +func Syscall_socket(domain int, typ int, proto int) (fd int, err error) { + return simulation.SyscallSysSocket(domain, typ, proto) +} + +func Syscall_Uname(buf *simulation.Utsname) (err error) { + return simulation.SyscallSysUname(buf) +} + +func Syscall_unlinkat(dirfd int, path string, flags int) (err error) { + return simulation.SyscallSysUnlinkat(dirfd, path, flags) +} + +func Syscall_write(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysWrite(fd, p) +} diff --git a/internal/hooks/go123/syscall_gensyscall_arm64.go b/internal/hooks/go123/syscall_gensyscall_arm64.go new file mode 100644 index 0000000..749c742 --- /dev/null +++ b/internal/hooks/go123/syscall_gensyscall_arm64.go @@ -0,0 +1,113 @@ +//go:build linux + +// Code generated by gensyscall. DO NOT EDIT. +package go123 + +import ( + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +// prevent unused imports +var _ unsafe.Pointer + +func Syscall_accept4(s int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen, flags int) (fd int, err error) { + return simulation.SyscallSysAccept4(s, rsa, addrlen, flags) +} + +func Syscall_bind(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysBind(s, addr, addrlen) +} + +func Syscall_Close(fd int) (err error) { + return simulation.SyscallSysClose(fd) +} + +func Syscall_connect(s int, addr unsafe.Pointer, addrlen simulation.Socklen) (err error) { + return simulation.SyscallSysConnect(s, addr, addrlen) +} + +func Syscall_fcntl(fd int, cmd int, arg int) (val int, err error) { + return simulation.SyscallSysFcntl(fd, cmd, arg) +} + +func Syscall_Fstat(fd int, stat *simulation.Stat_t) (err error) { + return simulation.SyscallSysFstat(fd, stat) +} + +func Syscall_fstatat(dirfd int, path string, stat *simulation.Stat_t, flags int) (err error) { + return simulation.SyscallSysFstatat(dirfd, path, stat, flags) +} + +func Syscall_Fsync(fd int) (err error) { + return simulation.SyscallSysFsync(fd) +} + +func Syscall_Ftruncate(fd int, length int64) (err error) { + return simulation.SyscallSysFtruncate(fd, length) +} + +func Syscall_Getdents(fd int, buf []byte) (n int, err error) { + return simulation.SyscallSysGetdents64(fd, buf) +} + +func Syscall_getpeername(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetpeername(fd, rsa, addrlen) +} + +func Syscall_Getpid() (pid int) { + return simulation.SyscallSysGetpid() +} + +func Syscall_getsockname(fd int, rsa *simulation.RawSockaddrAny, addrlen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockname(fd, rsa, addrlen) +} + +func Syscall_getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *simulation.Socklen) (err error) { + return simulation.SyscallSysGetsockopt(s, level, name, val, vallen) +} + +func Syscall_Listen(s int, n int) (err error) { + return simulation.SyscallSysListen(s, n) +} + +func Syscall_openat(dirfd int, path string, flags int, mode uint32) (fd int, err error) { + return simulation.SyscallSysOpenat(dirfd, path, flags, mode) +} + +func Syscall_pread(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPread64(fd, p, offset) +} + +func Syscall_pwrite(fd int, p []byte, offset int64) (n int, err error) { + return simulation.SyscallSysPwrite64(fd, p, offset) +} + +func Syscall_read(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysRead(fd, p) +} + +func Syscall_Renameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) { + return simulation.SyscallSysRenameat(olddirfd, oldpath, newdirfd, newpath) +} + +func Syscall_setsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + return simulation.SyscallSysSetsockopt(s, level, name, val, vallen) +} + +func Syscall_socket(domain int, typ int, proto int) (fd int, err error) { + return simulation.SyscallSysSocket(domain, typ, proto) +} + +func Syscall_Uname(buf *simulation.Utsname) (err error) { + return simulation.SyscallSysUname(buf) +} + +func Syscall_unlinkat(dirfd int, path string, flags int) (err error) { + return simulation.SyscallSysUnlinkat(dirfd, path, flags) +} + +func Syscall_write(fd int, p []byte) (n int, err error) { + return simulation.SyscallSysWrite(fd, p) +} diff --git a/internal/hooks/go123/time.go b/internal/hooks/go123/time.go new file mode 100644 index 0000000..437899e --- /dev/null +++ b/internal/hooks/go123/time.go @@ -0,0 +1,112 @@ +package go123 + +import ( + "time" + "unsafe" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/race" +) + +func Time_now() (sec int64, nsec int32, mono int64) { + now := gosimruntime.Nanotime() + + // TODO: somehow avoid this division? + sec = now / 1e9 + mono = now + nsec = int32(mono - sec*1e9) + + return +} + +func Time_runtimeNano() int64 { + return gosimruntime.Nanotime() +} + +func Time_Sleep(duration time.Duration) { + gosimruntime.Sleep(int64(duration)) +} + +//go:norace +func fireRuntimeTimer(t *gosimruntime.Timer) { + rt := t.Arg.(*timeTimer) + machine := t.Machine + + f := rt.f + arg := rt.arg + seq := rt.seq + + // TODO: only Go here in race mode; otherwise (always?) try to reuse some + // outer (per-machine?) goroutine + gosimruntime.GoFromTimerHandler(func() { + // TODO: in mean mode, insert delay here? + if race.Enabled { + race.Acquire(unsafe.Pointer(rt)) + } + delta := int64(0) // TODO: adjust? + f(arg, seq, delta) + // FIXME: this might be borked; standard library expects to run + // atomically instead of in a goroutine? + }, machine) + + if rt.period != 0 { + // FIXME: this might be borked; base off now? + rt.when += rt.period + t.Reset(rt.when) + } +} + +type timeTimer struct { + c unsafe.Pointer + init bool + timer +} + +type timer struct { + pp *gosimruntime.Timer // repurposed pp + when int64 + period int64 + f func(arg any, seq uintptr, delta int64) // NOTE: must not be closure + arg any // passed to f + seq uintptr // passed to f +} + +//go:norace +func Time_newTimer(when, period int64, f func(any, uintptr, int64), arg any, cp unsafe.Pointer) *timeTimer { + t := &timeTimer{ + init: true, // huh + } + + if race.Enabled { + race.Release(unsafe.Pointer(t)) + } + + // TODO: use cp to do async chan stuff? + + t.when = when + t.period = period + t.f = f + t.arg = arg + t.seq = 0 + + t.pp = gosimruntime.NewTimer(fireRuntimeTimer, t, gosimruntime.CurrentMachine(), t.when) + + return t +} + +//go:norace +func Time_stopTimer(t *timeTimer) bool { + return t.pp.Stop() +} + +//go:norace +func Time_resetTimer(t *timeTimer, when, period int64) bool { + if race.Enabled { + race.Release(unsafe.Pointer(t)) + } + + t.when = when + t.period = period + + return t.pp.Reset(when) +} diff --git a/internal/prettylog/prettylog.go b/internal/prettylog/prettylog.go new file mode 100644 index 0000000..b642138 --- /dev/null +++ b/internal/prettylog/prettylog.go @@ -0,0 +1,408 @@ +// MIT License +// +// # Copyright (c) 2017 Olivier Poitrey +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// Based on https://github.com/rs/zerolog/blob/master/console.go. +package prettylog + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + "path" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/mattn/go-isatty" +) + +const ( + colorBlack = iota + 30 + colorRed + colorGreen + colorYellow + colorBlue + colorMagenta + colorCyan + colorWhite + + colorBold = 1 + colorDarkGray = 90 +) + +// algorithm: +// - print well known fields in fixed order with standard formatters +// - print other fields in input order (XXX: sorted for now) +// - for fields that are big (original json >80 characters?) print them on separate lines (XXX: traceback only for now) + +type Writer struct { + out io.Writer + formatter formatter +} + +// NewWriter creates and initializes a new ConsoleWriter. +func NewWriter(out io.Writer) *Writer { + w := Writer{ + out: out, + } + + noColor := (os.Getenv("NO_COLOR") != "") || os.Getenv("TERM") == "dumb" || + (!isatty.IsTerminal(os.Stdout.Fd()) && !isatty.IsCygwinTerminal(os.Stdout.Fd())) + noColor = noColor && !(os.Getenv("FORCE_COLOR") != "") + w.formatter = formatter{noColor: noColor} + + return &w +} + +var writePool = sync.Pool{ + New: func() interface{} { + return bytes.NewBuffer(make([]byte, 0, 1024)) + }, +} + +// Write transforms the JSON input with formatters and appends to w.Out. +func (w *Writer) Write(p []byte) (n int, err error) { + buf := writePool.Get().(*bytes.Buffer) + defer func() { + buf.Reset() + writePool.Put(buf) + }() + + i := 0 + for i < len(p) && p[i] == ' ' { + i++ + } + prefix := p[:i] + p = p[i:] + + var evt map[string]interface{} + d := json.NewDecoder(bytes.NewReader(p)) + d.UseNumber() + err = d.Decode(&evt) + if err != nil { + // XXX: test that this is output + w.out.Write(prefix) + w.out.Write(p) + return n, fmt.Errorf("cannot decode event: %s", err) + } + // XXX: what happens with stuff after the message? fix and test + + for _, p := range []string{ + "step", + "machine", + // "goroutine", + slog.TimeKey, + slog.LevelKey, + slog.SourceKey, + slog.MessageKey, + } { + w.writePart(buf, evt, p) + } + + w.writeFields(evt, buf) + + err = buf.WriteByte('\n') + if err != nil { + return n, err + } + + // XXX: clean up somehow + first := true + buffer := buf.Bytes() + for { + idx := bytes.IndexByte(buffer, '\n') + if idx == -1 { + break + } + w.out.Write(prefix) + if !first { + w.out.Write([]byte(" ")) + } + w.out.Write(buffer[:idx+1]) + first = false + buffer = buffer[idx+1:] + } + if len(buffer) > 0 { + w.out.Write(prefix) + if !first { + w.out.Write([]byte(" ")) + } + w.out.Write(buffer) + } + + return len(p), err +} + +func jsonMarshal(v interface{}, indent bool) ([]byte, error) { + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + encoder.SetEscapeHTML(false) + if indent { + encoder.SetIndent(" ", " ") + } + if err := encoder.Encode(v); err != nil { + return nil, err + } + b := buf.Bytes() + if len(b) > 0 { + // Remove trailing \n which is added by Encode. + return b[:len(b)-1], nil + } + return b, nil +} + +// needsQuote returns true when the string s should be quoted in output. +func needsQuote(s string) bool { + for i := range s { + if s[i] < 0x20 || s[i] > 0x7e || s[i] == ' ' || s[i] == '\\' || s[i] == '"' { + return true + } + } + return false +} + +const errorKey = "err" + +// writeFields appends formatted key-value pairs to buf. +func (w Writer) writeFields(evt map[string]interface{}, buf *bytes.Buffer) { + fields := make([]string, 0, len(evt)) + for field := range evt { + // XXX: support skipping more? + switch field { + case "step", "machine", "goroutine", slog.LevelKey, slog.TimeKey, slog.MessageKey, slog.SourceKey: + continue + } + fields = append(fields, field) + } + sort.Strings(fields) + + // Move the "error" field to the front + ei := sort.Search(len(fields), func(i int) bool { return fields[i] >= errorKey }) + if ei < len(fields) && fields[ei] == errorKey { + fields = append(slices.Insert(fields[:ei], 0, errorKey), fields[ei+1:]...) + } + + for _, field := range fields { + if buf.Len() > 0 { + buf.WriteByte(' ') + } + buf.WriteString(w.formatter.fieldName(field)) + + if field == "traceback" { + b, err := jsonMarshal(evt[field], true) + if err != nil { + fmt.Fprintf(buf, w.formatter.colorize("[error: %v]", colorRed), err) + } else { + buf.WriteString(w.formatter.fieldValue(field, b)) + } + continue + } + + switch value := evt[field].(type) { + case string: + if needsQuote(value) { + buf.WriteString(w.formatter.fieldValue(field, strconv.Quote(value))) + } else { + buf.WriteString(w.formatter.fieldValue(field, value)) + } + case json.Number: + buf.WriteString(w.formatter.fieldValue(field, value)) + default: + b, err := jsonMarshal(value, false) + if err != nil { + fmt.Fprintf(buf, w.formatter.colorize("[error: %v]", colorRed), err) + } else { + buf.WriteString(w.formatter.fieldValue(field, b)) + } + } + } +} + +var pad = " " // hope you don't need more :) + +func padLeft(s string, n int) string { + if len(s) >= n { + return s + } + return pad[:n-len(s)] + s +} + +func padRight(s string, n int) string { + if len(s) >= n { + return s + } + return s + pad[:n-len(s)] +} + +// writePart appends a formatted part to buf. +func (w Writer) writePart(buf *bytes.Buffer, evt map[string]interface{}, p string) { + var s string + switch p { + case slog.LevelKey: + s = w.formatter.level(evt[p]) + case slog.TimeKey: + s = w.formatter.timestamp(evt[p]) + case slog.MessageKey: + // XXX: this level is a string so will cause type problems? + s = w.formatter.message(evt[slog.LevelKey], evt[p]) + case slog.SourceKey: + s = w.formatter.caller(evt[p]) + case "machine": + s = padRight(fmt.Sprintf("%s/%s", evt["machine"], evt["goroutine"]), 10) + case "step": + s = padLeft(fmt.Sprint(evt[p]), 5) + default: + s = w.formatter.fieldValue(p, evt[p]) + } + + if len(s) > 0 { + if buf.Len() > 0 { + buf.WriteByte(' ') // Write space only if not the first part + } + buf.WriteString(s) + } +} + +type formatter struct { + noColor bool +} + +// colorize returns the string s wrapped in ANSI code c, unless disabled is true or c is 0. +func (f *formatter) colorize(s interface{}, c ...int) string { + if len(c) == 0 || (len(c) == 1 && c[0] == 0) || f.noColor { + // XXX: %s or %v? + return fmt.Sprintf("%s", s) + } + for _, c := range c { + s = fmt.Sprintf("\x1b[%dm%v\x1b[0m", c, s) + } + return s.(string) +} + +const timeFormat = "15:04:05.000" + +func (f *formatter) timestamp(i interface{}) string { + if s, ok := i.(string); ok { + ts, err := time.ParseInLocation(time.RFC3339Nano, s, time.UTC) + if err != nil { + // ignore + } else { + i = ts.In(time.UTC).Format(timeFormat) + } + } + return f.colorize(i, colorDarkGray) +} + +var levelColors = map[slog.Level]int{ + slog.LevelDebug: colorMagenta, + slog.LevelInfo: colorGreen, + slog.LevelWarn: colorYellow, + slog.LevelError: colorRed, +} + +// formattedLevels are used by ConsoleWriter's consoleDefaultFormatLevel +// for a short level name. +var formattedLevels = map[slog.Level]string{ + slog.LevelDebug: "DBG", + slog.LevelInfo: "INF", + slog.LevelWarn: "WRN", + slog.LevelError: "ERR", +} + +func (f *formatter) level(i interface{}) string { + var l string + if ll, ok := i.(string); ok { + var level slog.Level + level.UnmarshalText([]byte(ll)) + fl, ok := formattedLevels[level] + if ok { + l = f.colorize(fl, levelColors[level]) + } else { + l = strings.ToUpper(ll)[0:3] + } + } else { + if i == nil { + l = "???" + } else { + l = strings.ToUpper(fmt.Sprintf("%s", i))[0:3] + } + } + return l +} + +func (f *formatter) caller(i interface{}) string { + m, ok := i.(map[string]any) + if !ok { + return "" + } + + // fn, _ := m["function"].(string) + file, _ := m["file"].(string) + line, _ := m["line"].(json.Number) + + name := path.Base(file) + dir := path.Base(path.Dir(file)) + file = fmt.Sprintf("%s/%s:%s", dir, name, line) + + c := file + if len(c) > 0 { + /* + if cwd, err := os.Getwd(); err == nil { + if rel, err := filepath.Rel(cwd, c); err == nil { + c = rel + } + } + */ + c = f.colorize(c, colorDarkGray) + f.colorize(" >", colorCyan) + } + return c +} + +func (f *formatter) message(level interface{}, i interface{}) string { + if i == nil || i == "" { + return "" + } + switch level { + case slog.LevelInfo, slog.LevelWarn, slog.LevelError: + return f.colorize(fmt.Sprintf("%s", i), colorBold) + default: + return fmt.Sprintf("%s", i) + } +} + +func (f *formatter) fieldName(i interface{}) string { + return f.colorize(fmt.Sprintf("%s=", i), colorCyan) +} + +func (f *formatter) fieldValue(field string, i interface{}) string { + if field == errorKey { + return f.colorize(fmt.Sprintf("%s", i), colorBold, colorRed) + } else { + return fmt.Sprintf("%s", i) + } +} diff --git a/internal/prettylog/prettylog_test.go b/internal/prettylog/prettylog_test.go new file mode 100644 index 0000000..1f18d5b --- /dev/null +++ b/internal/prettylog/prettylog_test.go @@ -0,0 +1,93 @@ +package prettylog_test + +import ( + "bytes" + "errors" + "flag" + "os" + "path" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/prettylog" + "github.com/jellevandenhooff/gosim/metatesting" +) + +var ( + rewriteInput = flag.Bool("rewriteInput", false, "rewrite input files") + rewrite = flag.Bool("rewrite", false, "rewrite golden files") +) + +func format(input []byte) []byte { + var buffer bytes.Buffer + writer := prettylog.NewWriter(&buffer) + + lines := bytes.SplitAfter(input, []byte("\n")) + for _, line := range lines { + if len(line) == 0 { + continue + } + writer.Write(line) + } + + return buffer.Bytes() +} + +func TestPrettyLog(t *testing.T) { + files, err := os.ReadDir("./testdata") + if err != nil { + t.Fatal(err) + } + + if *rewriteInput { + mt := metatesting.ForOtherPackage(t, gosimtool.Module+"/internal/tests/behavior") + out, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestLogForPrettyTest", + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path.Join("testdata", "simple.txt"), out.LogOutput, 0o644); err != nil { + t.Fatal(err) + } + } + + for _, entry := range files { + name := entry.Name() + if strings.HasSuffix(name, ".golden") { + continue + } + if !strings.HasSuffix(name, ".txt") { + continue + } + + t.Run(name, func(t *testing.T) { + input, err := os.ReadFile(path.Join("testdata", name)) + if err != nil { + t.Fatal(err) + } + output, err := os.ReadFile(path.Join("testdata", name+".golden")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + + expected := format(input) + if testing.Verbose() { + os.Stdout.Write(expected) + } + if *rewrite { + if err := os.WriteFile(path.Join("testdata", name+".golden"), expected, 0o644); err != nil { + t.Error(err) + } + } else { + if diff := cmp.Diff(string(output), string(expected)); diff != "" { + t.Error(diff) + } + } + }) + } +} diff --git a/internal/prettylog/testdata/simple.txt b/internal/prettylog/testdata/simple.txt new file mode 100644 index 0000000..9f7da3d --- /dev/null +++ b/internal/prettylog/testdata/simple.txt @@ -0,0 +1,5 @@ +=== RUN TestLogForPrettyTest +{"time":"2020-01-15T14:10:03.000001234Z","level":"INFO","source":{"function":"translated/github.com/jellevandenhooff/gosim/internal_/tests/behavior_test.ImplTestLogForPrettyTest","file":"/Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/tests/behavior/log_gosim_test.go","line":53},"msg":"hello info","machine":"main","foo":"bar","goroutine":4,"step":1} +{"time":"2020-01-15T14:10:23.000001234Z","level":"INFO","source":{"function":"translated/github.com/jellevandenhooff/gosim/internal_/tests/behavior_test.ImplTestLogForPrettyTest.func1","file":"/Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/tests/behavior/log_gosim_test.go","line":58},"msg":"warn","machine":"machine-2","ok":"now","delay":0,"goroutine":5,"step":2} +{"time":"2020-01-15T14:10:23.000001234Z","level":"INFO","source":{"function":"translated/github.com/jellevandenhooff/gosim/internal_/tests/behavior_test.ImplTestLogForPrettyTest.func1","file":"/Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/tests/behavior/log_gosim_test.go","line":59},"msg":"before","machine":"machine-2","goroutine":5,"step":3} +{"time":"2020-01-15T06:10:23.000001234-08:00","level":"ERROR","source":{"function":"github.com/jellevandenhooff/gosim/gosimruntime.(*goroutine).step","file":"/Users/jelle/hack/gosim/gosimruntime/runtime.go","line":656},"msg":"uncaught panic","traceback":["goroutine 9 [running]:","github.com/jellevandenhooff/gosim/gosimruntime.(*goroutine).exitpoint(0x1400045de40)"," /Users/jelle/hack/gosim/gosimruntime/runtime.go:728 +0x9c","panic({0x1011dc360?, 0x10147b090?})"," /opt/homebrew/Cellar/go/1.23.3/libexec/src/runtime/panic.go:785 +0x124","translated/github.com/jellevandenhooff/gosim/internal_/tests/behavior_test.ImplTestLogForPrettyTest.func1()"," /Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/tests/behavior/log_gosim_test.go:60 +0xe0","translated/github.com/jellevandenhooff/gosim/internal_/simulation.(*Simulation).startMachine.func1()"," /Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/simulation/simulation_gosim.go:96 +0x4c","github.com/jellevandenhooff/gosim/gosimruntime.(*goroutine).entrypoint(0x1400045de40, 0x14000461f50)"," /Users/jelle/hack/gosim/gosimruntime/runtime.go:712 +0x60","github.com/jellevandenhooff/gosim/gosimruntime.goroutineEntrypoint()"," /Users/jelle/hack/gosim/gosimruntime/runtime.go:705 +0x30","github.com/jellevandenhooff/gosim/internal/coro.(*Coro).Start.func1(0x0?)"," /Users/jelle/hack/gosim/internal/coro/coro_linkname.go:38 +0x24","created by github.com/jellevandenhooff/gosim/internal/coro.(*Coro).Start in goroutine 5"," /Users/jelle/hack/gosim/internal/coro/coro_linkname.go:36 +0x70",""],"panic":"help","machine":"machine-2","goroutine":5} diff --git a/internal/prettylog/testdata/simple.txt.golden b/internal/prettylog/testdata/simple.txt.golden new file mode 100644 index 0000000..10975b6 --- /dev/null +++ b/internal/prettylog/testdata/simple.txt.golden @@ -0,0 +1,24 @@ +=== RUN TestLogForPrettyTest + 1 main/4 14:10:03.000 INF behavior/log_gosim_test.go:53 > hello info foo=bar + 2 machine-2/5 14:10:23.000 INF behavior/log_gosim_test.go:58 > warn delay=0 ok=now + 3 machine-2/5 14:10:23.000 INF behavior/log_gosim_test.go:59 > before + machine-2/5 14:10:23.000 ERR gosimruntime/runtime.go:656 > uncaught panic panic=help traceback=[ + "goroutine 9 [running]:", + "github.com/jellevandenhooff/gosim/gosimruntime.(*goroutine).exitpoint(0x1400045de40)", + " /Users/jelle/hack/gosim/gosimruntime/runtime.go:728 +0x9c", + "panic({0x1011dc360?, 0x10147b090?})", + " /opt/homebrew/Cellar/go/1.23.3/libexec/src/runtime/panic.go:785 +0x124", + "translated/github.com/jellevandenhooff/gosim/internal_/tests/behavior_test.ImplTestLogForPrettyTest.func1()", + " /Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/tests/behavior/log_gosim_test.go:60 +0xe0", + "translated/github.com/jellevandenhooff/gosim/internal_/simulation.(*Simulation).startMachine.func1()", + " /Users/jelle/hack/gosim/.gosim/translated/linux_arm64/github.com/jellevandenhooff/gosim/internal_/simulation/simulation_gosim.go:96 +0x4c", + "github.com/jellevandenhooff/gosim/gosimruntime.(*goroutine).entrypoint(0x1400045de40, 0x14000461f50)", + " /Users/jelle/hack/gosim/gosimruntime/runtime.go:712 +0x60", + "github.com/jellevandenhooff/gosim/gosimruntime.goroutineEntrypoint()", + " /Users/jelle/hack/gosim/gosimruntime/runtime.go:705 +0x30", + "github.com/jellevandenhooff/gosim/internal/coro.(*Coro).Start.func1(0x0?)", + " /Users/jelle/hack/gosim/internal/coro/coro_linkname.go:38 +0x24", + "created by github.com/jellevandenhooff/gosim/internal/coro.(*Coro).Start in goroutine 5", + " /Users/jelle/hack/gosim/internal/coro/coro_linkname.go:36 +0x70", + "" + ] diff --git a/internal/race/doc.go b/internal/race/doc.go new file mode 100644 index 0000000..d93e461 --- /dev/null +++ b/internal/race/doc.go @@ -0,0 +1,2 @@ +// Package race is a copy of the built-in package internal/race. +package race diff --git a/internal/race/norace.go b/internal/race/norace.go new file mode 100644 index 0000000..b20ff01 --- /dev/null +++ b/internal/race/norace.go @@ -0,0 +1,44 @@ +// Copyright 2015 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +//go:build !race + +package race + +import ( + "unsafe" +) + +const Enabled = false + +func Acquire(addr unsafe.Pointer) { +} + +func Release(addr unsafe.Pointer) { +} + +func ReleaseMerge(addr unsafe.Pointer) { +} + +func Disable() { +} + +func Enable() { +} + +func Read(addr unsafe.Pointer) { +} + +func Write(addr unsafe.Pointer) { +} + +func ReadRange(addr unsafe.Pointer, len int) { +} + +func WriteRange(addr unsafe.Pointer, len int) { +} + +func Errors() int { + return 0 +} diff --git a/internal/race/race.go b/internal/race/race.go new file mode 100644 index 0000000..4d86a4d --- /dev/null +++ b/internal/race/race.go @@ -0,0 +1,54 @@ +// Copyright 2015 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +//go:build race + +package race + +import ( + "runtime" + "unsafe" +) + +const Enabled = true + +func Acquire(addr unsafe.Pointer) { + runtime.RaceAcquire(addr) +} + +func Release(addr unsafe.Pointer) { + runtime.RaceRelease(addr) +} + +func ReleaseMerge(addr unsafe.Pointer) { + runtime.RaceReleaseMerge(addr) +} + +func Disable() { + runtime.RaceDisable() +} + +func Enable() { + runtime.RaceEnable() +} + +func Read(addr unsafe.Pointer) { + runtime.RaceRead(addr) +} + +func Write(addr unsafe.Pointer) { + runtime.RaceWrite(addr) +} + +func ReadRange(addr unsafe.Pointer, len int) { + runtime.RaceReadRange(addr, len) +} + +func WriteRange(addr unsafe.Pointer, len int) { + runtime.RaceWriteRange(addr, len) +} + +func Errors() int { + return runtime.RaceErrors() +} diff --git a/internal/reflect/deepequal.go b/internal/reflect/deepequal.go new file mode 100644 index 0000000..2d81f20 --- /dev/null +++ b/internal/reflect/deepequal.go @@ -0,0 +1,251 @@ +// Copyright 2009 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Copied from +// https://go.googlesource.com/go/+/refs/heads/master/src/reflect/deepequal.go. + +// Deep equality test via reflection + +package reflect + +import ( + "bytes" +) + +// During deepValueEqual, must keep track of checks that are +// in progress. The comparison algorithm assumes that all +// checks in progress are true when it reencounters them. +// Visited comparisons are stored in a map indexed by visit. +type visit struct { + // a1 unsafe.Pointer + // a2 unsafe.Pointer + a1 uintptr + a2 uintptr + typ Type +} + +// Tests for deep equality using reflected types. The map argument tracks +// comparisons that have already been seen, which allows short circuiting on +// recursive types. +func deepValueEqual(v1, v2 Value, visited map[visit]bool) bool { + if !v1.IsValid() || !v2.IsValid() { + return v1.IsValid() == v2.IsValid() + } + if v1.Type() != v2.Type() { + return false + } + + // We want to avoid putting more in the visited map than we need to. + // For any possible reference cycle that might be encountered, + // hard(v1, v2) needs to return true for at least one of the types in the cycle, + // and it's safe and valid to get Value's internal pointer. + hard := func(v1, v2 Value) bool { + switch v1.Kind() { + case Pointer: + /* + if v1.typ().PtrBytes == 0 { + // not-in-heap pointers can't be cyclic. + // At least, all of our current uses of runtime/internal/sys.NotInHeap + // have that property. The runtime ones aren't cyclic (and we don't use + // DeepEqual on them anyway), and the cgo-generated ones are + // all empty structs. + return false + } + */ + fallthrough + case Map, Slice, Interface: + // Nil pointers cannot be cyclic. Avoid putting them in the visited map. + return !v1.IsNil() && !v2.IsNil() + } + return false + } + + // if hard(v1, v2) { + if v1.CanAddr() && v2.CanAddr() && hard(v1, v2) { + // For a Pointer or Map value, we need to check flagIndir, + // which we do by calling the pointer method. + // For Slice or Interface, flagIndir is always set, + // and using v.ptr suffices. + /* + ptrval := func(v Value) unsafe.Pointer { + switch v.Kind() { + case Pointer, Map: + return v.pointer() + default: + return v.ptr + } + } + addr1 := ptrval(v1) + addr2 := ptrval(v2) + */ + addr1 := v1.UnsafeAddr() + addr2 := v2.UnsafeAddr() + if uintptr(addr1) > uintptr(addr2) { + // Canonicalize order to reduce number of entries in visited. + // Assumes non-moving garbage collector. + addr1, addr2 = addr2, addr1 + } + + // Short circuit if references are already seen. + typ := v1.Type() + v := visit{addr1, addr2, typ} + if visited[v] { + return true + } + + // Remember for later. + visited[v] = true + } + + switch v1.Kind() { + case Array: + for i := 0; i < v1.Len(); i++ { + if !deepValueEqual(v1.Index(i), v2.Index(i), visited) { + return false + } + } + return true + case Slice: + if v1.IsNil() != v2.IsNil() { + return false + } + if v1.Len() != v2.Len() { + return false + } + if v1.UnsafePointer() == v2.UnsafePointer() { + return true + } + // Special case for []byte, which is common. + if v1.Type().Elem().Kind() == Uint8 { + return bytes.Equal(v1.Bytes(), v2.Bytes()) + // return bytealg.Equal(v1.Bytes(), v2.Bytes()) + } + for i := 0; i < v1.Len(); i++ { + if !deepValueEqual(v1.Index(i), v2.Index(i), visited) { + return false + } + } + return true + case Interface: + if v1.IsNil() || v2.IsNil() { + return v1.IsNil() == v2.IsNil() + } + return deepValueEqual(v1.Elem(), v2.Elem(), visited) + case Pointer: + if v1.UnsafePointer() == v2.UnsafePointer() { + return true + } + return deepValueEqual(v1.Elem(), v2.Elem(), visited) + case Struct: + for i, n := 0, v1.NumField(); i < n; i++ { + if !deepValueEqual(v1.Field(i), v2.Field(i), visited) { + return false + } + } + return true + case Map: + if v1.IsNil() != v2.IsNil() { + return false + } + if v1.Len() != v2.Len() { + return false + } + if v1.UnsafePointer() == v2.UnsafePointer() { + return true + } + for _, k := range v1.MapKeys() { + val1 := v1.MapIndex(k) + val2 := v2.MapIndex(k) + if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(val1, val2, visited) { + return false + } + } + return true + case Func: + if v1.IsNil() && v2.IsNil() { + return true + } + // Can't do better than this: + return false + case Int, Int8, Int16, Int32, Int64: + return v1.Int() == v2.Int() + case Uint, Uint8, Uint16, Uint32, Uint64, Uintptr: + return v1.Uint() == v2.Uint() + case String: + return v1.String() == v2.String() + case Bool: + return v1.Bool() == v2.Bool() + case Float32, Float64: + return v1.Float() == v2.Float() + case Complex64, Complex128: + return v1.Complex() == v2.Complex() + default: + // Normal equality suffices + // return valueInterface(v1, false) == valueInterface(v2, false) + return v1.Interface() == v2.Interface() + } +} + +// DeepEqual reports whether x and y are “deeply equal,” defined as follows. +// Two values of identical type are deeply equal if one of the following cases applies. +// Values of distinct types are never deeply equal. +// +// Array values are deeply equal when their corresponding elements are deeply equal. +// +// Struct values are deeply equal if their corresponding fields, +// both exported and unexported, are deeply equal. +// +// Func values are deeply equal if both are nil; otherwise they are not deeply equal. +// +// Interface values are deeply equal if they hold deeply equal concrete values. +// +// Map values are deeply equal when all of the following are true: +// they are both nil or both non-nil, they have the same length, +// and either they are the same map object or their corresponding keys +// (matched using Go equality) map to deeply equal values. +// +// Pointer values are deeply equal if they are equal using Go's == operator +// or if they point to deeply equal values. +// +// Slice values are deeply equal when all of the following are true: +// they are both nil or both non-nil, they have the same length, +// and either they point to the same initial entry of the same underlying array +// (that is, &x[0] == &y[0]) or their corresponding elements (up to length) are deeply equal. +// Note that a non-nil empty slice and a nil slice (for example, []byte{} and []byte(nil)) +// are not deeply equal. +// +// Other values - numbers, bools, strings, and channels - are deeply equal +// if they are equal using Go's == operator. +// +// In general DeepEqual is a recursive relaxation of Go's == operator. +// However, this idea is impossible to implement without some inconsistency. +// Specifically, it is possible for a value to be unequal to itself, +// either because it is of func type (uncomparable in general) +// or because it is a floating-point NaN value (not equal to itself in floating-point comparison), +// or because it is an array, struct, or interface containing +// such a value. +// On the other hand, pointer values are always equal to themselves, +// even if they point at or contain such problematic values, +// because they compare equal using Go's == operator, and that +// is a sufficient condition to be deeply equal, regardless of content. +// DeepEqual has been defined so that the same short-cut applies +// to slices and maps: if x and y are the same slice or the same map, +// they are deeply equal regardless of content. +// +// As DeepEqual traverses the data values it may find a cycle. The +// second and subsequent times that DeepEqual compares two pointer +// values that have been compared before, it treats the values as +// equal rather than examining the values to which they point. +// This ensures that DeepEqual terminates. +func DeepEqual(x, y any) bool { + if x == nil || y == nil { + return x == y + } + v1 := ValueOf(x) + v2 := ValueOf(y) + if v1.Type() != v2.Type() { + return false + } + return deepValueEqual(v1, v2, make(map[visit]bool)) +} diff --git a/internal/reflect/doc.go b/internal/reflect/doc.go new file mode 100644 index 0000000..be74fb0 --- /dev/null +++ b/internal/reflect/doc.go @@ -0,0 +1,6 @@ +// Package reflect is a shim of the built-in reflect package. +// +// The built-in reflect package does not know how to handle gosim's replacement +// Map type. This shim package special-cases gosim's replaced types and +// otherwise forwards to the built-in reflect package. +package reflect diff --git a/internal/reflect/makefunc.go b/internal/reflect/makefunc.go new file mode 100644 index 0000000..26e185e --- /dev/null +++ b/internal/reflect/makefunc.go @@ -0,0 +1,10 @@ +package reflect + +import "reflect" //gosim:notranslate + +func MakeFunc(typ Type, fn func(args []Value) (results []Value)) Value { + wrapped := func(args []reflect.Value) (results []reflect.Value) { + return unwrapValues(fn(wrapValues(args))) + } + return Value{inner: reflect.MakeFunc(typ.(typeImpl).inner, wrapped)} +} diff --git a/internal/reflect/swapper.go b/internal/reflect/swapper.go new file mode 100644 index 0000000..6d6cf78 --- /dev/null +++ b/internal/reflect/swapper.go @@ -0,0 +1,7 @@ +package reflect + +import "reflect" //gosim:notranslate + +func Swapper(slice any) func(i, j int) { + return reflect.Swapper(slice) +} diff --git a/internal/reflect/type.go b/internal/reflect/type.go new file mode 100644 index 0000000..3427f50 --- /dev/null +++ b/internal/reflect/type.go @@ -0,0 +1,410 @@ +// Copyright 2009 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Based on https://go.googlesource.com/go/+/refs/heads/master/src/reflect/type.go. + +package reflect + +import ( + "reflect" //gosim:notranslate + "sync" //gosim:notranslate + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func makeMapType() reflect.Type { + // TODO: this is in a function instead of inlined because translate does not + // handle the reflect notranslate well otherwise. + return reflect.TypeFor[gosimruntime.ReflectMap]() +} + +var mapInterfaceType reflect.Type = makeMapType() + +type Type interface { + Align() int + FieldAlign() int + Method(int) Method + MethodByName(string) (Method, bool) + NumMethod() int + Name() string + PkgPath() string + Size() uintptr + String() string + Kind() Kind + Implements(u Type) bool + AssignableTo(u Type) bool + ConvertibleTo(u Type) bool + Comparable() bool + Bits() int + ChanDir() ChanDir + IsVariadic() bool + Elem() Type + Field(i int) StructField + FieldByIndex(index []int) StructField + FieldByName(name string) (StructField, bool) + FieldByNameFunc(match func(string) bool) (StructField, bool) + In(i int) Type + Key() Type + Len() int + NumField() int + NumIn() int + NumOut() int + Out(i int) Type + OverflowComplex(x complex128) bool + OverflowFloat(x float64) bool + OverflowInt(x int64) bool + OverflowUint(x uint64) bool + // CanSeq() bool + // CanSeq2() bool + isType() + // common() *abi.Type + // uncommon() *uncommonType +} + +func unwrapTypes(x []Type) []reflect.Type { + inner := make([]reflect.Type, len(x)) + for i, v := range x { + inner[i] = v.(typeImpl).inner + } + return inner +} + +type typeImpl struct { + inner reflect.Type + kind wrapKind +} + +var jankHashMap sync.Map + +func wrapType(typ reflect.Type) Type { + gosimruntime.BeginControlledNondeterminism() + defer gosimruntime.EndControlledNondeterminism() + + // XXX: prevent allocations here, cache? + if typ == nil { + return nil + } + + known, ok := jankHashMap.Load(typ) + if ok { + return known.(Type) + } + + if typ.Kind() == reflect.Struct && typ.NumField() == 1 { + field := typ.Field(0) + if field.Name == "Impl" && field.Type.Implements(mapInterfaceType) { + impl := typeImpl{inner: typ, kind: wrappedMap} + jankHashMap.Store(typ, impl) + return impl + } + } + impl := typeImpl{inner: typ} + jankHashMap.Store(typ, impl) + return impl +} + +var _ Type = typeImpl{} + +func (t typeImpl) Align() int { + return t.inner.Align() +} + +func (t typeImpl) FieldAlign() int { + return t.inner.FieldAlign() +} + +func (t typeImpl) Method(idx int) Method { + return wrapMethod(t.inner.Method(idx)) +} + +func (t typeImpl) MethodByName(name string) (Method, bool) { + method, ok := t.inner.MethodByName(name) + return wrapMethod(method), ok +} + +func (t typeImpl) NumMethod() int { + return t.inner.NumMethod() +} + +func (t typeImpl) Name() string { + return t.inner.Name() +} + +func (t typeImpl) PkgPath() string { + return t.inner.PkgPath() +} + +func (t typeImpl) Size() uintptr { + return t.inner.Size() +} + +func (t typeImpl) String() string { + return t.inner.String() +} + +func (t typeImpl) Kind() Kind { + switch t.kind { + case normal: + return t.inner.Kind() + case wrappedMap: + return Map + default: + panic("help") + } +} + +func (t typeImpl) Implements(u Type) bool { + return t.inner.Implements(u.(typeImpl).inner) +} + +func (t typeImpl) AssignableTo(u Type) bool { + return t.inner.AssignableTo(u.(typeImpl).inner) +} + +func (t typeImpl) ConvertibleTo(u Type) bool { + return t.inner.ConvertibleTo(u.(typeImpl).inner) +} + +func (t typeImpl) Comparable() bool { + return t.inner.Comparable() +} + +func (t typeImpl) Bits() int { + return t.inner.Bits() +} + +func (t typeImpl) ChanDir() ChanDir { + return t.inner.ChanDir() +} + +func (t typeImpl) IsVariadic() bool { + return t.inner.IsVariadic() +} + +func (t typeImpl) Elem() Type { + switch t.kind { + case wrappedMap: + descriptor := getDecriptor(t) + return wrapType(descriptor.ElemType()) + default: + return wrapType(t.inner.Elem()) + } +} + +func (t typeImpl) Field(i int) StructField { + return wrapStructField(t.inner.Field(i)) +} + +func (t typeImpl) FieldByIndex(index []int) StructField { + return wrapStructField(t.inner.FieldByIndex(index)) +} + +func (t typeImpl) FieldByName(name string) (StructField, bool) { + field, ok := t.inner.FieldByName(name) + return wrapStructField(field), ok +} + +func (t typeImpl) FieldByNameFunc(match func(string) bool) (StructField, bool) { + field, ok := t.inner.FieldByNameFunc(match) + return wrapStructField(field), ok +} + +func (t typeImpl) In(i int) Type { + return wrapType(t.inner.In(i)) +} + +func (t typeImpl) Key() Type { + switch t.kind { + case wrappedMap: + descriptor := getDecriptor(t) + return wrapType(descriptor.KeyType()) + default: + return wrapType(t.inner.Key()) + } +} + +func (t typeImpl) Len() int { + return t.inner.Len() +} + +func (t typeImpl) NumField() int { + return t.inner.NumField() +} + +func (t typeImpl) NumIn() int { + return t.inner.NumIn() +} + +func (t typeImpl) NumOut() int { + return t.inner.NumOut() +} + +func (t typeImpl) Out(i int) Type { + return wrapType(t.inner.Out(i)) +} + +func (t typeImpl) OverflowComplex(x complex128) bool { + return t.inner.OverflowComplex(x) +} + +func (t typeImpl) OverflowFloat(x float64) bool { + return t.inner.OverflowFloat(x) +} + +func (t typeImpl) OverflowInt(x int64) bool { + return t.inner.OverflowInt(x) +} + +func (t typeImpl) OverflowUint(x uint64) bool { + return t.inner.OverflowUint(x) +} + +func (t typeImpl) isType() {} + +type Method struct { + Name string + PkgPath string + Type Type + Func Value + Index int +} + +func wrapMethod(inner reflect.Method) Method { + return Method{ + Name: inner.Name, + PkgPath: inner.PkgPath, + Type: wrapType(inner.Type), + Func: wrapValue(inner.Func), + Index: inner.Index, + } +} + +func (m Method) IsExported() bool { + return m.PkgPath == "" +} + +type StructField struct { + Name string + PkgPath string + Type Type + Tag StructTag + Offset uintptr + Index []int + Anonymous bool +} + +func wrapStructField(inner reflect.StructField) StructField { + return StructField{ + Name: inner.Name, + + PkgPath: inner.PkgPath, + + Type: wrapType(inner.Type), + Tag: inner.Tag, + Offset: inner.Offset, + Index: inner.Index, + Anonymous: inner.Anonymous, + } +} + +func wrapStructFields(inner []reflect.StructField) []StructField { + outer := make([]StructField, len(inner)) + for i, v := range inner { + outer[i] = wrapStructField(v) + } + return outer +} + +func (f StructField) IsExported() bool { + return f.PkgPath == "" +} + +type StructTag = reflect.StructTag + +type ChanDir = reflect.ChanDir + +const ( + RecvDir ChanDir = 1 << iota + SendDir + BothDir = RecvDir | SendDir +) + +type Kind = reflect.Kind + +type ValueError = reflect.ValueError + +const ( + Invalid Kind = iota + Bool + Int + Int8 + Int16 + Int32 + Int64 + Uint + Uint8 + Uint16 + Uint32 + Uint64 + Uintptr + Float32 + Float64 + Complex64 + Complex128 + Array + Chan + Func + Interface + Map + Pointer + Slice + String + Struct + UnsafePointer +) + +const Ptr = Pointer + +func TypeOf(i any) Type { + return wrapType(reflect.TypeOf(i)) +} + +func PtrTo(t Type) Type { return PointerTo(t) } + +func PointerTo(t Type) Type { + return wrapType(reflect.PointerTo(t.(typeImpl).inner)) +} + +func ChanOf(dir ChanDir, t Type) Type { + panic("missing") + // return wrapType(reflect.ChanOf(dir, t.(typeImpl).inner)) +} + +func MapOf(key, elem Type) Type { + panic("missing") +} + +func FuncOf(in, out []Type, variadic bool) Type { + return wrapType(reflect.FuncOf(unwrapTypes(in), unwrapTypes(out), variadic)) +} + +func SliceOf(t Type) Type { + return wrapType(reflect.SliceOf(t.(typeImpl).inner)) +} + +func StructOf(fields []StructField) Type { + panic("missing") +} + +func ArrayOf(length int, elem Type) Type { + return wrapType(reflect.ArrayOf(length, elem.(typeImpl).inner)) +} + +func TypeFor[T any]() Type { + var v T + if t := TypeOf(v); t != nil { + return t // optimize for T being a non-interface kind + } + return TypeOf((*T)(nil)).Elem() // only for an interface kind +} diff --git a/internal/reflect/value.go b/internal/reflect/value.go new file mode 100644 index 0000000..0d521c7 --- /dev/null +++ b/internal/reflect/value.go @@ -0,0 +1,555 @@ +// Copyright 2009 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Based on https://go.googlesource.com/go/+/refs/heads/master/src/reflect/value.go. + +package reflect + +import ( + "reflect" //gosim:notranslate + "unsafe" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +type wrapKind byte + +const ( + normal wrapKind = iota + wrappedMap + wrappedChan +) + +type Value struct { + inner reflect.Value + kind wrapKind +} + +func wrapValue(inner reflect.Value) Value { + if inner.Kind() == reflect.Struct && inner.NumField() == 1 { + field := inner.Type().Field(0) + if field.Name == "Impl" && field.Type.Implements(mapInterfaceType) { + return Value{ + inner: inner, + kind: wrappedMap, + } + } + } + + return Value{ + inner: inner, + } +} + +func wrapValues(x []reflect.Value) []Value { + outer := make([]Value, len(x)) + for i, v := range x { + outer[i] = wrapValue(v) + } + return outer +} + +func unwrapValues(x []Value) []reflect.Value { + inner := make([]reflect.Value, len(x)) + for i, v := range x { + inner[i] = v.inner + } + return inner +} + +func Append(s Value, x ...Value) Value { + return wrapValue(reflect.Append(s.inner, unwrapValues(x)...)) +} + +func AppendSlice(s, t Value) Value { + return wrapValue(reflect.AppendSlice(s.inner, t.inner)) +} + +func Copy(dst, src Value) int { + return reflect.Copy(dst.inner, src.inner) +} + +type SelectDir int + +// NOTE: These values must match ../runtime/select.go:/selectDir. + +const ( + _ SelectDir = iota + SelectSend + SelectRecv + SelectDefault +) + +type SelectCase struct { + Dir SelectDir + Chan Value + Send Value +} + +func Select(cases []SelectCase) (chosen int, recv Value, recvOK bool) { + panic("missing") +} + +func MakeSlice(typ Type, len, cap int) Value { + return wrapValue(reflect.MakeSlice(typ.(typeImpl).inner, len, cap)) +} + +func MakeChan(typ Type, buffer int) Value { + panic("missing") +} + +func getDecriptor(typ Type) gosimruntime.ReflectMapType { + implPtrTyp := typ.(typeImpl).inner.Field(0).Type + zeroImplPtr := reflect.Zero(implPtrTyp) + mapInterface := zeroImplPtr.Interface().(gosimruntime.ReflectMap) + return mapInterface.Type() +} + +func MakeMap(typ Type) Value { + if typ.Kind() != reflect.Map { + reflect.MakeMap(typ.(typeImpl).inner) + panic("unreachable") + } + + descriptor := getDecriptor(typ) + return ValueOf(descriptor.Make()) +} + +func MakeMapWithSize(typ Type, n int) Value { + // XXX: n? + return MakeMap(typ) +} + +func Indirect(v Value) Value { + return wrapValue(reflect.Indirect(v.inner)) +} + +func ValueOf(i any) Value { + return wrapValue(reflect.ValueOf(i)) +} + +func Zero(typ Type) Value { + return wrapValue(reflect.Zero(typ.(typeImpl).inner)) +} + +func New(typ Type) Value { + return wrapValue(reflect.New(typ.(typeImpl).inner)) +} + +func NewAt(typ Type, p unsafe.Pointer) Value { + return wrapValue(reflect.NewAt(typ.(typeImpl).inner, p)) +} + +func (v Value) Addr() Value { + return wrapValue(v.inner.Addr()) +} + +func (v Value) Bool() bool { + return v.inner.Bool() +} + +func (v Value) Bytes() []byte { + return v.inner.Bytes() +} + +func (v Value) CanAddr() bool { + return v.inner.CanAddr() +} + +func (v Value) CanSet() bool { + return v.inner.CanSet() +} + +func (v Value) Call(in []Value) []Value { + return wrapValues(v.inner.Call(unwrapValues(in))) +} + +func (v Value) CallSlice(in []Value) []Value { + return wrapValues(v.inner.CallSlice(unwrapValues(in))) +} + +func (v Value) Cap() int { + return v.inner.Cap() +} + +func (v Value) Close() { + v.inner.Close() +} + +func (v Value) CanComplex() bool { + return v.inner.CanComplex() +} + +func (v Value) Complex() complex128 { + return v.inner.Complex() +} + +func (v Value) Elem() Value { + return wrapValue(v.inner.Elem()) +} + +func (v Value) Field(i int) Value { + return wrapValue(v.inner.Field(i)) +} + +func (v Value) FieldByIndex(index []int) Value { + return wrapValue(v.inner.FieldByIndex(index)) +} + +func (v Value) FieldByIndexErr(index []int) (Value, error) { + inner, err := v.inner.FieldByIndexErr(index) + return wrapValue(inner), err +} + +func (v Value) FieldByName(name string) Value { + return wrapValue(v.inner.FieldByName(name)) +} + +func (v Value) FieldByNameFunc(match func(string) bool) Value { + return wrapValue(v.inner.FieldByNameFunc(match)) +} + +func (v Value) CanFloat() bool { + return v.inner.CanFloat() +} + +func (v Value) Float() float64 { + return v.inner.Float() +} + +func (v Value) Index(i int) Value { + return wrapValue(v.inner.Index(i)) +} + +func (v Value) CanInt() bool { + return v.inner.CanInt() +} + +func (v Value) Int() int64 { + return v.inner.Int() +} + +func (v Value) CanInterface() bool { + return v.inner.CanInterface() +} + +func (v Value) Interface() (i any) { + return v.inner.Interface() +} + +func (v Value) InterfaceData() [2]uintptr { + return v.inner.InterfaceData() +} + +func (v Value) IsNil() bool { + switch v.kind { + case wrappedMap: + return v.Field(0).IsNil() + default: + return v.inner.IsNil() + } +} + +func (v Value) IsValid() bool { + return v.inner.IsValid() +} + +func (v Value) IsZero() bool { + return v.inner.IsZero() +} + +func (v Value) SetZero() { + v.inner.SetZero() +} + +func (v Value) Kind() Kind { + switch v.kind { + case normal: + return v.inner.Kind() + case wrappedMap: + return Map + default: + panic("help") + } +} + +func (v Value) Len() int { + switch v.kind { + case normal: + return v.inner.Len() + case wrappedMap: + return getMapInterface(v).Len() + default: + panic("help") + } +} + +func (v Value) MapIndex(key Value) Value { + value, _ := getMapInterface(v).GetIndex(key.inner) + return wrapValue(value) +} + +func (v Value) MapKeys() []Value { + mapIface := getMapInterface(v) + values := make([]Value, 0, mapIface.Len()) + iter := mapIface.Iter() + for iter.Next() { + values = append(values, wrapValue(iter.Key())) + } + return values +} + +type MapIter struct { + inner gosimruntime.ReflectMapIter +} + +func (iter *MapIter) Key() Value { + return wrapValue(iter.inner.Key()) +} + +func (v Value) SetIterKey(iter *MapIter) { + v.Set(iter.Key()) +} + +func (iter *MapIter) Value() Value { + return wrapValue(iter.inner.Value()) +} + +func (v Value) SetIterValue(iter *MapIter) { + v.Set(iter.Value()) +} + +func (iter *MapIter) Next() bool { + return iter.inner.Next() +} + +type zeroIter struct{} + +func (z zeroIter) Value() reflect.Value { + panic("missing") +} + +func (z zeroIter) Key() reflect.Value { + panic("missing") +} + +func (z zeroIter) Next() bool { + return false +} + +func (iter *MapIter) Reset(v Value) { + if v.IsZero() { + iter.inner = zeroIter{} + } + + switch v.kind { + case wrappedMap: + inner := v.Field(0).Interface().(gosimruntime.ReflectMap).Iter() + iter.inner = inner + default: + var iter reflect.MapIter + iter.Reset(v.inner) // will panic because this is not a map + panic("unreachable") + } +} + +func getMapInterface(v Value) gosimruntime.ReflectMap { + return v.Field(0).Interface().(gosimruntime.ReflectMap) +} + +func (v Value) MapRange() *MapIter { + switch v.kind { + case wrappedMap: + inner := getMapInterface(v).Iter() + return &MapIter{inner: inner} + default: + v.inner.MapRange() // will panic because this is not a map + panic("unreachable") + } +} + +func (v Value) Method(i int) Value { + return wrapValue(v.inner.Method(i)) +} + +func (v Value) NumMethod() int { + return v.inner.NumMethod() +} + +func (v Value) MethodByName(name string) Value { + return wrapValue(v.inner.MethodByName(name)) +} + +func (v Value) NumField() int { + return v.inner.NumField() +} + +func (v Value) OverflowComplex(x complex128) bool { + return v.inner.OverflowComplex(x) +} + +func (v Value) OverflowFloat(x float64) bool { + return v.inner.OverflowFloat(x) +} + +func (v Value) OverflowInt(x int64) bool { + return v.inner.OverflowInt(x) +} + +func (v Value) OverflowUint(x uint64) bool { + return v.inner.OverflowUint(x) +} + +// This prevents inlining Value.Pointer when -d=checkptr is enabled, +// which ensures cmd/compile can recognize unsafe.Pointer(v.Pointer()) +// and make an exception. +// +//go:nocheckptr +func (v Value) Pointer() uintptr { + switch v.kind { + case wrappedMap: + return uintptr(getMapInterface(v).UnsafePointer()) + default: + return v.inner.Pointer() + } +} + +func (v Value) Recv() (x Value, ok bool) { + panic("missing") +} + +func (v Value) Send(x Value) { + panic("missing") +} + +func (v Value) Set(x Value) { + v.inner.Set(x.inner) +} + +func (v Value) SetBool(x bool) { + v.inner.SetBool(x) +} + +func (v Value) SetBytes(x []byte) { + v.inner.SetBytes(x) +} + +func (v Value) SetComplex(x complex128) { + v.inner.SetComplex(x) +} + +func (v Value) SetFloat(x float64) { + v.inner.SetFloat(x) +} + +func (v Value) SetInt(x int64) { + v.inner.SetInt(x) +} + +func (v Value) SetLen(n int) { + v.inner.SetLen(n) +} + +func (v Value) SetCap(n int) { + v.inner.SetCap(n) +} + +func (v Value) SetMapIndex(key, elem Value) { + getMapInterface(v).SetIndex(key.inner, elem.inner) +} + +func (v Value) SetUint(x uint64) { + v.inner.SetUint(x) +} + +func (v Value) SetPointer(x unsafe.Pointer) { + v.inner.SetPointer(x) +} + +func (v Value) SetString(x string) { + v.inner.SetString(x) +} + +func (v Value) Slice(i, j int) Value { + return wrapValue(v.inner.Slice(i, j)) +} + +func (v Value) Slice3(i, j, k int) Value { + return wrapValue(v.inner.Slice3(i, j, k)) +} + +func (v Value) String() string { + return v.inner.String() +} + +func (v Value) TryRecv() (x Value, ok bool) { + panic("missing") +} + +func (v Value) TrySend(x Value) bool { + panic("missing") +} + +func (v Value) Type() Type { + return wrapType(v.inner.Type()) +} + +func (v Value) CanUint() bool { + return v.inner.CanUint() +} + +func (v Value) Uint() uint64 { + return v.inner.Uint() +} + +// This prevents inlining Value.UnsafeAddr when -d=checkptr is enabled, +// which ensures cmd/compile can recognize unsafe.Pointer(v.UnsafeAddr()) +// and make an exception. +// +//go:nocheckptr +func (v Value) UnsafeAddr() uintptr { + switch v.kind { + case wrappedMap: + return uintptr(getMapInterface(v).UnsafePointer()) + default: + return v.inner.UnsafeAddr() + } +} + +func (v Value) UnsafePointer() unsafe.Pointer { + switch v.kind { + case wrappedMap: + return getMapInterface(v).UnsafePointer() + default: + return v.inner.UnsafePointer() + } +} + +type StringHeader = reflect.StringHeader + +type SliceHeader = reflect.SliceHeader + +func (v Value) Grow(n int) { + v.inner.Grow(n) +} + +func (v Value) Clear() { + v.inner.Clear() +} + +func (v Value) Convert(t Type) Value { + return wrapValue(v.inner.Convert(t.(typeImpl).inner)) +} + +func (v Value) CanConvert(t Type) bool { + return v.inner.CanConvert(t.(typeImpl).inner) +} + +func (v Value) Comparable() bool { + return v.inner.Comparable() +} + +func (v Value) Equal(u Value) bool { + return v.inner.Equal(u.inner) +} diff --git a/internal/reflect/visiblefields.go b/internal/reflect/visiblefields.go new file mode 100644 index 0000000..16ddedd --- /dev/null +++ b/internal/reflect/visiblefields.go @@ -0,0 +1,7 @@ +package reflect + +import "reflect" //gosim:notranslate + +func VisibleFields(t Type) []StructField { + return wrapStructFields(reflect.VisibleFields(t.(typeImpl).inner)) +} diff --git a/internal/simulation/fs/chunkedfile.go b/internal/simulation/fs/chunkedfile.go new file mode 100644 index 0000000..3ad2f72 --- /dev/null +++ b/internal/simulation/fs/chunkedfile.go @@ -0,0 +1,272 @@ +package fs + +import ( + "slices" + "sync" + + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +// A chunkedFile is the backing storage for files. It supports efficient reads +// and writes with multiple versions of the file existing by storing the data in +// fixed-sized reference-counted copy-on-write chunks. +// +// The goals are as follows: +// +// To simulate crashes we store two versions of each file: The latest in-memory +// representation with all writes faithfully applied, and the on-disk version +// that does not yet have all writes. We want to share storage and work as much +// as possible between these two versions. +// +// When a file gets modified, we would like to allocate as little memory as +// possible. Common scenarios are resizing files, appending to a file, and +// rewriting parts of a file. +// +// Tricky combinations of these requirements are, for example, rewriting a small +// part of a large file. Do we make a copy of the entire file and modify just +// the small part? When we append to a file and its backing storage needs to +// grow, do we copy all existing data? +// +// To support these operations efficiently, the chunkedFile stores data in +// fixed-sized reference-counted chunks: Each file is split in chunks of +// chunkSize, and all operations modify those chunks. When copying a large file, +// we share the chunks. If some chunks get modified later, only those get +// copied. When we append to a file, we similarly keep all the old chunks. +// +// When a write comes in over a syscall, we read the data into correctly-aligned +// chunks. In the happy case, that is the only time data is copied byte-by-byte +// and afterwards we only move references to the chunk around. +// +// Chunks are referenced counted so we can re-use no longer needed chunks in a +// sync.Pool and reduce the number of memory allocations. +type chunkedFile struct { + // TODO: inline a fixed size array of chunks that covers most file sizes? + chunks []*refCountedChunk + size int +} + +const chunkSize = 1024 + +type refCountedChunk struct { + // XXX: inline data to skip an object? + refs int + data []byte +} + +var chunkPool = sync.Pool{ + New: func() any { + return &refCountedChunk{ + refs: 0, + data: make([]byte, chunkSize), + } + }, +} + +func allocRefCountedChunk() *refCountedChunk { + chunk := chunkPool.Get().(*refCountedChunk) + if chunk.refs != 0 { + panic("help") + } + chunk.refs++ + return chunk +} + +func (c *refCountedChunk) incRef() { + if c.refs <= 0 { + panic("help") + } + c.refs++ +} + +func (c *refCountedChunk) decRef() { + if c.refs <= 0 { + panic("help") + } + c.refs-- + if c.refs == 0 { + chunkPool.Put(c) + } +} + +// TODO: when do we free files? need to track open inodes. (should be not too +// bad? close does get called?) + +// TODO: when do we free the filesystem? need to track alive machines. (can do +// this at end of simulation?) + +// small helper to make testing possible +func requiredCount(chunkPos int, len int) int { + if len == 0 { + return 0 + } + count := 0 + if chunkPos != 0 { + count++ + len -= min(chunkSize-chunkPos, len) + } + count += (len + chunkSize - 1) / chunkSize + return count +} + +func makeChunks(pos int, view syscallabi.ByteSliceView) []*refCountedChunk { + if view.Len() == 0 { + return nil + } + chunkPos := pos % chunkSize + count := requiredCount(chunkPos, view.Len()) + out := make([]*refCountedChunk, 0, count) + if chunkPos != 0 { + chunk := allocRefCountedChunk() + n := view.Read(chunk.data[:chunkSize-chunkPos]) + view = view.SliceFrom(n) + out = append(out, chunk) + } + for view.Len() > 0 { + chunk := allocRefCountedChunk() + n := view.Read(chunk.data) + view = view.SliceFrom(n) + out = append(out, chunk) + } + return out +} + +// TODO: add initializer? + +var zeroChunk = &refCountedChunk{ + refs: -1e6, // should trigger asserts hopefully + data: make([]byte, chunkSize), +} + +func (w *chunkedFile) Resize(newSize int) { + oldCount := len(w.chunks) + newCount := (newSize + chunkSize - 1) / chunkSize + + if newCount > oldCount { + w.chunks = append(w.chunks, make([]*refCountedChunk, newCount-oldCount)...) + } else { + for i := newCount; i < oldCount; i++ { + w.releaseChunk(i) + } + // TODO: free memory? unclear + w.chunks = w.chunks[:newCount] + } + + // TODO: document guarantees about zeroing? + + // zero tail of last chunk if needed + oldSize := w.size + if newSize < oldSize { + if chunkPos := newSize % chunkSize; chunkPos != 0 { + if chunkIdx := newCount - 1; w.chunks[chunkIdx] != nil { + chunk := w.ensureWritableChunk(chunkIdx) + // TODO: only have to zero up to oldSize, really? + copy(chunk.data[chunkPos:], zeroes) + } + } + } + + // TODO: help zero (part) of chunk? + w.size = newSize +} + +func (w *chunkedFile) releaseChunk(idx int) { + chunk := w.chunks[idx] + if chunk != nil { + chunk.decRef() + w.chunks[idx] = nil + } +} + +func (w *chunkedFile) ensureWritableChunk(idx int) *refCountedChunk { + if chunk := w.chunks[idx]; chunk != nil && chunk.refs == 1 { + return chunk + } + // TODO: should we guarantee that chunks in pool are zeroed out? + oldChunk := w.getReadableChunk(idx) + newChunk := allocRefCountedChunk() + copy(newChunk.data, oldChunk.data) + w.chunks[idx] = newChunk + return newChunk +} + +func (w *chunkedFile) getReadableChunk(idx int) *refCountedChunk { + chunk := w.chunks[idx] + if chunk == nil { + return zeroChunk + } + return chunk +} + +func (w *chunkedFile) Clone() *chunkedFile { + chunks := slices.Clone(w.chunks) + for _, chunk := range chunks { + if chunk != nil { + chunk.incRef() + } + } + return &chunkedFile{ + chunks: chunks, + size: w.size, + } +} + +func (w *chunkedFile) Free() { + for i, chunk := range w.chunks { + if chunk != nil { + chunk.decRef() + } + w.chunks[i] = nil + } + // set to nil to prevent reuse + w.chunks = nil + w.size = 0 +} + +func (w *chunkedFile) Write(pos int, size int, chunks []*refCountedChunk) { + if size == 0 { + // TODO: is this reasonable? + return + } + + // partial chunks on either end + chunkIdx := pos / chunkSize + if chunkPos := pos % chunkSize; chunkPos != 0 { + // TODO: what if this partial on both sides? handle and test. + chunk := w.ensureWritableChunk(chunkIdx) + n := copy(chunk.data[chunkPos:min(chunkPos+size, chunkSize)], chunks[0].data) + chunks = chunks[1:] + size -= n + chunkIdx++ + } + for size >= chunkSize { + w.releaseChunk(chunkIdx) + chunks[0].incRef() + w.chunks[chunkIdx] = chunks[0] + chunks = chunks[1:] + size -= chunkSize + chunkIdx++ + } + if size > 0 { + chunk := w.ensureWritableChunk(chunkIdx) + copy(chunk.data[:size], chunks[0].data) + } +} + +func (w *chunkedFile) Read(pos int, out syscallabi.ByteSliceView) { + // TODO: precondition that read fits in file? + + chunkIdx := pos / chunkSize + if chunkPos := pos % chunkSize; chunkPos != 0 { + // TODO: what if this partial on both sides? handle and test. + chunk := w.getReadableChunk(chunkIdx) + n := out.Write(chunk.data[chunkPos:]) + out = out.SliceFrom(n) + chunkIdx++ + } + for out.Len() > 0 { + chunk := w.getReadableChunk(chunkIdx) + n := out.Write(chunk.data) + out = out.SliceFrom(n) + chunkIdx++ + } +} diff --git a/internal/simulation/fs/chunkedfile_test.go b/internal/simulation/fs/chunkedfile_test.go new file mode 100644 index 0000000..33ae141 --- /dev/null +++ b/internal/simulation/fs/chunkedfile_test.go @@ -0,0 +1,238 @@ +package fs + +import ( + "bytes" + "crypto/rand" + "fmt" + "slices" + "testing" + + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +func TestMakeChunks(t *testing.T) { + data := make([]byte, chunkSize*8) + if _, err := rand.Read(data); err != nil { + t.Fatal(err) + } + + interesting := []int{ + 0, 10, chunkSize - 10, chunkSize, chunkSize + 10, + 2*chunkSize - 10, 2 * chunkSize, 2*chunkSize + 10, + 7*chunkSize - 10, 7 * chunkSize, 7*chunkSize + 10, + len(data), + } + + for _, start := range interesting { + for _, end := range interesting { + if start > end { + continue + } + t.Run(fmt.Sprintf("%d-%d", start, end), func(t *testing.T) { + view := syscallabi.ByteSliceView{Ptr: data[start:end]} + chunks := makeChunks(start, view) + if count := requiredCount(start%chunkSize, end-start); count != len(chunks) { + t.Errorf("calculated count %d but got %d", count, len(chunks)) + } + if start == end { + return + } + chunkPos := start % chunkSize + if chunkPos != 0 { + n := min(end-start, chunkSize-chunkPos) + if !bytes.Equal(chunks[0].data[:n], data[start:start+n]) { + t.Errorf("problem for %d-%d", start, start+n) + } + chunks = chunks[1:] + start += n + } + for start < end { + n := min(end-start, chunkSize) + if !bytes.Equal(chunks[0].data[:n], data[start:start+n]) { + t.Errorf("problem for %d-%d", start, start+n) + } + chunks = chunks[1:] + start += n + } + }) + } + } +} + +// TODO: have another wrapper around chunkedFile that does explicit []byte +// reads/writes so we can check those manually + +type checker struct { + f *chunkedFile + b []byte +} + +func newChecker() *checker { + return &checker{ + f: &chunkedFile{}, + } +} + +func (c *checker) resize(newN int) { + oldN := len(c.b) + if newN > oldN { + newB := append(c.b, make([]byte, newN-oldN)...) + clear(newB[oldN:newN]) + c.b = newB + } else { + c.b = c.b[:newN] + } + if len(c.b) != newN { + panic("help") + } + + c.f.Resize(newN) +} + +func (c *checker) write(t *testing.T, from, to int) { + t.Helper() + + if _, err := rand.Read(c.b[from:to]); err != nil { + t.Fatal(err) + } + + chunks := makeChunks(from, syscallabi.ByteSliceView{Ptr: c.b[from:to]}) + c.f.Write(from, to-from, chunks) + for _, chunk := range chunks { + chunk.decRef() + } +} + +func (c *checker) read(t *testing.T, from, to int) { + t.Helper() + + buf := make([]byte, to-from) + c.f.Read(from, syscallabi.ByteSliceView{Ptr: buf}) + if !bytes.Equal(buf, c.b[from:to]) { + firstBad := 0 + for buf[firstBad] == c.b[from+firstBad] { + firstBad++ + } + lastBad := (to - from) + for buf[lastBad-1] == c.b[from+lastBad-1] { + lastBad-- + } + t.Errorf("read %d-%d: bad data %d-%d", from, to, from+firstBad, from+lastBad) + } +} + +func TestFileRead(t *testing.T) { + data := make([]byte, chunkSize*8) + if _, err := rand.Read(data); err != nil { + t.Fatal(err) + } + + interesting := []int{ + 0, 10, chunkSize - 10, chunkSize, chunkSize + 10, + 2*chunkSize - 10, 2 * chunkSize, 2*chunkSize + 10, + 7*chunkSize - 10, 7 * chunkSize, 7*chunkSize + 10, + len(data), + } + + check := func(size int, file *chunkedFile) { + t.Helper() + + c := &checker{ + f: file, + b: data[:size], + } + + for _, start := range interesting { + for _, end := range interesting { + if start > end || end > size { + continue + } + + c.read(t, start, end) + } + } + } + + for _, size := range interesting { + chunks := makeChunks(0, syscallabi.ByteSliceView{Ptr: data}) + file := &chunkedFile{size: size, chunks: chunks} + check(size, file) + } + + // for the second iteration, zero out a chunk of the input and put a nil chunk there + copy(data[chunkSize:chunkSize*2], make([]byte, chunkSize)) + + for _, size := range interesting { + chunks := makeChunks(0, syscallabi.ByteSliceView{Ptr: data}) + if len(chunks) > 2 { + chunks[1] = nil + } + file := &chunkedFile{size: size, chunks: chunks} + check(size, file) + } +} + +func (f *checker) clone() *checker { + return &checker{ + f: f.f.Clone(), + b: slices.Clone(f.b), + } +} + +func (f *checker) free() { + f.f.Free() + f.f = nil + f.b = nil +} + +func TestFileWrite(t *testing.T) { + f := newChecker() + f.resize(chunkSize * 3) + f.read(t, 0, 2*chunkSize) + f.read(t, chunkSize-10, chunkSize+10) + f.read(t, chunkSize-10, 2*chunkSize+10) + f.write(t, 10, chunkSize-10) + f.read(t, 10, chunkSize-10) + f.read(t, 0, chunkSize) + f.read(t, 0, 2*chunkSize) + f.read(t, 0, chunkSize*3) + f.write(t, 0, chunkSize*3) + f.read(t, 0, chunkSize*3) + f.resize(chunkSize*3 - 10) + f.resize(chunkSize * 3) + f.read(t, 0, chunkSize*3) + + // XXX: zero-sized writes? + f.write(t, 10, 10) + f.write(t, chunkSize, chunkSize) + f.write(t, 0, 0) + + f.resize(chunkSize*5 - 10) + f.read(t, 0, chunkSize*5-10) + f.write(t, 0, chunkSize*5-10) + f.read(t, 0, chunkSize*5-10) + + t.Logf("%d %p", f.f.chunks[0].refs, f.f.chunks[0]) + g := f.clone() + t.Logf("%d %p", f.f.chunks[0].refs, f.f.chunks[0]) + t.Logf("%d %p", g.f.chunks[0].refs, g.f.chunks[0]) + g.read(t, 0, chunkSize*5-10) + g.write(t, 10, 20) + g.read(t, 0, chunkSize) + f.read(t, 0, chunkSize) + + g.free() + + f.read(t, 0, chunkSize) + f.resize(10 * chunkSize) + g = f.clone() + g.write(t, 0, chunkSize*10) + f.read(t, 0, 10*chunkSize) + g.read(t, 0, 10*chunkSize) + + // TODO: check ref count? + // TODO: check for nil? + + // TODO: file write releasing empty chunk. but how? poll pool? maybe check + // allocs for a write cycle? or check that a chunk is reused? +} diff --git a/internal/simulation/fs/depkind_string.go b/internal/simulation/fs/depkind_string.go new file mode 100644 index 0000000..02368c9 --- /dev/null +++ b/internal/simulation/fs/depkind_string.go @@ -0,0 +1,47 @@ +// Code generated by "stringer -type=depKind,depKeyKind"; DO NOT EDIT. + +package fs + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[depStoreAs-0] + _ = x[depStoreAsAndDependsOn-1] + _ = x[depDependsOn-2] +} + +const _depKind_name = "depStoreAsdepStoreAsAndDependsOndepDependsOn" + +var _depKind_index = [...]uint8{0, 10, 32, 44} + +func (i depKind) String() string { + if i >= depKind(len(_depKind_index)-1) { + return "depKind(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _depKind_name[_depKind_index[i]:_depKind_index[i+1]] +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[depKeyNone-0] + _ = x[depKeyAlloc-1] + _ = x[depKeyWriteFile-2] + _ = x[depKeyWriteDir-3] + _ = x[depKeyResizeFile-4] + _ = x[depKeyWriteDirEntry-5] +} + +const _depKeyKind_name = "depKeyNonedepKeyAllocdepKeyWriteFiledepKeyWriteDirdepKeyResizeFiledepKeyWriteDirEntry" + +var _depKeyKind_index = [...]uint8{0, 10, 21, 36, 50, 66, 85} + +func (i depKeyKind) String() string { + if i >= depKeyKind(len(_depKeyKind_index)-1) { + return "depKeyKind(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _depKeyKind_name[_depKeyKind_index[i]:_depKeyKind_index[i+1]] +} diff --git a/internal/simulation/fs/filesystem.go b/internal/simulation/fs/filesystem.go new file mode 100644 index 0000000..9b4b334 --- /dev/null +++ b/internal/simulation/fs/filesystem.go @@ -0,0 +1,851 @@ +package fs + +import ( + "cmp" + "errors" + "fmt" + "maps" + "os" + "slices" + "sync" + "syscall" + + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +// next: +// - delete on test end, somehow (but pool isn't shared between tests? maybe we could/should make it?) +// - think about where we should invoke GC and why that's safe +// -- eg. when we create/apply multiple ops. should there be an API for doing that atomically? + +// after a crash, iterate over all files and check for zeroes and if so, delete (optimize later: track orphans?) + +// XXX: handle bad requests somehow? + +type backingFile struct { + inode int + file *chunkedFile + + linkCount int +} + +var zeroes = make([]byte, 1024) + +func (f *backingFile) read(pos int64, buffer syscallabi.ByteSliceView) int { + if int(pos) < f.file.size { + n := min(buffer.Len(), f.file.size-int(pos)) + f.file.Read(int(pos), buffer.Slice(0, n)) + return n + } + return 0 +} + +func (f *backingFile) len() int { + return f.file.size +} + +func (f *backingFile) resize(size int) { + f.file.Resize(size) +} + +func (f *backingFile) write(pos int, size int, chunks []*refCountedChunk) { + f.file.Write(pos, size, chunks) +} + +func (f *backingFile) clone() *backingFile { + return &backingFile{ + inode: f.inode, + file: f.file.Clone(), + linkCount: f.linkCount, // XXX: test that link count is maintained + } +} + +type backingDir struct { + inode int + entries map[string]int +} + +const rootInode = 1 + +type filesystemState struct { + objects map[int]any + next int +} + +func (fss *filesystemState) clone() *filesystemState { + objects := make(map[int]any) + for inode, obj := range fss.objects { + switch obj := obj.(type) { + case *backingFile: + objects[inode] = obj.clone() + case *backingDir: + objects[inode] = &backingDir{ + inode: obj.inode, + entries: maps.Clone(obj.entries), + } + } + } + + return &filesystemState{ + objects: objects, + next: fss.next, + } +} + +type Filesystem struct { + mu sync.Mutex + + // for now: flat filesystem, only one directory + + pendingOps *pendingOps + + // XXX: jank... + openCountByInode map[int]int + pendingCountByInode map[int]int + + // XXX: make a different, higher-level type? + mem *filesystemState + persisted *filesystemState // XXX: rename to "disk"? +} + +func (fs *Filesystem) apply(op fsOp) { + /* + switch op := op.(type) { + case *resizeOp: + slog.Info("resize", "size", op.size) + case *writeOp: + slog.Info("write", "size", len(op.data), "pos", op.pos) + } + */ + fs.pendingOps.addOp(op) + op.touchedInodesIter(func(inode int) bool { + fs.pendingCountByInode[inode]++ + return true + }) + op.apply(fs.mem, fs) +} + +func (fs *Filesystem) releaseOpRefs(ops []*pendingOp) { + for _, op := range ops { + op.fsOp.touchedInodesIter(func(inode int) bool { + newCount := fs.pendingCountByInode[inode] - 1 + if newCount == 0 { + delete(fs.pendingCountByInode, inode) + // XXX: this check here does double work. any file whose links + // hit zero MUST eventually be flushed by an op that refers to + // it, which will be this op, and so here we can delete it. + fs.maybeGC(inode) + } else { + fs.pendingCountByInode[inode] = newCount + } + return true + }) + } +} + +func (fs *Filesystem) flushInode(inode int) { + ops := fs.pendingOps.flushInode(inode, fs.pendingOps.buffer) + for _, op := range ops { + op.fsOp.apply(fs.persisted, fs) + // XXX: janky but works for now. release chunks. + if op, ok := op.fsOp.(*writeOp); ok { + for i, chunk := range op.chunks { + chunk.decRef() + op.chunks[i] = nil // set to nil just in case? + } + op.chunks = nil // set to nil just in case? + } + } + + fs.releaseOpRefs(ops) + fs.pendingOps.releaseBuffer(ops) +} + +type gcer interface { + maybeGC(inode int) +} + +type fsOp interface { + apply(fs *filesystemState, gcer gcer) + addDeps(p *pendingOps, op *pendingOp) + touchedInodesIter(func(inode int) bool) + String() string +} + +func inititalState() *filesystemState { + return &filesystemState{ + objects: map[int]any{ + rootInode: &backingDir{ + inode: rootInode, + entries: make(map[string]int), + }, + }, + next: rootInode + 1, + } +} + +func NewFilesystem() *Filesystem { + return &Filesystem{ + pendingOps: newPendingOps(), + mem: inititalState(), + persisted: inititalState(), + openCountByInode: make(map[int]int), + pendingCountByInode: make(map[int]int), + } +} + +func (fs *filesystemState) getFile(idx int) *backingFile { + return fs.objects[idx].(*backingFile) +} + +func (fs *filesystemState) getDir(idx int) *backingDir { + return fs.objects[idx].(*backingDir) +} + +type writeOp struct { + // XXX: should this be an inode number? + inode int + + pos int + size int + chunks []*refCountedChunk +} + +func (o *writeOp) apply(fs *filesystemState, gcer gcer) { + file := fs.getFile(o.inode) + file.write(o.pos, o.size, o.chunks) +} + +func (o *writeOp) touchedInodesIter(iter func(inode int) bool) { + iter(o.inode) +} + +func (o *writeOp) String() string { + return fmt.Sprintf("write file:%d pos:%d size:%d", o.inode, o.pos, o.size) +} + +type resizeOp struct { + inode int + size int +} + +func (o *resizeOp) apply(fs *filesystemState, gcer gcer) { + file := fs.getFile(o.inode) + file.resize(o.size) +} + +func (o *resizeOp) touchedInodesIter(iter func(inode int) bool) { + iter(o.inode) +} + +func (o *resizeOp) String() string { + return fmt.Sprintf("resize file:%d size:%d", o.inode, o.size) +} + +type dirOpPart struct { + dir int + name string + file int +} + +func (p *dirOpPart) String() string { + return fmt.Sprintf("dir:%d name:%s file:%d", p.dir, p.name, p.file) +} + +type dirOp struct { + parts []dirOpPart +} + +func (o *dirOp) apply(fs *filesystemState, gcer gcer) { + for _, p := range o.parts { + dir := fs.getDir(p.dir) + + old := dir.entries[p.name] + if file, ok := fs.objects[old].(*backingFile); ok { + file.linkCount-- + // slog.Info("fs link count", "inode", file.inode, "count", file.linkCount) + if file.linkCount == 0 { + gcer.maybeGC(old) // XXX: jank. think about locking/ordering/help + } + } + + if p.file == -1 { + delete(dir.entries, p.name) + } else { + dir.entries[p.name] = p.file + if file, ok := fs.objects[p.file].(*backingFile); ok { + file.linkCount++ + // slog.Info("fs link count", "inode", file.inode, "count", file.linkCount) + } + } + } +} + +func (o *dirOp) touchedInodesIter(iter func(inode int) bool) { + for _, p := range o.parts { + if !iter(p.file) || !iter(p.dir) { + return + } + } +} + +func (o *dirOp) String() string { + return fmt.Sprintf("dirop %v", o.parts) +} + +type allocOp struct { + dir bool + + inode int +} + +func (o *allocOp) apply(fs *filesystemState, gcer gcer) { + if _, ok := fs.objects[o.inode]; ok { + panic("double alloc") + } + if o.dir { + fs.objects[o.inode] = &backingDir{ + inode: o.inode, + entries: make(map[string]int), + } + } else { + fs.objects[o.inode] = &backingFile{ + inode: o.inode, + file: &chunkedFile{}, + } + } + if o.inode >= fs.next { + fs.next = o.inode + 1 + } +} + +func (o *allocOp) touchedInodesIter(iter func(inode int) bool) { + iter(o.inode) +} + +func (o *allocOp) String() string { + return fmt.Sprintf("alloc inode:%d dir:%v", o.inode, o.dir) +} + +func (o *allocOp) addDeps(p *pendingOps, op *pendingOp) { + p.addDep(op, depKey{kind: depKeyAlloc, inode: int32(o.inode)}, depStoreAs) + p.addDep(op, depKey{kind: depKeyResizeFile, inode: int32(o.inode)}, depStoreAs) // optimization? +} + +func (o *writeOp) addDeps(p *pendingOps, op *pendingOp) { + p.addDep(op, depKey{kind: depKeyWriteFile, inode: int32(o.inode)}, depStoreAs) + // p.addDep(op, depKey{kind: depKeyAlloc, inode: int32(o.inode)}, depDependsOn) // optimization? + p.addDep(op, depKey{kind: depKeyResizeFile, inode: int32(o.inode)}, depDependsOn) +} + +func (o *resizeOp) addDeps(p *pendingOps, op *pendingOp) { + // p.addDep(op, depKey{kind: depKeyAlloc, inode: int32(o.inode)}, depDependsOn) // optimization? + p.addDep(op, depKey{kind: depKeyResizeFile, inode: int32(o.inode)}, depStoreAsAndDependsOn) +} + +func (o *dirOp) addDeps(p *pendingOps, op *pendingOp) { + for _, part := range o.parts { + p.addDep(op, depKey{kind: depKeyAlloc, inode: int32(part.file)}, depDependsOn) + p.addDep(op, depKey{kind: depKeyAlloc, inode: int32(part.dir)}, depDependsOn) + p.addDep(op, depKey{kind: depKeyWriteDir, inode: int32(part.dir)}, depStoreAs) + p.addDep(op, depKey{kind: depKeyWriteDirEntry, inode: int32(part.dir), entry: part.name}, depStoreAsAndDependsOn) + } +} + +type nilgc struct{} + +func (nilgc) maybeGC(inode int) {} + +type crashIterator struct { + workCh chan bool + nextCh chan []fsOp +} + +func (c *crashIterator) run(ops *pendingOps) { + c.workCh = make(chan bool) + c.nextCh = make(chan []fsOp) + go func() { + if !<-c.workCh { + return + } + ops.iterAllCrashes(func(ops []fsOp) bool { + c.nextCh <- ops + return <-c.workCh + }) + close(c.nextCh) + }() +} + +func (c *crashIterator) next() ([]fsOp, bool) { + c.workCh <- true + next, ok := <-c.nextCh + return next, ok +} + +/* +func (c *crashIterator) stop() { + c.workCh <- false + <-c.nextCh +} +*/ + +func (fs *Filesystem) Release() { + // XXX: somehow free resources here. +} + +func (fs *Filesystem) FlushEverything() { + fs.mu.Lock() + defer fs.mu.Unlock() + + // XXX: these methods for handling stop/crash/restart are kinda janky... + // how do we free things? how do we not needlessly copy (eg. after a restart + // where we don't care about the old? etc.) + + fs.persisted = fs.mem.clone() + fs.pendingOps = newPendingOps() + fs.pendingCountByInode = make(map[int]int) +} + +func (fs *Filesystem) CrashClone(partialDisk bool) *Filesystem { + fs.mu.Lock() + defer fs.mu.Unlock() + + // XXX: think about if we do the right thing with the next inode number + + persisted := fs.persisted.clone() + if partialDisk { + pickedOps := fs.pendingOps.randomCrashSubset() + for _, op := range pickedOps { + op.apply(persisted, nilgc{}) // XXX: jank think about gc + } + } + mem := persisted.clone() + + return &Filesystem{ + persisted: persisted, + mem: mem, + pendingOps: newPendingOps(), + openCountByInode: make(map[int]int), + pendingCountByInode: make(map[int]int), + } +} + +func (fs *Filesystem) IterCrashes() func() (*Filesystem, bool) { + var iter crashIterator + iter.run(fs.pendingOps) + + // XXX: handle leaking?? + + return func() (*Filesystem, bool) { + ops, ok := iter.next() + if !ok { + return nil, false + } + + newPersisted := fs.persisted.clone() + for _, op := range ops { + op.apply(newPersisted, nilgc{}) // XXX: jank think about gc + } + // XXX: find zero-ref files and delete them + return &Filesystem{ + persisted: newPersisted, + mem: newPersisted.clone(), + pendingOps: newPendingOps(), + openCountByInode: make(map[int]int), + pendingCountByInode: make(map[int]int), + }, true + } +} + +/* +func simCrash(fs *filesystem) { + // XXX: return a new *filesystem instead? + // not entirely necessary in this case and maybe less efficient since we'd have to clone persisted + + ops := fs.pendingOps.randomCrashSubset() + for _, op := range ops { + op.apply(fs.persisted) + } + + fs.mem = fs.persisted.clone() + fs.pendingOps = newPendingOps() +} +*/ + +func tryGetLinkCount(s *filesystemState, inode int) (int, bool) { + switch file := s.objects[inode].(type) { + case *backingFile: + return file.linkCount, true + } + return 0, false +} + +type InodeInfo struct { + MemLinks int + MemExists bool + DiskLinks int + DiskExists bool + Handles int + Ops int +} + +func (fs *Filesystem) getInodeInfoLocked(inode int) InodeInfo { + mem, memOk := tryGetLinkCount(fs.mem, inode) + disk, diskOk := tryGetLinkCount(fs.persisted, inode) + return InodeInfo{ + // XXX: also return if exists or not + Handles: fs.openCountByInode[inode], + Ops: fs.pendingCountByInode[inode], + MemLinks: mem, + MemExists: memOk, + DiskLinks: disk, + DiskExists: diskOk, + } +} + +func (fs *Filesystem) maybeGC(inode int) { + if inode == 2 { + // slog.Info("considering gc", "inode", inode, "info", fs.getInodeInfoLocked(inode)) + } + + // haha take taht!!! + if _, ok := fs.openCountByInode[inode]; ok { + return + } + if _, ok := fs.pendingCountByInode[inode]; ok { + return + } + switch file := fs.mem.objects[inode].(type) { + case *backingFile: + if file.linkCount == 0 { + // XXX: release the file here + file.file.Free() + file.file = nil + delete(fs.mem.objects, inode) + // delete? but what about persisted????????????????????????????????????? (must be zero also or we are in a strange situation) + // slog.Info("fs freeing mem", "inode", inode) + } + } + switch file := fs.persisted.objects[inode].(type) { + case *backingFile: + if file.linkCount == 0 { + // XXX: release the file here + file.file.Free() + file.file = nil + delete(fs.persisted.objects, inode) + // delete? but what about persisted????????????????????????????????????? (must be zero also or we are in a strange situation) + // slog.Info("fs freeing persisted", "inode", inode) + } + } +} + +func (fs *Filesystem) Read(inode int, pos int64, buffer syscallabi.ByteSliceView) int { + fs.mu.Lock() + defer fs.mu.Unlock() + + file := fs.mem.getFile(inode) + return file.read(pos, buffer) +} + +func (fs *Filesystem) Write(inode int, pos int64, buffer syscallabi.ByteSliceView) int { + fs.mu.Lock() + defer fs.mu.Unlock() + + file := fs.mem.getFile(inode) + + if int(pos)+buffer.Len() > file.len() { + op := &resizeOp{ + inode: inode, + size: int(pos) + buffer.Len(), + } + fs.apply(op) + } + + chunks := makeChunks(int(pos), buffer) + len := buffer.Len() + + op := &writeOp{ + inode: inode, + pos: int(pos), + size: buffer.Len(), + chunks: chunks, + // XXX: if we're running in some kind of non-fail mode, no need to make an (expensive?) copy? + // data: data, // XXX: test that we clone this please somehow (also other clones) + } + fs.apply(op) + + return len +} + +func (fs *Filesystem) Truncate(inode, pos int) { + fs.mu.Lock() + defer fs.mu.Unlock() + + file := fs.mem.getFile(inode) + + if pos != file.len() { + op := &resizeOp{ + inode: inode, + size: pos, + } + fs.apply(op) + } +} + +func (fs *Filesystem) Sync(inode int) { + fs.mu.Lock() + defer fs.mu.Unlock() + + fs.flushInode(inode) +} + +func (fs *Filesystem) Mkdir(name string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + if _, ok := fs.mem.getDir(rootInode).entries[name]; ok { + return errors.New("help already exists") + } + + inode := fs.mem.next + op := &allocOp{ + dir: true, + inode: inode, + } + fs.apply(op) + + dOp := &dirOp{ + parts: []dirOpPart{ + { + dir: rootInode, + name: name, + file: inode, + }, + }, + } + fs.apply(dOp) + + return nil +} + +type StatResp struct { + Inode int + IsDir bool +} + +// XXX: these two stats should share implementation + +func (fs *Filesystem) statLocked(inode int) (StatResp, error) { + obj := fs.mem.objects[inode] + switch obj.(type) { + case *backingDir: + return StatResp{ + Inode: inode, + IsDir: true, + }, nil + + case *backingFile: + return StatResp{ + Inode: inode, + IsDir: false, + }, nil + + default: + panic("invalid") + } +} + +func (fs *Filesystem) Stat(path string) (StatResp, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir := fs.mem.getDir(rootInode) + inode, ok := dir.entries[path] + if !ok { + return StatResp{}, syscall.ENOENT + } + return fs.statLocked(inode) +} + +func (fs *Filesystem) Statfd(inode int) (StatResp, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + return fs.statLocked(inode) +} + +func (fs *Filesystem) Remove(path string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir := fs.mem.getDir(rootInode) + if _, ok := dir.entries[path]; !ok { + return syscall.ENOENT + } + + dOp := &dirOp{ + parts: []dirOpPart{ + { + dir: rootInode, + name: path, + file: -1, + }, + }, + } + // XXX: clean up backing? + fs.apply(dOp) + + return nil +} + +func (fs *Filesystem) Rename(from, to string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir := fs.mem.getDir(rootInode) + inode, ok := dir.entries[from] + if !ok { + return syscall.ENOENT + } + + // slog.Info("fs rename", "from", from, "to", to, "inode", inode) + + dOp := &dirOp{ + parts: []dirOpPart{ + { + dir: rootInode, + name: from, + file: -1, + }, + { + dir: rootInode, + name: to, + file: inode, + }, + }, + } + fs.apply(dOp) + + return nil +} + +func (fs *Filesystem) OpenFile(path string, flag int) (int, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + inode, ok := fs.mem.getDir(rootInode).entries[path] + if path == "." { + // XXX: JANK + inode = rootInode + ok = true + } + if !ok { + if flag&os.O_CREATE == 0 { + return 0, syscall.ENOENT + } + + inode = fs.mem.next + op := &allocOp{ + dir: false, + inode: inode, + } + fs.apply(op) + + dOp := &dirOp{ + parts: []dirOpPart{ + { + dir: rootInode, + name: path, + file: inode, + }, + }, + } + fs.apply(dOp) + + // slog.Info("fs create", "path", args.Path, "inode", inode) + } + if ok && flag&os.O_EXCL != 0 { + return 0, errors.New("file already existed") + } + if ok && flag&os.O_TRUNC != 0 { + file := fs.mem.getFile(inode) + if file.len() != 0 { + op := &resizeOp{ + inode: inode, + size: 0, + } + fs.apply(op) + } + } + fs.openCountByInode[inode]++ + + return inode, nil +} + +func (fs *Filesystem) CloseFile(inode int) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + // XXX: make this API handle based so it's less brittle? + newCount := fs.openCountByInode[inode] - 1 + if newCount == 0 { + delete(fs.openCountByInode, inode) + fs.maybeGC(inode) + } else { + fs.openCountByInode[inode] = newCount + } + + return nil +} + +type ReadDirEntry struct { + Name string + IsDir bool +} + +func (fs *Filesystem) ReadDir(name string) ([]ReadDirEntry, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + // XXX: jank + if name != "." { + return nil, errors.New("bad dir") + } + dir := fs.mem.getDir(rootInode) + + var entries []ReadDirEntry + for name, inode := range dir.entries { + var de ReadDirEntry + + obj := fs.mem.objects[inode] + switch obj.(type) { + case *backingDir: + de = ReadDirEntry{ + Name: name, // XXX? + IsDir: true, + } + + case *backingFile: + de = ReadDirEntry{ + Name: name, // XXX? + IsDir: false, + } + + default: + panic("invalid") + } + + entries = append(entries, de) + } + slices.SortFunc(entries, func(a, b ReadDirEntry) int { + return cmp.Compare(a.Name, b.Name) + }) + + return entries, nil +} + +func (fs *Filesystem) GetInodeInfo(inode int) InodeInfo { + fs.mu.Lock() + defer fs.mu.Unlock() + return fs.getInodeInfoLocked(inode) +} diff --git a/internal/simulation/fs/pendingops.go b/internal/simulation/fs/pendingops.go new file mode 100644 index 0000000..fd3e503 --- /dev/null +++ b/internal/simulation/fs/pendingops.go @@ -0,0 +1,417 @@ +package fs + +import ( + "cmp" + "fmt" + "math/rand" + "slices" + "sync" +) + +//go:generate go run golang.org/x/tools/cmd/stringer -type=depKind,depKeyKind + +// A depKey is used to find fsOp dependencies. + +type depKeyKind byte + +const ( + depKeyNone depKeyKind = iota + depKeyAlloc + depKeyWriteFile + depKeyWriteDir + depKeyResizeFile // XXX: combine with alloc? + depKeyWriteDirEntry +) + +// A depKey is used to find fsOp dependencies. +// +// Plain struct to reduce allocations. +type depKey struct { + kind depKeyKind + inode int32 + entry string // for dir entry write, "" means nothing +} + +func (k *depKey) String() string { + return fmt.Sprintf("kind:%s inode:%d entry:%s", k.kind, k.inode, k.entry) +} + +type depKind byte + +const ( + depStoreAs depKind = iota + depStoreAsAndDependsOn + depDependsOn +) + +func (k depKind) isStoreAs() bool { + return k != depDependsOn +} + +func (k depKind) isDependsOn() bool { + return k != depStoreAs +} + +type pendingOp struct { + first *pendingDep + pos int + fsOp fsOp +} + +var opPool = &sync.Pool{ + New: func() any { + return &pendingOp{} + }, +} + +var depPool = &sync.Pool{ + New: func() any { + return &pendingDep{} + }, +} + +var ( + flagOpEnqueued = &pendingOp{} + flagOpWalked = &pendingOp{} +) + +type pendingDep struct { + key depKey + kind depKind + op *pendingOp + + prev, next *pendingDep // in dep order + + sibling *pendingDep // from dep +} + +// pendingOps tracks a filesystem operations that are not yet flushed to disk. +// Dependencies between are organized using dependency keys (depKeys): Any +// operation can be "stored as" one or more dependency keys, and "depend on" on +// one or more keys. An operation depends on all operations stored under keys it +// depends. +// +// For example, a file inode allocation is stored with `depKey{kind: +// depKeyAlloc, inode: inode}` with `depStoreAs`. A later write to that file +// depends on that allocation using `depKey{kind: depKeyAlloc, inode: inode}` +// with `depDependsOn`. +// +// These operations and their dependencies are tracked in a graph. Every +// operation is held in *pendingOp. Each *pendingOp has one or more *pendingDep, +// stored in a linked list under pendingOp.first and pendingDep.sibling. Each +// pendingDep is stored in a linked list for its correspending depKey in +// pendingDep.prev and pendingDep.next, with the last pendingDep for each key +// stored in pendingOps.lastDepForKey. +// +// To flush an operation to disk, that operation and all its dependencies must +// be removed from the graph: For each operation we enqueue all store-as +// predecessors of its depends-on dependencies. Then we do that recursively +// using a BFS. +// +// This processing is really a walk over the pendingDep graph, but the queue is +// tracked with pendingOp nodes. To make sure each pendingOp is enqueued and +// processed exactly once, the status of a pendingDep is tracked in +// pendingDep.op with flagOpEnqueued and flagOpWalked. A dependency is marked +// flagOpEnqueued when its corresponding pendingOp has been enqueued, while a +// dep is marked flagOpWalked when it and all the deps before it (dep.prev, +// dep.prev.prev, etc.) have been enqueued. +// +// A dep and its ops are removed from the graph when the op is flushed to disk. +type pendingOps struct { + nextPos int + lastDepForKey map[depKey]*pendingDep // used for graph + buffer []*pendingOp // used for flush inodes +} + +func newPendingOps() *pendingOps { + return &pendingOps{ + nextPos: 1, // must be > 0 for iterAllCrashes + lastDepForKey: make(map[depKey]*pendingDep), + } +} + +func (p *pendingOps) addDep(op *pendingOp, key depKey, kind depKind) { + prev := p.lastDepForKey[key] + + if prev == nil && kind == depDependsOn { + // fast path for dependencies that have already been flushed, eg. a + // write to an already allocated and resized file + return + } + if prev != nil && prev.op == op && prev.kind == kind { + // fast path for eg. rename in dir + return + } + + dep := depPool.Get().(*pendingDep) + dep.key = key // XXX: make unnecessary? + dep.kind = kind + dep.op = op + + dep.prev = prev + dep.next = nil + if dep.prev != nil { + dep.prev.next = dep + } + p.lastDepForKey[key] = dep + + dep.sibling = op.first + op.first = dep + + // log.Printf("adding dep %s %s", key.String(), kind) +} + +func (p *pendingOps) addOpInternal() *pendingOp { + op := opPool.Get().(*pendingOp) + + op.pos = p.nextPos + p.nextPos++ + + return op +} + +func (p *pendingOps) maybeEnqueueOpFromDep(dep *pendingDep, out []*pendingOp) []*pendingOp { + if dep.op == flagOpEnqueued || dep.op == flagOpWalked { + // dep is already enqueued + return out + } + out = append(out, dep.op) + // mark parent as nil in nodes so we know op is enqueued. nodes will be + // removed in flushOp + for cur := dep.op.first; cur != nil; cur = cur.sibling { + cur.op = flagOpEnqueued + } + return out +} + +func (p *pendingOps) enqueuePrevDeps(dep *pendingDep, out []*pendingOp) []*pendingOp { + for cur := dep; cur != nil && cur.op != flagOpWalked; cur = cur.prev { + if cur.kind.isStoreAs() { + out = p.maybeEnqueueOpFromDep(cur, out) + } else { + // XXX: how do we unhook this guy??????????????????????????????????? + // do we just leave it??? but then its prev and next are wrong!!!!! + // (actually i think they are right, just other depends-on deps at + // the start of the chain) + } + cur.op = flagOpWalked + } + return out +} + +func (p *pendingOps) processOp(op *pendingOp, out []*pendingOp) []*pendingOp { + cur := op.first + op.first = nil + for cur != nil { + if cur.kind.isDependsOn() { + out = p.enqueuePrevDeps(cur, out) + } + + // unhook and free node + if cur.prev != nil { + cur.prev.next = cur.next + } + if cur.next != nil { + cur.next.prev = cur.prev + } else { + // XXX: do this with a sentinel instead and we can skip the key field (and map lookup)? + if cur.prev == nil { + delete(p.lastDepForKey, cur.key) + } else { + p.lastDepForKey[cur.key] = cur.prev + } + } + + sibling := cur.sibling + cur.prev, cur.next, cur.sibling, cur.op = nil, nil, nil, nil + depPool.Put(cur) + cur = sibling + } + return out +} + +func (p *pendingOps) processQueue(out []*pendingOp) []*pendingOp { + for idx := 0; idx < len(out); idx++ { + cur := out[idx] + out = p.processOp(cur, out) + } + slices.SortFunc(out, byOpPos) + return out +} + +func (p *pendingOps) addOp(fsOp fsOp) { + op := p.addOpInternal() + op.fsOp = fsOp + fsOp.addDeps(p, op) +} + +func (p *pendingOps) flushInode(inode int, out []*pendingOp) []*pendingOp { + // file + out = p.enqueuePrevDeps(p.lastDepForKey[depKey{kind: depKeyAlloc, inode: int32(inode)}], out) + out = p.enqueuePrevDeps(p.lastDepForKey[depKey{kind: depKeyResizeFile, inode: int32(inode)}], out) + out = p.enqueuePrevDeps(p.lastDepForKey[depKey{kind: depKeyWriteFile, inode: int32(inode)}], out) + // dir + out = p.enqueuePrevDeps(p.lastDepForKey[depKey{kind: depKeyWriteDir, inode: int32(inode)}], out) + + out = p.processQueue(out) + + return out +} + +func (p *pendingOps) releaseBuffer(ops []*pendingOp) { + for i, op := range ops { + ops[i] = nil + op.fsOp = nil + /* + if op.first != nil || op.fsOp != nil { + panic("bad pool") + } + */ + opPool.Put(op) + } + p.buffer = ops[:0] +} + +func byOpPos(a, b *pendingOp) int { + return cmp.Compare(a.pos, b.pos) +} + +func (p *pendingOps) allOps() []*pendingOp { + // collect all nodes + var allOps []*pendingOp + for _, head := range p.lastDepForKey { + cur := head + for cur != nil { + // use pos as a little marker + if cur.op.pos > 0 { + cur.op.pos = -cur.op.pos + allOps = append(allOps, cur.op) + // log.Println(cur.parent) + } + cur = cur.prev + } + } + + // undo pos marking + for _, op := range allOps { + op.pos = -op.pos + } + + // sort by pos + slices.SortFunc(allOps, byOpPos) + + return allOps +} + +func (p *pendingOps) iterAllCrashes(yield func([]fsOp) bool) { + // XXX: if anything gets modified while iterAllCrashes runs we have a + // problem. do a clone? + + allOps := p.allOps() + + // walk subsets: + // - if we ever skip an op, mark all its store dep keys unavailable + // - only allow adding a new op if key is available + + unavailable := make(map[depKey]int) + var picked []fsOp + + var search func(idx int) bool + search = func(idx int) bool { + if idx == len(allOps) { + return yield(picked) + } + + op := allOps[idx] + + // consider picking + cur := op.first + ok := true + for cur != nil { + if cur.kind.isDependsOn() && unavailable[cur.key] > 0 { + ok = false + break + } + cur = cur.sibling + } + + // do pick + if ok { + // mark picked + picked = append(picked, op.fsOp) + if !search(idx + 1) { + return false + } + // unmark picked + picked = picked[:len(picked)-1] + } + + // don't pick + // mark not picked + cur = op.first + for cur != nil { + if cur.kind.isStoreAs() { + unavailable[cur.key]++ + } + cur = cur.sibling + } + if !search(idx + 1) { + return false + } + // unmark not picked + cur = op.first + for cur != nil { + if cur.kind.isStoreAs() { + unavailable[cur.key]-- + } + cur = cur.sibling + } + return true + } + + search(0) +} + +func (p *pendingOps) randomCrashSubset() []fsOp { + allOps := p.allOps() + + // similar to iterAllCrashes + // - if we ever skip an op, mark all its store dep keys unavailable + // - only allow adding a new op if key is available + + unavailable := make(map[depKey]bool) + var picked []fsOp + + for _, op := range allOps { + // consider picking + cur := op.first + ok := true + for cur != nil { + if cur.kind.isDependsOn() && unavailable[cur.key] { + ok = false + break + } + cur = cur.sibling + } + + // consider pick + // XXX: this could really benefit from fuzzing guideance + pick := ok && rand.Int31n(3) >= 1 + + if pick { + // do pick + picked = append(picked, op.fsOp) + } else { + // mark unavailable + cur = op.first + for cur != nil { + if cur.kind.isStoreAs() { + unavailable[cur.key] = true + } + cur = cur.sibling + } + } + } + + return picked +} diff --git a/internal/simulation/fs/pendingops_test.go b/internal/simulation/fs/pendingops_test.go new file mode 100644 index 0000000..85ebdae --- /dev/null +++ b/internal/simulation/fs/pendingops_test.go @@ -0,0 +1,169 @@ +package fs + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestPendingOps(t *testing.T) { + type testStep struct { + flushDeps []int // flush these deps + flushOps []int // expect these ops + + addOp int // add this op + addDependsOn []int + addStoreAs []int + addStoreAsAndDependsOn []int + } + + testCases := []struct { + name string + steps []testStep + }{ + { + name: "basic", + steps: []testStep{ + {addOp: 1, addStoreAs: []int{1}}, + {flushDeps: []int{1}, flushOps: []int{1}}, + {flushDeps: []int{1}, flushOps: []int{}}, + }, + }, + { + name: "chain at once", + steps: []testStep{ + {addOp: 1, addStoreAs: []int{1}}, + {addOp: 2, addStoreAs: []int{2}, addDependsOn: []int{1}}, + {addOp: 3, addStoreAs: []int{3}, addDependsOn: []int{2}}, + {addOp: 4, addStoreAs: []int{4}, addDependsOn: []int{3}}, + {flushDeps: []int{4}, flushOps: []int{1, 2, 3, 4}}, + {flushDeps: []int{2}, flushOps: []int{}}, + {flushDeps: []int{4}, flushOps: []int{}}, + }, + }, + { + name: "chain in parts", + steps: []testStep{ + {addOp: 1, addStoreAs: []int{1}}, + {addOp: 2, addStoreAs: []int{2}, addDependsOn: []int{1}}, + {addOp: 3, addStoreAs: []int{3}, addDependsOn: []int{2}}, + {addOp: 4, addStoreAs: []int{4}, addDependsOn: []int{3}}, + {flushDeps: []int{2}, flushOps: []int{1, 2}}, + {flushDeps: []int{4}, flushOps: []int{3, 4}}, + {flushDeps: []int{2}, flushOps: []int{}}, + {flushDeps: []int{4}, flushOps: []int{}}, + }, + }, + { + name: "chain at once with more after", + steps: []testStep{ + {addOp: 1, addStoreAs: []int{1}}, + {addOp: 2, addStoreAs: []int{2}, addDependsOn: []int{1}}, + {addOp: 3, addStoreAs: []int{3}, addDependsOn: []int{2}}, + {addOp: 4, addStoreAs: []int{4}, addDependsOn: []int{3}}, + {addOp: 5, addStoreAs: []int{1}}, + {addOp: 6, addStoreAs: []int{1}}, + {flushDeps: []int{4}, flushOps: []int{1, 2, 3, 4}}, + {flushDeps: []int{2}, flushOps: []int{}}, + {flushDeps: []int{1}, flushOps: []int{5, 6}}, + }, + }, + { + name: "no duplicates in output", + steps: []testStep{ + {addOp: 1, addStoreAsAndDependsOn: []int{1, 2}}, + {addOp: 2, addStoreAsAndDependsOn: []int{1, 2}}, + {addOp: 3, addStoreAsAndDependsOn: []int{1, 2}}, + {addOp: 4, addStoreAsAndDependsOn: []int{1, 2}}, + {flushDeps: []int{2}, flushOps: []int{1, 2, 3, 4}}, + {flushDeps: []int{1}, flushOps: []int{}}, + }, + }, + { + name: "fast path", + steps: []testStep{ + {addOp: 1, addDependsOn: []int{1}}, + {addOp: 2, addStoreAs: []int{1}}, + {addOp: 3, addDependsOn: []int{1}}, + {addOp: 4, addDependsOn: []int{1}, addStoreAs: []int{2}}, + {flushDeps: []int{2}, flushOps: []int{2, 4}}, + {flushDeps: []int{1}, flushOps: []int{}}, + }, + }, + { + name: "double deps", + steps: []testStep{ + {addOp: 1, addStoreAs: []int{1}, addDependsOn: []int{1}}, + {addOp: 2, addStoreAs: []int{2}, addDependsOn: []int{1, 1}}, + {addOp: 3, addStoreAsAndDependsOn: []int{2, 3, 3}}, + {addOp: 4, addStoreAsAndDependsOn: []int{3, 4}}, + {flushDeps: []int{4}, flushOps: []int{1, 2, 3, 4}}, + {flushDeps: []int{1}, flushOps: []int{}}, + }, + }, + { + name: "tricky release", + steps: []testStep{ + {addOp: 1, addStoreAsAndDependsOn: []int{1}}, + {addOp: 2, addStoreAsAndDependsOn: []int{1}}, + {addOp: 3, addStoreAsAndDependsOn: []int{1, 2}}, + {addOp: 4, addStoreAsAndDependsOn: []int{1}}, + {addOp: 5, addStoreAsAndDependsOn: []int{1}}, + {flushDeps: []int{2}, flushOps: []int{1, 2, 3}}, + {flushDeps: []int{1}, flushOps: []int{4, 5}}, + }, + }, + { + name: "less tricky release", + steps: []testStep{ + {addOp: 1, addStoreAsAndDependsOn: []int{1}}, + {addOp: 3, addStoreAsAndDependsOn: []int{1, 2}}, + {addOp: 4, addStoreAsAndDependsOn: []int{1}}, + {addOp: 5, addStoreAsAndDependsOn: []int{1}}, + {flushDeps: []int{2}, flushOps: []int{1, 3}}, + {flushDeps: []int{1}, flushOps: []int{4, 5}}, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + ops := newPendingOps() + + for i, step := range testCase.steps { + if step.addOp != 0 { + op := ops.addOpInternal() + op.fsOp = &allocOp{inode: step.addOp} + for _, dep := range step.addDependsOn { + ops.addDep(op, depKey{inode: int32(dep)}, depDependsOn) + } + for _, dep := range step.addStoreAs { + ops.addDep(op, depKey{inode: int32(dep)}, depStoreAs) + } + for _, dep := range step.addStoreAsAndDependsOn { + ops.addDep(op, depKey{inode: int32(dep)}, depStoreAsAndDependsOn) + } + + } + + if step.flushDeps != nil { + out := ops.buffer + for _, dep := range step.flushDeps { + out = ops.enqueuePrevDeps(ops.lastDepForKey[depKey{inode: int32(dep)}], out) + } + out = ops.processQueue(out) + + flushOps := []int{} + for _, op := range out { + flushOps = append(flushOps, op.fsOp.(*allocOp).inode) + } + ops.releaseBuffer(out) + + if diff := cmp.Diff(step.flushOps, flushOps); diff != "" { + t.Errorf("step %d: flush deps %v: diff: %s", i+1, step.flushDeps, diff) + } + } + } + }) + } +} diff --git a/internal/simulation/generate.go b/internal/simulation/generate.go new file mode 100644 index 0000000..ad6f1d7 --- /dev/null +++ b/internal/simulation/generate.go @@ -0,0 +1,5 @@ +//go:build generate + +package simulation + +//go:generate go run ./gensyscall diff --git a/internal/simulation/gensyscall/main.go b/internal/simulation/gensyscall/main.go new file mode 100644 index 0000000..6448789 --- /dev/null +++ b/internal/simulation/gensyscall/main.go @@ -0,0 +1,781 @@ +// Copyright 2018 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// This file is based on +// https://cs.opensource.google/go/x/sys/+/refs/tags/v0.26.0:unix/mksyscall.go + +// This program generates wrappers for system calls in gosim to ergonomically +// invoke them, implement them, and hook them in the go standard library. +package main + +import ( + "bufio" + "bytes" + "cmp" + "encoding/json" + "fmt" + "go/format" + "io" + "log" + "os" + "os/exec" + "path" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/translate" +) + +// findAndReadSources looks up the paths of files in go packages path using `go +// list` and reads them. +func findAndReadSources(files []string) map[string][]byte { + pkgs := make(map[string]string) + for _, file := range files { + pkg := path.Dir(file) + pkgs[pkg] = "" + } + + args := []string{"list", "-json=ImportPath,Dir"} + for pkg := range pkgs { + args = append(args, pkg) + } + cmd := exec.Command("go", args...) + out, err := cmd.Output() + if err != nil { + log.Fatal(err) + } + + decoder := json.NewDecoder(bytes.NewReader(out)) + for { + var output struct { + ImportPath string + Dir string + } + if err := decoder.Decode(&output); err != nil { + if err == io.EOF { + break + } + log.Fatal(err) + } + pkgs[output.ImportPath] = output.Dir + } + + contents := make(map[string][]byte) + for _, file := range files { + pkg := path.Dir(file) + dir := pkgs[pkg] + p := path.Join(dir, path.Base(file)) + bytes, err := os.ReadFile(p) + if err != nil { + log.Fatal(err) + } + contents[file] = bytes + } + + return contents +} + +type outputWithSortKey struct { + sortKey string + output string +} + +// outputSorter holds outputs to be sorted by key +type outputSorter struct { + outputs []outputWithSortKey +} + +func newOutputSorter() *outputSorter { + return &outputSorter{} +} + +func (o *outputSorter) append(key, value string) { + o.outputs = append(o.outputs, outputWithSortKey{sortKey: strings.ToLower(key), output: value}) +} + +func (o *outputSorter) output() string { + slices.SortFunc(o.outputs, func(a, b outputWithSortKey) int { + return cmp.Compare(a.sortKey, b.sortKey) + }) + var out strings.Builder + for _, output := range o.outputs { + out.WriteString(output.output) + } + return out.String() +} + +func writeFormattedGoFile(path, contents string) { + b := []byte("// Code generated by gensyscall. DO NOT EDIT.\n" + contents) + b, err := format.Source(b) + if err != nil { + log.Printf("error formatting %s: %s; continuing so you can inspect...", path, err) + } + b = bytes.Replace(b, []byte("//\npackage"), []byte("package"), 1) // who knows why we need this but otherwise go fmt is mad + if err := os.WriteFile(path, b, 0o644); err != nil { + log.Fatal(err) + } +} + +var ( + regexpSys = regexp.MustCompile(`^\/\/sys\t`) + regexpSysNonblock = regexp.MustCompile(`^\/\/sysnb\t`) + regexpSysDeclaration = regexp.MustCompile(`^\/\/sys(nb)?\t(\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*((?i)_?SYS_[A-Z0-9_]+))?$`) + regexpComma = regexp.MustCompile(`\s*,\s*`) + regexpSyscallName = regexp.MustCompile(`([a-z])([A-Z])`) + regexpParamKV = regexp.MustCompile(`^(\S*) (\S*)$`) +) + +type TypeKind string + +const ( + TypePlain TypeKind = "" + TypePointer TypeKind = "*" + TypeSlice TypeKind = "[]" +) + +// Param is function parameter +type Param struct { + Orig string + Name string + Kind TypeKind + Type string +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexpParamKV.FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + typ := ps[2] + kind := TypePlain + if strings.HasPrefix(typ, "[]") { + typ = typ[2:] + kind = TypeSlice + } + if strings.HasPrefix(typ, "*") { + typ = typ[1:] + kind = TypePointer + } + return Param{Orig: p, Name: ps[1], Kind: kind, Type: typ} +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []Param { + list = strings.TrimSpace(list) + if list == "" { + return nil + } + var params []Param + for _, paramStr := range regexpComma.Split(list, -1) { + params = append(params, parseParam(paramStr)) + } + return params +} + +type syscallInfo struct { + pkg string + funcName string + sysName string + sysVal int + inputsStr, outputsStr string + async bool +} + +var filterCommon = []string{ + "SYS_ACCEPT4", + "SYS_BIND", + "SYS_CLOSE", + "SYS_CONNECT", + "SYS_FCNTL", + "SYS_FSTAT", + "SYS_FSYNC", + "SYS_FTRUNCATE", + "SYS_GETDENTS64", + "SYS_GETPEERNAME", + "SYS_GETPID", + // "SYS_GETRANDOM", + "SYS_GETSOCKNAME", + "SYS_GETSOCKOPT", + "SYS_LISTEN", + "SYS_OPENAT", + "SYS_PREAD64", + "SYS_PWRITE64", + "SYS_READ", + "SYS_RENAMEAT", + "SYS_SETSOCKOPT", + "SYS_SOCKET", + "SYS_UNAME", + "SYS_UNLINKAT", + "SYS_WRITE", +} + +var filterArm64 = append(slices.Clip(filterCommon), []string{ + "SYS_FSTATAT", +}...) + +var filterAmd64 = append(slices.Clip(filterCommon), []string{ + "SYS_NEWFSTATAT", +}...) + +var filter = map[string][]string{ + "arm64": filterArm64, + "amd64": filterAmd64, +} + +func parseSyscalls(paths []string, filter []string) []syscallInfo { + sources := findAndReadSources(paths) + filterMap := make(map[string]int) + for _, name := range filter { + filterMap[name] = 0 + } + var parsedSyscalls []syscallInfo + for file, contents := range sources { + // fmt.Println(file) // , "\n", string(contents[:1024])) + + pkg := path.Dir(file) + + s := bufio.NewScanner(bytes.NewReader(contents)) + for s.Scan() { + t := s.Text() + if regexpSys.FindStringSubmatch(t) == nil && regexpSysNonblock.FindStringSubmatch(t) == nil { + continue + } + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, errno error) + // Split into name, in params, out params. + f := regexpSysDeclaration.FindStringSubmatch(t) + if f == nil { + log.Fatalf("%s:%s\nmalformed //sys declaration\n", file, t) + os.Exit(1) + } + + funcName, inputsStr, outputsStr, sysName := f[2], f[3], f[4], f[5] + + if sysName == "" { + sysName = "SYS_" + regexpSyscallName.ReplaceAllString(funcName, `${1}_$2`) + sysName = strings.ToUpper(sysName) + } + + if funcName == "readlen" || funcName == "writelen" { + // help different signatures, skip + continue + } + + if _, ok := filterMap[sysName]; !ok { + continue + } + filterMap[sysName]++ + + parsedSyscalls = append(parsedSyscalls, syscallInfo{ + pkg: pkg, + funcName: funcName, + sysName: sysName, + inputsStr: inputsStr, + outputsStr: outputsStr, + }) + } + } + for sysName, count := range filterMap { + if count == 0 { + log.Fatalf("help, did not see %s", sysName) + } + } + + // JANKY: Getrandom here for now because with the vDSO changes it no longer + // appears as a comment. + parsedSyscalls = append(parsedSyscalls, syscallInfo{ + pkg: "golang.org/x/sys/unix", + funcName: "Getrandom", + sysName: "SYS_GETRANDOM", + inputsStr: "buf []byte, flags int", + outputsStr: "n int, err error", + }) + + return parsedSyscalls +} + +func dedupSyscalls(syscalls []syscallInfo) []syscallInfo { + // TODO: dedup syscalls better??? + var deduped []syscallInfo + seen := make(map[string]syscallInfo) + for _, info := range syscalls { + pkg, sysName, inputsStr, outputsStr := info.pkg, info.sysName, info.inputsStr, info.outputsStr + + if pkg == "golang.org/x/sys/unix" && sysName == "SYS_FSTATAT" { + // slight difference w/ syscall, but ok + continue + } + if pkg == "golang.org/x/sys/unix" && sysName == "SYS_NEWFSTATAT" { + // slight difference w/ syscall, but ok + continue + } + + if previous, ok := seen[sysName]; ok { + if inputsStr != previous.inputsStr || outputsStr != previous.outputsStr { + log.Fatalf("help, %s has multiple different versions:\n%+v\n%+v", sysName, info, previous) + } + continue + } + seen[sysName] = info + deduped = append(deduped, info) + } + return deduped +} + +func ifaceName(sysName string) string { + parts := strings.Split(sysName, "_") + for i, part := range parts { + parts[i] = strings.ToUpper(part[:1]) + strings.ToLower(part[1:]) + } + return strings.Join(parts, "") +} + +var typemap = map[string]string{ + "RawSockaddrAny": "RawSockaddrAny", + "Utsname": "Utsname", + "Stat_t": "Stat_t", + "_Socklen": "Socklen", +} + +type Usecase string + +const ( + Proxy Usecase = "Proxy" + Switch Usecase = "Switch" + Iface Usecase = "Iface" +) + +func toUintptr(val string, typ Param) string { + if typ.Kind != TypePlain { + panic("help") + } + if typ.Type == "bool" { + return fmt.Sprintf("syscallabi.BoolToUintptr(%s)", val) + } else if typ.Type == "error" { + return fmt.Sprintf("syscallabi.ErrErrno(%s)", val) + } else { + return fmt.Sprintf("uintptr(%s)", val) + } +} + +func fromUintptr(val string, typ Param) string { + if typ.Kind != TypePlain { + panic("help") + } + if typ.Type == "bool" { + return fmt.Sprintf("syscallabi.BoolFromUintptr(%s)", val) + } else if typ.Type == "error" { + return fmt.Sprintf("syscallabi.ErrnoErr(%s)", val) + } else { + return fmt.Sprintf("%s(%s)", maybeMapType(Switch, typ.Type), val) + } +} + +func maybeMapType(usecase Usecase, typ string) string { + known, ok := typemap[typ] + if !ok { + return typ + } + if usecase == Proxy { + return "simulation." + known + } + return known +} + +func uppercase(s string) string { + return strings.ToUpper(s[0:1]) + s[1:] +} + +func writeProxies(rootDir string, pkgs []string, syscalls []syscallInfo, version string, arch string) { + proxyTexts := make(map[string]*outputSorter) + for _, pkg := range pkgs { + proxyTexts[pkg] = newOutputSorter() + } + translateTexts := newOutputSorter() + for _, info := range syscalls { + pkg, funcName, sysName, inputsStr, outputsStr := info.pkg, info.funcName, info.sysName, info.inputsStr, info.outputsStr + + in := parseParamList(inputsStr) + out := parseParamList(outputsStr) + + proxyName := translate.RewriteSelector(pkg, funcName) + ifaceName := ifaceName(sysName) + + var proxyArgs, proxyCallArgs []string + for _, p := range in { + proxyArgs = append(proxyArgs, fmt.Sprintf("%s %s%s", p.Name, p.Kind, maybeMapType(Proxy, p.Type))) + proxyCallArgs = append(proxyCallArgs, p.Name) + } + var proxyRets []string + for _, p := range out { + proxyRets = append(proxyRets, p.Orig) + } + var proxyRet string + if len(out) > 0 { + proxyRet = fmt.Sprintf(" (%s)", strings.Join(proxyRets, ", ")) + } + proxyText := "" + + fmt.Sprintf("func %s(%s)%s {\n", proxyName, strings.Join(proxyArgs, ", "), proxyRet) + + fmt.Sprintf("\treturn simulation.Syscall%s(%s)\n", ifaceName, strings.Join(proxyCallArgs, ", ")) + + "}\n\n" + proxyTexts[pkg].append(funcName, proxyText) + translateText := fmt.Sprintf("\t{Pkg: %s, Selector: %s}: {Pkg: hooks%sPackage},\n", strconv.Quote(pkg), strconv.Quote(funcName), uppercase(version)) + translateTexts.append(pkg+" "+funcName, translateText) + } + for _, pkg := range pkgs { + file := strings.Replace(pkg, ".", "", -1) + file = strings.Replace(file, "/", "_", -1) + path := path.Join(rootDir, "internal/hooks/"+version, file+"_gensyscall_"+arch+".go") + + writeFormattedGoFile(path, `//go:build linux +package `+version+` + +import ( + "unsafe" + + "`+gosimtool.Module+`/internal/simulation" +) + +// prevent unused imports +var _ unsafe.Pointer + +`+proxyTexts[pkg].output()) + } + writeFormattedGoFile(path.Join(rootDir, "internal/translate/hooks_"+version+"_"+arch+"_gensyscall.go"), `package translate + +func init() { + hooksGensyscall`+uppercase(version)+`ByArch["`+arch+`"] = map[packageSelector]packageSelector{ +`+translateTexts.output()+`} +} +`) +} + +func writeSyscalls(outputPath string, syscalls []syscallInfo, isMachine bool) { + osTypeName := "LinuxOS" + osImplName := "linuxOS" + osIfaceName := "linuxOSIface" + if isMachine { + osTypeName = "GosimOS" + osImplName = "gosimOS" + osIfaceName = "gosimOSIface" + } + + callers := newOutputSorter() + dispatches := newOutputSorter() + checks := newOutputSorter() + ifaces := newOutputSorter() + for _, info := range syscalls { + sysName, inputsStr, outputsStr := info.sysName, info.inputsStr, info.outputsStr + + in := parseParamList(inputsStr) + out := parseParamList(outputsStr) + ifaceName := ifaceName(sysName) + + var callerArgs, dispatchArgs, ifaceArgs []string + prepareCaller := "" + prepareDispatch := "" + cleanupCallerText := "" + + // XXX: thread through async + + for i, p := range in { + if p.Kind == TypePointer { + prepareCaller += fmt.Sprintf("\tsyscall.Ptr%d = %s\n", i, p.Name) + cleanupCallerText += fmt.Sprintf("\tsyscall.Ptr%d = nil\n", i) + prepareDispatch += fmt.Sprintf("\t\t%s := syscallabi.NewValueView(syscall.Ptr%d.(%s%s))\n", p.Name, i, p.Kind, maybeMapType(Switch, p.Type)) + ifaceArgs = append(ifaceArgs, fmt.Sprintf("%s syscallabi.ValueView[%s]", p.Name, maybeMapType(Iface, p.Type))) + } else if p.Kind == TypePlain && (p.Type == "string" || p.Type == "unsafe.Pointer" || p.Type == "any") { + prepareCaller += fmt.Sprintf("\tsyscall.Ptr%d = %s\n", i, p.Name) + cleanupCallerText += fmt.Sprintf("\tsyscall.Ptr%d = nil\n", i) + prepareDispatch += fmt.Sprintf("\t\t%s := syscall.Ptr%d.(%s)\n", p.Name, i, p.Type) + ifaceArgs = append(ifaceArgs, fmt.Sprintf("%s %s", p.Name, p.Type)) + } else if p.Kind == TypeSlice { + prepareCaller += fmt.Sprintf("\tsyscall.Ptr%d, syscall.Int%d = unsafe.SliceData(%s), uintptr(len(%s))\n", i, i, p.Name, p.Name) + cleanupCallerText += fmt.Sprintf("\tsyscall.Ptr%d = nil\n", i) + prepareDispatch += fmt.Sprintf("\t\t%s := syscallabi.NewSliceView(syscall.Ptr%d.(*%s), syscall.Int%d)\n", p.Name, i, maybeMapType(Switch, p.Type), i) + ifaceArgs = append(ifaceArgs, fmt.Sprintf("%s syscallabi.SliceView[%s]", p.Name, maybeMapType(Iface, p.Type))) + } else { + prepareCaller += fmt.Sprintf("\tsyscall.Int%d = %s\n", i, toUintptr(p.Name, p)) + prepareDispatch += fmt.Sprintf("\t\t%s := %s\n", p.Name, fromUintptr(fmt.Sprintf("syscall.Int%d", i), p)) + ifaceArgs = append(ifaceArgs, fmt.Sprintf("%s %s", p.Name, maybeMapType(Iface, p.Type))) + } + + callerArgs = append(callerArgs, fmt.Sprintf("%s %s%s", p.Name, p.Kind, maybeMapType(Iface, p.Type))) + dispatchArgs = append(dispatchArgs, p.Name) + } + + if info.async { + dispatchArgs = append(dispatchArgs, "syscall") + ifaceArgs = append(ifaceArgs, "syscall *syscallabi.Syscall") + } + + parseCallerText := "" + parseDispatchText := "" + + sysVal := fmt.Sprintf("unix.%s", sysName) + if info.sysVal != 0 { + sysVal = fmt.Sprint(info.sysVal) + } + + var callerRets, dispatchRets, ifaceRets []string + + // Assign return values. + for i, p := range out { + callerRets = append(callerRets, p.Orig) + dispatchRets = append(dispatchRets, p.Name) + ifaceRets = append(ifaceRets, p.Orig) + + if p.Type == "string" { + reg := fmt.Sprintf("syscall.RPtr%d", i) + parseCallerText += fmt.Sprintf("\t%s = %s.(string)\n", p.Name, reg) + parseDispatchText += fmt.Sprintf("\t\t%s = %s\n", reg, p.Name) + } else { + reg := fmt.Sprintf("syscall.R%d", i) + if p.Name == "err" { + reg = "syscall.Errno" + } + parseCallerText += fmt.Sprintf("\t%s = %s\n", p.Name, fromUintptr(reg, p)) + parseDispatchText += fmt.Sprintf("\t\t%s = %s\n", reg, toUintptr(p.Name, p)) + } + } + + var callerRet, dispatchRet, ifaceRet string + if len(out) > 0 { + callerRet = fmt.Sprintf(" (%s)", strings.Join(callerRets, ", ")) + dispatchRet = fmt.Sprintf("%s := ", strings.Join(dispatchRets, ", ")) + ifaceRet = fmt.Sprintf(" (%s)", strings.Join(ifaceRets, ", ")) + } + + if info.async { + dispatchRet = "" + parseDispatchText = "" + } else { + parseDispatchText += "\t\tsyscall.Complete()\n" + } + + dispatchText := fmt.Sprintf("\tcase %s:\n", sysVal) + + "\t\t// called by (for find references):\n" + + fmt.Sprintf("\t\t_ = %s\n", "Syscall"+ifaceName) + + prepareDispatch + + fmt.Sprintf("\t\t%sos.%s(%s)\n", dispatchRet, ifaceName, strings.Join(dispatchArgs, ", ")) + + parseDispatchText + dispatches.append(sysName, dispatchText) + + if info.sysVal == 0 { + checkText := fmt.Sprintf("\tcase %s:\n", sysVal) + + "\t\treturn true\n" + checks.append(sysName, checkText) + } + + ifaceText := fmt.Sprintf("\t%s(%s)%s\n", ifaceName, strings.Join(ifaceArgs, ", "), ifaceRet) + ifaces.append(sysName, ifaceText) + + callerText := "//go:norace\n" + + fmt.Sprintf("func %s(%s)%s {\n", "Syscall"+ifaceName, strings.Join(callerArgs, ", "), callerRet) + + "\t// invokes (for go to definition):\n" + + fmt.Sprintf("\t_ = (*%s).%s\n", osTypeName, ifaceName) + + "\tsyscall := syscallabi.GetGoroutineLocalSyscall()\n" + + fmt.Sprintf("\tsyscall.OS = %s\n", osImplName) + + fmt.Sprintf("\tsyscall.Trap = %s\n", sysVal) + + prepareCaller + + fmt.Sprintf("\t%s.dispatchSyscall(syscall)\n", osImplName) + + parseCallerText + + cleanupCallerText + + "\treturn\n" + + "}\n\n" + callers.append(sysName, callerText) + } + + output := "" + if !isMachine { + output += "//go:build linux\n" + } + output += fmt.Sprintf(`package simulation + +import ( + "unsafe" + "syscall" + "time" + + "golang.org/x/sys/unix" + + "`+gosimtool.Module+`/internal/simulation/fs" + "`+gosimtool.Module+`/internal/simulation/syscallabi" +) + +// prevent unused imports +var ( + _ unsafe.Pointer + _ time.Duration + _ syscallabi.ValueView[any] + _ syscall.Errno + _ fs.InodeInfo + _ unix.Errno +) + +// %s is the interface *%s must implement to work +// with HandleSyscall. The interface is not used but helpful for implementing +// new syscalls. +type %s interface { +`, osIfaceName, osTypeName, osIfaceName) + ifaces.output() + fmt.Sprintf(`} + +var _ %s = &%s{} + +//go:norace +func (os *%s) dispatchSyscall(s *syscallabi.Syscall) { + // XXX: help this happens for globals in os + if os == nil { + s.Errno = uintptr(syscall.ENOSYS) + return + } + os.dispatcher.Dispatch(s) +} + +//go:norace +func (os *%s) HandleSyscall(syscall *syscallabi.Syscall) { + switch (syscall.Trap) { +`, osIfaceName, osTypeName, osTypeName, osTypeName) + dispatches.output() + ` default: + panic("bad") + } +} + +` + callers.output() + + if !isMachine { + output += ` +func IsHandledSyscall(trap uintptr) bool { + switch (trap) { +` + checks.output() + ` default: + return false + } +} +` + } + + writeFormattedGoFile(outputPath, output) +} + +func main() { + rootDir, err := gosimtool.FindGoModDir() + if err != nil { + log.Fatal(err) + } + + for _, arch := range []string{"arm64", "amd64"} { + pkgs := []string{ + "syscall", + "golang.org/x/sys/unix", + } + + syscalls := parseSyscalls([]string{ + "syscall/syscall_linux.go", + "syscall/syscall_linux_" + arch + ".go", + "golang.org/x/sys/unix/syscall_linux.go", + "golang.org/x/sys/unix/syscall_linux_" + arch + ".go", + }, filter[arch]) + + writeProxies(rootDir, pkgs, syscalls, "go123", arch) + deduped := dedupSyscalls(syscalls) + + deduped = append(deduped, []syscallInfo{ + { + sysName: "POLL_OPEN", + sysVal: 1000, + inputsStr: "fd int, desc *syscallabi.PollDesc", + outputsStr: "code int", + }, + { + sysName: "POLL_CLOSE", + sysVal: 1001, + inputsStr: "fd int, desc *syscallabi.PollDesc", + outputsStr: "code int", + }, + }...) + + writeSyscalls(path.Join(rootDir, "internal/simulation/linux_gensyscall_"+arch+".go"), deduped, false) + } + + machineCalls := []syscallInfo{ + { + sysName: "SET_SIMULATION_TIMEOUT", + inputsStr: "timeout time.Duration", + outputsStr: "err error", + }, + { + sysName: "SET_CONNECTED", + inputsStr: "a string, b string, connected bool", + outputsStr: "err error", + }, + { + sysName: "SET_DELAY", + inputsStr: "a string, b string, delay time.Duration", + outputsStr: "err error", + }, + { + sysName: "MACHINE_NEW", + inputsStr: "label string, addr string, program any", + outputsStr: "machineID int", + }, + { + sysName: "MACHINE_STOP", + inputsStr: "machineID int, graceful bool", + }, + { + sysName: "MACHINE_INODE_INFO", + inputsStr: "machineID int, inodeID int, info *fs.InodeInfo", + }, + { + sysName: "MACHINE_WAIT", + inputsStr: "machineID int", + async: true, + }, + { + sysName: "MACHINE_RECOVER_INIT", + inputsStr: "machineID int, program any", + outputsStr: "iterID int", + }, + { + sysName: "MACHINE_RECOVER_NEXT", + inputsStr: "iterID int", + outputsStr: "machineID int, ok bool", + }, + { + sysName: "MACHINE_RECOVER_RELEASE", + inputsStr: "iterID int", + outputsStr: "", + }, + { + sysName: "MACHINE_RESTART", + inputsStr: "machineID int, partialDisk bool", + outputsStr: "err error", + }, + { + sysName: "MACHINE_GET_LABEL", + inputsStr: "machineID int", + outputsStr: "label string, err error", + }, + { + sysName: "MACHINE_SET_BOOT_PROGRAM", + inputsStr: "machineID int, program any", + outputsStr: "err error", + }, + { + sysName: "MACHINE_SET_SOMETIMES_CRASH_ON_SYNC", + inputsStr: "machineID int, crash bool", + outputsStr: "err error", + }, + } + + for i := range machineCalls { + call := &machineCalls[i] + call.sysVal = 1000 + i + } + + writeSyscalls(path.Join(rootDir, "internal/simulation/gosim_gensyscall.go"), machineCalls, true) +} diff --git a/internal/simulation/gosim.go b/internal/simulation/gosim.go new file mode 100644 index 0000000..1fa6895 --- /dev/null +++ b/internal/simulation/gosim.go @@ -0,0 +1,206 @@ +package simulation + +import ( + "fmt" + "net/netip" + "syscall" + "time" + + "github.com/jellevandenhooff/gosim/internal/simulation/fs" + "github.com/jellevandenhooff/gosim/internal/simulation/network" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +const defaultTimeout = 10 * time.Minute + +// GosimOS implements all simulation-level system calls, such as creating +// machines, messing with the network, etc. +type GosimOS struct { + dispatcher syscallabi.Dispatcher + + simulation *Simulation + + // TODO: this is currently protected by simulation.mu. + + // TODO: have a common by-ID scheme because I don't want infinite maps, please + nextCrashIterId int + crashItersById map[int]func() (int, bool) +} + +func NewGosimOS(s *Simulation, d syscallabi.Dispatcher) *GosimOS { + return &GosimOS{ + dispatcher: d, + + simulation: s, + + crashItersById: make(map[int]func() (int, bool)), + } +} + +func (g *GosimOS) SetSimulationTimeout(d time.Duration) error { + g.simulation.timeoutTimer.Reset(d) + return nil +} + +func (g *GosimOS) MachineNew(label string, addrStr string, program any) (machineId int) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + // TODO: handle failure but do allow an empty addrStr to indicate unset addr + addr, _ := netip.ParseAddr(addrStr) + + bootProgram := program.(func()) + + machine := g.simulation.newMachine(label, addr, fs.NewFilesystem(), bootProgram) + + // TODO: don't start machine??? + g.simulation.startMachine(machine) + + return machine.id +} + +func (g *GosimOS) MachineRecoverInit(machineID int, program any) int { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + fs := machine.filesystem + + // TODO: clone the filesystem or somehow prevent modifications + iter := fs.IterCrashes() + counter := 0 + + id := g.nextCrashIterId + g.nextCrashIterId++ + + bootProgram := program.(func()) + + g.crashItersById[id] = func() (int, bool) { + fs, ok := iter() + if !ok { + return 0, false + } + + label := fmt.Sprintf("%s-iter-recover-%d", machine.label, counter) + counter++ + + var addr netip.Addr + newMachine := g.simulation.newMachine(label, addr, fs, bootProgram) + g.simulation.startMachine(newMachine) + + return newMachine.id, true + } + return id +} + +func (g *GosimOS) MachineRecoverNext(crashIterId int) (int, bool) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + // XXX: jank + iter := g.crashItersById[crashIterId] + return iter() +} + +func (g *GosimOS) MachineRecoverRelease(crashIterId int) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + // XXX: jank + delete(g.crashItersById, crashIterId) +} + +func (g *GosimOS) MachineInodeInfo(machineID int, inode int, info syscallabi.ValueView[fs.InodeInfo]) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + info.Set(machine.filesystem.GetInodeInfo(inode)) +} + +func (g *GosimOS) MachineWait(machineID int, syscall *syscallabi.Syscall) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + if machine.runtimeMachine == nil { + // XXX: improve how we mark stopped please + syscall.Complete() + } else { + machine.waiters = append(machine.waiters, syscall) + } +} + +func (g *GosimOS) MachineStop(machineID int, graceful bool) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + g.simulation.stopMachine(machine, graceful) +} + +func (g *GosimOS) MachineRestart(machineID int, partialDisk bool) (err error) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + + if machine.runtimeMachine != nil { + return syscall.EALREADY + } + + machine.filesystem = machine.filesystem.CrashClone(partialDisk) + g.simulation.startMachine(machine) + + // XXX: return a new machine ID? + // what if a machine gets restarted multiple times? + // do we track some kind of status? + return nil +} + +func (g *GosimOS) MachineGetLabel(machineID int) (label string, err error) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + + return machine.label, nil +} + +func (g *GosimOS) MachineSetBootProgram(machineID int, program any) (err error) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + machine.bootProgram = program.(func()) + + return nil +} + +func (g *GosimOS) MachineSetSometimesCrashOnSync(machineID int, value bool) (err error) { + g.simulation.mu.Lock() + defer g.simulation.mu.Unlock() + + machine := g.simulation.machinesById[machineID] + // XXX: on restart how do we copy these? + machine.sometimesCrashOnSyncMu.Lock() + machine.sometimesCrashOnSync = true + machine.sometimesCrashOnSyncMu.Unlock() + return nil +} + +func mustHostPair(a, b string) network.HostPair { + return network.HostPair{ + SourceHost: netip.MustParseAddr(a), DestHost: netip.MustParseAddr(b), + } +} + +func (g *GosimOS) SetConnected(a, b string, connected bool) error { + g.simulation.network.SetConnected(mustHostPair(a, b), connected) + return nil +} + +func (g *GosimOS) SetDelay(a, b string, delay time.Duration) error { + g.simulation.network.SetDelay(mustHostPair(a, b), delay) + return nil +} diff --git a/internal/simulation/gosim_gensyscall.go b/internal/simulation/gosim_gensyscall.go new file mode 100644 index 0000000..6b3d27d --- /dev/null +++ b/internal/simulation/gosim_gensyscall.go @@ -0,0 +1,374 @@ +// Code generated by gensyscall. DO NOT EDIT. +package simulation + +import ( + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/jellevandenhooff/gosim/internal/simulation/fs" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +// prevent unused imports +var ( + _ unsafe.Pointer + _ time.Duration + _ syscallabi.ValueView[any] + _ syscall.Errno + _ fs.InodeInfo + _ unix.Errno +) + +// gosimOSIface is the interface *GosimOS must implement to work +// with HandleSyscall. The interface is not used but helpful for implementing +// new syscalls. +type gosimOSIface interface { + MachineGetLabel(machineID int) (label string, err error) + MachineInodeInfo(machineID int, inodeID int, info syscallabi.ValueView[fs.InodeInfo]) + MachineNew(label string, addr string, program any) (machineID int) + MachineRecoverInit(machineID int, program any) (iterID int) + MachineRecoverNext(iterID int) (machineID int, ok bool) + MachineRecoverRelease(iterID int) + MachineRestart(machineID int, partialDisk bool) (err error) + MachineSetBootProgram(machineID int, program any) (err error) + MachineSetSometimesCrashOnSync(machineID int, crash bool) (err error) + MachineStop(machineID int, graceful bool) + MachineWait(machineID int, syscall *syscallabi.Syscall) + SetConnected(a string, b string, connected bool) (err error) + SetDelay(a string, b string, delay time.Duration) (err error) + SetSimulationTimeout(timeout time.Duration) (err error) +} + +var _ gosimOSIface = &GosimOS{} + +//go:norace +func (os *GosimOS) dispatchSyscall(s *syscallabi.Syscall) { + // XXX: help this happens for globals in os + if os == nil { + s.Errno = uintptr(syscall.ENOSYS) + return + } + os.dispatcher.Dispatch(s) +} + +//go:norace +func (os *GosimOS) HandleSyscall(syscall *syscallabi.Syscall) { + switch syscall.Trap { + case 1011: + // called by (for find references): + _ = SyscallMachineGetLabel + machineID := int(syscall.Int0) + label, err := os.MachineGetLabel(machineID) + syscall.RPtr0 = label + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case 1005: + // called by (for find references): + _ = SyscallMachineInodeInfo + machineID := int(syscall.Int0) + inodeID := int(syscall.Int1) + info := syscallabi.NewValueView(syscall.Ptr2.(*fs.InodeInfo)) + os.MachineInodeInfo(machineID, inodeID, info) + syscall.Complete() + case 1003: + // called by (for find references): + _ = SyscallMachineNew + label := syscall.Ptr0.(string) + addr := syscall.Ptr1.(string) + program := syscall.Ptr2.(any) + machineID := os.MachineNew(label, addr, program) + syscall.R0 = uintptr(machineID) + syscall.Complete() + case 1007: + // called by (for find references): + _ = SyscallMachineRecoverInit + machineID := int(syscall.Int0) + program := syscall.Ptr1.(any) + iterID := os.MachineRecoverInit(machineID, program) + syscall.R0 = uintptr(iterID) + syscall.Complete() + case 1008: + // called by (for find references): + _ = SyscallMachineRecoverNext + iterID := int(syscall.Int0) + machineID, ok := os.MachineRecoverNext(iterID) + syscall.R0 = uintptr(machineID) + syscall.R1 = syscallabi.BoolToUintptr(ok) + syscall.Complete() + case 1009: + // called by (for find references): + _ = SyscallMachineRecoverRelease + iterID := int(syscall.Int0) + os.MachineRecoverRelease(iterID) + syscall.Complete() + case 1010: + // called by (for find references): + _ = SyscallMachineRestart + machineID := int(syscall.Int0) + partialDisk := syscallabi.BoolFromUintptr(syscall.Int1) + err := os.MachineRestart(machineID, partialDisk) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case 1012: + // called by (for find references): + _ = SyscallMachineSetBootProgram + machineID := int(syscall.Int0) + program := syscall.Ptr1.(any) + err := os.MachineSetBootProgram(machineID, program) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case 1013: + // called by (for find references): + _ = SyscallMachineSetSometimesCrashOnSync + machineID := int(syscall.Int0) + crash := syscallabi.BoolFromUintptr(syscall.Int1) + err := os.MachineSetSometimesCrashOnSync(machineID, crash) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case 1004: + // called by (for find references): + _ = SyscallMachineStop + machineID := int(syscall.Int0) + graceful := syscallabi.BoolFromUintptr(syscall.Int1) + os.MachineStop(machineID, graceful) + syscall.Complete() + case 1006: + // called by (for find references): + _ = SyscallMachineWait + machineID := int(syscall.Int0) + os.MachineWait(machineID, syscall) + case 1001: + // called by (for find references): + _ = SyscallSetConnected + a := syscall.Ptr0.(string) + b := syscall.Ptr1.(string) + connected := syscallabi.BoolFromUintptr(syscall.Int2) + err := os.SetConnected(a, b, connected) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case 1002: + // called by (for find references): + _ = SyscallSetDelay + a := syscall.Ptr0.(string) + b := syscall.Ptr1.(string) + delay := time.Duration(syscall.Int2) + err := os.SetDelay(a, b, delay) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case 1000: + // called by (for find references): + _ = SyscallSetSimulationTimeout + timeout := time.Duration(syscall.Int0) + err := os.SetSimulationTimeout(timeout) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + default: + panic("bad") + } +} + +//go:norace +func SyscallMachineGetLabel(machineID int) (label string, err error) { + // invokes (for go to definition): + _ = (*GosimOS).MachineGetLabel + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1011 + syscall.Int0 = uintptr(machineID) + gosimOS.dispatchSyscall(syscall) + label = syscall.RPtr0.(string) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallMachineInodeInfo(machineID int, inodeID int, info *fs.InodeInfo) { + // invokes (for go to definition): + _ = (*GosimOS).MachineInodeInfo + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1005 + syscall.Int0 = uintptr(machineID) + syscall.Int1 = uintptr(inodeID) + syscall.Ptr2 = info + gosimOS.dispatchSyscall(syscall) + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallMachineNew(label string, addr string, program any) (machineID int) { + // invokes (for go to definition): + _ = (*GosimOS).MachineNew + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1003 + syscall.Ptr0 = label + syscall.Ptr1 = addr + syscall.Ptr2 = program + gosimOS.dispatchSyscall(syscall) + machineID = int(syscall.R0) + syscall.Ptr0 = nil + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallMachineRecoverInit(machineID int, program any) (iterID int) { + // invokes (for go to definition): + _ = (*GosimOS).MachineRecoverInit + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1007 + syscall.Int0 = uintptr(machineID) + syscall.Ptr1 = program + gosimOS.dispatchSyscall(syscall) + iterID = int(syscall.R0) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallMachineRecoverNext(iterID int) (machineID int, ok bool) { + // invokes (for go to definition): + _ = (*GosimOS).MachineRecoverNext + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1008 + syscall.Int0 = uintptr(iterID) + gosimOS.dispatchSyscall(syscall) + machineID = int(syscall.R0) + ok = syscallabi.BoolFromUintptr(syscall.R1) + return +} + +//go:norace +func SyscallMachineRecoverRelease(iterID int) { + // invokes (for go to definition): + _ = (*GosimOS).MachineRecoverRelease + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1009 + syscall.Int0 = uintptr(iterID) + gosimOS.dispatchSyscall(syscall) + return +} + +//go:norace +func SyscallMachineRestart(machineID int, partialDisk bool) (err error) { + // invokes (for go to definition): + _ = (*GosimOS).MachineRestart + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1010 + syscall.Int0 = uintptr(machineID) + syscall.Int1 = syscallabi.BoolToUintptr(partialDisk) + gosimOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallMachineSetBootProgram(machineID int, program any) (err error) { + // invokes (for go to definition): + _ = (*GosimOS).MachineSetBootProgram + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1012 + syscall.Int0 = uintptr(machineID) + syscall.Ptr1 = program + gosimOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallMachineSetSometimesCrashOnSync(machineID int, crash bool) (err error) { + // invokes (for go to definition): + _ = (*GosimOS).MachineSetSometimesCrashOnSync + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1013 + syscall.Int0 = uintptr(machineID) + syscall.Int1 = syscallabi.BoolToUintptr(crash) + gosimOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallMachineStop(machineID int, graceful bool) { + // invokes (for go to definition): + _ = (*GosimOS).MachineStop + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1004 + syscall.Int0 = uintptr(machineID) + syscall.Int1 = syscallabi.BoolToUintptr(graceful) + gosimOS.dispatchSyscall(syscall) + return +} + +//go:norace +func SyscallMachineWait(machineID int) { + // invokes (for go to definition): + _ = (*GosimOS).MachineWait + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1006 + syscall.Int0 = uintptr(machineID) + gosimOS.dispatchSyscall(syscall) + return +} + +//go:norace +func SyscallSetConnected(a string, b string, connected bool) (err error) { + // invokes (for go to definition): + _ = (*GosimOS).SetConnected + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1001 + syscall.Ptr0 = a + syscall.Ptr1 = b + syscall.Int2 = syscallabi.BoolToUintptr(connected) + gosimOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr0 = nil + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSetDelay(a string, b string, delay time.Duration) (err error) { + // invokes (for go to definition): + _ = (*GosimOS).SetDelay + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1002 + syscall.Ptr0 = a + syscall.Ptr1 = b + syscall.Int2 = uintptr(delay) + gosimOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr0 = nil + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSetSimulationTimeout(timeout time.Duration) (err error) { + // invokes (for go to definition): + _ = (*GosimOS).SetSimulationTimeout + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = gosimOS + syscall.Trap = 1000 + syscall.Int0 = uintptr(timeout) + gosimOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} diff --git a/internal/simulation/linux_gensyscall_amd64.go b/internal/simulation/linux_gensyscall_amd64.go new file mode 100644 index 0000000..d359885 --- /dev/null +++ b/internal/simulation/linux_gensyscall_amd64.go @@ -0,0 +1,837 @@ +//go:build linux + +// Code generated by gensyscall. DO NOT EDIT. +package simulation + +import ( + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/jellevandenhooff/gosim/internal/simulation/fs" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +// prevent unused imports +var ( + _ unsafe.Pointer + _ time.Duration + _ syscallabi.ValueView[any] + _ syscall.Errno + _ fs.InodeInfo + _ unix.Errno +) + +// linuxOSIface is the interface *LinuxOS must implement to work +// with HandleSyscall. The interface is not used but helpful for implementing +// new syscalls. +type linuxOSIface interface { + PollClose(fd int, desc syscallabi.ValueView[syscallabi.PollDesc]) (code int) + PollOpen(fd int, desc syscallabi.ValueView[syscallabi.PollDesc]) (code int) + SysAccept4(s int, rsa syscallabi.ValueView[RawSockaddrAny], addrlen syscallabi.ValueView[Socklen], flags int) (fd int, err error) + SysBind(s int, addr unsafe.Pointer, addrlen Socklen) (err error) + SysClose(fd int) (err error) + SysConnect(s int, addr unsafe.Pointer, addrlen Socklen) (err error) + SysFcntl(fd int, cmd int, arg int) (val int, err error) + SysFstat(fd int, stat syscallabi.ValueView[Stat_t]) (err error) + SysFsync(fd int) (err error) + SysFtruncate(fd int, length int64) (err error) + SysGetdents64(fd int, buf syscallabi.SliceView[byte]) (n int, err error) + SysGetpeername(fd int, rsa syscallabi.ValueView[RawSockaddrAny], addrlen syscallabi.ValueView[Socklen]) (err error) + SysGetpid() (pid int) + SysGetrandom(buf syscallabi.SliceView[byte], flags int) (n int, err error) + SysGetsockname(fd int, rsa syscallabi.ValueView[RawSockaddrAny], addrlen syscallabi.ValueView[Socklen]) (err error) + SysGetsockopt(s int, level int, name int, val unsafe.Pointer, vallen syscallabi.ValueView[Socklen]) (err error) + SysListen(s int, n int) (err error) + SysNewfstatat(fd int, path string, stat syscallabi.ValueView[Stat_t], flags int) (err error) + SysOpenat(dirfd int, path string, flags int, mode uint32) (fd int, err error) + SysPread64(fd int, p syscallabi.SliceView[byte], offset int64) (n int, err error) + SysPwrite64(fd int, p syscallabi.SliceView[byte], offset int64) (n int, err error) + SysRead(fd int, p syscallabi.SliceView[byte]) (n int, err error) + SysRenameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) + SysSetsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) + SysSocket(domain int, typ int, proto int) (fd int, err error) + SysUname(buf syscallabi.ValueView[Utsname]) (err error) + SysUnlinkat(dirfd int, path string, flags int) (err error) + SysWrite(fd int, p syscallabi.SliceView[byte]) (n int, err error) +} + +var _ linuxOSIface = &LinuxOS{} + +//go:norace +func (os *LinuxOS) dispatchSyscall(s *syscallabi.Syscall) { + // XXX: help this happens for globals in os + if os == nil { + s.Errno = uintptr(syscall.ENOSYS) + return + } + os.dispatcher.Dispatch(s) +} + +//go:norace +func (os *LinuxOS) HandleSyscall(syscall *syscallabi.Syscall) { + switch syscall.Trap { + case 1001: + // called by (for find references): + _ = SyscallPollClose + fd := int(syscall.Int0) + desc := syscallabi.NewValueView(syscall.Ptr1.(*syscallabi.PollDesc)) + code := os.PollClose(fd, desc) + syscall.R0 = uintptr(code) + syscall.Complete() + case 1000: + // called by (for find references): + _ = SyscallPollOpen + fd := int(syscall.Int0) + desc := syscallabi.NewValueView(syscall.Ptr1.(*syscallabi.PollDesc)) + code := os.PollOpen(fd, desc) + syscall.R0 = uintptr(code) + syscall.Complete() + case unix.SYS_ACCEPT4: + // called by (for find references): + _ = SyscallSysAccept4 + s := int(syscall.Int0) + rsa := syscallabi.NewValueView(syscall.Ptr1.(*RawSockaddrAny)) + addrlen := syscallabi.NewValueView(syscall.Ptr2.(*Socklen)) + flags := int(syscall.Int3) + fd, err := os.SysAccept4(s, rsa, addrlen, flags) + syscall.R0 = uintptr(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_BIND: + // called by (for find references): + _ = SyscallSysBind + s := int(syscall.Int0) + addr := syscall.Ptr1.(unsafe.Pointer) + addrlen := Socklen(syscall.Int2) + err := os.SysBind(s, addr, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_CLOSE: + // called by (for find references): + _ = SyscallSysClose + fd := int(syscall.Int0) + err := os.SysClose(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_CONNECT: + // called by (for find references): + _ = SyscallSysConnect + s := int(syscall.Int0) + addr := syscall.Ptr1.(unsafe.Pointer) + addrlen := Socklen(syscall.Int2) + err := os.SysConnect(s, addr, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FCNTL: + // called by (for find references): + _ = SyscallSysFcntl + fd := int(syscall.Int0) + cmd := int(syscall.Int1) + arg := int(syscall.Int2) + val, err := os.SysFcntl(fd, cmd, arg) + syscall.R0 = uintptr(val) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FSTAT: + // called by (for find references): + _ = SyscallSysFstat + fd := int(syscall.Int0) + stat := syscallabi.NewValueView(syscall.Ptr1.(*Stat_t)) + err := os.SysFstat(fd, stat) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FSYNC: + // called by (for find references): + _ = SyscallSysFsync + fd := int(syscall.Int0) + err := os.SysFsync(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FTRUNCATE: + // called by (for find references): + _ = SyscallSysFtruncate + fd := int(syscall.Int0) + length := int64(syscall.Int1) + err := os.SysFtruncate(fd, length) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETDENTS64: + // called by (for find references): + _ = SyscallSysGetdents64 + fd := int(syscall.Int0) + buf := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + n, err := os.SysGetdents64(fd, buf) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETPEERNAME: + // called by (for find references): + _ = SyscallSysGetpeername + fd := int(syscall.Int0) + rsa := syscallabi.NewValueView(syscall.Ptr1.(*RawSockaddrAny)) + addrlen := syscallabi.NewValueView(syscall.Ptr2.(*Socklen)) + err := os.SysGetpeername(fd, rsa, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETPID: + // called by (for find references): + _ = SyscallSysGetpid + pid := os.SysGetpid() + syscall.R0 = uintptr(pid) + syscall.Complete() + case unix.SYS_GETRANDOM: + // called by (for find references): + _ = SyscallSysGetrandom + buf := syscallabi.NewSliceView(syscall.Ptr0.(*byte), syscall.Int0) + flags := int(syscall.Int1) + n, err := os.SysGetrandom(buf, flags) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETSOCKNAME: + // called by (for find references): + _ = SyscallSysGetsockname + fd := int(syscall.Int0) + rsa := syscallabi.NewValueView(syscall.Ptr1.(*RawSockaddrAny)) + addrlen := syscallabi.NewValueView(syscall.Ptr2.(*Socklen)) + err := os.SysGetsockname(fd, rsa, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETSOCKOPT: + // called by (for find references): + _ = SyscallSysGetsockopt + s := int(syscall.Int0) + level := int(syscall.Int1) + name := int(syscall.Int2) + val := syscall.Ptr3.(unsafe.Pointer) + vallen := syscallabi.NewValueView(syscall.Ptr4.(*Socklen)) + err := os.SysGetsockopt(s, level, name, val, vallen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_LISTEN: + // called by (for find references): + _ = SyscallSysListen + s := int(syscall.Int0) + n := int(syscall.Int1) + err := os.SysListen(s, n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_NEWFSTATAT: + // called by (for find references): + _ = SyscallSysNewfstatat + fd := int(syscall.Int0) + path := syscall.Ptr1.(string) + stat := syscallabi.NewValueView(syscall.Ptr2.(*Stat_t)) + flags := int(syscall.Int3) + err := os.SysNewfstatat(fd, path, stat, flags) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_OPENAT: + // called by (for find references): + _ = SyscallSysOpenat + dirfd := int(syscall.Int0) + path := syscall.Ptr1.(string) + flags := int(syscall.Int2) + mode := uint32(syscall.Int3) + fd, err := os.SysOpenat(dirfd, path, flags, mode) + syscall.R0 = uintptr(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_PREAD64: + // called by (for find references): + _ = SyscallSysPread64 + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + offset := int64(syscall.Int2) + n, err := os.SysPread64(fd, p, offset) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_PWRITE64: + // called by (for find references): + _ = SyscallSysPwrite64 + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + offset := int64(syscall.Int2) + n, err := os.SysPwrite64(fd, p, offset) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_READ: + // called by (for find references): + _ = SyscallSysRead + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + n, err := os.SysRead(fd, p) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_RENAMEAT: + // called by (for find references): + _ = SyscallSysRenameat + olddirfd := int(syscall.Int0) + oldpath := syscall.Ptr1.(string) + newdirfd := int(syscall.Int2) + newpath := syscall.Ptr3.(string) + err := os.SysRenameat(olddirfd, oldpath, newdirfd, newpath) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_SETSOCKOPT: + // called by (for find references): + _ = SyscallSysSetsockopt + s := int(syscall.Int0) + level := int(syscall.Int1) + name := int(syscall.Int2) + val := syscall.Ptr3.(unsafe.Pointer) + vallen := uintptr(syscall.Int4) + err := os.SysSetsockopt(s, level, name, val, vallen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_SOCKET: + // called by (for find references): + _ = SyscallSysSocket + domain := int(syscall.Int0) + typ := int(syscall.Int1) + proto := int(syscall.Int2) + fd, err := os.SysSocket(domain, typ, proto) + syscall.R0 = uintptr(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_UNAME: + // called by (for find references): + _ = SyscallSysUname + buf := syscallabi.NewValueView(syscall.Ptr0.(*Utsname)) + err := os.SysUname(buf) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_UNLINKAT: + // called by (for find references): + _ = SyscallSysUnlinkat + dirfd := int(syscall.Int0) + path := syscall.Ptr1.(string) + flags := int(syscall.Int2) + err := os.SysUnlinkat(dirfd, path, flags) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_WRITE: + // called by (for find references): + _ = SyscallSysWrite + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + n, err := os.SysWrite(fd, p) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + default: + panic("bad") + } +} + +//go:norace +func SyscallPollClose(fd int, desc *syscallabi.PollDesc) (code int) { + // invokes (for go to definition): + _ = (*LinuxOS).PollClose + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = 1001 + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = desc + linuxOS.dispatchSyscall(syscall) + code = int(syscall.R0) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallPollOpen(fd int, desc *syscallabi.PollDesc) (code int) { + // invokes (for go to definition): + _ = (*LinuxOS).PollOpen + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = 1000 + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = desc + linuxOS.dispatchSyscall(syscall) + code = int(syscall.R0) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysAccept4(s int, rsa *RawSockaddrAny, addrlen *Socklen, flags int) (fd int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysAccept4 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_ACCEPT4 + syscall.Int0 = uintptr(s) + syscall.Ptr1 = rsa + syscall.Ptr2 = addrlen + syscall.Int3 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + fd = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysBind(s int, addr unsafe.Pointer, addrlen Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysBind + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_BIND + syscall.Int0 = uintptr(s) + syscall.Ptr1 = addr + syscall.Int2 = uintptr(addrlen) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysClose(fd int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysClose + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_CLOSE + syscall.Int0 = uintptr(fd) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysConnect(s int, addr unsafe.Pointer, addrlen Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysConnect + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_CONNECT + syscall.Int0 = uintptr(s) + syscall.Ptr1 = addr + syscall.Int2 = uintptr(addrlen) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysFcntl(fd int, cmd int, arg int) (val int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFcntl + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FCNTL + syscall.Int0 = uintptr(fd) + syscall.Int1 = uintptr(cmd) + syscall.Int2 = uintptr(arg) + linuxOS.dispatchSyscall(syscall) + val = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysFstat(fd int, stat *Stat_t) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFstat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FSTAT + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = stat + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysFsync(fd int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFsync + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FSYNC + syscall.Int0 = uintptr(fd) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysFtruncate(fd int, length int64) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFtruncate + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FTRUNCATE + syscall.Int0 = uintptr(fd) + syscall.Int1 = uintptr(length) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysGetdents64(fd int, buf []byte) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetdents64 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETDENTS64 + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(buf), uintptr(len(buf)) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysGetpeername(fd int, rsa *RawSockaddrAny, addrlen *Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetpeername + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETPEERNAME + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = rsa + syscall.Ptr2 = addrlen + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysGetpid() (pid int) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetpid + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETPID + linuxOS.dispatchSyscall(syscall) + pid = int(syscall.R0) + return +} + +//go:norace +func SyscallSysGetrandom(buf []byte, flags int) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetrandom + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETRANDOM + syscall.Ptr0, syscall.Int0 = unsafe.SliceData(buf), uintptr(len(buf)) + syscall.Int1 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr0 = nil + return +} + +//go:norace +func SyscallSysGetsockname(fd int, rsa *RawSockaddrAny, addrlen *Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetsockname + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETSOCKNAME + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = rsa + syscall.Ptr2 = addrlen + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysGetsockopt(s int, level int, name int, val unsafe.Pointer, vallen *Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetsockopt + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETSOCKOPT + syscall.Int0 = uintptr(s) + syscall.Int1 = uintptr(level) + syscall.Int2 = uintptr(name) + syscall.Ptr3 = val + syscall.Ptr4 = vallen + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr3 = nil + syscall.Ptr4 = nil + return +} + +//go:norace +func SyscallSysListen(s int, n int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysListen + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_LISTEN + syscall.Int0 = uintptr(s) + syscall.Int1 = uintptr(n) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysNewfstatat(fd int, path string, stat *Stat_t, flags int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysNewfstatat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_NEWFSTATAT + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = path + syscall.Ptr2 = stat + syscall.Int3 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysOpenat(dirfd int, path string, flags int, mode uint32) (fd int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysOpenat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_OPENAT + syscall.Int0 = uintptr(dirfd) + syscall.Ptr1 = path + syscall.Int2 = uintptr(flags) + syscall.Int3 = uintptr(mode) + linuxOS.dispatchSyscall(syscall) + fd = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysPread64(fd int, p []byte, offset int64) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysPread64 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_PREAD64 + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + syscall.Int2 = uintptr(offset) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysPwrite64(fd int, p []byte, offset int64) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysPwrite64 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_PWRITE64 + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + syscall.Int2 = uintptr(offset) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysRead(fd int, p []byte) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysRead + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_READ + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysRenameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysRenameat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_RENAMEAT + syscall.Int0 = uintptr(olddirfd) + syscall.Ptr1 = oldpath + syscall.Int2 = uintptr(newdirfd) + syscall.Ptr3 = newpath + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr3 = nil + return +} + +//go:norace +func SyscallSysSetsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysSetsockopt + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_SETSOCKOPT + syscall.Int0 = uintptr(s) + syscall.Int1 = uintptr(level) + syscall.Int2 = uintptr(name) + syscall.Ptr3 = val + syscall.Int4 = uintptr(vallen) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr3 = nil + return +} + +//go:norace +func SyscallSysSocket(domain int, typ int, proto int) (fd int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysSocket + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_SOCKET + syscall.Int0 = uintptr(domain) + syscall.Int1 = uintptr(typ) + syscall.Int2 = uintptr(proto) + linuxOS.dispatchSyscall(syscall) + fd = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysUname(buf *Utsname) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysUname + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_UNAME + syscall.Ptr0 = buf + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr0 = nil + return +} + +//go:norace +func SyscallSysUnlinkat(dirfd int, path string, flags int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysUnlinkat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_UNLINKAT + syscall.Int0 = uintptr(dirfd) + syscall.Ptr1 = path + syscall.Int2 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysWrite(fd int, p []byte) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysWrite + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_WRITE + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +func IsHandledSyscall(trap uintptr) bool { + switch trap { + case unix.SYS_ACCEPT4: + return true + case unix.SYS_BIND: + return true + case unix.SYS_CLOSE: + return true + case unix.SYS_CONNECT: + return true + case unix.SYS_FCNTL: + return true + case unix.SYS_FSTAT: + return true + case unix.SYS_FSYNC: + return true + case unix.SYS_FTRUNCATE: + return true + case unix.SYS_GETDENTS64: + return true + case unix.SYS_GETPEERNAME: + return true + case unix.SYS_GETPID: + return true + case unix.SYS_GETRANDOM: + return true + case unix.SYS_GETSOCKNAME: + return true + case unix.SYS_GETSOCKOPT: + return true + case unix.SYS_LISTEN: + return true + case unix.SYS_NEWFSTATAT: + return true + case unix.SYS_OPENAT: + return true + case unix.SYS_PREAD64: + return true + case unix.SYS_PWRITE64: + return true + case unix.SYS_READ: + return true + case unix.SYS_RENAMEAT: + return true + case unix.SYS_SETSOCKOPT: + return true + case unix.SYS_SOCKET: + return true + case unix.SYS_UNAME: + return true + case unix.SYS_UNLINKAT: + return true + case unix.SYS_WRITE: + return true + default: + return false + } +} diff --git a/internal/simulation/linux_gensyscall_arm64.go b/internal/simulation/linux_gensyscall_arm64.go new file mode 100644 index 0000000..c982f54 --- /dev/null +++ b/internal/simulation/linux_gensyscall_arm64.go @@ -0,0 +1,837 @@ +//go:build linux + +// Code generated by gensyscall. DO NOT EDIT. +package simulation + +import ( + "syscall" + "time" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/jellevandenhooff/gosim/internal/simulation/fs" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +// prevent unused imports +var ( + _ unsafe.Pointer + _ time.Duration + _ syscallabi.ValueView[any] + _ syscall.Errno + _ fs.InodeInfo + _ unix.Errno +) + +// linuxOSIface is the interface *LinuxOS must implement to work +// with HandleSyscall. The interface is not used but helpful for implementing +// new syscalls. +type linuxOSIface interface { + PollClose(fd int, desc syscallabi.ValueView[syscallabi.PollDesc]) (code int) + PollOpen(fd int, desc syscallabi.ValueView[syscallabi.PollDesc]) (code int) + SysAccept4(s int, rsa syscallabi.ValueView[RawSockaddrAny], addrlen syscallabi.ValueView[Socklen], flags int) (fd int, err error) + SysBind(s int, addr unsafe.Pointer, addrlen Socklen) (err error) + SysClose(fd int) (err error) + SysConnect(s int, addr unsafe.Pointer, addrlen Socklen) (err error) + SysFcntl(fd int, cmd int, arg int) (val int, err error) + SysFstat(fd int, stat syscallabi.ValueView[Stat_t]) (err error) + SysFstatat(dirfd int, path string, stat syscallabi.ValueView[Stat_t], flags int) (err error) + SysFsync(fd int) (err error) + SysFtruncate(fd int, length int64) (err error) + SysGetdents64(fd int, buf syscallabi.SliceView[byte]) (n int, err error) + SysGetpeername(fd int, rsa syscallabi.ValueView[RawSockaddrAny], addrlen syscallabi.ValueView[Socklen]) (err error) + SysGetpid() (pid int) + SysGetrandom(buf syscallabi.SliceView[byte], flags int) (n int, err error) + SysGetsockname(fd int, rsa syscallabi.ValueView[RawSockaddrAny], addrlen syscallabi.ValueView[Socklen]) (err error) + SysGetsockopt(s int, level int, name int, val unsafe.Pointer, vallen syscallabi.ValueView[Socklen]) (err error) + SysListen(s int, n int) (err error) + SysOpenat(dirfd int, path string, flags int, mode uint32) (fd int, err error) + SysPread64(fd int, p syscallabi.SliceView[byte], offset int64) (n int, err error) + SysPwrite64(fd int, p syscallabi.SliceView[byte], offset int64) (n int, err error) + SysRead(fd int, p syscallabi.SliceView[byte]) (n int, err error) + SysRenameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) + SysSetsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) + SysSocket(domain int, typ int, proto int) (fd int, err error) + SysUname(buf syscallabi.ValueView[Utsname]) (err error) + SysUnlinkat(dirfd int, path string, flags int) (err error) + SysWrite(fd int, p syscallabi.SliceView[byte]) (n int, err error) +} + +var _ linuxOSIface = &LinuxOS{} + +//go:norace +func (os *LinuxOS) dispatchSyscall(s *syscallabi.Syscall) { + // XXX: help this happens for globals in os + if os == nil { + s.Errno = uintptr(syscall.ENOSYS) + return + } + os.dispatcher.Dispatch(s) +} + +//go:norace +func (os *LinuxOS) HandleSyscall(syscall *syscallabi.Syscall) { + switch syscall.Trap { + case 1001: + // called by (for find references): + _ = SyscallPollClose + fd := int(syscall.Int0) + desc := syscallabi.NewValueView(syscall.Ptr1.(*syscallabi.PollDesc)) + code := os.PollClose(fd, desc) + syscall.R0 = uintptr(code) + syscall.Complete() + case 1000: + // called by (for find references): + _ = SyscallPollOpen + fd := int(syscall.Int0) + desc := syscallabi.NewValueView(syscall.Ptr1.(*syscallabi.PollDesc)) + code := os.PollOpen(fd, desc) + syscall.R0 = uintptr(code) + syscall.Complete() + case unix.SYS_ACCEPT4: + // called by (for find references): + _ = SyscallSysAccept4 + s := int(syscall.Int0) + rsa := syscallabi.NewValueView(syscall.Ptr1.(*RawSockaddrAny)) + addrlen := syscallabi.NewValueView(syscall.Ptr2.(*Socklen)) + flags := int(syscall.Int3) + fd, err := os.SysAccept4(s, rsa, addrlen, flags) + syscall.R0 = uintptr(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_BIND: + // called by (for find references): + _ = SyscallSysBind + s := int(syscall.Int0) + addr := syscall.Ptr1.(unsafe.Pointer) + addrlen := Socklen(syscall.Int2) + err := os.SysBind(s, addr, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_CLOSE: + // called by (for find references): + _ = SyscallSysClose + fd := int(syscall.Int0) + err := os.SysClose(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_CONNECT: + // called by (for find references): + _ = SyscallSysConnect + s := int(syscall.Int0) + addr := syscall.Ptr1.(unsafe.Pointer) + addrlen := Socklen(syscall.Int2) + err := os.SysConnect(s, addr, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FCNTL: + // called by (for find references): + _ = SyscallSysFcntl + fd := int(syscall.Int0) + cmd := int(syscall.Int1) + arg := int(syscall.Int2) + val, err := os.SysFcntl(fd, cmd, arg) + syscall.R0 = uintptr(val) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FSTAT: + // called by (for find references): + _ = SyscallSysFstat + fd := int(syscall.Int0) + stat := syscallabi.NewValueView(syscall.Ptr1.(*Stat_t)) + err := os.SysFstat(fd, stat) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FSTATAT: + // called by (for find references): + _ = SyscallSysFstatat + dirfd := int(syscall.Int0) + path := syscall.Ptr1.(string) + stat := syscallabi.NewValueView(syscall.Ptr2.(*Stat_t)) + flags := int(syscall.Int3) + err := os.SysFstatat(dirfd, path, stat, flags) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FSYNC: + // called by (for find references): + _ = SyscallSysFsync + fd := int(syscall.Int0) + err := os.SysFsync(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_FTRUNCATE: + // called by (for find references): + _ = SyscallSysFtruncate + fd := int(syscall.Int0) + length := int64(syscall.Int1) + err := os.SysFtruncate(fd, length) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETDENTS64: + // called by (for find references): + _ = SyscallSysGetdents64 + fd := int(syscall.Int0) + buf := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + n, err := os.SysGetdents64(fd, buf) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETPEERNAME: + // called by (for find references): + _ = SyscallSysGetpeername + fd := int(syscall.Int0) + rsa := syscallabi.NewValueView(syscall.Ptr1.(*RawSockaddrAny)) + addrlen := syscallabi.NewValueView(syscall.Ptr2.(*Socklen)) + err := os.SysGetpeername(fd, rsa, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETPID: + // called by (for find references): + _ = SyscallSysGetpid + pid := os.SysGetpid() + syscall.R0 = uintptr(pid) + syscall.Complete() + case unix.SYS_GETRANDOM: + // called by (for find references): + _ = SyscallSysGetrandom + buf := syscallabi.NewSliceView(syscall.Ptr0.(*byte), syscall.Int0) + flags := int(syscall.Int1) + n, err := os.SysGetrandom(buf, flags) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETSOCKNAME: + // called by (for find references): + _ = SyscallSysGetsockname + fd := int(syscall.Int0) + rsa := syscallabi.NewValueView(syscall.Ptr1.(*RawSockaddrAny)) + addrlen := syscallabi.NewValueView(syscall.Ptr2.(*Socklen)) + err := os.SysGetsockname(fd, rsa, addrlen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_GETSOCKOPT: + // called by (for find references): + _ = SyscallSysGetsockopt + s := int(syscall.Int0) + level := int(syscall.Int1) + name := int(syscall.Int2) + val := syscall.Ptr3.(unsafe.Pointer) + vallen := syscallabi.NewValueView(syscall.Ptr4.(*Socklen)) + err := os.SysGetsockopt(s, level, name, val, vallen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_LISTEN: + // called by (for find references): + _ = SyscallSysListen + s := int(syscall.Int0) + n := int(syscall.Int1) + err := os.SysListen(s, n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_OPENAT: + // called by (for find references): + _ = SyscallSysOpenat + dirfd := int(syscall.Int0) + path := syscall.Ptr1.(string) + flags := int(syscall.Int2) + mode := uint32(syscall.Int3) + fd, err := os.SysOpenat(dirfd, path, flags, mode) + syscall.R0 = uintptr(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_PREAD64: + // called by (for find references): + _ = SyscallSysPread64 + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + offset := int64(syscall.Int2) + n, err := os.SysPread64(fd, p, offset) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_PWRITE64: + // called by (for find references): + _ = SyscallSysPwrite64 + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + offset := int64(syscall.Int2) + n, err := os.SysPwrite64(fd, p, offset) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_READ: + // called by (for find references): + _ = SyscallSysRead + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + n, err := os.SysRead(fd, p) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_RENAMEAT: + // called by (for find references): + _ = SyscallSysRenameat + olddirfd := int(syscall.Int0) + oldpath := syscall.Ptr1.(string) + newdirfd := int(syscall.Int2) + newpath := syscall.Ptr3.(string) + err := os.SysRenameat(olddirfd, oldpath, newdirfd, newpath) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_SETSOCKOPT: + // called by (for find references): + _ = SyscallSysSetsockopt + s := int(syscall.Int0) + level := int(syscall.Int1) + name := int(syscall.Int2) + val := syscall.Ptr3.(unsafe.Pointer) + vallen := uintptr(syscall.Int4) + err := os.SysSetsockopt(s, level, name, val, vallen) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_SOCKET: + // called by (for find references): + _ = SyscallSysSocket + domain := int(syscall.Int0) + typ := int(syscall.Int1) + proto := int(syscall.Int2) + fd, err := os.SysSocket(domain, typ, proto) + syscall.R0 = uintptr(fd) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_UNAME: + // called by (for find references): + _ = SyscallSysUname + buf := syscallabi.NewValueView(syscall.Ptr0.(*Utsname)) + err := os.SysUname(buf) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_UNLINKAT: + // called by (for find references): + _ = SyscallSysUnlinkat + dirfd := int(syscall.Int0) + path := syscall.Ptr1.(string) + flags := int(syscall.Int2) + err := os.SysUnlinkat(dirfd, path, flags) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + case unix.SYS_WRITE: + // called by (for find references): + _ = SyscallSysWrite + fd := int(syscall.Int0) + p := syscallabi.NewSliceView(syscall.Ptr1.(*byte), syscall.Int1) + n, err := os.SysWrite(fd, p) + syscall.R0 = uintptr(n) + syscall.Errno = syscallabi.ErrErrno(err) + syscall.Complete() + default: + panic("bad") + } +} + +//go:norace +func SyscallPollClose(fd int, desc *syscallabi.PollDesc) (code int) { + // invokes (for go to definition): + _ = (*LinuxOS).PollClose + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = 1001 + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = desc + linuxOS.dispatchSyscall(syscall) + code = int(syscall.R0) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallPollOpen(fd int, desc *syscallabi.PollDesc) (code int) { + // invokes (for go to definition): + _ = (*LinuxOS).PollOpen + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = 1000 + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = desc + linuxOS.dispatchSyscall(syscall) + code = int(syscall.R0) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysAccept4(s int, rsa *RawSockaddrAny, addrlen *Socklen, flags int) (fd int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysAccept4 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_ACCEPT4 + syscall.Int0 = uintptr(s) + syscall.Ptr1 = rsa + syscall.Ptr2 = addrlen + syscall.Int3 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + fd = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysBind(s int, addr unsafe.Pointer, addrlen Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysBind + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_BIND + syscall.Int0 = uintptr(s) + syscall.Ptr1 = addr + syscall.Int2 = uintptr(addrlen) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysClose(fd int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysClose + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_CLOSE + syscall.Int0 = uintptr(fd) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysConnect(s int, addr unsafe.Pointer, addrlen Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysConnect + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_CONNECT + syscall.Int0 = uintptr(s) + syscall.Ptr1 = addr + syscall.Int2 = uintptr(addrlen) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysFcntl(fd int, cmd int, arg int) (val int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFcntl + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FCNTL + syscall.Int0 = uintptr(fd) + syscall.Int1 = uintptr(cmd) + syscall.Int2 = uintptr(arg) + linuxOS.dispatchSyscall(syscall) + val = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysFstat(fd int, stat *Stat_t) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFstat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FSTAT + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = stat + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysFstatat(dirfd int, path string, stat *Stat_t, flags int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFstatat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FSTATAT + syscall.Int0 = uintptr(dirfd) + syscall.Ptr1 = path + syscall.Ptr2 = stat + syscall.Int3 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysFsync(fd int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFsync + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FSYNC + syscall.Int0 = uintptr(fd) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysFtruncate(fd int, length int64) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysFtruncate + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_FTRUNCATE + syscall.Int0 = uintptr(fd) + syscall.Int1 = uintptr(length) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysGetdents64(fd int, buf []byte) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetdents64 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETDENTS64 + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(buf), uintptr(len(buf)) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysGetpeername(fd int, rsa *RawSockaddrAny, addrlen *Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetpeername + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETPEERNAME + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = rsa + syscall.Ptr2 = addrlen + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysGetpid() (pid int) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetpid + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETPID + linuxOS.dispatchSyscall(syscall) + pid = int(syscall.R0) + return +} + +//go:norace +func SyscallSysGetrandom(buf []byte, flags int) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetrandom + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETRANDOM + syscall.Ptr0, syscall.Int0 = unsafe.SliceData(buf), uintptr(len(buf)) + syscall.Int1 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr0 = nil + return +} + +//go:norace +func SyscallSysGetsockname(fd int, rsa *RawSockaddrAny, addrlen *Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetsockname + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETSOCKNAME + syscall.Int0 = uintptr(fd) + syscall.Ptr1 = rsa + syscall.Ptr2 = addrlen + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr2 = nil + return +} + +//go:norace +func SyscallSysGetsockopt(s int, level int, name int, val unsafe.Pointer, vallen *Socklen) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysGetsockopt + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_GETSOCKOPT + syscall.Int0 = uintptr(s) + syscall.Int1 = uintptr(level) + syscall.Int2 = uintptr(name) + syscall.Ptr3 = val + syscall.Ptr4 = vallen + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr3 = nil + syscall.Ptr4 = nil + return +} + +//go:norace +func SyscallSysListen(s int, n int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysListen + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_LISTEN + syscall.Int0 = uintptr(s) + syscall.Int1 = uintptr(n) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysOpenat(dirfd int, path string, flags int, mode uint32) (fd int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysOpenat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_OPENAT + syscall.Int0 = uintptr(dirfd) + syscall.Ptr1 = path + syscall.Int2 = uintptr(flags) + syscall.Int3 = uintptr(mode) + linuxOS.dispatchSyscall(syscall) + fd = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysPread64(fd int, p []byte, offset int64) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysPread64 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_PREAD64 + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + syscall.Int2 = uintptr(offset) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysPwrite64(fd int, p []byte, offset int64) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysPwrite64 + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_PWRITE64 + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + syscall.Int2 = uintptr(offset) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysRead(fd int, p []byte) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysRead + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_READ + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysRenameat(olddirfd int, oldpath string, newdirfd int, newpath string) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysRenameat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_RENAMEAT + syscall.Int0 = uintptr(olddirfd) + syscall.Ptr1 = oldpath + syscall.Int2 = uintptr(newdirfd) + syscall.Ptr3 = newpath + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + syscall.Ptr3 = nil + return +} + +//go:norace +func SyscallSysSetsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysSetsockopt + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_SETSOCKOPT + syscall.Int0 = uintptr(s) + syscall.Int1 = uintptr(level) + syscall.Int2 = uintptr(name) + syscall.Ptr3 = val + syscall.Int4 = uintptr(vallen) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr3 = nil + return +} + +//go:norace +func SyscallSysSocket(domain int, typ int, proto int) (fd int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysSocket + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_SOCKET + syscall.Int0 = uintptr(domain) + syscall.Int1 = uintptr(typ) + syscall.Int2 = uintptr(proto) + linuxOS.dispatchSyscall(syscall) + fd = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + return +} + +//go:norace +func SyscallSysUname(buf *Utsname) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysUname + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_UNAME + syscall.Ptr0 = buf + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr0 = nil + return +} + +//go:norace +func SyscallSysUnlinkat(dirfd int, path string, flags int) (err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysUnlinkat + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_UNLINKAT + syscall.Int0 = uintptr(dirfd) + syscall.Ptr1 = path + syscall.Int2 = uintptr(flags) + linuxOS.dispatchSyscall(syscall) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +//go:norace +func SyscallSysWrite(fd int, p []byte) (n int, err error) { + // invokes (for go to definition): + _ = (*LinuxOS).SysWrite + syscall := syscallabi.GetGoroutineLocalSyscall() + syscall.OS = linuxOS + syscall.Trap = unix.SYS_WRITE + syscall.Int0 = uintptr(fd) + syscall.Ptr1, syscall.Int1 = unsafe.SliceData(p), uintptr(len(p)) + linuxOS.dispatchSyscall(syscall) + n = int(syscall.R0) + err = syscallabi.ErrnoErr(syscall.Errno) + syscall.Ptr1 = nil + return +} + +func IsHandledSyscall(trap uintptr) bool { + switch trap { + case unix.SYS_ACCEPT4: + return true + case unix.SYS_BIND: + return true + case unix.SYS_CLOSE: + return true + case unix.SYS_CONNECT: + return true + case unix.SYS_FCNTL: + return true + case unix.SYS_FSTAT: + return true + case unix.SYS_FSTATAT: + return true + case unix.SYS_FSYNC: + return true + case unix.SYS_FTRUNCATE: + return true + case unix.SYS_GETDENTS64: + return true + case unix.SYS_GETPEERNAME: + return true + case unix.SYS_GETPID: + return true + case unix.SYS_GETRANDOM: + return true + case unix.SYS_GETSOCKNAME: + return true + case unix.SYS_GETSOCKOPT: + return true + case unix.SYS_LISTEN: + return true + case unix.SYS_OPENAT: + return true + case unix.SYS_PREAD64: + return true + case unix.SYS_PWRITE64: + return true + case unix.SYS_READ: + return true + case unix.SYS_RENAMEAT: + return true + case unix.SYS_SETSOCKOPT: + return true + case unix.SYS_SOCKET: + return true + case unix.SYS_UNAME: + return true + case unix.SYS_UNLINKAT: + return true + case unix.SYS_WRITE: + return true + default: + return false + } +} diff --git a/internal/simulation/network/delayqueue.go b/internal/simulation/network/delayqueue.go new file mode 100644 index 0000000..14288ca --- /dev/null +++ b/internal/simulation/network/delayqueue.go @@ -0,0 +1,111 @@ +package network + +import ( + "container/heap" + "sync" + "time" +) + +type timeIndex struct { + // XXX: can we pack these together somehow? + time int64 + index int +} + +func (ti timeIndex) Less(other timeIndex) bool { + if ti.time != other.time { + return ti.time < other.time + } + return ti.index < other.index +} + +type timedHeap []*Packet + +func (h timedHeap) Len() int { return len(h) } +func (h timedHeap) Less(i, j int) bool { return h[i].ArrivalTime.Less(h[j].ArrivalTime) } +func (h timedHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *timedHeap) Push(x any) { + panic("don't use") + // *h = append(*h, x.(timedPacket)) +} + +func (h *timedHeap) Pop() any { + panic("don't use") + // old := *h + // n := len(old) + // x := old[n-1] + // *h = old[0 : n-1] + // return x +} + +type delayQueue struct { + mu sync.Mutex + items timedHeap + + timer *time.Timer + cond *sync.Cond +} + +func newDelayQueue() *delayQueue { + q := &delayQueue{ + items: make(timedHeap, 0, 16), + } + q.cond = sync.NewCond(&q.mu) + q.timer = time.AfterFunc(time.Hour, func() { + q.mu.Lock() + defer q.mu.Unlock() + q.cond.Broadcast() + }) + q.timer.Stop() + return q +} + +func (q *delayQueue) updateTimerLocked() { + if len(q.items) == 0 { + q.timer.Stop() + return + } + delay := time.Until(time.Unix(0, q.items[0].ArrivalTime.time)) + if delay > 0 { + q.timer.Reset(delay) + } else { + q.timer.Stop() + q.cond.Broadcast() + } +} + +func (q *delayQueue) enqueue(item *Packet) { + q.mu.Lock() + defer q.mu.Unlock() + // xxx: skip an allocation + // heap.Push(&q.items, item) + n := len(q.items) + q.items = append(q.items, item) + heap.Fix(&q.items, n) + q.updateTimerLocked() +} + +func (q *delayQueue) dequeue() *Packet { + q.mu.Lock() + defer q.mu.Unlock() + for { + now := time.Now().UnixNano() + if len(q.items) > 0 && q.items[0].ArrivalTime.time <= now { //.After(time.Now()) { + // xxx: skip an allocation + // next := heap.Pop(&q.items).(timedPacket) + + next := q.items[0] + n := len(q.items) - 1 + q.items[0] = q.items[n] + q.items = q.items[:n] + heap.Fix(&q.items, 0) + + q.updateTimerLocked() + return next + } + q.cond.Wait() + } +} diff --git a/internal/simulation/network/net.go b/internal/simulation/network/net.go new file mode 100644 index 0000000..613fba8 --- /dev/null +++ b/internal/simulation/network/net.go @@ -0,0 +1,228 @@ +package network + +import ( + "net/netip" + "sync" + "time" +) + +type HostPair struct { + SourceHost, DestHost netip.Addr +} + +func (id HostPair) Flip() HostPair { + return HostPair{SourceHost: id.DestHost, DestHost: id.SourceHost} +} + +type PortPair struct { + SourcePort uint16 + DestPort uint16 +} + +func (id PortPair) Flip() PortPair { + return PortPair{ + SourcePort: id.DestPort, + DestPort: id.SourcePort, + } +} + +type ConnId struct { + Hosts HostPair + Ports PortPair +} + +func (id ConnId) Source() netip.AddrPort { + return netip.AddrPortFrom(id.Hosts.SourceHost, id.Ports.SourcePort) +} + +func (id ConnId) Dest() netip.AddrPort { + return netip.AddrPortFrom(id.Hosts.DestHost, id.Ports.DestPort) +} + +func (id ConnId) Flip() ConnId { + return ConnId{ + Hosts: id.Hosts.Flip(), + Ports: id.Ports.Flip(), + } +} + +type Endpoint struct { + net *Network + addr netip.Addr + stack *Stack + + // XXX: make this reorder prevention happen at the conn ID level? + // XXX: make the stack take care of reordering? + last map[netip.Addr]timeIndex + + closed bool +} + +func (e *Endpoint) Addr() netip.Addr { + return e.addr +} + +func (e *Endpoint) Send(packet *Packet) { + if packet.ConnId.Hosts.SourceHost != e.addr { + panic("bad packet") + } + + e.net.mu.Lock() + defer e.net.mu.Unlock() + + if e.closed { + return + // log.Printf("closed") + // return errors.New("closed") + } + + _, ok := e.net.endpoints[packet.ConnId.Hosts.DestHost] + if !ok { + return + // log.Printf("no such dest") + // return errors.New("no such dest") + } + + if e.net.disconnected[packet.ConnId.Hosts] { + return + // XXX: silently fail -- but maybe fail loudly optionally? ENOPATH or something? + // return nil + // return errors.New("disconnected") + } + + arrival := timeIndex{ + time: time.Now().Add(e.net.delay[packet.ConnId.Hosts]).UnixNano(), + index: 0, + } + if last := e.last[packet.ConnId.Hosts.DestHost]; last.time >= arrival.time { + arrival.time = last.time + arrival.index = last.index + 1 + } + e.last[packet.ConnId.Hosts.DestHost] = arrival + + packet.ArrivalTime = arrival + e.net.queue.enqueue(packet) +} + +type Network struct { + mu sync.Mutex + endpoints map[netip.Addr]*Endpoint + nextIp netip.Addr + + disconnected map[HostPair]bool + delay map[HostPair]time.Duration + + queue *delayQueue +} + +func NewNetwork() *Network { + return &Network{ + endpoints: make(map[netip.Addr]*Endpoint), + nextIp: netip.AddrFrom4([4]byte{11, 0, 0, 1}), // XXX: randomize? + + disconnected: make(map[HostPair]bool), + delay: make(map[HostPair]time.Duration), + + queue: newDelayQueue(), + } +} + +func (n *Network) SetConnected(p HostPair, connected bool) { + n.mu.Lock() + defer n.mu.Unlock() + + if !connected { + n.disconnected[p] = true + n.disconnected[p.Flip()] = true + } else { + delete(n.disconnected, p) + delete(n.disconnected, p.Flip()) + } +} + +func (n *Network) SetDelay(p HostPair, delay time.Duration) { + n.mu.Lock() + defer n.mu.Unlock() + + if delay != 0 { + n.delay[p] = delay + n.delay[p.Flip()] = delay + } else { + delete(n.delay, p) + delete(n.delay, p.Flip()) + } +} + +func (n *Network) NextIP() netip.Addr { + n.mu.Lock() + defer n.mu.Unlock() + + addr := n.nextIp + n.nextIp = n.nextIp.Next() + + return addr +} + +func (n *Network) AttachStack(addr netip.Addr, stack *Stack) { + n.mu.Lock() + defer n.mu.Unlock() + + if _, ok := n.endpoints[addr]; ok { + panic(addr) + } + + ep := &Endpoint{ + net: n, + addr: addr, + stack: stack, + + last: make(map[netip.Addr]timeIndex), + } + + n.endpoints[addr] = ep + stack.setEndpoint(ep) +} + +func (n *Network) DetachStack(stack *Stack) { + n.mu.Lock() + defer n.mu.Unlock() + + endpoint, ok := n.endpoints[stack.endpoint.addr] + if !ok { + panic(stack.endpoint.addr) + } + + endpoint.closed = true + delete(n.endpoints, stack.endpoint.addr) + stack.setEndpoint(nil) // XXX: good idea? +} + +func (n *Network) getStack(packet *Packet) *Stack { + n.mu.Lock() + defer n.mu.Unlock() + + endpoint, ok := n.endpoints[packet.ConnId.Hosts.DestHost] + if !ok { + // XXX: dropped, log? + return nil + } + + if endpoint.stack == nil { + // XXX: dropped, log + return nil + } + + return endpoint.stack +} + +func (n *Network) Run() { + for { + packet := n.queue.dequeue() + if stack := n.getStack(packet); stack != nil { + // call stack without holding n.mu so stack can send packets which + // grabs n.mu + stack.processIncomingPacket(packet) + } + freePacket(packet) + } +} diff --git a/internal/simulation/network/stack.go b/internal/simulation/network/stack.go new file mode 100644 index 0000000..01ec61c --- /dev/null +++ b/internal/simulation/network/stack.go @@ -0,0 +1,720 @@ +package network + +import ( + "errors" + "net/netip" + "sync" + "syscall" + "time" + + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +type PacketKind byte + +const ( + PacketKindOpenStream PacketKind = iota // uses InBuffer, OutBuffer + PacketKindOpenedStream // uses nothing + PacketKindStreamClose // uses nothing + PacketKindStreamData // uses Offset, Count + PacketKindStreamAck // uses Offset + PacketKindStreamWindow // uses Count +) + +// XXX: for connId, add a secret extra field so we can gracefully handle delayed +// packets when machines restart (replay chaos). otherwise, implement sequence +// numbers, tcp wait, ... + +type Packet struct { + ConnId ConnId + Kind PacketKind + InBuffer, OutBuffer *circularBuffer + Offset, Count int + ArrivalTime timeIndex // used by delayQueue +} + +var packetPool = &sync.Pool{ + New: func() any { + return &Packet{} + }, +} + +func allocPacket(kind PacketKind, id ConnId) *Packet { + p := packetPool.Get().(*Packet) + p.Kind = kind + p.ConnId = id + return p +} + +func freePacket(p *Packet) { + // XXX: poison values so we catch reuse? + p.ConnId = ConnId{} + p.Kind = 255 + p.InBuffer = nil + p.OutBuffer = nil + p.Offset = -1 + p.Count = -1 + packetPool.Put(p) +} + +type Stack struct { + mu sync.Mutex + + // XXX: check shutdown everywhere? + shutdown bool + + streams map[ConnId]*Stream + listeners map[uint16]*Listener + + endpoint *Endpoint + + nextPort uint16 +} + +func (s *Stack) Shutdown(graceful bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + + for _, stream := range s.streams { + s.endpoint.Send(allocPacket(PacketKindStreamClose, stream.id)) + + // clean up timers + if stream.ackFailedTimer != nil { + stream.ackFailedTimer.Stop() + stream.ackFailedTimer = nil + } + if stream.openFailedTimer != nil { + stream.openFailedTimer.Stop() + stream.openFailedTimer = nil + } + if stream.keepaliveTimer != nil { + stream.keepaliveTimer.Stop() + stream.keepaliveTimer = nil + } + + // nil out buffers to catch lingering code + stream.inBuffer = nil + stream.outBuffer = nil + } + + // nil out maps to catch lingering code + s.streams = nil + s.listeners = nil +} + +func (b *Stack) Endpoint() *Endpoint { + return b.endpoint +} + +func (b *Stack) setEndpoint(ep *Endpoint) { + b.endpoint = ep +} + +func NewStack() *Stack { + b := &Stack{ + streams: make(map[ConnId]*Stream), + listeners: make(map[uint16]*Listener), + + nextPort: 10000, + } + return b +} + +func (n *Stack) allocatePort() uint16 { + port := n.nextPort + n.nextPort++ + return port +} + +func (b *Stack) processIncomingPacket(packet *Packet) { + b.mu.Lock() + defer b.mu.Unlock() + if b.shutdown { + return + } + + switch packet.Kind { + case PacketKindOpenStream: + // XXX: pass to listener? + id := packet.ConnId.Flip() + + listener, ok := b.listeners[packet.ConnId.Ports.DestPort] + if !ok { + // reject, not listening + b.endpoint.Send(allocPacket(PacketKindStreamClose, id)) + break + } + + if listener.incomingCount == len(listener.incoming) { + // reject, too many pending + b.endpoint.Send(allocPacket(PacketKindStreamClose, id)) + break + } + + s := NewStream(id, packet.InBuffer, packet.OutBuffer) + s.opened = true + b.resetKeepaliveTimerLocked(s) + // XXX: check if id already exists? + b.streams[id] = s + + b.endpoint.Send(allocPacket(PacketKindOpenedStream, id)) + + pos := listener.incomingPos + listener.incomingCount + if pos >= len(listener.incoming) { + pos -= len(listener.incoming) + } + listener.incoming[pos] = s + listener.incomingCount++ + b.updateListenerPollers(listener) + + case PacketKindOpenedStream: + id := packet.ConnId.Flip() + + stream, ok := b.streams[id] + if !ok { + break // maybe send go away instead? + } + + stream.processIncomingPacket(b, packet) + + case PacketKindStreamAck: + id := packet.ConnId.Flip() + + stream, ok := b.streams[id] + if !ok { + break // maybe send go away instead? + } + + stream.processIncomingPacket(b, packet) + + case PacketKindStreamData: + id := packet.ConnId.Flip() + + stream, ok := b.streams[id] + if !ok { + break // maybe send go away instead? + } + + stream.processIncomingPacket(b, packet) + + case PacketKindStreamWindow: + id := packet.ConnId.Flip() + + stream, ok := b.streams[id] + if !ok { + break // maybe send go away instead? + } + + stream.processIncomingPacket(b, packet) + + case PacketKindStreamClose: + id := packet.ConnId.Flip() + + stream, ok := b.streams[id] + if !ok { + break // maybe send go away instead? + } + + if err := b.closeStream(stream, false); err != nil { + // XXX + panic(err) + } + + default: + panic(packet) + } +} + +// listeners, streams + +type Listener struct { + incoming []*Stream + incomingPos, incomingCount int + + closed bool + + port uint16 + + pollers syscallabi.Pollers +} + +var ErrPortAlreadyInUse = errors.New("port already in use") + +func (n *Stack) OpenListener(port uint16) (*Listener, error) { + n.mu.Lock() + defer n.mu.Unlock() + + if _, ok := n.listeners[port]; ok { + return nil, ErrPortAlreadyInUse + } + + l := &Listener{ + incoming: make([]*Stream, 5), + + closed: false, + + port: port, + } + n.listeners[port] = l + + return l, nil +} + +var ErrListenerClosed = errors.New("listener closed") + +func (n *Stack) Accept(listener *Listener) (*Stream, error) { + n.mu.Lock() + defer n.mu.Unlock() + + l := listener + + if l.closed { + return nil, ErrListenerClosed + } + + if l.incomingCount == 0 { + return nil, syscall.EWOULDBLOCK + } + + stream := l.incoming[l.incomingPos] + l.incoming[l.incomingPos] = nil + l.incomingPos++ + l.incomingCount-- + if l.incomingPos == len(l.incoming) { + l.incomingPos = 0 + } + n.updateListenerPollers(l) + return stream, nil +} + +func (n *Stack) updateListenerPollers(listener *Listener) { + listener.pollers.NotifyCanRead(listener.closed || listener.incomingCount > 0) +} + +func (n *Stack) CloseListener(listener *Listener) error { + n.mu.Lock() + defer n.mu.Unlock() + + l := listener + + if l.closed { + return ErrListenerClosed + } + + l.closed = true + + for i := 0; i < l.incomingCount; i++ { + n.closeStream(l.incoming[l.incomingPos], true) + l.incoming[l.incomingPos] = nil + l.incomingPos++ + if l.incomingPos == len(l.incoming) { + l.incomingPos = 0 + } + l.incomingCount-- + } + + delete(n.listeners, l.port) + + n.updateListenerPollers(l) + + return nil +} + +func (n *Stack) closeStream(stream *Stream, sendClose bool) error { + if stream.closed { + panic(stream) + } + + // fmt.Fprintf(gosimruntime.LogOut, "close stream\n") + + if sendClose { + n.endpoint.Send(allocPacket(PacketKindStreamClose, stream.id)) + } + + delete(n.streams, stream.id) + stream.closed = true + n.updateStreamPollers(stream) + + if stream.openFailedTimer != nil { + stream.openFailedTimer.Stop() + stream.openFailedTimer = nil + } + if stream.ackFailedTimer != nil { + stream.ackFailedTimer.Stop() + stream.ackFailedTimer = nil + } + if stream.keepaliveTimer != nil { + stream.keepaliveTimer.Stop() + stream.keepaliveTimer = nil + } + + return nil +} + +// A circularBuffer is a buffer for data in a TCP stream to minimize allocations +// and/or copies. The stream implementation passes buffer pointers on stream +// open, and afterwards all "data" packets are merely notifications that data +// can be read from the appropriate buffer. This way TCP reads and writes copy +// the data only twice (once from userspace to the circular buffer, and then +// from the circular buffer back to userspace), and with zero allocations. +type circularBuffer struct { + data []byte + read, write int + used int +} + +func newCircularBuffer(n int) *circularBuffer { + return &circularBuffer{ + data: make([]byte, n), + } +} + +func (c *circularBuffer) free() int { + return len(c.data) - c.used +} + +func (c *circularBuffer) Write(data syscallabi.ByteSliceView) int { + if free := c.free(); free < data.Len() { + data = data.Slice(0, free) + } + n := data.Read(c.data[c.write:]) + c.write += n + c.used += n + if c.write == len(c.data) { + c.write = 0 + m := data.SliceFrom(n).Read(c.data[c.write:]) + c.write += m + c.used += m + n += m + } + return n +} + +func (c *circularBuffer) Read(data syscallabi.ByteSliceView) int { + if c.used < data.Len() { + data = data.Slice(0, c.used) + } + n := data.Write(c.data[c.read:]) + c.read += n + c.used -= n + if c.read == len(c.data) { + c.read = 0 + m := data.SliceFrom(n).Write(c.data[c.read:]) + c.read += m + c.used -= m + n += m + } + return n +} + +type Stream struct { + id ConnId + + opened bool + closed bool + + openFailedTimer *time.Timer + ackFailedTimer *time.Timer + keepaliveTimer *time.Timer + + sendPos int + ackPos int + recvPos int + + // in-memory shared between both ends to skip allocating + // todo: stick these in a pool somehow + inBuffer, outBuffer *circularBuffer + + incomingAvailable int + outgoingWindow int + + pollers syscallabi.Pollers +} + +func (s *Stream) ID() ConnId { + return s.id +} + +// reset every time we send something that might trigger an ack +func (n *Stack) resetKeepaliveTimerLocked(s *Stream) { + if s.keepaliveTimer == nil { + s.keepaliveTimer = time.AfterFunc(30*time.Second, func() { + n.mu.Lock() + defer n.mu.Unlock() + if n.shutdown || s.closed { + return + } + + // only send keepalive if we have not already send any data + if s.ackPos != s.sendPos { + return + } + + packet := allocPacket(PacketKindStreamData, s.id) + packet.Offset = s.sendPos + packet.Count = 0 + n.endpoint.Send(packet) + + n.resetKeepaliveTimerLocked(s) + n.resetAckFailedTimerLocked(s) + }) + } else { + s.keepaliveTimer.Reset(15 * time.Second) + } +} + +func (n *Stack) resetAckFailedTimerLocked(s *Stream) { + if s.ackFailedTimer == nil { + // TODO: this time is way too low, it breaks when latency goes to 2.5s? + s.ackFailedTimer = time.AfterFunc(5*time.Second, func() { + n.mu.Lock() + defer n.mu.Unlock() + if n.shutdown || s.closed { + return + } + + // fmt.Fprintf(gosimruntime.LogOut, "ack failed\n") + + // fail no matter the ack pos; for keep alives we have ackpos == + // sendpos. when we do get the data, this timer is stopped, so this + // if should only help in case of a race + + // if s.ackPos != s.sendPos { + // stream.failureReason? + // stream.openFailed? + n.closeStream(s, false) + // } + }) + } else { + s.ackFailedTimer.Reset(5 * time.Second) + } +} + +func NewStream(id ConnId, inBuffer, outBuffer *circularBuffer) *Stream { + return &Stream{ + id: id, + + inBuffer: inBuffer, + outBuffer: outBuffer, + outgoingWindow: len(outBuffer.data), + } +} + +func (n *Stack) OpenStream(addr netip.AddrPort) (*Stream, error) { + n.mu.Lock() + defer n.mu.Unlock() + + localPort := n.allocatePort() + id := ConnId{ + Hosts: HostPair{ + SourceHost: n.endpoint.Addr(), + DestHost: addr.Addr(), + }, + Ports: PortPair{ + SourcePort: localPort, + DestPort: addr.Port(), + }, + } + + inBuffer := newCircularBuffer(1024) + outBuffer := newCircularBuffer(1024) + stream := NewStream(id, inBuffer, outBuffer) + + n.streams[stream.id] = stream + // XXX: check stream id doesn't already exist + + packet := allocPacket(PacketKindOpenStream, id) + packet.InBuffer = outBuffer + packet.OutBuffer = inBuffer + n.endpoint.Send(packet) + + stream.openFailedTimer = time.AfterFunc(time.Second, func() { + n.mu.Lock() + defer n.mu.Unlock() + if n.shutdown { + return + } + if stream.opened || stream.closed { + return + } + // stream.failureReason? + // stream.openFailed? + n.closeStream(stream, false) + }) + + return stream, nil +} + +var ErrStreamClosed = errors.New("stream closed") + +func (n *Stack) StreamClose(stream *Stream) error { + n.mu.Lock() + defer n.mu.Unlock() + + if stream.closed { + return ErrStreamClosed + } + + if err := n.closeStream(stream, true); err != nil { + return err + } + + return nil +} + +func (s *Stream) processIncomingPacket(b *Stack, packet *Packet) { + switch packet.Kind { + case PacketKindOpenedStream: + s.opened = true + s.openFailedTimer.Stop() + s.openFailedTimer = nil + b.resetKeepaliveTimerLocked(s) + b.updateStreamPollers(s) + + case PacketKindStreamData: + if packet.Offset != s.recvPos { + // XXX reject + return + } + + s.recvPos += packet.Count + + s.incomingAvailable += packet.Count + + response := allocPacket(PacketKindStreamAck, s.id) + response.Offset = packet.Offset + packet.Count + b.endpoint.Send(response) + b.updateStreamPollers(s) + + case PacketKindStreamAck: + if packet.Offset < s.ackPos { + // XXX: reject + return + } + s.ackPos = packet.Offset + if s.ackPos == s.sendPos { + s.ackFailedTimer.Stop() + } + + case PacketKindStreamWindow: + s.outgoingWindow += packet.Count + b.updateStreamPollers(s) + } +} + +func (n *Stack) updateStreamPollers(stream *Stream) { + stream.pollers.NotifyCanWrite(stream.closed || stream.outgoingWindow > 0) + stream.pollers.NotifyCanRead(stream.closed || stream.incomingAvailable > 0) +} + +func (n *Stack) StreamSend(stream *Stream, data syscallabi.ByteSliceView) (int, error) { + n.mu.Lock() + defer n.mu.Unlock() + + s := stream + if s.closed { + return 0, ErrStreamClosed + } + + if !s.opened { + return 0, syscall.EWOULDBLOCK + } + + if s.outgoingWindow == 0 { + return 0, syscall.EWOULDBLOCK + } + + m := min(s.outgoingWindow, data.Len()) + written := stream.outBuffer.Write(data.Slice(0, m)) + if written != m { + panic("bad") + } + + offset := s.sendPos + s.sendPos += written + + packet := allocPacket(PacketKindStreamData, s.id) + packet.Offset = offset + packet.Count = written + n.endpoint.Send(packet) + + s.outgoingWindow -= written + + n.resetAckFailedTimerLocked(s) + n.resetKeepaliveTimerLocked(s) + + n.updateStreamPollers(s) + + return written, nil +} + +func (n *Stack) StreamRecv(s *Stream, data syscallabi.ByteSliceView) (int, error) { + n.mu.Lock() + defer n.mu.Unlock() + + if s.incomingAvailable == 0 { + if s.closed { + return 0, ErrStreamClosed + } + return 0, syscall.EWOULDBLOCK + } + + m := min(s.incomingAvailable, data.Len()) + read := s.inBuffer.Read(data.Slice(0, m)) + if read != m { + panic("bad") + } + s.incomingAvailable -= m + + packet := allocPacket(PacketKindStreamWindow, s.id) + packet.Count = m + n.endpoint.Send(packet) + + n.updateStreamPollers(s) + + return m, nil +} + +type StreamStatus struct { + Open bool + Closed bool +} + +func (n *Stack) StreamStatus(stream *Stream) StreamStatus { + n.mu.Lock() + defer n.mu.Unlock() + + return StreamStatus{ + Open: stream.opened, + Closed: stream.closed, + } +} + +func (n *Stack) RegisterStreamPoller(poller *syscallabi.PollDesc, stream *Stream) { + n.mu.Lock() + defer n.mu.Unlock() + + stream.pollers.Add(poller) +} + +func (n *Stack) DeregisterStreamPoller(poller *syscallabi.PollDesc, stream *Stream) { + n.mu.Lock() + defer n.mu.Unlock() + + stream.pollers.Remove(poller) +} + +func (n *Stack) RegisterListenerPoller(poller *syscallabi.PollDesc, listener *Listener) { + n.mu.Lock() + defer n.mu.Unlock() + + listener.pollers.Add(poller) +} + +func (n *Stack) DeregisterListenerPoller(poller *syscallabi.PollDesc, listener *Listener) { + n.mu.Lock() + defer n.mu.Unlock() + + listener.pollers.Remove(poller) +} diff --git a/internal/simulation/os_linux.go b/internal/simulation/os_linux.go new file mode 100644 index 0000000..100e7d2 --- /dev/null +++ b/internal/simulation/os_linux.go @@ -0,0 +1,1180 @@ +//go:build linux + +package simulation + +import ( + "encoding/binary" + "errors" + "log" + "log/slog" + "math/rand" + "net/netip" + "os" + "sync" + "syscall" + "unsafe" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/simulation/fs" + "github.com/jellevandenhooff/gosim/internal/simulation/network" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +// LinuxOS implements gosim's versions of Linux system calls. +type LinuxOS struct { + dispatcher syscallabi.Dispatcher + + simulation *Simulation + + machine *Machine + + mu sync.Mutex + shutdown bool + + files map[int]interface{} + nextFd int +} + +func NewLinuxOS(simulation *Simulation, machine *Machine, dispatcher syscallabi.Dispatcher) *LinuxOS { + return &LinuxOS{ + dispatcher: dispatcher, + + simulation: simulation, + + machine: machine, + + files: make(map[int]interface{}), + nextFd: 5, + } +} + +func (l *LinuxOS) doShutdown() { + l.mu.Lock() + defer l.mu.Unlock() + + l.shutdown = true + + // nil out pointers to catch any use after shutdown + l.machine = nil + l.files = nil +} + +// XXX: var causes init issues???? +const logEnabled = false + +func logf(fmt string, args ...any) { + if logEnabled { + log.Printf(fmt, args...) + } +} + +// used to be... +// //go:nocheckptr +// //go:uintptrescapes +func RawSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { + if IsHandledSyscall(trap) { + // complain loudly if syscall that we support arrives + // here instead of through our renames + panic("syscall should have been rewritten somewhere") + } + + logf("unknown=%d %d %d %d %d %d %d", trap, a1, a2, a3, a4, a5, a6) + + return 0, 0, syscall.ENOSYS +} + +func (l *LinuxOS) allocFd() int { + fd := l.nextFd + l.nextFd++ + return fd +} + +func (l *LinuxOS) PollOpen(fd int, desc syscallabi.ValueView[syscallabi.PollDesc]) int { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + // XXX: what do we return here? shouldn't matter? + return 0 + } + + f := l.files[fd] + if socket, ok := f.(*Socket); ok { + if socket.Stream != nil { + l.machine.netstack.RegisterStreamPoller((*syscallabi.PollDesc)(desc.UnsafePointer()), socket.Stream) + } + if socket.Listener != nil { + l.machine.netstack.RegisterListenerPoller((*syscallabi.PollDesc)(desc.UnsafePointer()), socket.Listener) + } + } + return 0 +} + +func (l *LinuxOS) PollClose(fd int, desc syscallabi.ValueView[syscallabi.PollDesc]) int { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + // XXX: what do we return here? shouldn't matter? + return 0 + } + + f := l.files[fd] + if socket, ok := f.(*Socket); ok { + if socket.Stream != nil { + l.machine.netstack.DeregisterStreamPoller((*syscallabi.PollDesc)(desc.UnsafePointer()), socket.Stream) + } + if socket.Listener != nil { + l.machine.netstack.DeregisterListenerPoller((*syscallabi.PollDesc)(desc.UnsafePointer()), socket.Listener) + } + } + return 0 +} + +type ( + RawSockaddrAny = syscall.RawSockaddrAny + Utsname = syscall.Utsname + Stat_t = syscall.Stat_t +) + +type Socklen uint32 + +const ( + _AT_FDCWD = -0x64 + _AT_REMOVEDIR = 0x200 +) + +// TODO: make functions instead return Errno again? + +// for disk: +// to sim +// - random op errors +// - latency +// - writes get split in parts + +// behavior tests +// - assert that behavior shows up in (some) runs +// - assert that behavior never shows up +// - assert that we have all possible behavior (exact) + +// brokenness that might happen: +// - file entries are not are persisted until fsync on the directory [done] +// - writes to a file might be reordered until fsync [done] +// - writes to different files might be reordered [done] +// - an appended-to file might have zeroes [done] +// - after truncate a file might still have old data afterwards (zeroes not guaranteed) + +// XXX: +// - (later:) alert on cross-machine interactions (eg. chan, memory, ...?) + +type OsFile struct { + inode int + name string + pos int64 + flagRead, flagWrite bool + + didReaddir bool // XXX JANK +} + +type Socket struct { + IsBound bool + BoundAddr netip.AddrPort + + Listener *network.Listener + Stream *network.Stream + + RecvBuffer []byte + + // pending socket + // tcp listener + // file + // tcp conn +} + +// persist dir: +// all links into this dir we persist + +func (l *LinuxOS) SysOpenat(dirfd int, path string, flags int, mode uint32) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + if dirfd != _AT_FDCWD { + return 0, syscall.EINVAL // XXX? + } + + // just get rid of this + flags &= ^syscall.O_CLOEXEC + + logf("openat %d %s %d %d", dirfd, path, flags, mode) + + // TODO: some rules about paths; component length; total length; allow characters? + // TODO: check mode + + const flagSupported = os.O_RDONLY | os.O_WRONLY | os.O_RDWR | + os.O_CREATE | os.O_EXCL | os.O_TRUNC + + if flags&(^flagSupported) != 0 { + return 0, syscall.EINVAL // XXX? + } + + var flagRead, flagWrite bool + if flags&os.O_WRONLY != 0 { + if flags&(os.O_RDWR) != 0 { + return 0, syscall.EINVAL // XXX? + } + flagWrite = true + } else if flags&os.O_RDWR != 0 { + flagRead = true + flagWrite = true + } else { + flagRead = true + } + + inode, err := l.machine.filesystem.OpenFile(path, flags) + if err != nil { + if err == syscall.ENOENT { + return 0, syscall.ENOENT + } + return 0, syscall.EINVAL // XXX + } + + // TODO: add reference here? or in OpenFile? help this is strange. + + // TODO: this pos is wrong for writing maybe? what should happen when you + // open a file that already exists and start writing to it? + f := &OsFile{ + name: path, + inode: inode, + pos: 0, + flagRead: flagRead, + flagWrite: flagWrite, + } + + fd := l.allocFd() + l.files[fd] = f + + return fd, nil +} + +func (l *LinuxOS) SysRenameat(olddirfd int, oldpath string, newdirfd int, newpath string) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + logf("renameat %d %s %d %s", olddirfd, oldpath, newdirfd, newpath) + + if olddirfd != _AT_FDCWD { + return syscall.EINVAL // XXX? + } + + if newdirfd != _AT_FDCWD { + return syscall.EINVAL // XXX? + } + + if err := l.machine.filesystem.Rename(oldpath, newpath); err != nil { + if err == syscall.ENOENT { + return syscall.ENOENT + } + return syscall.EINVAL // XXX? + } + + return nil +} + +func (l *LinuxOS) SysGetdents64(fd int, data syscallabi.ByteSliceView) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + logf("getdents64 %d %d", fd, data.Len()) + + fdInternal, ok := l.files[fd] + if !ok { + return 0, syscall.EBADFD + } + + f, ok := fdInternal.(*OsFile) + if !ok { + return 0, syscall.ENOTDIR + } + + if f.didReaddir { + return 0, nil + } + + entries, err := l.machine.filesystem.ReadDir(f.name) + if err != nil { + return 0, syscall.EINVAL // XXX? + } + + var localbuffer [4096]byte + + retn := 0 + for _, entry := range entries { + name := entry.Name + + inode := int64(1) + offset := int64(1) + reclen := int(19) + len(name) + 1 + + if reclen > len(localbuffer) { + panic("help") + } + if retn+reclen > data.Len() { + break + } + + buffer := localbuffer[:reclen] + typ := syscall.DT_REG + if entry.IsDir { + typ = syscall.DT_DIR + } + + binary.LittleEndian.PutUint64(buffer[0:8], uint64(inode)) + binary.LittleEndian.PutUint64(buffer[8:16], uint64(offset)) + binary.LittleEndian.PutUint16(buffer[16:18], uint16(reclen)) + buffer[18] = byte(typ) + copy(buffer[19:], name) + buffer[19+len(name)] = 0 + + data.Slice(retn, retn+reclen).Write(buffer) + retn += reclen + } + + logf("getdents64 %d %d", fd, data.Len() /*, data*/) + + f.didReaddir = true + + return retn, nil +} + +func (l *LinuxOS) SysWrite(fd int, data syscallabi.ByteSliceView) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + if fd == syscall.Stdout || fd == syscall.Stderr { + buf := make([]byte, data.Len()) + data.Read(buf) + // TODO: use a custom machine here? + + var source string + if fd == syscall.Stdout { + source = "stdout" + } else { + source = "stderr" + } + + slog.Info(string(buf), "method", source, "from", l.machine.label) + return data.Len(), nil + } + + fdInternal, ok := l.files[fd] + if !ok { + logf("write %d badfd", fd) + return 0, syscall.EBADFD + } + + switch f := fdInternal.(type) { + case *OsFile: + if !f.flagWrite { + return 0, syscall.EBADFD + } + + retn := l.machine.filesystem.Write(f.inode, f.pos, data) + f.pos += int64(retn) + + logf("write %d %d %q", fd, data.Len(), data) + return retn, nil + + case *Socket: + if f.Stream == nil { + return 0, syscall.EBADFD + } + + retn, err := l.machine.netstack.StreamSend(f.Stream, data) + if err != nil { + if err == syscall.EWOULDBLOCK { + return retn, syscall.EWOULDBLOCK + } + if err == network.ErrStreamClosed { + return retn, syscall.EPIPE + } + return retn, syscall.EBADFD + } + + logf("write %d %d %q", fd, data.Len(), data) + return retn, nil + + default: + return 0, syscall.EBADFD + } +} + +func (l *LinuxOS) SysRead(fd int, data syscallabi.ByteSliceView) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return 0, syscall.EBADFD + } + + switch f := fdInternal.(type) { + case *OsFile: + if !f.flagRead { + return 0, syscall.EBADFD // XXX?? + } + + retn := l.machine.filesystem.Read(f.inode, f.pos, data) + f.pos += int64(retn) + + logf("read %d %d", fd, data.Len() /*, data[:retn]*/) + + return retn, nil + + case *Socket: + if f.Stream == nil { + return 0, syscall.EBADFD + } + + retn, err := l.machine.netstack.StreamRecv(f.Stream, data) + if err != nil { + if err == syscall.EWOULDBLOCK { + return 0, syscall.EWOULDBLOCK + } + if err == network.ErrStreamClosed { + return 0, syscall.EPIPE + } + return 0, syscall.EBADFD + } + + logf("read %d %d", fd, data.Len() /*, data[:retn]*/) + + return retn, nil + + default: + return 0, syscall.EBADFD + } +} + +func (l *LinuxOS) SysPwrite64(fd int, data syscallabi.ByteSliceView, offset int64) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return 0, syscall.EBADFD + } + + f, ok := fdInternal.(*OsFile) + if !ok { + return 0, syscall.EBADFD + } + + if !f.flagWrite { + return 0, syscall.EBADFD + } + + retn := l.machine.filesystem.Write(f.inode, offset, data) + + logf("writeat %d %d %q %d", fd, retn, data, offset) + + return retn, nil +} + +func (l *LinuxOS) SysPread64(fd int, data syscallabi.ByteSliceView, offset int64) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return 0, syscall.EBADFD + } + + f, ok := fdInternal.(*OsFile) + if !ok { + return 0, syscall.EBADFD + } + + if !f.flagRead { + return 0, syscall.EBADFD // XXX?? + } + + retn := l.machine.filesystem.Read(f.inode, offset, data) + + logf("readat %d %d %d", fd, data.Len() /*, data[:retn]*/, offset) + + return retn, nil +} + +func (l *LinuxOS) SysFsync(fd int) error { + // XXX: janky way to crash on sync + l.machine.sometimesCrashOnSyncMu.Lock() + maybeCrash := l.machine.sometimesCrashOnSync + l.machine.sometimesCrashOnSyncMu.Unlock() + if maybeCrash && rand.Intn(100) > 90 { + if rand.Int()&1 == 0 { + // XXX: include fd's file somehow??? + slog.Info("crashing machine before sync", "machine", l.machine.label) + l.simulation.mu.Lock() + defer l.simulation.mu.Unlock() + l.simulation.stopMachine(l.machine, false) + return syscall.EINTR // XXX whatever this should never be seen + } else { + defer func() { + slog.Info("crashing machine after sync", "machine", l.machine.label) + l.simulation.mu.Lock() + defer l.simulation.mu.Unlock() + l.simulation.stopMachine(l.machine, false) + }() + } + } + + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + f, ok := fdInternal.(*OsFile) + if !ok { + return syscall.EBADFD + } + + l.machine.filesystem.Sync(f.inode) + + logf("fsync %d", fd) + + return nil +} + +func fillStat(view syscallabi.ValueView[syscall.Stat_t], in fs.StatResp) { + var out syscall.Stat_t + out.Ino = uint64(in.Inode) + if in.IsDir { + out.Mode = syscall.S_IFDIR + } else { + out.Mode = syscall.S_IFREG + } + view.Set(out) +} + +func (l *LinuxOS) SysFstat(fd int, statBuf syscallabi.ValueView[syscall.Stat_t]) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + logf("fstat %d", fd) + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + // XXX: handle dir + f, ok := fdInternal.(*OsFile) + if !ok { + return syscall.EBADFD + } + + fsStat, err := l.machine.filesystem.Statfd(f.inode) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return syscall.ENOENT + } + return syscall.EINVAL // XXX? + } + + fillStat(statBuf, fsStat) + + return nil +} + +// arm64 +func (l *LinuxOS) SysFstatat(dirfd int, path string, statBuf syscallabi.ValueView[syscall.Stat_t], flags int) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + if dirfd != _AT_FDCWD { + return syscall.EINVAL // XXX? + } + + if flags != 0 { + return syscall.EINVAL // XXX? + } + + logf("fstatat %d %s %d", dirfd, path, flags) + + fsStat, err := l.machine.filesystem.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return syscall.ENOENT + } + return syscall.EINVAL // XXX? + } + + fillStat(statBuf, fsStat) + + return nil +} + +// amd64 +func (l *LinuxOS) SysNewfstatat(dirfd int, path string, statBuf syscallabi.ValueView[syscall.Stat_t], flags int) error { + return l.SysFstatat(dirfd, path, statBuf, flags) +} + +func (l *LinuxOS) SysUnlinkat(dirfd int, path string, flags int) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + if dirfd != _AT_FDCWD { + return syscall.EINVAL // XXX? + } + + logf("unlinkat %d %s %d", dirfd, path, flags) + + switch flags { + case 0: + if err := l.machine.filesystem.Remove(path); err != nil { + if err == syscall.ENOENT { + return syscall.ENOENT + } + return syscall.EINVAL + } + return nil + case _AT_REMOVEDIR: + // XXX: should special case dir vs file and also ENOTDIR + if err := l.machine.filesystem.Remove(path); err != nil { + if errors.Is(err, os.ErrNotExist) { + return syscall.ENOENT + } + return syscall.EINVAL + } + + return nil + + default: + return syscall.EINVAL + } +} + +func (l *LinuxOS) SysFtruncate(fd int, n int64) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + f, ok := fdInternal.(*OsFile) + if !ok { + return syscall.EBADFD + } + + if !f.flagWrite { + return syscall.EBADFD // XXX? + } + + l.machine.filesystem.Truncate(f.inode, int(n)) + + logf("truncate %d %d", fd, n) + + return nil +} + +func (l *LinuxOS) SysClose(fd int) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + logf("close %d", fd) + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + switch fdInternal := fdInternal.(type) { + case *OsFile: + if err := l.machine.filesystem.CloseFile(fdInternal.inode); err != nil { + // XXX? + return syscall.EBADFD + } + + case *Socket: + // TBD + // XXX: wowwwwwww we should get this one. better send a close! + + default: + return syscall.EBADFD + } + + delete(l.files, fd) + + return nil +} + +func (l *LinuxOS) SysSocket(net, flags, proto int) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + // handle sock_posix func (p *ipStackCapabilities) probe() + + logf("socket %d %d %d", net, flags, proto) + + // ipv4 close on exec non blocking tcp streams ONLY + if net != syscall.AF_INET { + return 0, syscall.EINVAL // XXX? + } + if flags != syscall.SOCK_STREAM|syscall.SOCK_CLOEXEC|syscall.SOCK_NONBLOCK { + return 0, syscall.EINVAL // XXX? + } + if proto != 0 { // XXX: no ipv6 + return 0, syscall.EINVAL // XXX? + } + + fd := l.allocFd() + l.files[fd] = &Socket{} + + return fd, nil +} + +func (l *LinuxOS) SysBind(fd int, addrPtr unsafe.Pointer, addrlen Socklen) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return syscall.ENOTSOCK + } + + if sock.IsBound { + return syscall.EINVAL + } + + addr, err := readAddr(addrPtr, addrlen) + if err != 0 { + return err + } + + logf("bind %d %s", fd, addr) + + switch { + case addr.Addr().Is4(): + default: + return syscall.EINVAL + } + + sock.IsBound = true + sock.BoundAddr = addr + return nil +} + +func (l *LinuxOS) SysListen(fd, backlog int) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return syscall.ENOTSOCK + } + + if !sock.IsBound { + return syscall.EBADFD + } + + if sock.Listener != nil { + return syscall.EBADFD + } + + // XXX + listener, err := l.machine.netstack.OpenListener(sock.BoundAddr.Port()) + if err != nil { + return syscall.EINVAL + } + sock.Listener = listener + + logf("listen %d %d", fd, backlog) + + return nil +} + +func readAddr(addr unsafe.Pointer, addrlen Socklen) (netip.AddrPort, syscall.Errno) { + rsa := syscallabi.NewValueView((*syscall.RawSockaddr)(addr)) + + // XXX: check addrlen here? + switch rsa.Get().Family { + case syscall.AF_INET: + // AF_INET + if addrlen != syscall.SizeofSockaddrInet4 { + return netip.AddrPort{}, syscall.EINVAL + } + rsa4 := syscallabi.NewValueView((*syscall.RawSockaddrInet4)(addr)).Get() + ip := netip.AddrFrom4(rsa4.Addr) + // XXX: HELP + portBytes := (*[2]byte)(unsafe.Pointer(&rsa4.Port)) + port := uint16(int(portBytes[0])<<8 + int(portBytes[1])) + for i := 0; i < 8; i++ { + if rsa4.Zero[i] != 0 { + return netip.AddrPort{}, syscall.EINVAL + } + } + return netip.AddrPortFrom(ip, port), 0 + + default: + return netip.AddrPort{}, syscall.EINVAL + } +} + +func writeAddr(outbuf syscallabi.ValueView[RawSockaddrAny], outLen syscallabi.ValueView[Socklen], addr netip.AddrPort) syscall.Errno { + switch { + case addr.Addr().Is4(): + outbuf := syscallabi.NewValueView((*syscall.RawSockaddrInet4)(outbuf.UnsafePointer())) + addrPort := addr.Port() + portBytes := (*[2]byte)(unsafe.Pointer(&addrPort)) + port := uint16(int(portBytes[0])<<8 + int(portBytes[1])) + outbuf.Set(syscall.RawSockaddrInet4{ + Family: syscall.AF_INET, + Port: port, + Addr: addr.Addr().As4(), + }) + outLen.Set(syscall.SizeofSockaddrInet4) + return 0 + + default: + return syscall.EADDRNOTAVAIL // XXX + } +} + +func (l *LinuxOS) SysAccept4(fd int, rsa syscallabi.ValueView[RawSockaddrAny], len syscallabi.ValueView[Socklen], flags int) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + logf("accept %d %d %d", fd, len.Get(), flags) + + if flags != syscall.SOCK_CLOEXEC|syscall.SOCK_NONBLOCK { + // XXX: check that flags are CLOEXEC and NONBLOCK (just like sys_socket) + return 0, syscall.EINVAL // XXX? + } + + fdInternal, ok := l.files[fd] + if !ok { + return 0, syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return 0, syscall.ENOTSOCK + } + + if sock.Listener == nil { + return 0, syscall.EBADFD + } + + stream, err := l.machine.netstack.Accept(sock.Listener) + if err != nil { + if err == syscall.EWOULDBLOCK { + return 0, syscall.EWOULDBLOCK + } + return 0, syscall.EBADFD + } + + newFd := l.allocFd() + l.files[newFd] = &Socket{ + Stream: stream, + } + + if err := writeAddr(rsa, len, stream.ID().Dest()); err != 0 { + return 0, err + } + + return newFd, nil +} + +func (l *LinuxOS) SysGetsockopt(fd int, level int, name int, ptr unsafe.Pointer, outlen syscallabi.ValueView[Socklen]) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + logf("getsockopt %d %d %d %d", fd, level, name, outlen.Get()) + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return syscall.ENOTSOCK + } + + switch { + case sock.Stream != nil: + if level != syscall.SOL_SOCKET || name != syscall.SO_ERROR { + return syscall.ENOSYS + } + + if outlen.Get() != 4 { + return syscall.EINVAL + } + + // this is used to determine connection status + + status := l.machine.netstack.StreamStatus(sock.Stream) + + var v uint32 + if !status.Open && !status.Closed { + v = uint32(syscall.EINPROGRESS) + } else if !status.Open && status.Closed { + // XXX: disambiguate later + v = uint32(syscall.ETIMEDOUT) + } else { + // XXX: this is where could also say ECONNREFUSED etc? + // XXX: supposed to clear error afterwards + v = 0 + } + syscallabi.NewValueView[uint32]((*uint32)(ptr)).Set(v) + return nil + + case sock.Listener != nil: + // XXX: seems bad response but + return syscall.ENOSYS + + default: + return syscall.EBADFD + } +} + +func (l *LinuxOS) SysSetsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + // XXX: yep + return nil +} + +func (l *LinuxOS) SysGetsockname(fd int, rsa syscallabi.ValueView[RawSockaddrAny], len syscallabi.ValueView[Socklen]) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + logf("getsockname %d %d", fd, len.Get()) + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return syscall.ENOTSOCK + } + + switch { + case sock.Stream != nil: + if err := writeAddr(rsa, len, sock.Stream.ID().Source()); err != 0 { + return err + } + return nil + + case sock.Listener != nil: + if err := writeAddr(rsa, len, netip.AddrPortFrom(netip.IPv4Unspecified(), 1)); err != 0 { // XXX TBD + return err + } + return nil + + default: + return syscall.EBADFD + } +} + +func (l *LinuxOS) SysGetpeername(fd int, rsa syscallabi.ValueView[RawSockaddrAny], len syscallabi.ValueView[Socklen]) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + logf("getpeername %d %d", fd, len.Get()) + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return syscall.ENOTSOCK + } + + if sock.Stream == nil { + return syscall.EBADFD + } + + if err := writeAddr(rsa, len, sock.Stream.ID().Dest()); err != 0 { + return err + } + + return nil +} + +func (l *LinuxOS) SysFcntl(fd int, cmd int, arg int) (val int, err error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + return 0, syscall.ENOSYS +} + +func (l *LinuxOS) SysConnect(fd int, addrPtr unsafe.Pointer, addrLen Socklen) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return syscall.EINVAL + } + + fdInternal, ok := l.files[fd] + if !ok { + return syscall.EBADFD + } + + sock, ok := fdInternal.(*Socket) + if !ok { + return syscall.ENOTSOCK + } + + addr, errno := readAddr(addrPtr, addrLen) + if errno != 0 { + return errno + } + + logf("connect %d %s", fd, addr) + + switch { + case addr.Addr().Is4(): + default: + return syscall.EINVAL + } + + stream, err := l.machine.netstack.OpenStream(addr) + if err != nil { + return syscall.EINVAL + } + + // XXX + sock.Stream = stream + + // XXX: non blocking now + return syscall.EINPROGRESS +} + +func (l *LinuxOS) SysGetrandom(ptr syscallabi.ByteSliceView, flags int) (int, error) { + // TODO: implement entirely in userspace? + + l.mu.Lock() + defer l.mu.Unlock() + if l.shutdown { + return 0, syscall.EINVAL + } + + logf("getrandom %d %d", ptr.Len(), flags) + + if flags != 0 { + return 0, syscall.EINVAL + } + + n := 0 + for ptr.Len() > 0 { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], gosimruntime.Fastrand64()) + m := ptr.Write(buf[:]) + ptr = ptr.SliceFrom(m) + n += m + } + return n, nil +} + +func (l *LinuxOS) SysGetpid() int { + // TODO: return some random number instead? + return 42 +} + +func (l *LinuxOS) SysUname(buf syscallabi.ValueView[Utsname]) (err error) { + ptr := (*Utsname)(buf.UnsafePointer()) + name := syscallabi.NewSliceView(&ptr.Nodename[0], uintptr(len(ptr.Nodename))) + var nameArray [65]int8 + n := min(len(l.machine.label), 64) + for i := 0; i < n; i++ { + nameArray[i] = int8(l.machine.label[i]) + } + nameArray[n] = 0 + name.Write(nameArray[:n+1]) + return nil +} diff --git a/internal/simulation/os_other.go b/internal/simulation/os_other.go new file mode 100644 index 0000000..f1af2dc --- /dev/null +++ b/internal/simulation/os_other.go @@ -0,0 +1,20 @@ +//go:build !linux + +// This file lets the os package compile on non-linux go builds. +// +// TODO: Fix build tags somehow to make this file unnecessary. + +package simulation + +import "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" + +type LinuxOS struct{} + +type linuxOSIface interface{} + +func NewLinuxOS(simulation *Simulation, machine *Machine, dispatcher syscallabi.Dispatcher) *LinuxOS { + return &LinuxOS{} +} + +func (l *LinuxOS) doShutdown() { +} diff --git a/internal/simulation/simulation.go b/internal/simulation/simulation.go new file mode 100644 index 0000000..cd0d5ce --- /dev/null +++ b/internal/simulation/simulation.go @@ -0,0 +1,180 @@ +package simulation + +import ( + "fmt" + "log/slog" + "net/netip" + "sync" + "time" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/simulation/fs" + "github.com/jellevandenhooff/gosim/internal/simulation/network" + "github.com/jellevandenhooff/gosim/internal/simulation/syscallabi" +) + +type Machine struct { + id int + label string + addr netip.Addr + bootProgram func() + + // TODO: decide how this is shared with LinuxOS exactly and how to lock it + filesystem *fs.Filesystem + netstack *network.Stack + + gosimOS *GosimOS + linuxOS *LinuxOS + + runtimeMachine *gosimruntime.Machine + + waiters []*syscallabi.Syscall + + sometimesCrashOnSyncMu sync.Mutex + sometimesCrashOnSync bool +} + +type Simulation struct { + dispatcher syscallabi.Dispatcher + + mu sync.Mutex + + nextMachineID int + machinesById map[int]*Machine + main *Machine + + network *network.Network + + timeoutTimer *time.Timer +} + +func (s *Simulation) newMachine(label string, addr netip.Addr, filesystem *fs.Filesystem, bootProgram func()) *Machine { + id := s.nextMachineID + s.nextMachineID++ + + if label == "" { + label = fmt.Sprintf("machine-%d", id) + } + + if !addr.IsValid() { + addr = s.network.NextIP() + } + + m := &Machine{ + id: id, + label: label, + addr: addr, + bootProgram: bootProgram, + + filesystem: filesystem, + netstack: nil, + + runtimeMachine: nil, + } + + s.machinesById[id] = m + + return m +} + +func (s *Simulation) startMachine(machine *Machine) { + machine.runtimeMachine = gosimruntime.NewMachine(machine.label) + + machine.netstack = network.NewStack() + s.network.AttachStack(machine.addr, machine.netstack) + + machine.linuxOS = NewLinuxOS(s, machine, s.dispatcher) + machine.gosimOS = NewGosimOS(s, s.dispatcher) + + linuxOS := machine.linuxOS + gosimOS := machine.gosimOS + bootProgram := machine.bootProgram + + gosimruntime.GoWithMachine(func() { + setupUserspace(gosimOS, linuxOS, machine.id, machine.label) + bootProgram() + // not using defer because we want an unhandled panic to bubble up; Stop + // prevents all code from running afterwards + SyscallMachineStop(machine.id, true) + }, machine.runtimeMachine) +} + +func (s *Simulation) stopMachine(machine *Machine, graceful bool) { + if machine.runtimeMachine == nil { + // XXX: already stopped + return + } + + // first, stop all the work the machine is currently doing. this atomically + // stops all goroutines, unhooks all selects, and prevents any future timers + // from firing. + gosimruntime.StopMachine(machine.runtimeMachine) + + // then, wait for current syscalls to finish and prevent all work on any + // pending future syscalls. + machine.linuxOS.doShutdown() + machine.linuxOS = nil + + // TODO: add shutdown? or share? + machine.gosimOS = nil + + // make sure the network stack won't do anything in the future + // graceful shutdown also sends close packets + machine.netstack.Shutdown(graceful) + + // stop sending packets to this netstack + s.network.DetachStack(machine.netstack) + machine.netstack = nil + + if graceful { + // XXX: flush all writes to disk + machine.filesystem.FlushEverything() + } + + // XXX: what about polldescs? any other comms between machine and the world? + + // XXX: shut down any background flusher in the filesystem (tbd, does not exist yet) + // XXX: keep a snapshot of the filesystem? + // XXX: mark machine as bad? don't want to run code in it again. + machine.runtimeMachine = nil + + for _, waiter := range machine.waiters { + waiter.Complete() + } + machine.waiters = nil + + if machine == s.main { + // XXX: should we distinguish a test ending with a clean return vs + // shutdown being called on the machine? + gosimruntime.SetAbortError(gosimruntime.ErrMainReturned) + } +} + +func Runtime(fun func()) { + setupSlog("os") + + timeoutTimer := time.AfterFunc(defaultTimeout, func() { + slog.Error("test timeout") + gosimruntime.SetAbortError(gosimruntime.ErrTimeout) + }) + + dispatcher := syscallabi.NewDispatcher() + + s := &Simulation{ + dispatcher: dispatcher, + + nextMachineID: 1, + machinesById: make(map[int]*Machine), + + network: network.NewNetwork(), + + timeoutTimer: timeoutTimer, + } + go s.network.Run() + + addr := s.network.NextIP() + s.main = s.newMachine("main", addr, fs.NewFilesystem(), fun) + s.startMachine(s.main) + + dispatcher.Run() +} diff --git a/internal/simulation/syscallabi/errno.go b/internal/simulation/syscallabi/errno.go new file mode 100644 index 0000000..379ee16 --- /dev/null +++ b/internal/simulation/syscallabi/errno.go @@ -0,0 +1,44 @@ +// Copyright 2019 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Based on https://go.googlesource.com/go/+/refs/heads/master/src/internal/poll/errno_unix.go. + +package syscallabi + +import "syscall" + +// Do the interface allocations only once for common +// Errno values. +var ( + errEAGAIN error = syscall.EAGAIN + errEINVAL error = syscall.EINVAL + errENOENT error = syscall.ENOENT +) + +// ErrnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func ErrnoErr(e uintptr) error { + switch syscall.Errno(e) { + case 0: + return nil + case syscall.EAGAIN: + return errEAGAIN + case syscall.EINVAL: + return errEINVAL + case syscall.ENOENT: + return errENOENT + } + return syscall.Errno(e) +} + +func ErrErrno(err error) uintptr { + if err == nil { + return 0 + } + syscallErr, ok := err.(syscall.Errno) + if !ok { + panic(syscallErr) + } + return uintptr(syscallErr) +} diff --git a/internal/simulation/syscallabi/poll.go b/internal/simulation/syscallabi/poll.go new file mode 100644 index 0000000..2359138 --- /dev/null +++ b/internal/simulation/syscallabi/poll.go @@ -0,0 +1,259 @@ +package syscallabi + +import ( + "sync" + "time" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +// TODO: clean/change this API? +// - document who owns what fields more clearly? +// - document which functions are called from os and which from userspace? +// - race.Disable/race.Enable to prevent dependency edges? +// - try to get rid of norace? + +// PollDesc is a poll descriptor like the go runtime's netpoll descriptors. +// +// Blocking operations pass poll descriptors to the OS, which tracks those +// descriptors and marks them as readable or writable using a Pollers. +type PollDesc struct { + fd int + registeredIndex int + + mu sync.Mutex + + readable gosimruntime.Uint32Futex + writable gosimruntime.Uint32Futex + + readTimer *time.Timer + readTimeout time.Time + + writeTimer *time.Timer + writeTimeout time.Time // XXX: need this? implicit in timer somehow? +} + +func (pd *PollDesc) FD() int { + return pd.fd +} + +var freePollDescs []*PollDesc + +//go:norace +func AllocPollDesc(fd int) *PollDesc { + if len(freePollDescs) == 0 { + desc := &PollDesc{ + fd: fd, + registeredIndex: -1, + } + return desc + } else { + desc := freePollDescs[len(freePollDescs)-1] + freePollDescs = freePollDescs[:len(freePollDescs)-1] + desc.fd = fd + return desc + } +} + +//go:norace +func (pd *PollDesc) Close() { + pd.mu.Lock() + defer pd.mu.Unlock() + + if pd.registeredIndex != -1 { + // make sure we are unhooked on the sim side + panic(pd) + } + + pd.fd = -1 + pd.readable.Set(0) + pd.writable.Set(0) + pd.readTimeout = time.Time{} + if pd.readTimer != nil { + pd.readTimer.Stop() + } + pd.writeTimeout = time.Time{} + if pd.writeTimer != nil { + pd.writeTimer.Stop() + } + + freePollDescs = append(freePollDescs, pd) +} + +func (pd *PollDesc) WaitCanceled(mode int) { + panic("not implemented") // only used on windows +} + +//go:norace +func (pd *PollDesc) Reset(mode int) int { + // called before wait + + var value uint32 + switch mode { + case 'r': + value = pd.readable.Get() + case 'w': + value = pd.writable.Get() + default: + panic(mode) + } + + switch { + case value&(1< 0 { + if pd.readTimer == nil { + pd.readTimer = time.AfterFunc(time.Duration(d), func() { + pd.mu.Lock() + defer pd.mu.Unlock() + if !pd.readTimeout.IsZero() && !time.Now().Before(pd.readTimeout) { + pd.readable.SetBit(pollErrTimeout, true) + } + }) + } else { + pd.readTimer.Reset(time.Duration(d)) + } + } else { + if pd.readTimer != nil { + pd.readTimer.Stop() + } + } + } + if mode == 'w' || mode == 'r'+'w' { + pd.writeTimeout = deadline + pd.writable.SetBit(pollErrTimeout, d < 0) + if d > 0 { + if pd.writeTimer == nil { + pd.writeTimer = time.AfterFunc(time.Duration(d), func() { + pd.mu.Lock() + defer pd.mu.Unlock() + if !pd.writeTimeout.IsZero() && !time.Now().Before(pd.writeTimeout) { + pd.writable.SetBit(pollErrTimeout, true) + } + }) + } else { + pd.writeTimer.Reset(time.Duration(d)) + } + } else { + if pd.writeTimer != nil { + pd.writeTimer.Stop() + } + } + } +} + +// Error values returned by runtime_pollReset and runtime_pollWait. +// These must match the values in runtime/netpoll.go. +const ( + pollNoError = 0 + pollErrClosing = 1 + pollErrTimeout = 2 + pollErrNotPollable = 3 +) + +//go:norace +func (pd *PollDesc) Unblock() { + pd.mu.Lock() + defer pd.mu.Unlock() + + // logf("unblock %d", ctx.fd) + + pd.readable.SetBit(pollErrClosing, true) + pd.writable.SetBit(pollErrClosing, true) +} + +type Pollers struct { + canWrite bool + canRead bool + pollers []*PollDesc +} + +//go:norace +func (ps *Pollers) Add(p *PollDesc) { + if p.registeredIndex != -1 { + panic("help poller already registered") + } + p.writable.SetBit(pollNoError, ps.canWrite) + p.readable.SetBit(pollNoError, ps.canRead) + p.registeredIndex = len(ps.pollers) + ps.pollers = append(ps.pollers, p) +} + +//go:norace +func (ps *Pollers) Remove(p *PollDesc) { + if ps.pollers[p.registeredIndex] != p { + panic("help") + } + last := len(ps.pollers) - 1 + if last != p.registeredIndex { + ps.pollers[p.registeredIndex] = ps.pollers[last] + ps.pollers[p.registeredIndex].registeredIndex = p.registeredIndex + } + ps.pollers[last] = nil + ps.pollers = ps.pollers[:last] + p.registeredIndex = -1 +} + +func (ps *Pollers) NotifyCanRead(can bool) { + if ps.canRead == can { + return + } + ps.canRead = can + for _, p := range ps.pollers { + p.readable.SetBit(pollNoError, ps.canRead) + } +} + +func (ps *Pollers) NotifyCanWrite(can bool) { + if ps.canWrite == can { + return + } + ps.canWrite = can + for _, p := range ps.pollers { + p.writable.SetBit(pollNoError, ps.canWrite) + } +} diff --git a/internal/simulation/syscallabi/syscall.go b/internal/simulation/syscallabi/syscall.go new file mode 100644 index 0000000..f206874 --- /dev/null +++ b/internal/simulation/syscallabi/syscall.go @@ -0,0 +1,200 @@ +package syscallabi + +import ( + "unsafe" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/race" +) + +// Setup configures the gosim runtime to allocate a Syscall struct +// for each goroutine. +func Setup() { + gosimruntime.SetSyscallAllocator(func() unsafe.Pointer { + return unsafe.Pointer(&Syscall{}) + }) +} + +// GetGoroutineLocalSyscall returns the per-goroutine pre-allocated +// Syscall struct. +// +// Sharing Syscalls is safe because each goroutine invokes at most +// one syscall at a time. +// +// TODO: Just pool them instead? +func GetGoroutineLocalSyscall() *Syscall { + return (*Syscall)(gosimruntime.GetGoroutineLocalSyscall()) +} + +// Syscall holds the arguments and return values of gosim syscalls. +// +// Syscalls are calls from user code to high-level system calls in the os +// package. To prevent allocations, each goroutine has a single Syscall that +// gets reused and be retrieved using GetGoroutineLocalSyscall. +// +// Syscalls are different from upcalls, which calls in this package from user +// goroutines that execute code on the scheduler goroutine. +type Syscall struct { + OS OS + + Trap uintptr + Int0, Int1, Int2, Int3, Int4 uintptr + Ptr0, Ptr1, Ptr2, Ptr3, Ptr4 any + R0, R1 uintptr + RPtr0 any + Errno uintptr + + Sema uint32 +} + +// OS is the interface for types that implement the syscall logic. +// +// Implementations of this interface are generated by gensyscall. +type OS interface { + HandleSyscall(*Syscall) +} + +// Wait waits for the system call to be completed. +func (u *Syscall) Wait() { + gosimruntime.Semacquire(&u.Sema, false) +} + +// Complete marks the system call completed and lets Wait return. +func (u *Syscall) Complete() { + gosimruntime.Semrelease(&u.Sema) +} + +// The Dispatcher is the bridge between user space code and OS code. User space +// goroutines call Dispatcher.Dispatch and the dispatcher runs OS.HandleSyscall +// on the OS goroutine. +type Dispatcher struct { + syscalls chan *Syscall +} + +// NewDispatcher creates a new Dispatcher. +func NewDispatcher() Dispatcher { + return Dispatcher{ + syscalls: make(chan *Syscall), + } +} + +// handleSyscall is a wrapper around HandleSyscall, extracted from Run +// to mark it norace. +// +//go:norace +func handleSyscall(syscall *Syscall) { + syscall.OS.HandleSyscall(syscall) +} + +func (b Dispatcher) Run() { + for syscall := range b.syscalls { + // for-over-func erases go:norace inside the for loop, so + // use a helper function + handleSyscall(syscall) + } +} + +//go:norace +func (b Dispatcher) Dispatch(syscall *Syscall) { + // XXX: how does this interact with the deps we set up (maybe) in yield? + // XXX: should we instead not create the deps in Select? + race.Disable() + b.syscalls <- syscall + // XXX: HELP what happens if this is reused from a crashed goroutine??? + syscall.Wait() + race.Enable() +} + +// BoolToUintptr stores a boolean as a uintptr for syscall arguments +// or return values. +func BoolToUintptr(v bool) uintptr { + if v { + return 1 + } + return 0 +} + +// BoolFromUintptr extracts a boolean stored as a uintptr for syscall +// arguments or return values. +func BoolFromUintptr(v uintptr) bool { + return v != 0 +} + +// TODO: for the Views below we could register read/write race detector deps in +// userspace? + +// SliceView is a OS-wrapper around slices passed from user space. It lets +// the OS access user space memory without triggering the race detector. +type SliceView[T any] struct { + Ptr []T +} + +func NewSliceView[T any](ptr *T, len uintptr) SliceView[T] { + return SliceView[T]{ + Ptr: unsafe.Slice(ptr, len), + } +} + +func (b SliceView[T]) Len() int { + return len(b.Ptr) +} + +//go:norace +func (b SliceView[T]) Read(into []T) int { + n := len(into) + if len(b.Ptr) < n { + n = len(b.Ptr) + } + for i := range b.Ptr[:n] { + into[i] = b.Ptr[i] + } + return n +} + +//go:norace +func (b SliceView[T]) Write(from []T) int { + n := len(b.Ptr) + if len(from) < n { + n = len(from) + } + for i := range from[:n] { + b.Ptr[i] = from[i] + } + return n +} + +func (b SliceView[T]) SliceFrom(from int) SliceView[T] { + return SliceView[T]{Ptr: b.Ptr[from:]} +} + +func (b SliceView[T]) Slice(from, to int) SliceView[T] { + return SliceView[T]{Ptr: b.Ptr[from:to]} +} + +type ByteSliceView = SliceView[byte] + +// ValueView is a OS-wrapper around pointers passed from user space. It lets the +// OS access user space memory without triggering the race detector. +type ValueView[T any] struct { + underlying *T +} + +func NewValueView[T any](ptr *T) ValueView[T] { + return ValueView[T]{ + underlying: ptr, + } +} + +func (s ValueView[T]) UnsafePointer() unsafe.Pointer { + return unsafe.Pointer(s.underlying) +} + +//go:norace +func (s ValueView[T]) Get() T { + return *s.underlying +} + +//go:norace +func (s ValueView[T]) Set(v T) { + *s.underlying = v +} diff --git a/internal/simulation/userspace.go b/internal/simulation/userspace.go new file mode 100644 index 0000000..82ba962 --- /dev/null +++ b/internal/simulation/userspace.go @@ -0,0 +1,100 @@ +package simulation + +import ( + "context" + "log" + "log/slog" + "os" + "time" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +// Per-machine globals initialized in setupUserspace: + +var ( + linuxOS *LinuxOS + gosimOS *GosimOS // XXX: elsewhere? in machine itself? + currentMachineID int +) + +func CurrentMachineID() int { + return currentMachineID +} + +type gosimSlogHandler struct { + inner slog.Handler +} + +func (w gosimSlogHandler) Enabled(ctx context.Context, level slog.Level) bool { + return w.inner.Enabled(ctx, level) +} + +func (w gosimSlogHandler) Handle(ctx context.Context, r slog.Record) error { + r.AddAttrs(slog.Int("goroutine", gosimruntime.GetGoroutine())) + r.AddAttrs(slog.Int("step", gosimruntime.Step())) + return w.inner.Handle(ctx, r) +} + +func (w gosimSlogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return gosimSlogHandler{ + inner: w.inner.WithAttrs(attrs), + } +} + +func (w gosimSlogHandler) WithGroup(name string) slog.Handler { + return gosimSlogHandler{ + inner: w.inner.WithGroup(name), + } +} + +type gosimLogWriter struct{} + +func (w gosimLogWriter) Write(b []byte) (n int, err error) { + gosimruntime.WriteLog(b) + return len(b), nil +} + +func setupSlog(machineLabel string) { + // We play a funny game with the logger. There exists a default slog.Logger in + // every machine, since they all have their own set of globals. All loggers + // point to the same LogOut writer and are set up using the configuration set + // here. + + // TODO racedetector: make sure that using slog doesn't introduce sneaky + // happens-before (this currently happens in the default handler which + // uses both a pool and a lock) + + time.Local = time.UTC + + var level slog.Level + if err := level.UnmarshalText([]byte(os.Getenv("GOSIM_LOG_LEVEL"))); err != nil { + panic(err) + } + + // TODO: capture stdout? stderr? + + ho := slog.HandlerOptions{ + Level: level, + AddSource: true, + } + handler := slog.NewJSONHandler(gosimLogWriter{}, &ho) + + // set short file flag so that we'll capture source info. see slog.SetDefault internals + // XXX: test this? + log.SetFlags(log.Lshortfile) + slog.SetDefault(slog.New(gosimSlogHandler{inner: handler}).With("machine", machineLabel)) +} + +func setupUserspace(gosimOS_ *GosimOS, linuxOS_ *LinuxOS, machineID int, label string) { + // XXX: does logging work during global init? + gosimruntime.InitGlobals(false, false) + + // yikes... how do we order this? should machine exist first? should these happen in an init() in here SOMEHOW? short-circuit this package? + // XXX: provide directoyr + gosimOS = gosimOS_ + linuxOS = linuxOS_ + currentMachineID = machineID + + setupSlog(label) +} diff --git a/internal/testing/match.go b/internal/testing/match.go new file mode 100644 index 0000000..c1a18a1 --- /dev/null +++ b/internal/testing/match.go @@ -0,0 +1,320 @@ +// Copyright 2015 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Based on +// https://go.googlesource.com/go/+/refs/heads/master/src/testing/match.go + +package testing + +import ( + "fmt" + "os" + "strconv" + "strings" + "sync" +) + +// matcher sanitizes, uniques, and filters names of subtests and subbenchmarks. +type matcher struct { + filter filterMatch + skip filterMatch + matchFunc func(pat, str string) (bool, error) + + mu sync.Mutex + + // subNames is used to deduplicate subtest names. + // Each key is the subtest name joined to the deduplicated name of the parent test. + // Each value is the count of the number of occurrences of the given subtest name + // already seen. + subNames map[string]int32 +} + +type filterMatch interface { + // matches checks the name against the receiver's pattern strings using the + // given match function. + matches(name []string, matchString func(pat, str string) (bool, error)) (ok, partial bool) + + // verify checks that the receiver's pattern strings are valid filters by + // calling the given match function. + verify(name string, matchString func(pat, str string) (bool, error)) error +} + +// simpleMatch matches a test name if all of the pattern strings match in +// sequence. +type simpleMatch []string + +// alternationMatch matches a test name if one of the alternations match. +type alternationMatch []filterMatch + +// TODO: fix test_main to avoid race and improve caching, also allowing to +// eliminate this Mutex. +var matchMutex sync.Mutex + +func allMatcher() *matcher { + return newMatcher(nil, "", "", "") +} + +func newMatcher(matchString func(pat, str string) (bool, error), patterns, name, skips string) *matcher { + var filter, skip filterMatch + if patterns == "" { + filter = simpleMatch{} // always partial true + } else { + filter = splitRegexp(patterns) + if err := filter.verify(name, matchString); err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for %s\n", err) + os.Exit(1) + } + } + if skips == "" { + skip = alternationMatch{} // always false + } else { + skip = splitRegexp(skips) + if err := skip.verify("-test.skip", matchString); err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for %v\n", err) + os.Exit(1) + } + } + return &matcher{ + filter: filter, + skip: skip, + matchFunc: matchString, + subNames: map[string]int32{}, + } +} + +func (m *matcher) fullName(c *common, subname string) (name string, ok, partial bool) { + name = subname + + m.mu.Lock() + defer m.mu.Unlock() + + if c != nil && c.level > 0 { + name = m.unique(c.name, rewrite(subname)) + } + + matchMutex.Lock() + defer matchMutex.Unlock() + + // We check the full array of paths each time to allow for the case that a pattern contains a '/'. + elem := strings.Split(name, "/") + + // filter must match. + // accept partial match that may produce full match later. + ok, partial = m.filter.matches(elem, m.matchFunc) + if !ok { + return name, false, false + } + + // skip must not match. + // ignore partial match so we can get to more precise match later. + skip, partialSkip := m.skip.matches(elem, m.matchFunc) + if skip && !partialSkip { + return name, false, false + } + + return name, ok, partial +} + +// clearSubNames clears the matcher's internal state, potentially freeing +// memory. After this is called, T.Name may return the same strings as it did +// for earlier subtests. +func (m *matcher) clearSubNames() { + m.mu.Lock() + defer m.mu.Unlock() + clear(m.subNames) +} + +func (m simpleMatch) matches(name []string, matchString func(pat, str string) (bool, error)) (ok, partial bool) { + for i, s := range name { + if i >= len(m) { + break + } + if ok, _ := matchString(m[i], s); !ok { + return false, false + } + } + return true, len(name) < len(m) +} + +func (m simpleMatch) verify(name string, matchString func(pat, str string) (bool, error)) error { + for i, s := range m { + m[i] = rewrite(s) + } + // Verify filters before doing any processing. + for i, s := range m { + if _, err := matchString(s, "non-empty"); err != nil { + return fmt.Errorf("element %d of %s (%q): %s", i, name, s, err) + } + } + return nil +} + +func (m alternationMatch) matches(name []string, matchString func(pat, str string) (bool, error)) (ok, partial bool) { + for _, m := range m { + if ok, partial = m.matches(name, matchString); ok { + return ok, partial + } + } + return false, false +} + +func (m alternationMatch) verify(name string, matchString func(pat, str string) (bool, error)) error { + for i, m := range m { + if err := m.verify(name, matchString); err != nil { + return fmt.Errorf("alternation %d of %s", i, err) + } + } + return nil +} + +func splitRegexp(s string) filterMatch { + a := make(simpleMatch, 0, strings.Count(s, "/")) + b := make(alternationMatch, 0, strings.Count(s, "|")) + cs := 0 + cp := 0 + for i := 0; i < len(s); { + switch s[i] { + case '[': + cs++ + case ']': + if cs--; cs < 0 { // An unmatched ']' is legal. + cs = 0 + } + case '(': + if cs == 0 { + cp++ + } + case ')': + if cs == 0 { + cp-- + } + case '\\': + i++ + case '/': + if cs == 0 && cp == 0 { + a = append(a, s[:i]) + s = s[i+1:] + i = 0 + continue + } + case '|': + if cs == 0 && cp == 0 { + a = append(a, s[:i]) + s = s[i+1:] + i = 0 + b = append(b, a) + a = make(simpleMatch, 0, len(a)) + continue + } + } + i++ + } + + a = append(a, s) + if len(b) == 0 { + return a + } + return append(b, a) +} + +// unique creates a unique name for the given parent and subname by affixing it +// with one or more counts, if necessary. +func (m *matcher) unique(parent, subname string) string { + base := parent + "/" + subname + + for { + n := m.subNames[base] + if n < 0 { + panic("subtest count overflow") + } + m.subNames[base] = n + 1 + + if n == 0 && subname != "" { + prefix, nn := parseSubtestNumber(base) + if len(prefix) < len(base) && nn < m.subNames[prefix] { + // This test is explicitly named like "parent/subname#NN", + // and #NN was already used for the NNth occurrence of "parent/subname". + // Loop to add a disambiguating suffix. + continue + } + return base + } + + name := fmt.Sprintf("%s#%02d", base, n) + if m.subNames[name] != 0 { + // This is the nth occurrence of base, but the name "parent/subname#NN" + // collides with the first occurrence of a subtest *explicitly* named + // "parent/subname#NN". Try the next number. + continue + } + + return name + } +} + +// parseSubtestNumber splits a subtest name into a "#%02d"-formatted int32 +// suffix (if present), and a prefix preceding that suffix (always). +func parseSubtestNumber(s string) (prefix string, nn int32) { + i := strings.LastIndex(s, "#") + if i < 0 { + return s, 0 + } + + prefix, suffix := s[:i], s[i+1:] + if len(suffix) < 2 || (len(suffix) > 2 && suffix[0] == '0') { + // Even if suffix is numeric, it is not a possible output of a "%02" format + // string: it has either too few digits or too many leading zeroes. + return s, 0 + } + if suffix == "00" { + if !strings.HasSuffix(prefix, "/") { + // We only use "#00" as a suffix for subtests named with the empty + // string — it isn't a valid suffix if the subtest name is non-empty. + return s, 0 + } + } + + n, err := strconv.ParseInt(suffix, 10, 32) + if err != nil || n < 0 { + return s, 0 + } + return prefix, int32(n) +} + +// rewrite rewrites a subname to having only printable characters and no white +// space. +func rewrite(s string) string { + b := []byte{} + for _, r := range s { + switch { + case isSpace(r): + b = append(b, '_') + case !strconv.IsPrint(r): + s := strconv.QuoteRune(r) + b = append(b, s[1:len(s)-1]...) + default: + b = append(b, string(r)...) + } + } + return string(b) +} + +func isSpace(r rune) bool { + if r < 0x2000 { + switch r { + // Note: not the same as Unicode Z class. + case '\t', '\n', '\v', '\f', '\r', ' ', 0x85, 0xA0, 0x1680: + return true + } + } else { + if r <= 0x200a { + return true + } + switch r { + case 0x2028, 0x2029, 0x202f, 0x205f, 0x3000: + return true + } + } + return false +} diff --git a/internal/testing/missing.go b/internal/testing/missing.go new file mode 100644 index 0000000..0784e5c --- /dev/null +++ b/internal/testing/missing.go @@ -0,0 +1,88 @@ +package testing + +func AllocsPerRun(runs int, f func()) (avg float64) { + panic("not implemented") +} + +func CoverMode() { + panic("not implemented") +} + +func Coverage() float64 { + panic("not implemented") +} + +func Init() { + panic("not implemented") +} + +func Main(matchString func(pat, str string) (bool, error), tests []InternalTest, benchmarks []InternalBenchmark, examples []InternalExample) { + panic("not implemented") +} + +func RegisterCover(c Cover) { + panic("not implemented") +} + +func RunBenchmarks(matchString func(pat, str string) (bool, error), benchmarks []InternalBenchmark) { + panic("not implemented") +} + +func RunExamples(matchString func(pat, str string) (bool, error), examples []InternalExample) (ok bool) { + panic("not implemented") +} + +func RunTests(matchString func(pat, str string) (bool, error), tests []InternalTest) (ok bool) { + panic("not implemented") +} + +func Testing() bool { + panic("not implemented") +} + +func Verbose() bool { + panic("not implemented") +} + +type Cover struct { + Mode string + Counters map[string][]uint32 + Blocks map[string][]CoverBlock + CoveredPackages string +} + +type CoverBlock struct { + Line0 uint32 // Line number for block start. + Col0 uint16 // Column number for block start. + Line1 uint32 // Line number for block end. + Col1 uint16 // Column number for block end. + Stmts uint16 // Number of statements included in this block. +} + +type InternalBenchmark struct { + Name string + F func(b *B) +} + +type InternalExample struct { + Name string + F func() + Output string + Unordered bool +} + +type InternalFuzzTarget struct { + Name string + Fn func(f *F) +} + +type InternalTest struct { + Name string + F func(*T) +} + +type F struct{} + +type PB struct{} + +// TODO: missing methods diff --git a/internal/testing/testing.go b/internal/testing/testing.go new file mode 100644 index 0000000..dbab593 --- /dev/null +++ b/internal/testing/testing.go @@ -0,0 +1,1022 @@ +// Copyright 2009 The Go Authors. All rights reserved. Use of this source code +// is governed by a BSD-style license that can be found at +// https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +// Based on +// https://go.googlesource.com/go/+/refs/heads/master/src/testing/testing.go + +// TODO: support gosim test [-v] +// TODO: decide what to do with non-testing.T logs in gosim CLI + +// TODO: support gosim test -list +// TODO: support gosim test success/fail counts? + +// TODO: somehow translate testing package instead of this copy-paste mess? + +package testing + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "regexp" + "runtime" + "sync" + "sync/atomic" + "time" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/race" +) + +type writer struct{} + +func (w writer) Write(p []byte) (n int, err error) { + gosimruntime.WriteLog(p) + return len(p), nil +} + +var running sync.Map + +// highPrecisionTime represents a single point in time. +// On all systems except Windows, using time.Time is fine. +type highPrecisionTime struct { + now time.Time +} + +// highPrecisionTimeNow returns high precision time for benchmarking. +func highPrecisionTimeNow() highPrecisionTime { + return highPrecisionTime{now: time.Now()} +} + +// highPrecisionTimeSince returns duration since b. +func highPrecisionTimeSince(b highPrecisionTime) time.Duration { + return time.Since(b.now) +} + +// fmtDuration returns a string representing d in the form "87.00s". +func fmtDuration(d time.Duration) string { + return fmt.Sprintf("%.2fs", d.Seconds()) +} + +type TB interface { + Cleanup(func()) + Error(args ...any) + Errorf(format string, args ...any) + Fail() + FailNow() + Failed() bool + Fatal(args ...any) + Fatalf(format string, args ...any) + Helper() + Log(args ...any) + Logf(format string, args ...any) + Name() string + Setenv(key, value string) + Skip(args ...any) + SkipNow() + Skipf(format string, args ...any) + Skipped() bool + TempDir() string + + private() +} + +var ( + _ TB = (*T)(nil) + _ TB = (*B)(nil) +) + +const marker = byte(0x16) // ^V for framing + +type chattyPrinter struct { + w io.Writer + lastNameMu sync.Mutex // guards lastName + lastName string // last printed test name in chatty mode + json bool // -v=json output mode +} + +func newChattyPrinter(w io.Writer) *chattyPrinter { + return &chattyPrinter{w: w, json: false /* chatty.json */} +} + +// prefix is like chatty.prefix but using p.json instead of chatty.json. +// Using p.json allows tests to check the json behavior without modifying +// the global variable. For convenience, we allow p == nil and treat +// that as not in json mode (because it's not chatty at all). +func (p *chattyPrinter) prefix() string { + if p != nil && p.json { + return string(marker) + } + return "" +} + +// Updatef prints a message about the status of the named test to w. +// +// The formatted message must include the test name itself. +func (p *chattyPrinter) Updatef(testName, format string, args ...any) { + p.lastNameMu.Lock() + defer p.lastNameMu.Unlock() + + // Since the message already implies an association with a specific new test, + // we don't need to check what the old test name was or log an extra NAME line + // for it. (We're updating it anyway, and the current message already includes + // the test name.) + p.lastName = testName + fmt.Fprintf(p.w, p.prefix()+format, args...) +} + +// Printf prints a message, generated by the named test, that does not +// necessarily mention that tests's name itself. +func (p *chattyPrinter) Printf(testName, format string, args ...any) { + p.lastNameMu.Lock() + defer p.lastNameMu.Unlock() + + if p.lastName == "" { + p.lastName = testName + } else if p.lastName != testName { + fmt.Fprintf(p.w, "%s=== NAME %s\n", p.prefix(), testName) + p.lastName = testName + } + + fmt.Fprintf(p.w, format, args...) +} + +type common struct { + mu sync.Mutex + helperPCs map[uintptr]struct{} + helperNames map[string]struct{} // helperPCs converted to function names + + barrier chan bool + signal chan bool + + parent *common + + chatty *chattyPrinter // A copy of chattyPrinter, if the chatty flag is set. + + cleanupName string + cleanupPc []uintptr + + ran bool + failed bool + skipped bool + done bool + finished bool + + level int + creator []uintptr + + isParallel bool + + cleanups []func() + + cleanupStarted atomic.Bool // Registered cleanup callbacks have started to execute + + hasSub atomic.Bool // whether there are sub-benchmarks. + sub []*T // Queue of subtests to be run in parallel. + + lastRaceErrors atomic.Int64 // Max value of race.Errors seen during the test or its subtests. + raceErrorLogged atomic.Bool + + name string + + runner string + + duration time.Duration + start highPrecisionTime // Time test or benchmark started +} + +// resetRaces updates c.parent's count of data race errors (or the global count, +// if c has no parent), and updates c.lastRaceErrors to match. +// +// Any races that occurred prior to this call to resetRaces will +// not be attributed to c. +func (c *common) resetRaces() { + if c.parent == nil { + c.lastRaceErrors.Store(int64(race.Errors())) + } else { + c.lastRaceErrors.Store(c.parent.checkRaces()) + } +} + +// checkRaces checks whether the global count of data race errors has increased +// since c's count was last reset. +// +// If so, it marks c as having failed due to those races (logging an error for +// the first such race), and updates the race counts for the parents of c so +// that if they are currently suspended (such as in a call to T.Run) they will +// not log separate errors for the race(s). +// +// Note that multiple tests may be marked as failed due to the same race if they +// are executing in parallel. +func (c *common) checkRaces() (raceErrors int64) { + raceErrors = int64(race.Errors()) + for { + last := c.lastRaceErrors.Load() + if raceErrors <= last { + // All races have already been reported. + return raceErrors + } + if c.lastRaceErrors.CompareAndSwap(last, raceErrors) { + break + } + } + + if c.raceErrorLogged.CompareAndSwap(false, true) { + // This is the first race we've encountered for this test. + // Mark the test as failed, and log the reason why only once. + // (Note that the race detector itself will still write a goroutine + // dump for any further races it detects.) + c.Errorf("race detected during execution of test") + } + + // Update the parent(s) of this test so that they don't re-report the race. + parent := c.parent + for parent != nil { + for { + last := parent.lastRaceErrors.Load() + if raceErrors <= last { + // This race was already reported by another (likely parallel) subtest. + return raceErrors + } + if parent.lastRaceErrors.CompareAndSwap(last, raceErrors) { + break + } + } + parent = parent.parent + } + + return raceErrors +} + +type T struct { + common + context *testContext +} + +func (t *T) report() { + if t.parent == nil { + return + } + + dstr := fmtDuration(t.duration) + format := "--- %s: %s (%s)\n" + if t.Failed() { + t.flushToParent(t.name, format, "FAIL", t.name, dstr) + } else if t.chatty != nil { + if t.Skipped() { + t.flushToParent(t.name, format, "SKIP", t.name, dstr) + } else { + t.flushToParent(t.name, format, "PASS", t.name, dstr) + } + } +} + +//go:norace +func (c *common) Fail() { + if c.parent != nil { + c.parent.Fail() + } + c.mu.Lock() + defer c.mu.Unlock() + if c.done { + panic("Fail in goroutine after " + c.name + " has completed") + } + c.failed = true +} + +func (c *common) setRan() { + if c.parent != nil { + c.parent.setRan() + } + c.mu.Lock() + defer c.mu.Unlock() + c.ran = true +} + +//go:norace +func (c *common) Failed() bool { + c.mu.Lock() + defer c.mu.Unlock() + + // TODO: race error count? + + return c.failed +} + +func (c *common) FailNow() { + c.Fail() + + c.mu.Lock() + c.finished = true + c.mu.Unlock() + + runtime.Goexit() + + // XXX: this is interesting... do we abort here or all the way up? do we make t.Run() mandatory + // gosimruntime.SetAbortError(gosimruntime.ErrAborted) +} + +func (c *common) log(level slog.Level, msg string, attrs ...slog.Attr) { + ctx := context.Background() + l := slog.Default() + + if !l.Enabled(ctx, level) { + return + } + + frame := c.frameSkip(2) // skip this function, this function's caller + pc := frame.PC // XXX: this is lossy if a helper got inlined. help can't do anything about that. + + // var pc uintptr + // if !internal.IgnorePC { + // var pcs [1]uintptr + // skip [runtime.Callers, this function, this function's caller] + // runtime.Callers(3, pcs[:]) + // pc = pcs[0] + // } + r := slog.NewRecord(time.Now(), level, msg, pc) + r.AddAttrs(attrs...) + l.Handler().Handle(ctx, r) +} + +func (c *common) Log(a ...any) { + // XXX: important, use fmt.Sprintln, that will be translated, to handle eg. reflect + c.log(slog.LevelInfo, fmt.Sprint(a...), slog.String("method", "t.Log")) +} + +func (c *common) Logf(format string, a ...any) { + c.log(slog.LevelInfo, fmt.Sprintf(format, a...), slog.String("method", "t.Logf")) +} + +func (c *common) Error(a ...any) { + c.log(slog.LevelError, fmt.Sprint(a...), slog.String("method", "t.Error")) + c.Fail() +} + +func (c *common) Errorf(format string, a ...any) { + c.log(slog.LevelError, fmt.Sprintf(format, a...), slog.String("method", "t.Errorf")) + c.Fail() +} + +func (c *common) Fatal(a ...any) { + c.log(slog.LevelError, fmt.Sprint(a...), slog.String("method", "t.Fatal")) + c.FailNow() +} + +func (c *common) Fatalf(format string, a ...any) { + c.log(slog.LevelError, fmt.Sprintf(format, a...), slog.String("method", "t.Fatalf")) + c.FailNow() +} + +func (c *common) Skip(a ...any) { + c.log(slog.LevelInfo, fmt.Sprint(a...), slog.String("method", "t.Skip")) + c.SkipNow() +} + +func (c *common) Skipf(format string, a ...any) { + c.log(slog.LevelInfo, fmt.Sprintf(format, a...), slog.String("method", "t.Skipf")) + c.SkipNow() +} + +func (c *common) Skipped() bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.skipped +} + +func (c *common) SkipNow() { + c.mu.Lock() + c.skipped = true + c.finished = true + c.mu.Unlock() + runtime.Goexit() +} + +func (c *common) Name() string { + return c.name +} + +// XXX: copied from standard library... think about correctness? + +// The maximum number of stack frames to go through when skipping helper functions for +// the purpose of decorating log messages. +const maxStackLen = 50 + +// callerName gives the function name (qualified with a package path) +// for the caller after skip frames (where 0 means the current function). +func callerName(skip int) string { + var pc [1]uintptr + n := runtime.Callers(skip+2, pc[:]) // skip + runtime.Callers + callerName + if n == 0 { + panic("testing: zero callers found") + } + return pcToName(pc[0]) +} + +func pcToName(pc uintptr) string { + pcs := []uintptr{pc} + frames := runtime.CallersFrames(pcs) + frame, _ := frames.Next() + return frame.Function +} + +// frameSkip searches, starting after skip frames, for the first caller frame +// in a function not marked as a helper and returns that frame. +// The search stops if it finds a tRunner function that +// was the entry point into the test and the test is not a subtest. +// This function must be called with c.mu held. +func (c *common) frameSkip(skip int) runtime.Frame { + // If the search continues into the parent test, we'll have to hold + // its mu temporarily. If we then return, we need to unlock it. + shouldUnlock := false + defer func() { + if shouldUnlock { + c.mu.Unlock() + } + }() + var pc [maxStackLen]uintptr + // Skip two extra frames to account for this function + // and runtime.Callers itself. + n := runtime.Callers(skip+2, pc[:]) + if n == 0 { + panic("testing: zero callers found") + } + frames := runtime.CallersFrames(pc[:n]) + var firstFrame, prevFrame, frame runtime.Frame + for more := true; more; prevFrame = frame { + frame, more = frames.Next() + if frame.Function == "runtime.gopanic" { + continue + } + if frame.Function == c.cleanupName { + frames = runtime.CallersFrames(c.cleanupPc) + continue + } + if firstFrame.PC == 0 { + firstFrame = frame + } + if frame.Function == c.runner { + // We've gone up all the way to the tRunner calling + // the test function (so the user must have + // called tb.Helper from inside that test function). + // If this is a top-level test, only skip up to the test function itself. + // If we're in a subtest, continue searching in the parent test, + // starting from the point of the call to Run which created this subtest. + if c.level > 1 { + frames = runtime.CallersFrames(c.creator) + parent := c.parent + // We're no longer looking at the current c after this point, + // so we should unlock its mu, unless it's the original receiver, + // in which case our caller doesn't expect us to do that. + if shouldUnlock { + c.mu.Unlock() + } + c = parent + // Remember to unlock c.mu when we no longer need it, either + // because we went up another nesting level, or because we + // returned. + shouldUnlock = true + c.mu.Lock() + continue + } + return prevFrame + } + // If more helper PCs have been added since we last did the conversion + c.mu.Lock() + if c.helperNames == nil { + c.helperNames = make(map[string]struct{}) + for pc := range c.helperPCs { + c.helperNames[pcToName(pc)] = struct{}{} + } + } + if _, ok := c.helperNames[frame.Function]; !ok { + c.mu.Unlock() + // Found a frame that wasn't inside a helper function. + return frame + } + c.mu.Unlock() + } + return firstFrame +} + +// flushToParent writes c.output to the parent after first writing the header +// with the given format and arguments. +func (c *common) flushToParent(testName, format string, args ...any) { + p := c.parent + p.mu.Lock() + defer p.mu.Unlock() + + c.mu.Lock() + defer c.mu.Unlock() + + /* + if len(c.output) > 0 { + // Add the current c.output to the print, + // and then arrange for the print to replace c.output. + // (This displays the logged output after the --- FAIL line.) + format += "%s" + args = append(args[:len(args):len(args)], c.output) + c.output = c.output[:0] + } + */ + + if c.chatty != nil /* && (p.w == c.chatty.w || c.chatty.json) */ { + // We're flushing to the actual output, so track that this output is + // associated with a specific test (and, specifically, that the next output + // is *not* associated with that test). + // + // Moreover, if c.output is non-empty it is important that this write be + // atomic with respect to the output of other tests, so that we don't end up + // with confusing '=== NAME' lines in the middle of our '--- PASS' block. + // Neither humans nor cmd/test2json can parse those easily. + // (See https://go.dev/issue/40771.) + // + // If test2json is used, we never flush to parent tests, + // so that the json stream shows subtests as they finish. + // (See https://go.dev/issue/29811.) + c.chatty.Updatef(testName, format, args...) + } else { + // We're flushing to the output buffer of the parent test, which will + // itself follow a test-name header when it is finally flushed to stdout. + // fmt.Fprintf(p.w, c.chatty.prefix()+format, args...) + } +} + +func (c *common) Helper() { + c.mu.Lock() + defer c.mu.Unlock() + if c.helperPCs == nil { + c.helperPCs = make(map[uintptr]struct{}) + } + // repeating code from callerName here to save walking a stack frame + var pc [1]uintptr + n := runtime.Callers(2, pc[:]) // skip runtime.Callers + Helper + if n == 0 { + panic("testing: zero callers found") + } + if _, found := c.helperPCs[pc[0]]; !found { + c.helperPCs[pc[0]] = struct{}{} + c.helperNames = nil // map will be recreated next time it is needed + } +} + +func (c *common) Cleanup(f func()) { + var pc [maxStackLen]uintptr + // Skip two extra frames to account for this function and runtime.Callers itself. + n := runtime.Callers(2, pc[:]) + cleanupPc := pc[:n] + + fn := func() { + defer func() { + c.mu.Lock() + defer c.mu.Unlock() + c.cleanupName = "" + c.cleanupPc = nil + }() + + name := callerName(0) + c.mu.Lock() + c.cleanupName = name + c.cleanupPc = cleanupPc + c.mu.Unlock() + + f() + } + + c.mu.Lock() + defer c.mu.Unlock() + c.cleanups = append(c.cleanups, fn) +} + +// panicHandling controls the panic handling used by runCleanup. +type panicHandling int + +const ( + normalPanic panicHandling = iota + recoverAndReturnPanic +) + +// runCleanup is called at the end of the test. +// If ph is recoverAndReturnPanic, it will catch panics, and return the +// recovered value if any. +func (c *common) runCleanup(ph panicHandling) (panicVal any) { + c.cleanupStarted.Store(true) + defer c.cleanupStarted.Store(false) + + if ph == recoverAndReturnPanic { + defer func() { + panicVal = recover() + }() + } + + // Make sure that if a cleanup function panics, + // we still run the remaining cleanup functions. + defer func() { + c.mu.Lock() + recur := len(c.cleanups) > 0 + c.mu.Unlock() + if recur { + c.runCleanup(normalPanic) + } + }() + + for { + var cleanup func() + c.mu.Lock() + if len(c.cleanups) > 0 { + last := len(c.cleanups) - 1 + cleanup = c.cleanups[last] + c.cleanups = c.cleanups[:last] + } + c.mu.Unlock() + if cleanup == nil { + return nil + } + cleanup() + } +} + +func (c *common) TempDir() string { + panic("not implemented") +} + +func (c *common) private() {} + +func (t *T) Parallel() { + if t.isParallel { + panic("testing: t.Parallel called multiple times") + } + t.isParallel = true + if t.parent.barrier == nil { + // T.Parallel has no effect when fuzzing. + // Multiple processes may run in parallel, but only one input can run at a + // time per process so we can attribute crashes to specific inputs. + return + } + + // We don't want to include the time we spend waiting for serial tests + // in the test duration. Record the elapsed time thus far and reset the + // timer afterwards. + t.duration += highPrecisionTimeSince(t.start) + + // Add to the list of tests to be released by the parent. + t.parent.sub = append(t.parent.sub, t) + + // Report any races during execution of this test up to this point. + // + // We will assume that any races that occur between here and the point where + // we unblock are not caused by this subtest. That assumption usually holds, + // although it can be wrong if the test spawns a goroutine that races in the + // background while the rest of the test is blocked on the call to Parallel. + // If that happens, we will misattribute the background race to some other + // test, or to no test at all — but that false-negative is so unlikely that it + // is not worth adding race-report noise for the common case where the test is + // completely suspended during the call to Parallel. + t.checkRaces() + + if t.chatty != nil { + t.chatty.Updatef(t.name, "=== PAUSE %s\n", t.name) + } + running.Delete(t.name) + + t.signal <- true // Release calling test. + <-t.parent.barrier // Wait for the parent test to complete. + t.context.waitParallel() + + if t.chatty != nil { + t.chatty.Updatef(t.name, "=== CONT %s\n", t.name) + } + running.Store(t.name, highPrecisionTimeNow()) + t.start = highPrecisionTimeNow() + + // Reset the local race counter to ignore any races that happened while this + // goroutine was blocked, such as in the parent test or in other parallel + // subtests. + // + // (Note that we don't call parent.checkRaces here: + // if other parallel subtests have already introduced races, we want to + // let them report those races instead of attributing them to the parent.) + t.lastRaceErrors.Store(int64(race.Errors())) +} + +func (t *T) Setenv(key, value string) { + panic("not implemented") +} + +type B struct { + common +} + +func (b *B) Setenv(key, value string) { + panic("not implemented") +} + +func (t *T) Deadline() (time.Time, bool) { + return time.Time{}, false +} + +var errNilPanicOrGoexit = errors.New("test executed panic(nil) or runtime.Goexit") + +func tRunner(t *T, fn func(t *T)) { + t.runner = callerName(0) + + defer func() { + t.checkRaces() + + // TODO(#61034): This is the wrong place for this check. + if t.Failed() { + // TODO: numFailed (numSkipped etc?) + // numFailed.Add(1) + } + + err := recover() + signal := true + + t.mu.Lock() + finished := t.finished + t.mu.Unlock() + + if !finished && err == nil { + err = errNilPanicOrGoexit + for p := t.parent; p != nil; p = p.parent { + p.mu.Lock() + finished = p.finished + p.mu.Unlock() + if finished { + // TODO: parallel check + signal = false + break + } + } + } + + // TODO: err != nil && fuzzing + + didPanic := false + defer func() { + if didPanic { + return + } + if err != nil { + panic(err) + } + running.Delete(t.name) + t.signal <- signal + }() + + doPanic := func(err any) { + t.Fail() + if r := t.runCleanup(recoverAndReturnPanic); r != nil { + t.Logf("cleanup panicked with %v", r) + } + for root := &t.common; root.parent != nil; root = root.parent { + root.mu.Lock() + root.duration += highPrecisionTimeSince(root.start) + d := root.duration + root.mu.Unlock() + root.flushToParent(root.name, "--- FAIL: %s (%s)\n", root.name, fmtDuration(d)) + if r := root.parent.runCleanup(recoverAndReturnPanic); r != nil { + // TODO: huh + // fmt.Fprintf(root.parent.w, "cleanup panicked with %v", r) + } + } + didPanic = true + panic(err) + } + if err != nil { + doPanic(err) + } + + t.duration += highPrecisionTimeSince(t.start) + + if len(t.sub) > 0 { + // Run parallel subtests. + + // Decrease the running count for this test and mark it as no longer running. + t.context.release() + running.Delete(t.name) + + // Release the parallel subtests. + close(t.barrier) + // Wait for subtests to complete. + for _, sub := range t.sub { + <-sub.signal + } + + // Run any cleanup callbacks, marking the test as running + // in case the cleanup hangs. + cleanupStart := highPrecisionTimeNow() + running.Store(t.name, cleanupStart) + err := t.runCleanup(recoverAndReturnPanic) + t.duration += highPrecisionTimeSince(cleanupStart) + if err != nil { + doPanic(err) + } + t.checkRaces() + if !t.isParallel { + // Reacquire the count for sequential tests. See comment in Run. + t.context.waitParallel() + } + } else if t.isParallel { + // Only release the count for this test if it was run as a parallel + // test. See comment in Run method. + t.context.release() + } + + t.report() + t.done = true // XXX: not locked because?? + if t.parent != nil && !t.hasSub.Load() { + t.setRan() + } + }() + defer func() { + if len(t.sub) == 0 { + t.runCleanup(normalPanic) + } + }() + + t.start = highPrecisionTimeNow() + t.resetRaces() // XXX: TODO + fn(t) + + // code beyond here will not be executed when FailNow is invoked + t.mu.Lock() + t.finished = true + t.mu.Unlock() +} + +// Run runs f as a subtest of t called name. It runs f in a separate goroutine +// and blocks until f returns or calls t.Parallel to become a parallel test. +// Run reports whether f succeeded (or at least did not fail before calling t.Parallel). +// +// Run may be called simultaneously from multiple goroutines, but all such calls +// must return before the outer test function for t returns. +func (t *T) Run(name string, f func(t *T)) bool { + // if t.cleanupStarted.Load() { + // panic("testing: t.Run called during t.Cleanup") + // } + + t.hasSub.Store(true) + testName, ok, _ := t.context.match.fullName(&t.common, name) + if !ok /* || shouldFailFast() */ { + return true + } + // Record the stack trace at the point of this call so that if the subtest + // function - which runs in a separate stack - is marked as a helper, we can + // continue walking the stack into the parent test. + var pc [maxStackLen]uintptr + n := runtime.Callers(2, pc[:]) + t = &T{ + common: common{ + barrier: make(chan bool), + signal: make(chan bool, 1), + name: testName, + parent: &t.common, + level: t.level + 1, + creator: pc[:n], + chatty: t.chatty, + }, + context: t.context, + } + // t.w = indenter{&t.common} + + if t.chatty != nil { + t.chatty.Updatef(t.name, "=== RUN %s\n", t.name) + } + running.Store(t.name, highPrecisionTimeNow()) + + // Instead of reducing the running count of this test before calling the + // tRunner and increasing it afterwards, we rely on tRunner keeping the + // count correct. This ensures that a sequence of sequential tests runs + // without being preempted, even when their parent is a parallel test. This + // may especially reduce surprises if *parallel == 1. + go tRunner(t, f) + + // The parent goroutine will block until the subtest either finishes or calls + // Parallel, but in general we don't know whether the parent goroutine is the + // top-level test function or some other goroutine it has spawned. + // To avoid confusing false-negatives, we leave the parent in the running map + // even though in the typical case it is blocked. + + if !<-t.signal { + // At this point, it is likely that FailNow was called on one of the + // parent tests by one of the subtests. Continue aborting up the chain. + runtime.Goexit() + } + + // if t.chatty != nil && t.chatty.json { + // t.chatty.Updatef(t.parent.name, "=== NAME %s\n", t.parent.name) + // } + return !t.failed +} + +// testContext holds all fields that are common to all tests. This includes +// synchronization primitives to run at most *parallel tests. +type testContext struct { + match *matcher + deadline time.Time + + // isFuzzing is true in the context used when generating random inputs + // for fuzz targets. isFuzzing is false when running normal tests and + // when running fuzz tests as unit tests (without -fuzz or when -fuzz + // does not match). + isFuzzing bool + + mu sync.Mutex + + // Channel used to signal tests that are ready to be run in parallel. + startParallel chan bool + + // running is the number of tests currently running in parallel. + // This does not include tests that are waiting for subtests to complete. + running int + + // numWaiting is the number tests waiting to be run in parallel. + numWaiting int + + // maxParallel is a copy of the parallel flag. + maxParallel int +} + +func newTestContext(maxParallel int, m *matcher) *testContext { + return &testContext{ + match: m, + startParallel: make(chan bool), + maxParallel: maxParallel, + running: 1, // Set the count to 1 for the main (sequential) test. + } +} + +func (c *testContext) waitParallel() { + c.mu.Lock() + if c.running < c.maxParallel { + c.running++ + c.mu.Unlock() + return + } + c.numWaiting++ + c.mu.Unlock() + <-c.startParallel +} + +func (c *testContext) release() { + c.mu.Lock() + if c.numWaiting == 0 { + c.running-- + c.mu.Unlock() + return + } + c.numWaiting-- + c.mu.Unlock() + c.startParallel <- true // Pick a waiting test to be run. +} + +func runTests(match string, skip string, tests []InternalTest) (ran, ok bool) { + ok = true + + parallel := 1 // XXX: pass later? + + ctx := newTestContext(parallel, newMatcher(regexp.MatchString, match, "-test.run", skip)) + t := &T{ + common: common{ + signal: make(chan bool, 1), + barrier: make(chan bool), + // w: os.Stdout, + chatty: newChattyPrinter(writer{}), + }, + context: ctx, + } + tRunner(t, func(t *T) { + for _, test := range tests { + t.Run(test.Name, test.F) + } + }) + select { + case <-t.signal: + default: + panic("internal error: tRunner exited without sending on t.signal") + } + ok = ok && !t.Failed() + ran = ran || t.ran + + return ran, ok +} + +func Entrypoint(match string, skip string, tests []gosimruntime.Test) bool { + var parsedTests []InternalTest + for _, test := range tests { + parsedTests = append(parsedTests, InternalTest{ + Name: test.Name, + F: test.Test.(func(*T)), + }) + } + _, ok := runTests(match, skip, parsedTests) + return ok +} + +func Short() bool { + return false +} diff --git a/internal/tests/Makefile b/internal/tests/Makefile new file mode 100644 index 0000000..7817a45 --- /dev/null +++ b/internal/tests/Makefile @@ -0,0 +1,5 @@ +all: + mkdir -p testpb + docker run --platform=linux/amd64 -v $(PWD):/defs namely/protoc-all -f testpb.proto -l go + mv gen/pb-go/github.com/jellevandenhooff/gosim/internal/tests/testpb/*.pb.go testpb/ + rm -rf gen diff --git a/internal/tests/behavior/chan_test.go b/internal/tests/behavior/chan_test.go new file mode 100644 index 0000000..f5a4c9a --- /dev/null +++ b/internal/tests/behavior/chan_test.go @@ -0,0 +1,336 @@ +package behavior_test + +import ( + "fmt" + "log" + "sync" + "testing" + "time" +) + +func TestChanSendRecvClose(t *testing.T) { + ch := make(chan int, 10) + + for i := 0; i < 5; i++ { + ch <- i + } + close(ch) + + for i := 0; i < 5; i++ { + v, ok := <-ch + if !ok || v != i { + t.Error(v, ok) + } + } + _, ok := <-ch + if ok { + t.Error(ok) + } +} + +func TestChanSelectRecvNilInterface(t *testing.T) { + ch := make(chan error, 1) + other := make(chan struct{}) + + ch <- nil + + select { + case err := <-ch: + if err != nil { + t.Fatal(err) + } + case <-other: + } +} + +func TestGoZeroChanSendRecvClose(t *testing.T) { + ch := make(chan int) + + sent0 := false + sent1 := false + + go func() { + ch <- 0 + sent0 = true + ch <- 1 + sent1 = true + close(ch) + }() + + // XXX: want to spin here... + if sent0 { + t.Error("should not have sent 0") + } + if v, ok := <-ch; !ok || v != 0 { + t.Error(v) + } + if sent1 { + t.Error("should not have sent 1") + } + if v, ok := <-ch; !ok || v != 1 { + t.Error(v) + } + if _, ok := <-ch; ok { + t.Error(ok) + } +} + +func TestGoChanBlock(t *testing.T) { + sent := make([]bool, 20) + sentCh := make([]chan struct{}, 20) + for i := 0; i < 20; i++ { + sentCh[i] = make(chan struct{}) + } + + ch := make(chan int, 10) + + go func() { + for i := 0; i < 20; i++ { + ch <- i + sent[i] = true + close(sentCh[i]) + } + }() + + <-sentCh[9] + // XXX: time.Sleep here, spin??? have introspection API that says other goroutine is blocked??? + if sent[10] { + t.Error("sent 10") + } + + v, ok := <-ch + if !ok || v != 0 { + t.Error(v) + } + + <-sentCh[10] + // XXX: time.Sleep here, spin??? + if sent[11] { + t.Error("sent 10") + } + + /* + // XXX: without this test fails because we have blocked goroutine that + // will never finish. handle somehow? + for i := 1; i < 20; i++ { + v, ok := <-ch + if !ok || v != i { + t.Error(v) + } + } + */ +} + +func TestGoChanDeterministic(t *testing.T) { + ch := make(chan int, 10) + + for i := 0; i < 5; i++ { + i := i + go func() { + for j := 0; j < 5; j++ { + ch <- i + } + }() + } + + result := "" + for i := 0; i < 5*5; i++ { + v, ok := <-ch + if !ok { + t.Error(ok) + } + result += fmt.Sprintf("%d", v) + } + + // XXX: assert this string is the same on every run + log.Print(result) + // return s +} + +func TestGoSelectBasicRecv(t *testing.T) { + ch1 := make(chan int, 10) + ch2 := make(chan int, 10) + + go func() { + for i := 0; i < 5; i++ { + ch1 <- 0 + } + }() + go func() { + for i := 0; i < 5; i++ { + ch2 <- 1 + } + }() + + result := "" + + for i := 0; i < 10; i++ { + select { + case v, ok := <-ch1: + if !ok { + t.Error(ok) + } + result += fmt.Sprint(v) + + case v, ok := <-ch2: + if !ok { + t.Error(ok) + } + result += fmt.Sprint(v) + } + } + + // XXX: assert this string is the same on every run + log.Print(result) + // return s +} + +func TestGoSelectZeroRecv(t *testing.T) { + ch1 := make(chan int) + ch2 := make(chan int) + + go func() { + for i := 0; i < 5; i++ { + ch1 <- 0 + } + }() + go func() { + for i := 0; i < 5; i++ { + ch2 <- 1 + } + }() + + result := "" + + for i := 0; i < 10; i++ { + select { + case v, ok := <-ch1: + if !ok { + t.Error(ok) + } + result += fmt.Sprint(v) + + case v, ok := <-ch2: + if !ok { + t.Error(ok) + } + result += fmt.Sprint(v) + } + } + + // XXX: assert this string is the same on every run + log.Print(result) + // return s +} + +func TestChanSendSelf(t *testing.T) { + ch := make(chan int) + + done := time.After(1 * time.Second) + + select { + case <-ch: + t.Error("help") + case ch <- 10: + t.Error("help") + case <-done: + } +} + +func TestChanMultipleZeroSelect(t *testing.T) { + a := make(chan int) + b := make(chan int) + + var mu sync.Mutex + count := 0 + + go func() { + a <- 0 + mu.Lock() + count++ + mu.Unlock() + }() + + go func() { + b <- 0 + mu.Lock() + count++ + mu.Unlock() + }() + + select { + case <-a: + case <-b: + } + + time.Sleep(10 * time.Millisecond) + + mu.Lock() + defer mu.Unlock() + if count != 1 { + t.Error("help", count) + } +} + +func TestChanRange(t *testing.T) { + ch := make(chan int) + + go func() { + for i := 0; i < 5; i++ { + ch <- i + } + close(ch) + }() + + j := 0 + for i := range ch { + if i != j { + t.Error("bad") + } + j++ + } + if j != 5 { + t.Error("bad") + } +} + +func TestChanLenCap(t *testing.T) { + ch := make(chan int) + if cap(ch) != 0 { + t.Error("bad") + } + + ch = make(chan int, 5) + if cap(ch) != 5 { + t.Error("bad") + } + + if len(ch) != 0 { + t.Error("bad") + } + + for i := 1; i <= 5; i++ { + ch <- i + if len(ch) != i { + t.Error("bad") + } + } + + for i := 1; i <= 5; i++ { + j := <-ch + if i != j { + t.Error("bad") + } + if len(ch) != 5-i { + t.Error("bad") + } + } +} + +// XXX: what if you do something crazy like send and recv on the same channel in a select? + +// XXX: what if you have a select on a zero-length channel that has a writer ready to go + +// XXX: test Select prefers non-default? + +// XXX: test Select randomness? diff --git a/internal/tests/behavior/context_test.go b/internal/tests/behavior/context_test.go new file mode 100644 index 0000000..09f5944 --- /dev/null +++ b/internal/tests/behavior/context_test.go @@ -0,0 +1,91 @@ +//go:build sim + +package behavior_test + +import ( + "context" + "testing" + "time" +) + +func TestContextCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + cancel() + }() + + <-ctx.Done() +} + +func TestContextCancelInner(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + inner, innerCancel := context.WithCancel(ctx) + defer innerCancel() + + cancelDone := make(chan struct{}) + + go func() { + cancel() + close(cancelDone) + }() + + <-inner.Done() + <-cancelDone +} + +func TestContextTimeout(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + before := time.Now() + <-ctx.Done() + after := time.Now() + + if delay := after.Sub(before); delay != 5*time.Second { + t.Error("bad delay", delay) + } +} + +func TestContextDeadline(t *testing.T) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(10*time.Second)) + defer cancel() + + before := time.Now() + <-ctx.Done() + after := time.Now() + + if delay := after.Sub(before); delay != 10*time.Second { + t.Error("bad delay", delay) + } +} + +func TestContextTimeoutInnerStricter(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + inner, innerCancel := context.WithTimeout(ctx, 2*time.Second) + defer innerCancel() + + before := time.Now() + <-inner.Done() + after := time.Now() + + if delay := after.Sub(before); delay != 2*time.Second { + t.Error("bad delay", delay) + } +} + +func TestContextTimeoutInnerLooser(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + inner, innerCancel := context.WithTimeout(ctx, 10*time.Second) + defer innerCancel() + + before := time.Now() + <-inner.Done() + after := time.Now() + + if delay := after.Sub(before); delay != 5*time.Second { + t.Error("bad delay", delay) + } +} diff --git a/internal/tests/behavior/crash_meta_test.go b/internal/tests/behavior/crash_meta_test.go new file mode 100644 index 0000000..cecc517 --- /dev/null +++ b/internal/tests/behavior/crash_meta_test.go @@ -0,0 +1,47 @@ +//go:build !sim + +package behavior_test + +import ( + "log" + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestCrashRestartFilesystemPartial(t *testing.T) { + contentsCount := make(map[string]int) + mt := metatesting.ForCurrentPackage(t) + for seed := int64(0); seed < 100; seed++ { + // XXX: this testing API is clunky... + // config := gosim.ConfigWithNSeeds(100) + // config.CaptureLog = true + // XXX: i don't like this log level override anymore + // config.LogLevelOverride = "INFO" + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestCrashRestartFilesystemPartialInner", + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + if run.Failed { + t.Fatal("run failed") + } + output := metatesting.ParseLog(run.LogOutput) + contents := metatesting.MustFindLogValue(output, "read", "contents").(string) + contentsCount[contents]++ + } + log.Println(contentsCount) + // expected either no file, a file that hasn't been resized, a resized file + // with no contents, or a file with its expected contents + for _, expected := range []string{"", "\x00\x00\x00\x00\x00", "hello", ""} { + if contentsCount[expected] == 0 { + t.Errorf("expected to see %q, got 0", expected) + } + delete(contentsCount, expected) + } + if len(contentsCount) > 0 { + t.Fatalf("unexpected results %v", contentsCount) + } +} diff --git a/internal/tests/behavior/crash_test.go b/internal/tests/behavior/crash_test.go new file mode 100644 index 0000000..9442ede --- /dev/null +++ b/internal/tests/behavior/crash_test.go @@ -0,0 +1,760 @@ +//go:build sim + +package behavior_test + +import ( + "errors" + "io" + "log" + "log/slog" + "net" + "net/netip" + "os" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/jellevandenhooff/gosim" + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func TestCrashTimer(t *testing.T) { + var before, after atomic.Bool + + // token to allow machine to read before, after + gosimruntime.TestRaceToken.Release() + + m1 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // test a timer can fire + time.AfterFunc(1*time.Second, func() { + before.Store(true) + }) + select {} + }) + + m2 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // test a crashed timer does not fire + time.AfterFunc(5*time.Second, func() { + after.Store(true) + panic("help") + }) + select {} + }) + + time.Sleep(2 * time.Second) + m1.Crash() + m2.Crash() + time.Sleep(10 * time.Second) + + if !before.Load() { + t.Error("expected before") + } + if after.Load() { + t.Error("did not expect after") + } +} + +func TestCrashTimersComplicatedHeap(t *testing.T) { + var timers []*time.Timer + for i := 0; i < 10; i++ { + timers = append(timers, time.NewTimer(time.Duration(i+1)*time.Second)) + } + + m := gosim.NewSimpleMachine(func() { + var timers []*time.Timer + for i := 0; i < 10; i++ { + timers = append(timers, time.NewTimer(time.Duration(i+1)*time.Second)) + } + select {} + }) + time.Sleep(time.Millisecond) + m.Crash() + for i, timer := range timers { + if i%2 == 0 { + timer.Stop() + } + } + // XXX: when the heap indexes are not adjusted this test times out??? + // (that is "h.timers[j].pos = j") + // XXX: check timer heap invariants +} + +func TestCrashSleep(t *testing.T) { + var before, after atomic.Bool + + // token to allow machine to read before, after + gosimruntime.TestRaceToken.Release() + + m1 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // test Sleep works + time.Sleep(1 * time.Second) + before.Store(true) + }) + + m2 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // test a crashed Sleep stops + time.Sleep(5 * time.Second) + after.Store(true) + panic("help") + }) + + time.Sleep(2 * time.Second) + m1.Crash() + m2.Crash() + time.Sleep(10 * time.Second) + + if !before.Load() { + t.Error("expected before") + } + if after.Load() { + t.Error("did not expect after") + } +} + +func TestCrashSemaphore(t *testing.T) { + var before, after atomic.Bool + var sema uint32 + + // token to allow machine to read before, after, sema + gosimruntime.TestRaceToken.Release() + + m1 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + + // test that we can acquire the semaphore + gosimruntime.Semacquire(&sema, false) + before.Store(true) + }) + + m2 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + + // test that an acquire can be crashed + time.Sleep(2 * time.Second) + gosimruntime.Semacquire(&sema, false) + after.Store(true) + }) + + time.Sleep(1 * time.Second) + // release for m1 + gosimruntime.Semrelease(&sema) + + time.Sleep(3 * time.Second) + m1.Crash() + m2.Crash() + + // after crash make sure sema still works + gosimruntime.Semrelease(&sema) + time.Sleep(time.Second) + gosimruntime.Semacquire(&sema, false) + + if !before.Load() { + t.Error("expected before") + } + if after.Load() { + t.Error("did not expect after") + } +} + +func TestCrashChan(t *testing.T) { + ch1 := make(chan int, 1) + ch2 := make(chan int, 1) + + m := gosim.NewSimpleMachine(func() { + // make sure we can recv and send channels + ch2 <- <-ch1 + 1 + // then hang + <-ch1 + }) + + time.Sleep(time.Second) + + // successful recv and send + ch1 <- 1 + if v := <-ch2; v != 2 { + t.Error(v) + } + + // wait a bit to make sure it's in the recv and crash + time.Sleep(2 * time.Second) + m.Crash() + time.Sleep(2 * time.Second) + + // test channel still works and recv won't consume our value + ch1 <- 3 + time.Sleep(5 * time.Second) + if v := <-ch1; v != 3 { + t.Error(v) + } +} + +func TestCrashSelect(t *testing.T) { + ch1 := make(chan int, 1) + ch2 := make(chan int, 0) + + m := gosim.NewSimpleMachine(func() { + // make sure that select works + select { + case x := <-ch1: + ch2 <- x + 1 + case <-ch2: + } + // then hang + select { + case <-ch1: + case ch2 <- 1: + } + }) + + time.Sleep(2 * time.Second) + ch1 <- 1 + if v := <-ch2; v != 2 { + t.Error(v) + } + + // make sure we are in the select and crash + time.Sleep(time.Second) + m.Crash() + time.Sleep(time.Second) + + // make sure chans still work + go func() { + ch2 <- <-ch1 + 1 + }() + ch1 <- 4 + if v := <-ch2; v != 5 { + t.Error(v) + } +} + +func TestCrashSyscall(t *testing.T) { + var before, after, finally atomic.Bool + + // sleeper is a helper machine that will successfully stop + sleeper := gosim.NewSimpleMachine(func() { + time.Sleep(5 * time.Second) + }) + + // working machine + gosimruntime.TestRaceToken.Release() + waiter := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // wait for sleeper, flag success + sleeper.Wait() + before.Store(true) + // then sleep, flag success after + time.Sleep(5 * time.Second) + finally.Store(true) + }) + + gosimruntime.TestRaceToken.Release() + crasher := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // wait for waiter, but will get crashed before it's done + waiter.Wait() + after.Store(true) + }) + + // see waiter work + time.Sleep(2 * time.Second) + if before.Load() { + t.Error("did not yet expect before") + } + time.Sleep(4 * time.Second) + if !before.Load() { + t.Error("expected before") + } + + // then crash crasher + crasher.Crash() + crasher.Wait() + // make sure after did not trigger + if after.Load() { + t.Error("did not expect after") + } + + // make sure waiter is still sleeping + if finally.Load() { + t.Error("did not yet expect finally") + } + // make sure wait works + waiter.Wait() + if !finally.Load() { + t.Error("did expect finally") + } +} + +func TestCrashStressLightly(t *testing.T) { + // small stress test that creates a bunch of machines interacting with + // goroutines, timers, and channels, and crashes them hopefully catching + // reuse problems + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + // create goroutines every 2 seconds + time.Sleep(2 * time.Second) + wg.Add(1) + go func() { + defer wg.Done() + var before, after atomic.Bool + gosimruntime.TestRaceToken.Release() + // spawn a new machine that we will run for a bit and then crash + m := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + // another goroutine for good measure + go func() { + // sleep a bit + time.Sleep(1 * time.Second) + ch := make(chan struct{}, 0) + // make a timer writer to a channel later + time.AfterFunc(1*time.Second, func() { + ch <- struct{}{} + }) + go func() { + // sleep a bit then try and then have a timer try to write after + time.Sleep(1 * time.Second) + time.AfterFunc(3*time.Second, func() { + after.Store(true) + }) + }() + go func() { + // sleep a bit and then wait for a chan to maybe write after. + // chan queues are FIFO so the code after will get to recv. + time.Sleep(1 * time.Second) + <-ch + after.Store(true) + }() + // recv and then flag success. + <-ch + before.Store(true) + }() + select {} + }) + // run for long enough to allow before to succeed. + time.Sleep(3 * time.Second) + m.Crash() + if !before.Load() { + t.Error("expected before") + } + // allow any lingering code to trigger + time.Sleep(10 * time.Second) + if after.Load() { + t.Error("did not expect after") + } + }() + } + // ensure all succeed + wg.Wait() +} + +// EchoServer is a small TCP server that listens on 10.0.0.1:8080 and echoes +// all incoming bytes. +func EchoServer() { + l, err := net.Listen("tcp", "10.0.0.1:8080") // XXX: need to specify IP? wtf? + if err != nil { + log.Fatal(err) + } + + for { + conn, err := l.Accept() + if err != nil { + log.Fatal(err) + } + go func() { + defer conn.Close() + for { + var buf [1024]byte + n, err := conn.Read(buf[:]) + if err != nil { + log.Println(err) + return + } + if _, err := conn.Write(buf[:n]); err != nil { + log.Println(err) + return + } + } + }() + } +} + +func TestCrashServerTCPConnBreaks(t *testing.T) { + // run echo server + m := gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.AddrFrom4([4]byte{10, 0, 0, 1}), + MainFunc: EchoServer, + }) + + // let it start listening + time.Sleep(time.Second) + + // dial it and see it echo + conn, err := net.Dial("tcp", "10.0.0.1:8080") + if err != nil { + t.Fatal(err) + } + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + var buf [5]byte + if _, err := io.ReadFull(conn, buf[:]); err != nil { + t.Fatal(err) + } + if string(buf[:]) != "hello" { + t.Fatal(string(buf[:])) + } + + // let it sit and cash it + time.Sleep(time.Second) + m.Crash() + + // XXX: test without any data sent? + + // XXX: should have network behavior be configurable: host not responding vs rst vs ... + + // then, expect read to eventually fail after write never gets acked + _, err = conn.Read(buf[:]) + if err == nil { + t.Error("expected error") + } + if !errors.Is(err, syscall.EPIPE) { + t.Errorf("expected syscall.EPIPE, got %s", err) + } +} + +// XXX: log.Fatal currently does not get reported/failed. fix that somehow? + +// EchoClient is a small TCP client that connects to on 10.0.0.1:8080 and writes +// "hello" once and expects it come back. +func EchoClient() { + conn, err := net.Dial("tcp", "10.0.0.1:8080") + if err != nil { + log.Fatal(err) + } + if _, err := conn.Write([]byte("hello")); err != nil { + log.Fatal(err) + } + var buf [5]byte + if _, err := io.ReadFull(conn, buf[:]); err != nil { + log.Fatal(err) + } + if string(buf[:]) != "hello" { + log.Fatal(string(buf[:])) + } +} + +func TestCrashClientTCPConnBreaks(t *testing.T) { + // run echo server + var done atomic.Bool + gosimruntime.TestRaceToken.Release() + gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.AddrFrom4([4]byte{10, 0, 0, 1}), + MainFunc: func() { + gosimruntime.TestRaceToken.Acquire() + + l, err := net.Listen("tcp", "10.0.0.1:8080") // XXX: need to specify IP? wtf? + if err != nil { + log.Fatal(err) + } + + conn, err := l.Accept() + if err != nil { + log.Fatal(err) + } + var buf [5]byte + n, err := io.ReadFull(conn, buf[:]) + if err != nil { + log.Println(err) + return + } + if _, err := conn.Write(buf[:n]); err != nil { + log.Println(err) + return + } + + // then, expect (eventually) an error because the client crashed and the network disconnected + _, err = conn.Read(buf[:]) + if err == nil { + t.Error("expected error") + } + if !errors.Is(err, syscall.EPIPE) { + t.Errorf("expected syscall.EPIPE, got %s", err) + } + done.Store(true) + }, + }) + + // let it start listening + time.Sleep(time.Second) + + client := gosim.NewSimpleMachine(EchoClient) + + // let it connect and do the echo + time.Sleep(time.Second) + + // then crash the client + client.Crash() + + // wait for the server to notice + time.Sleep(time.Minute) + + if !done.Load() { + t.Error("expected done") + } +} + +var crashTestGlobal int + +func TestCrashRestartGlobals(t *testing.T) { + var first atomic.Bool + gosimruntime.TestRaceToken.Release() + m := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + if crashTestGlobal != 0 { + t.Error("expect 0") + } + crashTestGlobal = 1 + if crashTestGlobal != 1 { + t.Error("expect 1") + } + first.Store(true) + }) + time.Sleep(time.Second) + m.Crash() + if !first.Load() { + t.Error("expected first") + } + var second atomic.Bool + gosimruntime.TestRaceToken.Release() + m.SetMainFunc(func() { + gosimruntime.TestRaceToken.Acquire() + if crashTestGlobal != 0 { + t.Error("expect 0") + } + crashTestGlobal = 1 + if crashTestGlobal != 1 { + t.Error("expect 1") + } + second.Store(true) + }) + m.Restart() + time.Sleep(time.Second) + if !second.Load() { + t.Error("expected second") + } +} + +func TestCrashRestartPendingDisk(t *testing.T) { + var first atomic.Bool + gosimruntime.TestRaceToken.Release() + m := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if _, err := a.Write([]byte("a")); err != nil { + t.Fatal(err) + } + if err := a.Sync(); err != nil { + t.Fatal(err) + } + if err := a.Close(); err != nil { + t.Fatal(err) + } + d, err := os.OpenFile(".", os.O_RDONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err := d.Sync(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile("b", []byte("b"), 0o644); err != nil { + t.Fatal(err) + } + first.Store(true) + gosim.CurrentMachine().Crash() + }) + m.Wait() + if !first.Load() { + t.Error("expected first") + } + var second atomic.Bool + gosimruntime.TestRaceToken.Release() + m.SetMainFunc(func() { + gosimruntime.TestRaceToken.Acquire() + contents, err := os.ReadFile("a") + if err != nil { + t.Fatal(err) + } + if string(contents) != "a" { + t.Fatal("bad contents") + } + _, err = os.ReadFile("b") + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("expected %v, got %v", os.ErrNotExist, err) + } + second.Store(true) + }) + m.Restart() + m.Wait() + if !second.Load() { + t.Error("expected second") + } +} + +func checkEcho(t *testing.T) { + t.Helper() + // dial it and see it echo + conn, err := net.Dial("tcp", "10.0.0.1:8080") + defer conn.Close() + + if err != nil { + t.Fatal(err) + } + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + var buf [5]byte + if _, err := io.ReadFull(conn, buf[:]); err != nil { + t.Fatal(err) + } + if string(buf[:]) != "hello" { + t.Fatal(string(buf[:])) + } +} + +func TestCrashRestartNetwork(t *testing.T) { + // run echo server + m := gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.AddrFrom4([4]byte{10, 0, 0, 1}), + MainFunc: EchoServer, + }) + + // let it start listening + time.Sleep(time.Second) + + // check we can dial + conn, err := net.Dial("tcp", "10.0.0.1:8080") + if err != nil { + t.Fatalf("expected dial success, got %v", err) + } + conn.Close() + + // check echo works + checkEcho(t) + + // crash + m.Crash() + + // check dialing fails + _, err = net.Dial("tcp", "10.0.0.1:8080") + if err == nil { + t.Fatalf("expected dial failure, got %v", err) + } + + // restart + m.Restart() // still runs the old EchoServer + + time.Sleep(time.Second) + + // check we can dial again + conn, err = net.Dial("tcp", "10.0.0.1:8080") + if err != nil { + t.Fatalf("expected dial success, got %v", err) + } + conn.Close() + + // check echo works again + checkEcho(t) + + // XXX: test what happens with in-flight packets +} + +func TestCrashRestartFilesystemPartialInner(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + if err := os.WriteFile("foo", []byte("hello"), 0o644); err != nil { + // XXX: could this be log.Fatal? + t.Fatal(err) + } + gosim.CurrentMachine().Crash() + }) + m.Wait() + // now see what we wrote to disk. see details below on scenarios. + m.SetMainFunc(func() { + contents, err := os.ReadFile("foo") + if err != nil { + if errors.Is(err, os.ErrNotExist) { + contents = []byte("") + } else { + t.Fatal(err) + } + } + slog.Info("read", "contents", string(contents)) + }) + m.RestartWithPartialDisk() + m.Wait() +} + +func TestCrashRestartBootProgram(t *testing.T) { + var done atomic.Bool + gosimruntime.TestRaceToken.Release() + + m := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + done.Store(true) + }) + + // give m time to change its boot program + time.Sleep(time.Second) + m.Crash() + if !done.Load() { + t.Fatal("expected done") + } + done.Store(false) + m.Restart() + + time.Sleep(time.Second) + if !done.Load() { + t.Fatal("expected done") + } +} + +func TestCrashRestartNewBootProgram(t *testing.T) { + var done atomic.Bool + gosimruntime.TestRaceToken.Release() + + m := gosim.NewSimpleMachine(func() { + gosim.CurrentMachine().SetMainFunc(func() { + gosimruntime.TestRaceToken.Acquire() + done.Store(true) + }) + }) + + // give m time to change its boot program + time.Sleep(time.Second) + m.Crash() + if done.Load() { + t.Fatal("did not expect done") + } + m.Restart() + + time.Sleep(time.Second) + if !done.Load() { + t.Fatal("expected done") + } +} + +// XXX: keep some ops on disk after crash? + +// XXX: set a "program" that automatically starts after every restart? + +// XXX: test that a grpc client will reconnect to a restarted server diff --git a/internal/tests/behavior/disk_crash_test.go b/internal/tests/behavior/disk_crash_test.go new file mode 100644 index 0000000..67dee84 --- /dev/null +++ b/internal/tests/behavior/disk_crash_test.go @@ -0,0 +1,615 @@ +//go:build sim + +package behavior + +import ( + "bytes" + "io" + "log" + "os" + "sync" + "testing" + + "github.com/jellevandenhooff/gosim" +) + +// XXX: check errors on all file ops? +// XXX: test / cause errors on all file ops? + +// XXX: check iterAllCrashes and randomCrashSubset behave the same way: +// add a helper that call Crash (many) times and compares scenarios + +func TestCrashDiskBasic(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + a.Write([]byte("a")) + a.Close() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "file a empty": {"a": {}}, + "file a single zero": {"a": {0}}, + "file a single a": {"a": {'a'}}, + "file a missing": {}, + }) +} + +func TestCrashDiskSingleFileMultipleWrite(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err := a.Truncate(2); err != nil { + t.Error(err) + return + } + if err := a.Sync(); err != nil { + t.Error(err) + return + } + d, err := os.OpenFile(".", os.O_RDONLY, 0o600) + if err != nil { + t.Error(err) + return + } + d.Sync() + a.Write([]byte("a")) + a.Write([]byte("b")) + a.Close() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "file a 00": {"a": {0, 0}}, + "file a a0": {"a": {'a', 0}}, + "file a 0b": {"a": {0, 'b'}}, + "file a ab": {"a": {'a', 'b'}}, + }) +} + +func TestCrashDiskMultipleFileMultipleWrite(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err := a.Truncate(2); err != nil { + t.Error(err) + } + if err := a.Sync(); err != nil { + t.Error(err) + } + b, err := os.OpenFile("b", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if err := b.Truncate(2); err != nil { + t.Error(err) + } + if err := b.Sync(); err != nil { + t.Error(err) + } + d, err := os.OpenFile(".", os.O_RDONLY, 0o600) + if err != nil { + t.Fatal(err) + } + d.Sync() + a.Write([]byte("a")) + a.Write([]byte("b")) + b.Write([]byte("c")) + b.Write([]byte("d")) + a.Close() + b.Close() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, crossProductContents(map[string]map[string][]byte{ + "file a 00": {"a": {0, 0}}, + "file a a0": {"a": {'a', 0}}, + "file a 0b": {"a": {0, 'b'}}, + "file a ab": {"a": {'a', 'b'}}, + }, map[string]map[string][]byte{ + "file b 00": {"b": {0, 0}}, + "file b c0": {"b": {'c', 0}}, + "file b 0d": {"b": {0, 'd'}}, + "file b cd": {"b": {'c', 'd'}}, + })) +} + +func TestCrashDiskFileSync(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + a.Write([]byte("a")) + a.Sync() + a.Close() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "file a single a": {"a": {'a'}}, + "file a missing": {}, + }) +} + +func TestCrashDiskWriteRename(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + a.Write([]byte("a")) + a.Close() + os.Rename("a", "b") + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "both missing": {}, + "file a empty": {"a": {}}, + "file a single zero": {"a": {0}}, + "file a single a": {"a": {'a'}}, + "file b empty": {"b": {}}, + "file b single zero": {"b": {0}}, + "file b single a": {"b": {'a'}}, + }) +} + +func TestCrashDiskWriteRenameSyncFile(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + // XXX: check errs? + a.Write([]byte("a")) + a.Sync() + a.Close() + os.Rename("a", "b") + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "both missing": {}, + "file a single a": {"a": {'a'}}, + "file b single a": {"b": {'a'}}, + }) +} + +func TestCrashDiskWriteRenameSyncDir(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + // XXX: check errs? + a.Write([]byte("a")) + a.Close() + os.Rename("a", "b") + d, err := os.OpenFile(".", os.O_RDONLY, 0o600) + if err != nil { + t.Fatal(err) + } + d.Sync() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + // XXX: is this correct? + "file b empty": {"b": {}}, + "file b single zero": {"b": {0}}, + "file b single a": {"b": {'a'}}, + }) +} + +// XXX: deps for reads/writes to same name??? what if rename a,b -> rename b,c??? + +func TestCrashDiskWriteRenameSyncFileAndDir(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + // XXX: check errs? + a.Write([]byte("a")) + a.Sync() + a.Close() + os.Rename("a", "b") + d, err := os.OpenFile(".", os.O_RDONLY, 0o600) + if err != nil { + t.Fatal(err) + } + d.Sync() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "file b single a": {"b": {'a'}}, + }) +} + +func TestCrashDiskWriteRenameTwiceSyncFile(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + // XXX: check errs? + a.Write([]byte("a")) + a.Sync() + a.Close() + os.Rename("a", "b") + os.Rename("b", "c") + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, map[string]map[string][]byte{ + "no file": {}, + "file a single a": {"a": {'a'}}, + "file b single a": {"b": {'a'}}, + "file c single a": {"c": {'a'}}, + }) +} + +func TestCrashDiskTwoFiles(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + a, err := os.OpenFile("a", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + a.Write([]byte("a")) + a.Close() + + b, err := os.OpenFile("b", os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + b.Write([]byte("b")) + b.Close() + gosim.CurrentMachine().Crash() + }) + m.Wait() + + disks := extractDisks(t, m) + + expectOutcomesExactly(t, disks, crossProductContents(map[string]map[string][]byte{ + "file a empty": {"a": {}}, + "file a single zero": {"a": {0}}, + "file a single a": {"a": {'a'}}, + "file a missing": {}, + }, map[string]map[string][]byte{ + "file b empty": {"b": {}}, + "file b single zero": {"b": {0}}, + "file b single b": {"b": {'b'}}, + "file b missing": {}, + })) +} + +func crossProductContents(a, b map[string]map[string][]byte) map[string]map[string][]byte { + c := make(map[string]map[string][]byte) + for a, aa := range a { + for b, bb := range b { + cc := make(map[string][]byte) + for k, v := range aa { + cc[k] = v + } + for k, v := range bb { + cc[k] = v + } + c[a+" and "+b] = cc + } + } + return c +} + +func readFiles(t *testing.T) map[string][]byte { + read := make(map[string][]byte) + + files, err := os.ReadDir(".") + if err != nil { + t.Error(err) + return nil + } + + for _, file := range files { + // XXX: os.Open, os.ReadFile? + f, err := os.OpenFile(file.Name(), os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return nil + } + bytes, err := io.ReadAll(f) + if err != nil { + f.Close() + t.Error(err) + return nil + } + + read[file.Name()] = bytes + } + + return read +} + +func matches(a, b map[string][]byte) bool { + for k := range a { + // distinguish missing from equal nils + if _, ok := b[k]; !ok { + return false + } + + if !bytes.Equal(a[k], b[k]) { + return false + } + } + + for k := range b { + if _, ok := a[k]; !ok { + return false + } + } + + return true +} + +// XXX: norace because mu and all are shared, m.Run has is its own context. should fix somehow +// +//go:norace +func extractDisks(t *testing.T, m gosim.Machine) []map[string][]byte { + var mu sync.Mutex + var all []map[string][]byte + + iter := m.IterDiskCrashStates(func() { + // XXX: this lock should (not) be necessary? + mu.Lock() + defer mu.Unlock() + read := readFiles(t) + all = append(all, read) + }) + + for m := range iter { + m.Wait() + // XXX: should stop m here so its background stuff doesnt clog us up + // m.Crash() // XXX: use m.Shutdown instead + } + // XXX: this lock should (not) be necessary? + mu.Lock() + defer mu.Unlock() + return all +} + +func expectOutcomesExactly(t *testing.T, disks []map[string][]byte, expected map[string]map[string][]byte) { + count := make(map[string]int) + for _, disk := range disks { + found := false + for name, contents := range expected { + if matches(disk, contents) { + count[name]++ + found = true + break + } + } + if !found { + t.Error("unexpected outcome") + for name, contents := range disk { + log.Print(name, contents) + } + return + } + } + + for name := range expected { + if count[name] == 0 { + t.Error("missing outcome", name) + } + log.Print(name, count[name]) + } +} + +/* +func missingFile(t *testing.T, name string) bool { + f, err := os.OpenFile(name, os.O_RDONLY, 0600) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true + } + t.Error(err) + return false + } + f.Close() + return false +} + +func fileContents(t *testing.T, name string, contents []byte) bool { + f, err := os.OpenFile(name, os.O_RDONLY, 0600) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return false + } + t.Error(err) + return false + } + b, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + f.Close() + return bytes.Equal(contents, b) +} + +func crossProduct(a, b map[string]bool) map[string]bool { + c := make(map[string]bool) + for a, aa := range a { + for b, bb := range b { + c[a+" and "+b] = aa && bb + } + } + return c +} + +func distinguish(t *testing.T, m map[string]bool) { + var keys []string + var found []string + for name, b := range m { + keys = append(keys, name) + if b { + found = append(found, name) + } + } + sort.Strings(keys) + sort.Strings(found) + + if len(found) != 1 { + t.Error(fmt.Sprintf("expected exactly 1 found, got %v", found)) + return + } + + s.Observe("result", "keys", keys) + s.Observe("result", "found", found[0]) +} +*/ + +// XXX: have a helper that enumerates all possible allowed disk outcomes? since we can clone anyway? +// - can also here include sync w/ crash backdated before in these scenarios for nice perf? + +// XXX: "test" or "inspect" disk option, clone machine and then see what happens? +// XXX: "allowed" and "not-allowed" and "expected/demanded" outcomes +// XXX: automatic restart, recovery of crashed machine (in a larger simulation?) + +// return or observe disk state + +// XXX: test disk deps? want some graph like api...; serialization of ops? + +/* +func TestCrashDiskFileSync(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskFileSync) + combineDistinguish(t, runs) +} + +func TestCrashDiskTwoFiles(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskTwoFiles) + combineDistinguish(t, runs) +} + +func TestCrashDiskWriteRename(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskWriteRename) + combineDistinguish(t, runs) +} + +func TestCrashDiskWriteRenameSyncFile(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskWriteRenameSyncFile) + combineDistinguish(t, runs) +} + +func TestCrashDiskWriteRenameTwiceSyncFile(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskWriteRenameTwiceSyncFile) + combineDistinguish(t, runs) +} + +func TestCrashDiskWriteRenameSyncDir(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskWriteRenameSyncDir) + combineDistinguish(t, runs) +} + +func TestCrashDiskWriteRenameSyncFileAndDir(t *testing.T) { + config := simtesting.ConfigWithNSeeds(1000) + + runs := config.RunSim(t, TestCrashDiskWriteRenameSyncFileAndDir) + combineDistinguish(t, runs) +} + +func combineDistinguish(t *testing.T, results []gosimruntime.RunResult) { + t.Helper() + + count := make(map[string]int) + + if len(results) == 0 { + t.Fatal("need at least one run") + } + + knownKeys := results[0].Observation("result", "keys").([]string) + for _, key := range knownKeys { + count[key] = 0 + } + + for _, result := range results { + keys := result.Observation("result", "keys").([]string) + found := result.Observation("result", "found").(string) + + if !slices.Equal(keys, knownKeys) { + t.Errorf("bad keys, got %v expected %v", keys, knownKeys) + } + + if _, ok := count[found]; !ok { + t.Errorf("bad found %s", found) + } + + count[found]++ + } + + for key, val := range count { + if val == 0 { + t.Errorf("missing outcome %s", key) + } + } + + keys := maps.Keys(count) + slices.Sort(keys) + for _, key := range keys { + t.Log(key, count[key]) + } +} +*/ + +// things to test: +// - file write then rename +// - partial writes +// - reordered writes (same file, different file) +// - sync works +// - reordered metadata ops diff --git a/internal/tests/behavior/disk_sim_test.go b/internal/tests/behavior/disk_sim_test.go new file mode 100644 index 0000000..3a6dca2 --- /dev/null +++ b/internal/tests/behavior/disk_sim_test.go @@ -0,0 +1,179 @@ +//go:build sim + +package behavior_test + +import ( + "errors" + "os" + "syscall" + "testing" + + "github.com/jellevandenhooff/gosim" +) + +// TODO: more scenarios for either triggering or not triggering GC (pending ops, rename, delete, etc.) + +func getInode(f *os.File) (int, error) { + info, err := f.Stat() + if err != nil { + return 0, err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, errors.New("not a syscall.Stat_t") + } + return int(stat.Ino), nil +} + +type gcTester struct { + *testing.T + dir *os.File +} + +func newGCTester(t *testing.T) *gcTester { + d, err := os.Open(".") + if err != nil { + t.Fatal(err) + } + return &gcTester{ + T: t, + dir: d, + } +} + +func (t *gcTester) mustCreate(name string) (*os.File, int) { + t.Helper() + f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + t.Fatal(err) + } + i, err := getInode(f) + if err != nil { + t.Fatal(err) + } + return f, i +} + +func (t *gcTester) mustWrite(f *os.File, b []byte) { + t.Helper() + if _, err := f.Write(b); err != nil { + t.Error(err) + } +} + +func (t *gcTester) mustClose(f *os.File) { + t.Helper() + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func (t *gcTester) mustSync(f *os.File) { + t.Helper() + if err := f.Sync(); err != nil { + t.Error(err) + } +} + +func (t *gcTester) checkMem(inode int, exists bool, links int) { + t.Helper() + got := gosim.CurrentMachine().GetInodeInfo(inode) + if got.MemExists != exists || got.MemLinks != links { + t.Errorf("inode %d: expected mem exists %v, links %d; got exists %v, links %d", inode, exists, links, got.MemExists, got.MemLinks) + } +} + +func (t *gcTester) checkDisk(inode int, exists bool, links int) { + t.Helper() + got := gosim.CurrentMachine().GetInodeInfo(inode) + if got.DiskExists != exists || got.DiskLinks != links { + t.Errorf("inode %d: expected disk exists %v, links %d; got exists %v, links %d", inode, exists, links, got.DiskExists, got.DiskLinks) + } +} + +func (t *gcTester) checkPendingOps(inode int, ops int) { + t.Helper() + got := gosim.CurrentMachine().GetInodeInfo(inode) + if got.Ops != ops { + t.Errorf("inode %d: expected ops %d, got %d", inode, ops, got.Ops) + } +} + +func (t *gcTester) checkOpenHandles(inode int, handles int) { + t.Helper() + got := gosim.CurrentMachine().GetInodeInfo(inode) + if got.Handles != handles { + t.Errorf("inode %d: expected handles %d, got %d", inode, handles, got.Handles) + } +} + +func (t *gcTester) rename(from, to string) { + t.Helper() + if err := os.Rename(from, to); err != nil { + t.Error(err) + } +} + +func TestDiskGC(t *testing.T) { + harness := newGCTester(t) + + // create a new file, expect mem but not disk representation + foo, fooInode := harness.mustCreate("foo") + harness.checkMem(fooInode, true, 1) + harness.checkDisk(fooInode, false, 0) + harness.checkOpenHandles(fooInode, 1) // f1 + harness.checkPendingOps(fooInode, 2) // alloc, dir write + + // write to the file, expect more pending ops + harness.mustWrite(foo, []byte("hello")) + harness.checkPendingOps(fooInode, 4) // alloc, dir write, resize, write + harness.checkDisk(fooInode, false, 0) // still not visible + + // test that sync creates file on disk but does not flush the dir write + harness.mustSync(foo) + harness.checkDisk(fooInode, true, 0) // not yet visible + harness.checkPendingOps(fooInode, 1) // dir write + + // check that handle changes after close + harness.mustClose(foo) + harness.checkOpenHandles(fooInode, 0) + + // create another file "bar", write to it, and flush the write + bar, barInode := harness.mustCreate("bar") + harness.mustWrite(bar, []byte("bye")) + harness.mustSync(bar) + harness.mustClose(bar) + harness.checkDisk(barInode, true, 0) // not yet visible + harness.checkPendingOps(barInode, 1) // dir write + + // sync dir to flush dir writes + harness.mustSync(harness.dir) + + // now dir entries are flushed to disk and link count is updated + harness.checkDisk(fooInode, true, 1) + harness.checkPendingOps(fooInode, 0) + harness.checkDisk(barInode, true, 1) + harness.checkPendingOps(barInode, 0) + + // rename "bar" to "foo", deleting "foo" + harness.rename("bar", "foo") + + // after rename in memory, memory GCs but disk is unchanged + harness.checkMem(fooInode, false, 0) // XXX: deleted from memory even though still on disk... problem? could it come back? + harness.checkDisk(fooInode, true, 1) + harness.checkPendingOps(fooInode, 0) // rename does not mention i1 + harness.checkMem(barInode, true, 1) + harness.checkDisk(barInode, true, 1) + harness.checkPendingOps(barInode, 1) // rename + + // flush rename to disk, expect "foo" to disappear from disk + harness.mustSync(harness.dir) + harness.checkDisk(fooInode, false, 0) + harness.checkDisk(barInode, true, 1) + harness.checkPendingOps(barInode, 0) +} + +// TODO: test delete using overwrite +// TODO: test delete using unlink +// TODO: test no delete until close, then delete +// TODO: test no delete until flush, then delete diff --git a/internal/tests/behavior/disk_test.go b/internal/tests/behavior/disk_test.go new file mode 100644 index 0000000..ac652c1 --- /dev/null +++ b/internal/tests/behavior/disk_test.go @@ -0,0 +1,911 @@ +package behavior_test + +import ( + "bytes" + "errors" + "io" + "log/slog" + "os" + "reflect" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/jellevandenhooff/gosim" +) + +func setupRealDisk(t *testing.T) { + if gosim.IsSim() { + return + } + + orig, err := os.Getwd() + if err != nil { + t.Error(err) + return + } + + tmp, err := os.MkdirTemp("", t.Name()) + if err != nil { + t.Error(err) + return + } + + t.Cleanup(func() { + if err := os.RemoveAll(tmp); err != nil { + panic(err) + } + }) + + if err := os.Chdir(tmp); err != nil { + t.Error(err) + return + } + + t.Cleanup(func() { + if err := os.Chdir(orig); err != nil { + panic(err) + } + }) +} + +func TestDiskBasic(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Sync(); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("hello")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskTruncateGrow(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Truncate(10); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("hello\x00\x00\x00\x00\x00")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskTruncateShrink(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Truncate(3); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("hel")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskOpenTruncate(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + t.Error(err) + return + } + if _, err := f.Write([]byte("goodbye")); err != nil { + t.Error(err) + return + } + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("goodbye")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskSyncDir(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile(".", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + // XXX: you canc sync a O_RDONLY dir? + if err := f.Sync(); err != nil { + t.Error(err) + return + } +} + +func TestDiskAfterClose(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + checkClosedError := func(err error, op string) { + t.Helper() + pathErr, ok := err.(*os.PathError) + if !ok { + t.Error("not a path error") + return + } + if pathErr.Op != op { + t.Errorf("expected op %q, got %q", op, pathErr.Op) + } + if pathErr.Err != os.ErrClosed { + t.Errorf("expected os.ErrClosed, got %v", pathErr.Err) + } + if pathErr.Path != "hello" { + t.Errorf("expected \"hello\", got %q", pathErr.Path) + } + } + + err = f.Close() + checkClosedError(err, "close") + slog.Info("return value of close after close", "err", err.Error()) + + n, err := f.Read(nil) + checkClosedError(err, "read") + slog.Info("return value of read after close", "n", n, "err", err.Error()) + + n, err = f.Write([]byte("hello")) + checkClosedError(err, "write") + slog.Info("return value of write after close", "n", n, "err", err.Error()) + + err = f.Sync() + checkClosedError(err, "sync") + slog.Info("return value of sync after close", "err", err.Error()) +} + +func TestDiskWriteAtEndOfFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 0); err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 3); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("helhello")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskWriteAtAfterEndOfFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 3); err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 3+5+3); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("\x00\x00\x00hello\x00\x00\x00hello")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskReadAtBasic(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 0); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + data := make([]byte, 5) + n, err := f.ReadAt(data, 0) + if err != nil { + t.Error(err) + } + if n != 5 || !bytes.Equal(data, []byte("hello")) { + t.Error("bad read", n, data) + } + + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskReadAtEndOfFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 0); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + { + data := make([]byte, 5) + n, err := f.ReadAt(data, 3) + if err != io.EOF { + t.Error(err) + } + if n != 2 || !bytes.Equal(data, []byte("lo\x00\x00\x00")) { + t.Error("bad read", n, data) + } + } + + { + data := make([]byte, 5) + n, err := f.ReadAt(data, 5) + if err != io.EOF { + t.Error(err) + } + if n != 0 || !bytes.Equal(data, []byte("\x00\x00\x00\x00\x00")) { + t.Error("bad read", n, data) + } + } + + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskReadAtAfterEndOfFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("hello", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.WriteAt([]byte("hello"), 0); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("hello", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + data := make([]byte, 5) + n, err := f.ReadAt(data, 10) + if err != io.EOF { + t.Error(err) + } + if n != 0 || !bytes.Equal(data, []byte("\x00\x00\x00\x00\x00")) { + t.Error("bad read", n, data) + } + + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskOpenMissingFile(t *testing.T) { + setupRealDisk(t) + + _, err := os.OpenFile("foo", os.O_RDONLY, 0o644) + if err == nil { + t.Error(err) + return + } + if !errors.Is(err, os.ErrNotExist) { + t.Error("should be not exist") + } + slog.Info("error opening a non-existing file", "err", err.Error()) +} + +func TestDiskStatMissingFile(t *testing.T) { + setupRealDisk(t) + + _, err := os.Stat("foo") + if err == nil { + t.Error(err) + return + } + if !errors.Is(err, os.ErrNotExist) { + t.Error("should be not exist") + } + slog.Info("error stat a non-existing file", "err", err.Error()) +} + +func TestDiskStatFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("foo", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + fi, err := os.Stat("foo") + if err != nil { + t.Error(err) + return + } + + if fi.IsDir() { + t.Error("should not be dir, not", fi.IsDir()) + } + if fi.Name() != "foo" { + t.Error("should be foo, not ", fi.Name()) + } +} + +func TestDiskStatFd(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("foo", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + fi, err := f.Stat() + if err != nil { + t.Error(err) + return + } + + if fi.IsDir() { + t.Error("should not be dir, not", fi.IsDir()) + } + if fi.Name() != "foo" { + t.Error("should be foo, not ", fi.Name()) + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + fi2, err := os.Stat("foo") + if err != nil { + t.Error(err) + return + } + + if !reflect.DeepEqual(fi, fi2) { + t.Error("different") + } +} + +func TestDiskDeleteMissingFile(t *testing.T) { + setupRealDisk(t) + + err := os.Remove("foo") + if err == nil { + t.Error(err) + return + } + if !errors.Is(err, os.ErrNotExist) { + t.Error("should be not exist", err) + } + slog.Info("error deleting a non-existing file", "err", err.Error()) +} + +func TestDiskDeleteFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("foo", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + if _, err := os.Stat("foo"); err != nil { + t.Error("missing file before delete", err) + return + } + + if err := os.Remove("foo"); err != nil { + t.Error("remove failed", err) + return + } + + if _, err := os.Stat("foo"); err == nil || !errors.Is(err, os.ErrNotExist) { + t.Error("delete did not work", err) + return + } +} + +func TestDiskRenameFile(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("foo", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + if _, err := os.Stat("foo"); err != nil { + t.Error("missing file before rename", err) + return + } + + if err := os.Rename("foo", "bar"); err != nil { + t.Error("rename failed", err) + return + } + + if _, err := os.Stat("foo"); err == nil || !errors.Is(err, os.ErrNotExist) { + t.Error("file still there after rename", err) + return + } + + f, err = os.OpenFile("bar", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("hello")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +func TestDiskRenameMissingFile(t *testing.T) { + setupRealDisk(t) + + err := os.Rename("foo", "bar") + if err == nil { + t.Error(err) + return + } + if !errors.Is(err, os.ErrNotExist) { + t.Error("should be not exist") + } + slog.Info("error renaming a non-existing file", "err", err.Error()) +} + +func TestDiskRenameFileReplace(t *testing.T) { + setupRealDisk(t) + + f, err := os.OpenFile("foo", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + f, err = os.OpenFile("bar", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + + if _, err := f.Write([]byte("old")); err != nil { + t.Error(err) + return + } + + if err := f.Close(); err != nil { + t.Error(err) + return + } + + if _, err := os.Stat("foo"); err != nil { + t.Error("missing file before rename", err) + return + } + + if _, err := os.Stat("bar"); err != nil { + t.Error("missing file before rename", err) + return + } + + if err := os.Rename("foo", "bar"); err != nil { + t.Error("rename failed", err) + return + } + + if _, err := os.Stat("foo"); err == nil || !errors.Is(err, os.ErrNotExist) { + t.Error("file still there after rename", err) + return + } + + f, err = os.OpenFile("bar", os.O_RDONLY, 0o644) + if err != nil { + t.Error(err) + return + } + data, err := io.ReadAll(f) + if err != nil { + t.Error(err) + } + if !bytes.Equal(data, []byte("hello")) { + t.Error("bad read", data) + } + if err := f.Close(); err != nil { + t.Error(err) + } +} + +type dirEntry struct { + Name string + IsDir bool +} + +func checkReadDir(t *testing.T, dir string, expected []dirEntry) { + t.Helper() + + entries, err := os.ReadDir(dir) + if err != nil { + t.Error(err) + return + } + + var converted []dirEntry + for _, entry := range entries { + converted = append(converted, dirEntry{ + Name: entry.Name(), + IsDir: entry.IsDir(), + }) + } + + if diff := cmp.Diff(expected, converted); diff != "" { + t.Error(diff) + } +} + +func TestDiskReadDir(t *testing.T) { + setupRealDisk(t) + + checkReadDir(t, ".", nil) + + f, err := os.OpenFile("foo", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + if err := f.Close(); err != nil { + t.Error(err) + return + } + + checkReadDir(t, ".", []dirEntry{ + { + Name: "foo", + IsDir: false, + }, + }) + + f, err = os.OpenFile("bar", os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Error(err) + return + } + if _, err := f.Write([]byte("hello")); err != nil { + t.Error(err) + return + } + if err := f.Close(); err != nil { + t.Error(err) + return + } + + checkReadDir(t, ".", []dirEntry{ + { + Name: "bar", + IsDir: false, + }, + { + Name: "foo", + IsDir: false, + }, + }) + + if err := os.Rename("bar", "zz"); err != nil { + t.Error(err) + return + } + + checkReadDir(t, ".", []dirEntry{ + { + Name: "foo", + IsDir: false, + }, + { + Name: "zz", + IsDir: false, + }, + }) + + if err := os.Remove("foo"); err != nil { + t.Error(err) + return + } + + checkReadDir(t, ".", []dirEntry{ + { + Name: "zz", + IsDir: false, + }, + }) +} + +// open dir +// rename dir +// delete open file +// stat dir? +// stat size? + +// fix duplication of writes in tests? + +// things to test: +// - error codes for... +// closed, eof, wrong mode +// - concurrent reads/writes (same handle, different handle) +// - read/write at end of file (writeat with offset at file, after file end) +// - read ending at file (err = nil or eof or both?) +// - ??? diff --git a/internal/tests/behavior/globals.go b/internal/tests/behavior/globals.go new file mode 100644 index 0000000..c3b25b1 --- /dev/null +++ b/internal/tests/behavior/globals.go @@ -0,0 +1,10 @@ +package behavior + +var privateGlobal int + +var PublicGlobal int + +func init() { + PublicGlobal = 20 + privateGlobal = 10 +} diff --git a/internal/tests/behavior/globals_for_test.go b/internal/tests/behavior/globals_for_test.go new file mode 100644 index 0000000..b5bf680 --- /dev/null +++ b/internal/tests/behavior/globals_for_test.go @@ -0,0 +1,13 @@ +package behavior + +func PrivateGlobalPtr() *int { + return &privateGlobal +} + +func init() { + CopiedPrivateGlobal = privateGlobal +} + +var InitializedPrivateGlobal = 35 + +var CopiedPrivateGlobal int diff --git a/internal/tests/behavior/globals_test.go b/internal/tests/behavior/globals_test.go new file mode 100644 index 0000000..a6bbb16 --- /dev/null +++ b/internal/tests/behavior/globals_test.go @@ -0,0 +1,85 @@ +//go:build sim + +package behavior_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim" + "github.com/jellevandenhooff/gosim/internal/tests/behavior" +) + +var sharedGlobal = 10 + +func TestGlobalNotShared(t *testing.T) { + // TODO: Ensure we run this test multiple times to proof we reset globals? + for i := 0; i < 2; i++ { + gosim.NewSimpleMachine(func() { + if sharedGlobal != 10 { + t.Errorf("expected 10, got %d", sharedGlobal) + } + sharedGlobal = 11 + }).Wait() + } +} + +var chain1 = globalReadFromMain + 1 + +var ( + globalReadFromForTest = behavior.CopiedPrivateGlobal + globalReadFromMain = behavior.PublicGlobal +) + +var ( + chain2 int + chain3 int +) + +func init() { + chain2 = chain1 * 2 +} + +func init() { + chain3 = chain2 + 3 +} + +func TestGlobals(t *testing.T) { + if behavior.PublicGlobal != 20 { + t.Errorf("expected 20, got %d", behavior.PublicGlobal) + } + if globalReadFromMain != 20 { + t.Errorf("expected 20, got %d", globalReadFromMain) + } + if chain1 != 21 { + t.Errorf("expected 21, got %d", chain1) + } + if chain2 != 42 { + t.Errorf("expected 42, got %d", chain2) + } + if chain3 != 45 { + t.Errorf("expected 45, got %d", chain3) + } + if behavior.CopiedPrivateGlobal != 10 { + t.Errorf("expected 10, got %d", behavior.CopiedPrivateGlobal) + } + if behavior.InitializedPrivateGlobal != 35 { + t.Errorf("expected 35, got %d", behavior.InitializedPrivateGlobal) + } + if globalReadFromForTest != 10 { + t.Errorf("expected 10, got %d", globalReadFromForTest) + } + if value := *behavior.PrivateGlobalPtr(); value != 10 { + t.Errorf("expected 10, got %d", value) + } +} + +// "compare observations" (expect subset?) +// vs +// "tests must pass exactly in both cases" (specificy observations) +// vs +// "when running a bunch of times, i want to observe both this and that" + +// - want to have a test that says... +// - sometimes this happens +// - sometimes this other things happens +// - BOTH do happen, given a reasonable number of runs diff --git a/internal/tests/behavior/log_meta_test.go b/internal/tests/behavior/log_meta_test.go new file mode 100644 index 0000000..80bfe27 --- /dev/null +++ b/internal/tests/behavior/log_meta_test.go @@ -0,0 +1,95 @@ +//go:build !sim + +package behavior_test + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func parseLog(t *testing.T, log []byte) []map[string]any { + var logs []map[string]any + for _, line := range bytes.Split(log, []byte("\n")) { + if bytes.HasPrefix(line, []byte("===")) || bytes.HasPrefix(line, []byte("---")) { + // XXX: test this also, later? + continue + } + if len(line) == 0 { + logs = append(logs, nil) + continue + } + var msg map[string]any + if err := json.Unmarshal(line, &msg); err != nil { + t.Fatal(line, err) + } + delete(msg, "source") + logs = append(logs, msg) + } + return logs +} + +func TestMetaLogMachineGoroutineTime(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestLogMachineGoroutineTime", + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } + + actual := parseLog(t, run.LogOutput) + expected := parseLog(t, []byte(`{"time":"2020-01-15T14:10:03.000001234Z","level":"INFO","msg":"hi there","machine":"main","goroutine":4,"step":1} +{"time":"2020-01-15T14:10:13.000001234Z","level":"INFO","msg":"hey","machine":"inside","goroutine":6,"step":2} +{"time":"2020-01-15T14:10:23.000001234Z","level":"INFO","msg":"propagated","machine":"inside","goroutine":7,"step":3} +{"time":"2020-01-15T14:10:23.000001234Z","level":"WARN","msg":"foo","machine":"inside","bar":"baz","counter":20,"goroutine":7,"step":4} +`)) + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Error("diff", diff) + } +} + +func TestMetaLogSLog(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestLogSLog", + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } + + actual := parseLog(t, run.LogOutput) + expected := parseLog(t, []byte(`{"time":"2020-01-15T14:10:03.000001234Z","level":"INFO","msg":"hello 10","machine":"main","goroutine":4,"step":1} +`)) + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Error("diff", diff) + } +} + +func TestMetaStdoutStderr(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestStdoutStderr", + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } + + actual := parseLog(t, run.LogOutput) + expected := parseLog(t, []byte(`{"time":"2020-01-15T14:10:03.000001234Z","level":"INFO","msg":"hello","machine":"os","method":"stdout","from":"main","goroutine":1,"step":1} +{"time":"2020-01-15T14:10:03.000001234Z","level":"INFO","msg":"goodbye","machine":"os","method":"stderr","from":"main","goroutine":1,"step":2} +`)) + + if diff := cmp.Diff(expected, actual); diff != "" { + t.Error("diff", diff) + } +} diff --git a/internal/tests/behavior/log_test.go b/internal/tests/behavior/log_test.go new file mode 100644 index 0000000..ef6f439 --- /dev/null +++ b/internal/tests/behavior/log_test.go @@ -0,0 +1,64 @@ +//go:build sim + +package behavior_test + +import ( + "log" + "log/slog" + "os" + "testing" + "time" + + "github.com/jellevandenhooff/gosim" +) + +func TestLogSLog(t *testing.T) { + slog.Info("hello 10") +} + +func TestLogMachineGoroutineTime(t *testing.T) { + done := make(chan struct{}) + + log.Printf("hi %s", "there") + go func() { + gosim.NewMachine(gosim.MachineConfig{ + Label: "inside", + MainFunc: func() { + time.Sleep(10 * time.Second) + log.Print("hey") + go func() { + time.Sleep(10 * time.Second) + log.Print("propagated") + slog.Warn("foo", "bar", "baz", "counter", 20) + close(done) + }() + <-done + }, + }) + }() + + <-done +} + +func TestStdoutStderr(t *testing.T) { + os.Stdout.WriteString("hello") + os.Stderr.WriteString("goodbye") +} + +func TestLogForPrettyTest(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + slog.Info("hello info", "foo", "bar") + start := time.Now() + time.Sleep(20 * time.Second) + + m := gosim.NewSimpleMachine(func() { + slog.Info("warn", "ok", "now", "delay", time.Since(start)) + log.Println("before") + panic("help") + }) + m.Wait() + log.Println("never") +} diff --git a/internal/tests/behavior/machine_meta_test.go b/internal/tests/behavior/machine_meta_test.go new file mode 100644 index 0000000..650641c --- /dev/null +++ b/internal/tests/behavior/machine_meta_test.go @@ -0,0 +1,42 @@ +//go:build !sim + +package behavior_test + +import ( + "log" + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestMachineCrashRuns(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + counts := make(map[int]int) + for seed := int64(0); seed < 50; seed++ { + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestMachineCrash", + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + if run.Failed { + t.Fatal("run failed") + } + var n int + for _, log := range metatesting.ParseLog(run.LogOutput) { + if log["msg"] == "output" { + n = int(log["n"].(float64)) + } + } + counts[n]++ + } + + for i := 0; i <= 3; i++ { + if counts[i] == 0 { + t.Errorf("expected some output %d, got %d", i, counts[i]) + } + } + + log.Println("counts", counts) +} diff --git a/internal/tests/behavior/machine_test.go b/internal/tests/behavior/machine_test.go new file mode 100644 index 0000000..3c10fb8 --- /dev/null +++ b/internal/tests/behavior/machine_test.go @@ -0,0 +1,209 @@ +//go:build sim + +package behavior_test + +import ( + "log/slog" + "sync" + "testing" + "time" + + "github.com/jellevandenhooff/gosim" + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func TestMachineCrash(t *testing.T) { + m := gosim.NewSimpleMachine(func() { + // sleep a second in both the machine and with the crash call to make + // sure they start "at the same time". all global initialization code + // runs before. + time.Sleep(time.Second) + for i := 1; i <= 3; i++ { + gosimruntime.Yield() + slog.Info("output", "n", i) + } + }) + time.Sleep(time.Second) + m.Crash() +} + +func TestMachineGo(t *testing.T) { + global := gosim.CurrentMachine() + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if global != gosim.CurrentMachine() { + t.Error("expected current machine to stay same") + } + }() + + wg.Add(1) + go func() { + go func() { + go func() { + go func() { + defer wg.Done() + if global != gosim.CurrentMachine() { + t.Error("expected current machine to stay same") + } + }() + }() + }() + }() + + wg.Add(1) + gosim.NewSimpleMachine(func() { + inner := gosim.CurrentMachine() + if inner == global { + t.Error("expected global to be different from inner") + } + + done := make(chan struct{}) + go func() { + defer close(done) + defer wg.Done() + if inner != gosim.CurrentMachine() { + t.Error("expected current machine to stay same") + } + }() + + // wait until done; if the machine main returns, all goroutines will stop + // XXX: test that also somehow??? + <-done + }) + + wg.Wait() +} + +func TestMachineCurrent(t *testing.T) { + global := gosim.CurrentMachine() + + if global != gosim.CurrentMachine() { + t.Error("expected current machine to stay same") + } + + var inner gosim.Machine + gosimruntime.TestRaceToken.Release() + m := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + inner = gosim.CurrentMachine() + // if inner == nil { + // t.Error("expected non-nil inner") + // } + if inner != gosim.CurrentMachine() { + t.Error("expected current machine to stay same") + } + if inner == global { + t.Error("expected different global and inner") + } + gosimruntime.TestRaceToken.Release() + }) + m.Wait() + gosimruntime.TestRaceToken.Acquire() + if inner != m { + t.Error("expected inner to match m") + } + + var inner2 gosim.Machine + gosimruntime.TestRaceToken.Release() + m2 := gosim.NewSimpleMachine(func() { + gosimruntime.TestRaceToken.Acquire() + inner2 = gosim.CurrentMachine() + if inner2 != gosim.CurrentMachine() { + t.Error("expected current machine to stay same") + } + // if inner2 == nil { + // t.Error("expected non-nil inner2") + // } + if inner2 == global { + t.Error("expected different global and inner2") + } + gosimruntime.TestRaceToken.Release() + }) + m2.Wait() + gosimruntime.TestRaceToken.Acquire() + if inner2 != m2 { + t.Error("expected inner2 to match m2") + } + + var inner3 gosim.Machine + m2.SetMainFunc(func() { + gosimruntime.TestRaceToken.Acquire() + inner3 = gosim.CurrentMachine() + gosimruntime.TestRaceToken.Release() + }) + gosimruntime.TestRaceToken.Release() + m2.Restart() + m2.Wait() + gosimruntime.TestRaceToken.Acquire() + if inner3 != m2 { + t.Error("expected current machine to stay same") + } +} + +func TestMachineTimer(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + gosim.NewSimpleMachine(func() { + inner := gosim.CurrentMachine() + go func() { + time.AfterFunc(time.Second, func() { + if gosim.CurrentMachine() != inner { + t.Error("expected timer machine to match inner") + } + wg.Done() + }) + }() + select {} + }) + + global := gosim.CurrentMachine() + wg.Add(1) + go func() { + time.AfterFunc(time.Second, func() { + if gosim.CurrentMachine() != global { + t.Error("expected timer machine to match global") + } + wg.Done() + }) + select {} + }() + + wg.Wait() +} + +var testGlobal = 0 + +func init() { + testGlobal = 3 +} + +func TestMachineGlobals(t *testing.T) { + if testGlobal != 3 { + t.Error("expected 3") + } + testGlobal = 1 + + gosim.NewSimpleMachine(func() { + if testGlobal != 3 { + t.Error("expected 3") + } + testGlobal = 2 + }).Wait() + + if testGlobal != 1 { + t.Error("expected 1") + } +} + +func TestMachineLabel(t *testing.T) { + m := gosim.NewMachine(gosim.MachineConfig{ + Label: "hello", + MainFunc: func() {}, + }) + if got := m.Label(); got != "hello" { + t.Errorf("expected %q, got %q", "hello", got) + } +} diff --git a/internal/tests/behavior/map_test.go b/internal/tests/behavior/map_test.go new file mode 100644 index 0000000..8ea42ac --- /dev/null +++ b/internal/tests/behavior/map_test.go @@ -0,0 +1,779 @@ +package behavior_test + +import ( + "cmp" + "maps" + "reflect" + "slices" + "testing" +) + +func TestMapSmallGetOK(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + if value, ok := m[2]; !ok || value != 5 { + t.Error("bad get") + } +} + +func TestMapGetOK(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[9] = 11 + m[16] = 7 + + if value, ok := m[2]; !ok || value != 5 { + t.Error("bad get") + } + if value, ok := m[16]; !ok || value != 7 { + t.Error("bad get") + } +} + +func TestMapRMW(t *testing.T) { + m := make(map[int]int) + m[1] += 5 + + m[1]++ + + if value, ok := m[1]; !ok || value != 6 { + t.Error("bad get") + } +} + +func TestMapSmallLen(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + delete(m, 2) + + if len(m) != 2 { + t.Error("bad len") + } +} + +func TestMapLen(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[9] = 11 + m[16] = 7 + m[32] = -1 + delete(m, 2) + + if len(m) != 5 { + t.Error("bad len") + } + + delete(m, 16) + if len(m) != 4 { + t.Error("bad len") + } +} + +func TestMapGetNil(t *testing.T) { + var m map[int]int + + if m[10] != 0 { + t.Error("bad get") + } + + if _, ok := m[10]; ok { + t.Error("bad get") + } +} + +func TestMapLenNil(t *testing.T) { + var m map[int]int + + if len(m) != 0 { + t.Error("bad len") + } +} + +func TestMapIterNil(t *testing.T) { + var m map[int]int + + for k, v := range m { + t.Log(k, v) + } +} + +func TestMapDeleteNil(t *testing.T) { + var m map[int]int + + delete(m, 0) +} + +// Lifted from golang.org/x/exp/maps: + +// Keys returns the keys of the map m. +// The keys will be in an indeterminate order. +func Keys[M ~map[K]V, K comparable, V any](m M) []K { + r := make([]K, 0, len(m)) + for k := range m { + r = append(r, k) + } + return r +} + +// Values returns the values of the map m. +// The values will be in an indeterminate order. +func Values[M ~map[K]V, K comparable, V any](m M) []V { + r := make([]V, 0, len(m)) + for _, v := range m { + r = append(r, v) + } + return r +} + +func TestMapsValues(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + delete(m, 2) + + values := Values(m) + slices.Sort(values) + + if !reflect.DeepEqual(values, []int{3, 8}) { + t.Error("bad values") + } +} + +func TestMapsKeys(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + delete(m, 2) + + keys := Keys(m) + slices.Sort(keys) + + if !reflect.DeepEqual(keys, []int{1, 3}) { + t.Error("bad keys") + } +} + +func TestMapsClone(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + delete(m, 2) + + clone := maps.Clone(m) + + for k, v := range m { + if clone[k] != v { + t.Error("bad value", k, v, clone[k]) + } + } + + for k, v := range clone { + if m[k] != v { + t.Error("bad value", k, v, m[k]) + } + } + + if v, ok := m[3]; !ok || v != 8 { + t.Error("bad get") + } + + if v, ok := clone[3]; !ok || v != 8 { + t.Error("bad get") + } + + delete(m, 3) + + if _, ok := m[3]; ok { + t.Error("bad get") + } + + if v, ok := clone[3]; !ok || v != 8 { + t.Error("bad get") + } +} + +func TestMapSmallGetMissing(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + if _, ok := m[5]; ok { + t.Error("bad get") + } +} + +func TestMapGetMissing(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[11] = 2 + m[17] = 9 + m[19] = 1 + + if _, ok := m[5]; ok { + t.Error("bad get") + } +} + +func TestMapSmallGetMissingThenSet(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + if _, ok := m[5]; ok { + t.Error("bad get") + } + + m[5] = 10 + + if value, ok := m[5]; !ok || value != 10 { + t.Error("bad get") + } +} + +func TestMapGetMissingThenSet(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[11] = 2 + m[17] = 9 + m[19] = 1 + + if _, ok := m[5]; ok { + t.Error("bad get") + } + + m[5] = 10 + + if value, ok := m[5]; !ok || value != 10 { + t.Error("bad get") + } +} + +func TestMapSmallDelete(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + if value, ok := m[2]; !ok || value != 5 { + t.Error("bad get") + } + + delete(m, 2) + + if _, ok := m[2]; ok { + t.Error("bad get") + } +} + +func TestMapDelete(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[11] = 2 + m[17] = 9 + m[19] = 1 + + if value, ok := m[2]; !ok || value != 5 { + t.Error("bad get") + } + + delete(m, 2) + + if _, ok := m[2]; ok { + t.Error("bad get") + } +} + +func TestMapSmallDeleteMissing(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + if _, ok := m[5]; ok { + t.Error("bad get") + } + + delete(m, 5) + + if _, ok := m[5]; ok { + t.Error("bad get") + } +} + +func TestMapDeleteMissing(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[11] = 2 + m[17] = 9 + m[19] = 1 + + if _, ok := m[5]; ok { + t.Error("bad get") + } + + delete(m, 5) + + if _, ok := m[5]; ok { + t.Error("bad get") + } +} + +type pair struct{ key, value int } + +func TestMapSmallSimpleIter(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + expected := []pair{ + {1, 3}, + {2, 5}, + {3, 8}, + } + + var seen []pair + for k, v := range m { + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapSimpleIter(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[11] = 9 + m[13] = 7 + + expected := []pair{ + {1, 3}, + {2, 5}, + {3, 8}, + {11, 9}, + {13, 7}, + } + + var seen []pair + for k, v := range m { + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapSmallRandomIter(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + var firstSeen []pair + for k, v := range m { + firstSeen = append(firstSeen, pair{key: k, value: v}) + } + + success := false + for i := 0; i < 20; i++ { + var seen []pair + for k, v := range m { + seen = append(seen, pair{key: k, value: v}) + } + + if !reflect.DeepEqual(seen, firstSeen) { + success = true + } + } + + if !success { + t.Error("bad") + } +} + +func TestMapRandomIter(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[11] = 9 + m[13] = 7 + + var firstSeen []pair + for k, v := range m { + firstSeen = append(firstSeen, pair{key: k, value: v}) + } + + success := false + for i := 0; i < 20; i++ { + var seen []pair + for k, v := range m { + seen = append(seen, pair{key: k, value: v}) + } + + if !reflect.DeepEqual(seen, firstSeen) { + success = true + } + } + + if !success { + t.Error("bad") + } +} + +func TestMapLiteral(t *testing.T) { + m := map[int]int{ + 1: 3, + 2: 5, + 3: 8, + } + + expected := []pair{ + {1, 3}, + {2, 5}, + {3, 8}, + } + + var seen []pair + for k, v := range m { + seen = append(seen, pair{key: k, value: v}) + } + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapSmallIterDelete(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[4] = 10 + + expected := []pair{ + {1, 3}, + {4, 10}, + } + + var seen []pair + for k, v := range m { + if k == 2 || k == 3 { + expected = append(expected, pair{k, v}) + } + + delete(m, 2) + delete(m, 3) + + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapIterDelete(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[4] = 10 + m[5] = 11 + m[6] = 13 + + expected := []pair{ + {1, 3}, + {4, 10}, + {5, 11}, + {6, 13}, + } + + var seen []pair + for k, v := range m { + if k == 2 || k == 3 { + expected = append(expected, pair{k, v}) + } + + delete(m, 2) + delete(m, 3) + + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapSmallIterAdd(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + expected := []pair{ + {1, 3}, + {2, 5}, + {3, 8}, + } + + var seen []pair + for k, v := range m { + if k == 4 { + expected = append(expected, pair{k, v}) + } + + m[4] = 10 + + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapIterAdd(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[5] = 11 + m[6] = 13 + + expected := []pair{ + {1, 3}, + {2, 5}, + {3, 8}, + {5, 11}, + {6, 13}, + } + + var seen []pair + for k, v := range m { + if k == 4 { + expected = append(expected, pair{k, v}) + } + + m[4] = 10 + + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapGrowingIterAdd(t *testing.T) { + // grow from small to not small during iteration + + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + expected := []pair{ + {1, 3}, + {2, 5}, + {3, 8}, + } + + var seen []pair + for k, v := range m { + if k >= 4 && k <= 7 { + expected = append(expected, pair{k, k + 6}) + } + + m[4] = 10 + m[5] = 11 + m[6] = 12 + m[7] = 13 + + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapIterDeleteAdd(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[4] = 10 + + expected := []pair{ + {1, 3}, + {4, 10}, + } + + var seen []pair + for k, v := range m { + if k == 2 || k == 3 || k == 5 { + expected = append(expected, pair{k, v}) + } + + delete(m, 2) + delete(m, 3) + m[5] = 13 + + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapIterDeleteRestore(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + expected := []pair{} + + var seen []pair + for k, v := range m { + delete(m, k) + m[k] = v + + expected = append(expected, pair{key: k, value: v}) + seen = append(seen, pair{key: k, value: v}) + } + + slices.SortFunc(seen, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + slices.SortFunc(expected, func(a pair, b pair) int { + return cmp.Compare(a.key, b.key) + }) + + if !reflect.DeepEqual(seen, expected) { + t.Error("bad", seen, expected) + } +} + +func TestMapSmallClear(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + + clear(m) + if len(m) != 0 { + t.Error("bad len") + } + if _, ok := m[2]; ok { + t.Error("bad get") + } + + m[8] = 10 + if value, ok := m[8]; !ok || value != 10 { + t.Error("bad get") + } +} + +func TestMapClear(t *testing.T) { + m := make(map[int]int) + m[1] = 3 + m[2] = 5 + m[3] = 8 + m[9] = 11 + m[16] = 7 + + clear(m) + if len(m) != 0 { + t.Error("bad len") + } + if _, ok := m[2]; ok { + t.Error("bad get") + } + + m[8] = 10 + if value, ok := m[8]; !ok || value != 10 { + t.Error("bad get") + } +} diff --git a/internal/tests/behavior/meta_test.go b/internal/tests/behavior/meta_test.go new file mode 100644 index 0000000..7ff5810 --- /dev/null +++ b/internal/tests/behavior/meta_test.go @@ -0,0 +1,24 @@ +//go:build !sim + +package behavior_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestMetaDeterministic(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + metatesting.CheckDeterministic(t, mt) +} + +func TestMetaSeeds(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + metatesting.CheckSeeds(t, mt, 5) +} + +func TestGosim(t *testing.T) { + runner := metatesting.ForCurrentPackage(t) + runner.RunAllTests(t) +} diff --git a/internal/tests/behavior/nemesis_meta_test.go b/internal/tests/behavior/nemesis_meta_test.go new file mode 100644 index 0000000..2f64579 --- /dev/null +++ b/internal/tests/behavior/nemesis_meta_test.go @@ -0,0 +1,56 @@ +//go:build !sim + +package behavior_test + +/* +func TestNemesisDelay(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + counts := make(map[float64]int) + for seed := int64(0); seed < 200; seed++ { + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestNemesisDelayInner", + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + if run.Failed { + t.Fatal("run failed") + } + delay := metatesting.MustFindLogValue(metatesting.ParseLog(run.LogOutput), "delay", "val").(float64) + counts[delay]++ + } + + t.Log(counts) + if counts[-1] == 0 || counts[0] == 0 || counts[1] == 0 { + t.Error("bad") + } +} + +func TestNemesisCrash(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + counts := make(map[float64]int) + for seed := int64(0); seed < 200; seed++ { + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestNemesisCrashInner", + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + if run.Failed { + t.Fatal("run failed") + } + logs := metatesting.ParseLog(run.LogOutput) + start := metatesting.MustFindLogValue(logs, "start", "now").(float64) + hello := metatesting.MustFindLogValue(logs, "hello", "now").(float64) + counts[hello-start]++ + } + + t.Log(counts) + // 5 seconds is the restart time + if counts[0] == 0 || counts[5] == 0 { + t.Error("bad") + } +} +*/ diff --git a/internal/tests/behavior/nemesis_test.go b/internal/tests/behavior/nemesis_test.go new file mode 100644 index 0000000..e732fac --- /dev/null +++ b/internal/tests/behavior/nemesis_test.go @@ -0,0 +1,46 @@ +//go:build sim + +package behavior_test + +/* +func TestNemesisDelayInner(t *testing.T) { + go nemesis.Nemesis(1) + + ch := make(chan struct{}, 2) + + var done1, done2 time.Time + + go func() { + defer func() { ch <- struct{}{} }() + time.Sleep(time.Second) + done1 = time.Now() + }() + + go func() { + defer func() { ch <- struct{}{} }() + time.Sleep(time.Second) + done2 = time.Now() + }() + + <-ch + <-ch + + slog.Info("delay", "val", float64(done2.Sub(done1)/time.Second)) +} + +func TestNemesisCrashInner(t *testing.T) { + slog.Info("start", "now", time.Now().Unix()) + f := func() { + for i := 0; i < 5; i++ { + gosim.Yield() + } + slog.Info("hello", "now", time.Now().Unix()) + // slog.Info("hello", "id", gosimruntime.CurrentMachine()) // logs current machine ID, reboot iteration? + // time.Sleep(time.Second) + } + m := gosim.NewSimpleMachine(f) + go nemesis.Restarter(0, []gosim.Machine{m}) + // let test do its thing + time.Sleep(time.Minute) +} +*/ diff --git a/internal/tests/behavior/net_test.go b/internal/tests/behavior/net_test.go new file mode 100644 index 0000000..765e1db --- /dev/null +++ b/internal/tests/behavior/net_test.go @@ -0,0 +1,1323 @@ +//go:build sim + +package behavior_test + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "net" + "net/http" + "net/netip" + "os" + "syscall" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/jellevandenhooff/gosim" + "github.com/jellevandenhooff/gosim/internal/tests/testpb" +) + +// TODO: at the end of each machine, also check that there are no more running goroutines/events/...? +// TODO: what happens if you message to something non-existent? +// TODO: what if the buffer overflows? + +var ( + aAddr = netip.AddrFrom4([4]byte{10, 1, 0, 1}).String() + bAddr = netip.AddrFrom4([4]byte{10, 1, 0, 2}).String() +) + +func TestNetTcpDial(t *testing.T) { + a := gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.MustParseAddr(aAddr), + MainFunc: func() { + listener, err := net.Listen("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + conn, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + log.Println(conn.LocalAddr()) + if conn.LocalAddr().String() != "10.1.0.1:1234" { + t.Error("bad addr") + } + log.Println(conn.RemoteAddr()) + if conn.RemoteAddr().String() != "10.1.0.2:10000" { + t.Error("bad addr") + } + + resp1 := make([]byte, 3) + if _, err := io.ReadFull(conn, resp1); err != nil { + t.Fatal(err) + } + + resp2 := make([]byte, 2) + if _, err := io.ReadFull(conn, resp2); err != nil { + t.Fatal(err) + } + + resp := string(resp1) + string(resp2) + + if resp != "hello" { + t.Fatal(resp) + } + + if _, err := conn.Write([]byte("world")); err != nil { + t.Fatal(err) + } + + if err := conn.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }, + }) + + b := gosim.NewMachine(gosim.MachineConfig{ + Label: "b", + Addr: netip.MustParseAddr(bAddr), + MainFunc: func() { + // give the other a second to start listening + time.Sleep(time.Second) + + conn, err := net.Dial("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + log.Println(conn.LocalAddr()) + if conn.LocalAddr().String() != "10.1.0.2:10000" { + t.Error("bad addr") + } + log.Println(conn.RemoteAddr()) + if conn.RemoteAddr().String() != "10.1.0.1:1234" { + t.Error("bad addr") + } + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + + resp1 := make([]byte, 3) + if _, err := io.ReadFull(conn, resp1); err != nil { + t.Fatal(err) + } + + resp2 := make([]byte, 2) + if _, err := io.ReadFull(conn, resp2); err != nil { + t.Fatal(err) + } + + resp := string(resp1) + string(resp2) + + if resp != "world" { + t.Fatal(resp) + } + + if err := conn.Close(); err != nil { + t.Fatal(err) + } + }, + }) + + a.Wait() + b.Wait() +} + +func TestNetTcpDisconnectBasic(t *testing.T) { + a := gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.MustParseAddr(aAddr), + MainFunc: func() { + listener, err := net.Listen("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + go func() { + time.Sleep(10 * time.Second) + listener.Close() + }() + + for { + conn, err := listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) { + return + } + t.Fatal(err) + } + + log.Println(conn.LocalAddr()) + log.Println(conn.RemoteAddr()) + + go func() { + buffer := make([]byte, 128) + for { + n, err := conn.Read(buffer) + if err != nil { + return + } + conn.Write(buffer[:n]) + } + }() + } + }, + }) + b := gosim.NewMachine(gosim.MachineConfig{ + Label: "b", + Addr: netip.MustParseAddr(bAddr), + MainFunc: func() { + // give the other a second to start listening + time.Sleep(time.Second) + + // connect works + conn, err := net.Dial("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + // echo works + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + resp := make([]byte, 5) + if _, err := io.ReadFull(conn, resp); err != nil { + t.Fatal(err) + } + respStr := string(resp) + if respStr != "hello" { + t.Fatal(respStr) + } + + // do write, don't read yet + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + time.Sleep(10 * time.Millisecond) + + gosim.SetConnected(aAddr, bAddr, false) + + // reading buffered data works + resp = make([]byte, 5) + if _, err := io.ReadFull(conn, resp); err != nil { + t.Fatal(err) + } + respStr = string(resp) + if respStr != "hello" { + t.Fatal(respStr) + } + + // another read times out, because there is no keep alive + resp = make([]byte, 5) + conn.SetReadDeadline(time.Now().Add(time.Second)) + if n, err := conn.Read(resp); n != 0 || !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatal(n, err) + } + conn.SetReadDeadline(time.Time{}) + + // write works, because there is no keep alive + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + // read without deadline now fails (because it blocks and the write above will cause an ack time out) + resp = make([]byte, 5) + if n, err := conn.Read(resp); n != 0 || !errors.Is(err, syscall.EPIPE) { + t.Fatal(err) + } + // writing now fails because the connection has timed out + if _, err := conn.Write([]byte("hello")); !errors.Is(err, syscall.EPIPE) { + t.Fatal(err) + } + + // a new connect now fails + if _, err := net.Dial("tcp", aAddr+":1234"); !errors.Is(err, syscall.ETIMEDOUT) { + t.Fatal(err) + } + + gosim.SetConnected(aAddr, bAddr, true) + + // new connection works again + conn, err = net.Dial("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + if _, err := conn.Write([]byte("hello")); err != nil { + t.Fatal(err) + } + resp = make([]byte, 5) + if _, err := io.ReadFull(conn, resp); err != nil { + t.Fatal(err) + } + respStr = string(resp) + if respStr != "hello" { + t.Fatal(respStr) + } + }, + }) + + a.Wait() + b.Wait() +} + +func TestNetTcpHttp(t *testing.T) { + a := gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.MustParseAddr(aAddr), + MainFunc: func() { + listener, err := net.Listen("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("hello world")) + }) + + srv := &http.Server{Handler: nil} + + go func() { + time.Sleep(10 * time.Second) + srv.Shutdown(context.Background()) + }() + + if err := srv.Serve(listener); err != nil { + if err != http.ErrServerClosed { + t.Fatal(err) + } + } + }, + }) + + b := gosim.NewMachine(gosim.MachineConfig{ + Label: "b", + Addr: netip.MustParseAddr(bAddr), + MainFunc: func() { + // give the other a second to start listening + time.Sleep(time.Second) + + resp, err := http.Get(fmt.Sprintf("http://%s:1234/", aAddr)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + log.Println(resp.StatusCode) + + bytes, err := io.ReadAll(resp.Body) + + log.Println(string(bytes), err) + + if string(bytes) != "hello world" { + t.Fatal(string(bytes)) + } + }, + }) + + a.Wait() + b.Wait() +} + +type testServer struct{} + +func (s *testServer) Echo(ctx context.Context, req *testpb.EchoRequest) (*testpb.EchoResponse, error) { + return &testpb.EchoResponse{ + Message: req.Message, + }, nil +} + +func TestNetTcpGrpc(t *testing.T) { + a := gosim.NewMachine(gosim.MachineConfig{ + Label: "a", + Addr: netip.MustParseAddr(aAddr), + MainFunc: func() { + listener, err := net.Listen("tcp", aAddr+":1234") + if err != nil { + t.Fatal(err) + } + + server := grpc.NewServer() + testpb.RegisterEchoServerServer(server, &testServer{}) + + go func() { + time.Sleep(10 * time.Second) + server.GracefulStop() + }() + + if err := server.Serve(listener); err != nil { + t.Fatal(err) + } + }, + }) + + b := gosim.NewMachine(gosim.MachineConfig{ + Label: "b", + Addr: netip.MustParseAddr(bAddr), + MainFunc: func() { + // give the other a second to start listening + time.Sleep(time.Second) + + client, err := grpc.Dial(fmt.Sprintf("%s:1234", aAddr), grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + + testClient := testpb.NewEchoServerClient(client) + + resp, err := testClient.Echo(context.Background(), &testpb.EchoRequest{Message: "hello world"}) + if err != nil { + t.Fatal(err) + } + if resp.Message != "hello world" { + t.Fatal(resp.Message) + } + + client.Close() + }, + }) + + a.Wait() + b.Wait() +} + +// TODO: test hosts crash? +// TODO: port tests below to TCP? + +/* +func TestNetBasic(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + if err := gosim.Send(bAddr, []byte("hello")); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "hello" { + t.Fatal(string(data)) + } + + if from != aAddr { + t.Fatal(from) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetPingPong(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + for i := 0; i < 10; i++ { + if err := gosim.Send(bAddr, []byte("ping")); err != nil { + t.Fatal(err) + } + + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "pong" { + t.Fatal(string(data)) + } + + if from != bAddr { + t.Fatal(from) + } + } + }) + + b.Run(func() { + for i := 0; i < 10; i++ { + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "ping" { + t.Fatal(string(data)) + } + + if from != aAddr { + t.Fatal(from) + } + if err := gosim.Send(aAddr, []byte("pong")); err != nil { + t.Fatal(err) + } + } + }) + + a.Wait() + b.Wait() +} + +func TestNetChain(t *testing.T) { + var machines []*gosim.Machine + + addr := func(i int) netip.Addr { + return netip.AddrFrom4([4]byte{10, 1, 0, 1 + byte(i)}) + } + + for i := 0; i < 10; i++ { + machines = append(machines, gosim.NewMachineWithLabel(fmt.Sprint(i), addr(i), func() {})) + } + + machines[0].Run(func() { + if err := gosim.Send(addr(1), []byte("ping from 0")); err != nil { + t.Fatal(err) + } + }) + + for i := 1; i < 9; i++ { + i := i + machines[i].Run(func() { + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "ping from "+fmt.Sprint(i-1) { + t.Fatal(string(data)) + } + + if from != addr(i-1) { + t.Fatal(from) + } + + if err := gosim.Send(addr(i+1), []byte("ping from "+fmt.Sprint(i))); err != nil { + t.Fatal(err) + } + }) + } + + machines[9].Run(func() { + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "ping from 8" { + t.Fatal(string(data)) + } + + if from != addr(8) { + t.Fatal(from) + } + }) + + for i := 0; i < 10; i++ { + machines[i].Wait() + } +} + +func TestNetAsync(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + go func() { + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "goodbye" { + t.Fatal(string(data)) + } + + if from != bAddr { + t.Fatal(from) + } + }() + + time.Sleep(time.Second) + + if err := gosim.Send(bAddr, []byte("hello")); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + data, from, err := gosim.Receive() + if err != nil { + t.Fatal(err) + } + + if string(data) != "hello" { + t.Fatal(string(data)) + } + + if from != aAddr { + t.Fatal(from) + } + + if err := gosim.Send(aAddr, []byte("goodbye")); err != nil { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} +*/ + +/* + +func TestDoubleListen(t *testing.T) { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + if _, err := gosim.OpenListener(80); err == nil || err.Error() != gosim.ErrPortAlreadyInUse.Error() { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + + listener2, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + if err := listener2.Close(); err != nil { + t.Fatal(err) + } +} + +func TestNetStreamOpenClose(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + if stream.Peer().Addr() != bAddr { + t.Fatal(stream.Peer()) + } + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + if stream.Peer().Addr() != aAddr { + t.Fatal(stream.Peer()) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamData(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + if err := stream.Send([]byte("ping")); err != nil { + t.Fatal(err) + } + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "pong" { + t.Fatal(resp) + } + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "ping" { + t.Fatal(resp) + } + + if err := stream.Send([]byte("pong")); err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamBackpressure(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + last := time.Now() + + for { + if err := stream.Send([]byte("ping")); err != nil { + t.Fatal(err) + } + if now := time.Now(); !now.Equal(last) { + gap := now.Sub(last) + if gap != time.Second { + s.Error("bad gap", gap) + } + break + } + } + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "pong" { + t.Fatal(resp) + } + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Second) + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + if string(resp) != "ping" { + t.Fatal(resp) + } + + if err := stream.Send([]byte("pong")); err != nil { + t.Fatal(err) + } + + for { + resp, err := stream.Recv() + if err != nil { + if err.Error() == gosim.ErrStreamClosed.Error() { + break + } + t.Fatal(err) + } + if string(resp) != "ping" { + t.Fatal(resp) + } + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamBlockRecvThenSend(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + + go func() { + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "pong" { + t.Fatal(resp) + } + + close(done) + }() + + time.Sleep(time.Second) + + if err := stream.Send([]byte("ping")); err != nil { + t.Fatal(err) + } + + <-done + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "ping" { + t.Fatal(resp) + } + + if err := stream.Send([]byte("pong")); err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamBlockSendThenRecv(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + + go func() { + last := time.Now() + + for { + if err := stream.Send([]byte("ping")); err != nil { + t.Fatal(err) + } + if now := time.Now(); !now.Equal(last) { + gap := now.Sub(last) + if gap != time.Second { + s.Error("bad gap", gap) + } + break + } + } + + close(done) + }() + + time.Sleep(time.Second) + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "pong" { + t.Fatal(resp) + } + + <-done + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Second) + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + if string(resp) != "ping" { + t.Fatal(resp) + } + + if err := stream.Send([]byte("pong")); err != nil { + t.Fatal(err) + } + + for { + resp, err := stream.Recv() + if err != nil { + if err.Error() == gosim.ErrStreamClosed.Error() { + break + } + t.Fatal(err) + } + if string(resp) != "ping" { + t.Fatal(resp) + } + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamCloseListenerClosesStream(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + time.Sleep(time.Second) + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamCloseListenerSide(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamCloseDialSide(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamTwoClientsQueue(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + bs := []*gosim.Machine{ + gosim.NewMachineWithLabel("b1", bAddr, func() {}), + gosim.NewMachineWithLabel("b2", bAddr.Next(), func() {}), + } + + a.Run(func() { + listener, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + // make sure streams are queued in listener + time.Sleep(time.Second) + + done := make(chan struct{}, 2) + + for i := 0; i < 2; i++ { + stream, err := listener.Accept() + if err != nil { + t.Fatal(err) + } + + go func() { + // make sure both streams exist for a while concurrently + time.Sleep(time.Second) + + if err := stream.Send([]byte("ping")); err != nil { + t.Fatal(err) + } + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "pong" { + t.Fatal(resp) + } + + if err := stream.Close(); err != nil { + t.Fatal(err) + } + + done <- struct{}{} + }() + } + + for i := 0; i < 2; i++ { + <-done + } + + if err := listener.Close(); err != nil { + t.Fatal(err) + } + }) + + for _, b := range bs { + b.Run(func() { + // make sure it's listening?? + time.Sleep(100 * time.Millisecond) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + resp, err := stream.Recv() + if err != nil { + t.Fatal(err) + } + + if string(resp) != "ping" { + t.Fatal(resp) + } + + if err := stream.Send([]byte("pong")); err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + } + + a.Wait() + for _, b := range bs { + b.Wait() + } +} + +func TestNetStreamListenerOverflow(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + _, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + time.Sleep(100 * time.Second) + }) + + b.Run(func() { + time.Sleep(time.Second) + + var mu sync.Mutex + var streams []*gosim.Stream + var done bool + + for { + mu.Lock() + if done { + mu.Unlock() + return + } + mu.Unlock() + + go func() { + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 80)) + if err != nil { + t.Fatal(err) + } + + mu.Lock() + if done { + mu.Unlock() + return + } + streams = append(streams, stream) + mu.Unlock() + + if _, err := stream.Recv(); err != nil { + if err.Error() == gosim.ErrStreamClosed.Error() { + mu.Lock() + done = true + for _, stream := range streams { + stream.Close() + } + mu.Unlock() + } + } + }() + + time.Sleep(100 * time.Millisecond) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamWrongPort(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + _, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + time.Sleep(100 * time.Second) + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(aAddr, 81)) + if err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} + +func TestNetStreamWrongHost(t *testing.T) { + a := gosim.NewMachineWithLabel("a", aAddr, func() {}) + b := gosim.NewMachineWithLabel("b", bAddr, func() {}) + + a.Run(func() { + _, err := gosim.OpenListener(80) + if err != nil { + t.Fatal(err) + } + + time.Sleep(100 * time.Second) + }) + + b.Run(func() { + time.Sleep(time.Second) + + stream, err := gosim.Dial(netip.AddrPortFrom(netip.AddrFrom4([4]byte{1, 2, 3, 4}), 80)) + if err != nil { + t.Fatal(err) + } + + if _, err := stream.Recv(); err == nil || err.Error() != gosim.ErrStreamClosed.Error() { + t.Fatal(err) + } + }) + + a.Wait() + b.Wait() +} +*/ diff --git a/internal/tests/behavior/os_sim_test.go b/internal/tests/behavior/os_sim_test.go new file mode 100644 index 0000000..bb4be3d --- /dev/null +++ b/internal/tests/behavior/os_sim_test.go @@ -0,0 +1,37 @@ +//go:build sim + +package behavior_test + +import ( + "os" + "testing" + + "github.com/jellevandenhooff/gosim" +) + +func TestMachineHostname(t *testing.T) { + name, err := os.Hostname() + if err != nil { + t.Fatal(err) + } + expected := "main" + if name != expected { + t.Errorf("bad name %q, expected %q", name, expected) + } + + m := gosim.NewMachine(gosim.MachineConfig{ + Label: "hello123", + MainFunc: func() { + name, err := os.Hostname() + if err != nil { + t.Fatal(err) + } + expected := "hello123" + if name != expected { + t.Errorf("bad name %q, expected %q", name, expected) + } + }, + }) + + m.Wait() +} diff --git a/internal/tests/behavior/os_test.go b/internal/tests/behavior/os_test.go new file mode 100644 index 0000000..eb3e894 --- /dev/null +++ b/internal/tests/behavior/os_test.go @@ -0,0 +1,21 @@ +package behavior_test + +import ( + "os" + "testing" +) + +func TestGetpid(t *testing.T) { + pid := os.Getpid() + if pid == 0 { + t.Error("got 0 pid") + } +} + +func TestHostname(t *testing.T) { + name, err := os.Hostname() + if err != nil { + t.Fatal(err) + } + t.Log(name) +} diff --git a/internal/tests/behavior/rand_meta_test.go b/internal/tests/behavior/rand_meta_test.go new file mode 100644 index 0000000..9a1f0c0 --- /dev/null +++ b/internal/tests/behavior/rand_meta_test.go @@ -0,0 +1,82 @@ +//go:build !sim + +package behavior_test + +import ( + "maps" + "strings" + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestMetaRandDifferentBySeed(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + tests, err := mt.ListTests() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + if !strings.HasPrefix(test, "TestRand") { + continue + } + + values := make(map[string]int) + for _, seed := range []int64{1, 2, 3, 4, 5} { + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: test, + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + value := metatesting.MustFindLogValue(metatesting.ParseLog(run.LogOutput), "rand", "val").(string) + values[value]++ + } + t.Logf("values=%v", values) + if len(values) != 5 { + t.Error("expected different seeds") + } + } +} + +func TestMetaRandDeterministic(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + tests, err := mt.ListTests() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + if !strings.HasPrefix(test, "TestRand") { + continue + } + + t.Run(test, func(t *testing.T) { + values := make(map[string]int) + for _, seed := range []int64{1, 2, 3, 4, 5} { + innerValues := make(map[string]int) + for range 2 { + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: test, + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + value := metatesting.MustFindLogValue(metatesting.ParseLog(run.LogOutput), "rand", "val").(string) + innerValues[value]++ + } + if len(innerValues) != 1 { + t.Error("expected determinism") + } + maps.Copy(values, innerValues) + } + t.Logf("values=%v", values) + if len(values) != 5 { + t.Error("expected different seeds") + } + }) + } +} diff --git a/internal/tests/behavior/rand_test.go b/internal/tests/behavior/rand_test.go new file mode 100644 index 0000000..d7643e5 --- /dev/null +++ b/internal/tests/behavior/rand_test.go @@ -0,0 +1,35 @@ +//go:build sim + +package behavior_test + +import ( + cryptorand "crypto/rand" + "encoding/hex" + "fmt" + "hash/maphash" + "log/slog" + mathrand "math/rand" + mathrandv2 "math/rand/v2" + "testing" +) + +func TestRandMapHash(t *testing.T) { + s := maphash.MakeSeed() + slog.Info("rand", "val", fmt.Sprint(maphash.String(s, "foo"))) +} + +func TestRandMathFloat64(t *testing.T) { + slog.Info("rand", "val", fmt.Sprint(mathrand.Float64())) +} + +func TestRandMathV2Float64(t *testing.T) { + slog.Info("rand", "val", fmt.Sprint(mathrandv2.Float64())) +} + +func TestRandCrypto(t *testing.T) { + var buf [32]byte + if _, err := cryptorand.Read(buf[:]); err != nil { + t.Fatal(err) + } + slog.Info("rand", "val", hex.EncodeToString(buf[:])) +} diff --git a/internal/tests/behavior/rand_v2_test.go b/internal/tests/behavior/rand_v2_test.go new file mode 100644 index 0000000..c51c542 --- /dev/null +++ b/internal/tests/behavior/rand_v2_test.go @@ -0,0 +1,16 @@ +package behavior_test + +import ( + "log" + "math/rand/v2" + "testing" +) + +func TestChacha8Rand(t *testing.T) { + rnd := rand.NewChaCha8([32]byte{}) + x := rnd.Uint64() + log.Println(x) + if x != 12432809566553409497 { + t.Error("bad") + } +} diff --git a/internal/tests/behavior/reflect_test.go b/internal/tests/behavior/reflect_test.go new file mode 100644 index 0000000..cac4274 --- /dev/null +++ b/internal/tests/behavior/reflect_test.go @@ -0,0 +1,212 @@ +package behavior_test + +import ( + "encoding/json" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestReflectFormatMap(t *testing.T) { + formatted := fmt.Sprint(map[string]int{"foo": 1, "bar": 2}) + if formatted != "map[bar:2 foo:1]" { + t.Error(formatted) + } +} + +func TestReflectMapValue(t *testing.T) { + m := make(map[int]string) + + value := reflect.ValueOf(m) + if value.Kind() != reflect.Map { + t.Error(value.Kind()) + } + + m[3] = "hello" + res := value.MapIndex(reflect.ValueOf(3)) + if res.Kind() != reflect.String || res.Interface() != "hello" { + t.Error(res) + } + + value.SetMapIndex(reflect.ValueOf(5), reflect.ValueOf("world")) + if m[5] != "world" { + t.Error(m[5]) + } + + mapKeys := value.MapKeys() + if len(mapKeys) != 2 { + t.Error(mapKeys) + } + + gotMapKeys := make(map[int]bool) + for _, key := range mapKeys { + gotMapKeys[key.Interface().(int)] = true + } + if len(gotMapKeys) != 2 || !gotMapKeys[3] || !gotMapKeys[5] { + t.Error(gotMapKeys) + } + + gotIter := make(map[int]string) + iter := value.MapRange() + for iter.Next() { + gotIter[iter.Key().Interface().(int)] = iter.Value().Interface().(string) + } + if len(gotIter) != 2 || gotIter[3] != "hello" || gotIter[5] != "world" { + t.Error(gotIter) + } + + // XXX: panics for bad behavior + // XXX: wrapping of items in interface (eg. map[string]any items have Kind reflect.Interface) +} + +func TestReflectMapType(t *testing.T) { + m := make(map[int]string) + + typ := reflect.TypeOf(m) + if typ.Kind() != reflect.Map { + t.Error(typ.Kind()) + } + + typ2 := reflect.TypeOf(map[int]string{2: "foo"}) + if typ != typ2 { + t.Error(typ, typ2) + } + + if typ.Key() != reflect.TypeOf(int(0)) { + t.Error(typ.Key().Kind()) + } + + if typ.Elem() != reflect.TypeOf(string("")) { + t.Error(typ.Elem().Kind()) + } + + // XXX: typ.String() +} + +func TestReflectJSON(t *testing.T) { + type Foo struct { + A int + B string `json:"buz"` + C map[string]any + } + type Bar struct { + D map[string]string + } + + x := Foo{ + A: 5, + B: "hello", + C: map[string]any{ + "x": "y", + "bar": Bar{ + D: map[string]string{ + "x": "y", + "z": "w", + }, + }, + }, + } + + bytes, err := json.Marshal(x) + if err != nil { + t.Error(err) + } + + if string(bytes) != `{"A":5,"buz":"hello","C":{"bar":{"D":{"x":"y","z":"w"}},"x":"y"}}` { + t.Error(string(bytes)) + } + + var out any + if err := json.Unmarshal(bytes, &out); err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(out, map[string]any{ + "A": 5.0, + "buz": "hello", + "C": map[string]any{ + "x": "y", + "bar": map[string]any{ + "D": map[string]any{ + "x": "y", + "z": "w", + }, + }, + }, + }) { + t.Error(out) + } + + newBytes, err := json.Marshal(out) + if err != nil { + t.Error(err) + } + + if string(newBytes) != `{"A":5,"C":{"bar":{"D":{"x":"y","z":"w"}},"x":"y"},"buz":"hello"}` { + t.Error(string(newBytes)) + } +} + +func TestReflectDeepEqual(t *testing.T) { + cases := []struct { + A, B any + Equal bool + }{ + { + A: "foo", + B: "bar", + Equal: false, + }, + { + A: "foo", + B: "foo", + Equal: true, + }, + { + A: map[string]int{"foo": 3, "bar": 4}, + B: map[string]int{"foo": 4, "bar": 3}, + Equal: false, + }, + { + A: map[string]int{"foo": 3, "bar": 4}, + B: map[string]int{"foo": 3, "bar": 4}, + Equal: true, + }, + } + + for _, c := range cases { + if reflect.DeepEqual(c.A, c.B) != c.Equal { + t.Error(c.A, c.B, c.Equal) + } + } +} + +func TestReflectGoCmp(t *testing.T) { + cases := []struct { + A, B any + MustContain []string + }{ + { + A: "foo", + B: "bar", + MustContain: []string{"foo", "bar"}, + }, + } + + for _, c := range cases { + diff := cmp.Diff(c.A, c.B) + ok := true + for _, part := range c.MustContain { + if !strings.Contains(diff, part) { + ok = false + } + } + if !ok { + t.Error(c.A, c.B, c.MustContain, strconv.Quote(diff)) + } + } +} diff --git a/internal/tests/behavior/sync_test.go b/internal/tests/behavior/sync_test.go new file mode 100644 index 0000000..48b8b60 --- /dev/null +++ b/internal/tests/behavior/sync_test.go @@ -0,0 +1,31 @@ +package behavior_test + +import ( + "fmt" + "log" + "sync" + "testing" +) + +func TestGoMutexWaitGroup(t *testing.T) { + var mu sync.Mutex + var wg sync.WaitGroup + var order string + + for i := 0; i < 5; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 5; j++ { + mu.Lock() + order += fmt.Sprint(i) + mu.Unlock() + } + }() + } + wg.Wait() + + // TODO: assert this string is the same on every run + log.Print(order) +} diff --git a/internal/tests/behavior/testing_meta_test.go b/internal/tests/behavior/testing_meta_test.go new file mode 100644 index 0000000..9fb4c4f --- /dev/null +++ b/internal/tests/behavior/testing_meta_test.go @@ -0,0 +1,218 @@ +//go:build !sim + +package behavior_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestTFatalAborts(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestTFatalAborts", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrTestFailed.Error() { // XXX: janky + t.Error("expected test failed err") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "ERROR help", + }); diff != "" { + t.Error(diff) + } +} + +func TestPanicAbortsOther(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestPanicAbortsOther", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrPaniced.Error() { // XXX: janky + t.Error("expected paniced err") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "ERROR uncaught panic", + }); diff != "" { + t.Error(diff) + } +} + +func TestPanic(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestPanic", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrPaniced.Error() { // XXX: janky + t.Error("expected paniced err") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "ERROR uncaught panic", + }); diff != "" { + t.Error(diff) + } +} + +func TestPanicInsideMachine(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestPanicInsideMachine", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrPaniced.Error() { // XXX: janky + t.Error("expected paniced err") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "ERROR uncaught panic", + }); diff != "" { + t.Error(diff) + } +} + +func TestTErrorDoesNotAbort(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestTErrorDoesNotAbort", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrTestFailed.Error() { // XXX: janky + t.Error("expected test failed err") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "ERROR help", + "INFO after", + "INFO done", + }); diff != "" { + t.Error(diff) + } +} + +func TestSimulationTimeout(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestSimulationTimeout", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrTimeout.Error() { // XXX: janky + t.Error("expected timeout") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "INFO 30s", + "INFO 1m30s", + "INFO 2m30s", + "INFO 3m30s", + "INFO 4m30s", + "INFO 5m30s", + "INFO 6m30s", + "INFO 7m30s", + "INFO 8m30s", + "INFO 9m30s", + "ERROR test timeout", + }); diff != "" { + t.Error(diff) + } +} + +func TestSimulationTimeoutOverride(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + + run, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestSimulationTimeoutOverride", + Seed: 1, + ExtraEnv: []string{"TESTINGFAIL=1"}, + }) + if err != nil { + t.Fatal(err) + } + + if !run.Failed { + t.Error("expected failure") + } + if run.Err != gosimruntime.ErrTimeout.Error() { // XXX: janky + t.Error("expected timeout") + } + + if diff := cmp.Diff(metatesting.SimplifyParsedLog(metatesting.ParseLog(run.LogOutput)), []string{ + "INFO before", + "INFO 30s", + "INFO 1m30s", + "INFO 2m30s", + "INFO 3m30s", + "INFO 4m30s", + "ERROR test timeout", + }); diff != "" { + t.Error(diff) + } +} diff --git a/internal/tests/behavior/testing_test.go b/internal/tests/behavior/testing_test.go new file mode 100644 index 0000000..cd536a9 --- /dev/null +++ b/internal/tests/behavior/testing_test.go @@ -0,0 +1,296 @@ +//go:build sim + +package behavior + +import ( + "log" + "os" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + + "github.com/jellevandenhooff/gosim" +) + +// TODO: t.Fail, t.FailNow, log.Fatal, fmt.Print, slog.Log, os.Exit + +func TestTFatalAborts(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + log.Println("before") + t.Fatal("help") + log.Println("after") +} + +func TestPanicAbortsOther(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + ch := make(chan struct{}) + go func() { + log.Println("before") + panic("help") + log.Println("after") + close(ch) + }() + + <-ch + log.Println("never") +} + +func TestPanic(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + log.Println("before") + panic("help") + log.Println("never") +} + +func TestPanicInsideMachine(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + m := gosim.NewSimpleMachine(func() { + log.Println("before") + panic("help") + log.Println("after") + }) + m.Wait() + log.Println("never") +} + +func TestTErrorDoesNotAbort(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + ch := make(chan struct{}) + + go func() { + log.Println("before") + t.Error("help") + log.Println("after") + close(ch) + }() + + <-ch + log.Println("done") +} + +/* +// TODO: Port +func TestNestedNondeterminismLog(t *testing.T) { + config := gosim.SimConfig{ + Seeds: []int64{1}, + DontReportFail: true, + CaptureLog: true, + LogLevelOverride: "INFO", + } + + result := gosim.RunNestedTest(t, config, gosim.Test{Test: func(t *testing.T) { + config := gosim.SimConfig{ + Seeds: []int64{1}, + EnableTracer: true, + CheckDeterminismRuns: 1, + CaptureLog: true, + LogLevelOverride: "INFO", + } + + result := gosim.RunNestedTest(t, config, gosim.Test{Test: func(t *testing.T) { + t.Log(gosimruntime.Nondeterministic()) + }}) + + if len(result) != 2 { + t.Error("expected 2 runs, got", len(result)) + } + if result[0].Failed || result[0].Err != nil { + t.Error("expected success for first run") + } + if result[1].Failed || result[1].Err != nil { + t.Error("expected success second run") + } + }}) + + if !result[0].Failed { + t.Error("expected failure") + } + + if diff := cmp.Diff(gosim.SimplifyParsedLog(gosim.ParseLog(result[0].LogOutput)), []string{ + "ERROR traces differ: non-determinism found", + "ERROR logs differ: non-determinism found", + }); diff != "" { + t.Error(diff) + } +} + +func TestNondeterminism(t *testing.T) { + if gosimruntime.Nondeterministic()%2 == 0 { + go func() { + }() + } +} + +func TestNestedNondeterminismTrace(t *testing.T) { + config := gosim.SimConfig{ + Seeds: []int64{1}, + DontReportFail: true, + CaptureLog: true, + LogLevelOverride: "INFO", + } + + result := gosim.RunNestedTest(t, config, gosim.Test{Test: func(t *testing.T) { + config := gosim.SimConfig{ + Seeds: []int64{1}, + EnableTracer: true, + CheckDeterminismRuns: 1, + CaptureLog: true, + LogLevelOverride: "INFO", + } + + result := gosim.RunNestedTest(t, config, gosim.Test{Test: func(t *testing.T) { + if gosimruntime.Nondeterministic()%2 == 0 { + go func() { + }() + } + }}) + + if len(result) != 2 { + t.Error("expected 2 runs, got", len(result)) + } + if result[0].Failed || result[0].Err != nil { + t.Error("expected success for first run") + } + if result[1].Failed || result[1].Err != nil { + t.Error("expected success second run") + } + }}) + + if !result[0].Failed { + t.Error("expected failure") + } + if result[0].Err != nil { + t.Error("expected no error") + } + + if diff := cmp.Diff(gosim.SimplifyParsedLog(gosim.ParseLog(result[0].LogOutput)), []string{ + "ERROR traces differ: non-determinism found", + }); diff != "" { + t.Error(diff) + } +} +*/ + +func TestTName(t *testing.T) { + // use Meta prefix so we don't run under TestDeterminism which gives a different name + if t.Name() != "TestTName" { + t.Error(t.Name()) + } +} + +func TestNestedTName(t *testing.T) { + t.Run("subtest", func(t *testing.T) { + if t.Name() != "TestNestedTName/subtest" { + t.Error(t.Name()) + } + }) +} + +func TestCleanup(t *testing.T) { + var order []int + + // TODO: help, what about a clean-up inside of an aborted machine? + + t.Run("subtest", func(t *testing.T) { + t.Cleanup(func() { + order = append(order, 2) + }) + t.Cleanup(func() { + order = append(order, 1) + }) + order = append(order, 0) + }) + + if diff := cmp.Diff([]int{0, 1, 2}, order); diff != "" { + t.Error(diff) + } +} + +func TestIsSim(t *testing.T) { + if !gosim.IsSim() { + t.Error("should be sim") + } +} + +/* +// TODO: Port +func TestDeadlock(t *testing.T) { + c := gosim.SimConfig{ + Seeds: []int64{1}, + DontReportFail: true, + } + + result := gosim.RunNestedTest(t, c, gosim.Test{Test: func(t *testing.T) { + var mu1, mu2 sync.Mutex + + go func() { + mu2.Lock() + time.Sleep(time.Second) + mu1.Lock() + }() + + mu1.Lock() + time.Sleep(time.Second) + mu2.Lock() + }}) + + if !result[0].Failed { + t.Error("need fail") + } + if result[0].Err != gosimruntime.ErrTimeout { + t.Error(result[0].Err) + } +} +*/ + +func TestHello(t *testing.T) { + t.Log(os.Getenv("HELLO")) +} + +func TestSimulationTimeout(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + log.Println("before") + start := time.Now() + time.Sleep(30 * time.Second) + for range 20 { + log.Println(time.Since(start)) + time.Sleep(time.Minute) + } + log.Println("after") +} + +func TestSimulationTimeoutOverride(t *testing.T) { + if os.Getenv("TESTINGFAIL") != "1" { + t.Skip() + } + + gosim.SetSimulationTimeout(5 * time.Minute) + + log.Println("before") + start := time.Now() + time.Sleep(30 * time.Second) + for range 20 { + log.Println(time.Since(start)) + time.Sleep(time.Minute) + } + log.Println("after") +} diff --git a/internal/tests/behavior/time_test.go b/internal/tests/behavior/time_test.go new file mode 100644 index 0000000..c633123 --- /dev/null +++ b/internal/tests/behavior/time_test.go @@ -0,0 +1,392 @@ +//go:build sim + +package behavior_test + +import ( + "log" + "log/slog" + "testing" + "time" +) + +// TODO: test these against real behavior with some slack built in for time +// TODO: test monotonic time with different machines/universes + +func TestTimeBasic(t *testing.T) { + a := time.Now() + + time.Sleep(10 * time.Second) + + b := time.Now() + if b.Sub(a) < 10*time.Second { + t.Error(a, b) + } + + time.Sleep(10 * time.Second) + + c := time.Now() + if c.Sub(b) < 10*time.Second { + t.Error(a, b) + } + + log.Println(a, b, c) +} + +func TestTimerPast(t *testing.T) { + a := time.Now() + timer := time.NewTimer(-10 * time.Second) + <-timer.C + b := time.Now() + if !a.Equal(b) { + t.Error(a, b) + } +} + +func TestTimerAfter(t *testing.T) { + a := time.Now() + + <-time.After(10 * time.Second) + + b := time.Now() + if b.Sub(a) != 10*time.Second { + t.Error(a, b) + } + + <-time.After(10 * time.Second) + + c := time.Now() + if c.Sub(b) != 10*time.Second { + t.Error(a, b) + } +} + +func TestTimerValue(t *testing.T) { + a := time.Now() + ret := <-time.After(10 * time.Second) + b := time.Now() + if b.Sub(a) != 10*time.Second { + t.Error(a, b) + } + if !ret.Equal(b) { + t.Error(ret, b) + } +} + +func TestTimerValueSleep(t *testing.T) { + a := time.Now() + ch := time.After(10 * time.Second) + time.Sleep(20 * time.Second) + ret := <-ch + b := time.Now() + if b.Sub(a) != 20*time.Second { + t.Error(a, b) + } + if ret.Sub(a) != 10*time.Second { + t.Error(ret, a) + } +} + +func TestTimerResetOk(t *testing.T) { + a := time.Now() + + timer := time.NewTimer(10 * time.Second) + if !timer.Reset(20 * time.Second) { + t.Error("bad reset") + } + <-timer.C + + b := time.Now() + if b.Sub(a) != 20*time.Second { + t.Error(a, b) + } +} + +func TestTimerStopOk(t *testing.T) { + a := time.Now() + + timer := time.NewTimer(10 * time.Second) + + if timer.Stop() { + timer.Reset(20 * time.Second) + } else { + t.Error("bad") + } + <-timer.C + + b := time.Now() + if b.Sub(a) != 20*time.Second { + t.Error(a, b) + } +} + +func TestTimerStopNotOk(t *testing.T) { + a := time.Now() + + timer := time.NewTimer(10 * time.Second) + + time.Sleep(20 * time.Second) + + if !timer.Stop() { + <-timer.C + timer.Reset(20 * time.Second) + } else { + t.Error("bad") + } + + <-timer.C + + b := time.Now() + if b.Sub(a) != 40*time.Second { + t.Error(a, b) + } +} + +func TestTimerAfterFunc(t *testing.T) { + a := time.Now() + + done := make(chan struct{}, 0) + time.AfterFunc(10*time.Second, func() { + done <- struct{}{} + }) + + <-done + b := time.Now() + if b.Sub(a) != 10*time.Second { + t.Error(a, b) + } +} + +func TestTimerAfterFuncReset(t *testing.T) { + a := time.Now() + + done := make(chan struct{}, 0) + timer := time.AfterFunc(10*time.Second, func() { + done <- struct{}{} + }) + + <-done + b := time.Now() + if b.Sub(a) != 10*time.Second { + t.Error(a, b) + } + + timer.Reset(10 * time.Second) + timer.Reset(5 * time.Second) + + <-done + c := time.Now() + if c.Sub(b) != 5*time.Second { + t.Error(b, c) + } +} + +func TestTimeSameStart(t *testing.T) { + time.Sleep(10 * time.Second) + + // TODO: need a test for determinism here? + slog.Info("after", "now", time.Now().String()) +} + +func TestTimeTickerStop(t *testing.T) { + start := time.Now() + + ticker := time.NewTicker(5 * time.Second) + + v := <-ticker.C + if now := time.Now(); now.Sub(start) != 5*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 5*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 10*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 10*time.Second { + t.Error(start, v) + } + + ticker.Stop() + time.Sleep(10 * time.Second) + select { + case v := <-ticker.C: + t.Error(v, start) + default: + } +} + +func TestTimeTickerReset(t *testing.T) { + start := time.Now() + + ticker := time.NewTicker(5 * time.Second) + + v := <-ticker.C + if now := time.Now(); now.Sub(start) != 5*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 5*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 10*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 10*time.Second { + t.Error(start, v) + } + + time.Sleep(2 * time.Second) + ticker.Reset(6 * time.Second) + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 18*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 18*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 24*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 24*time.Second { + t.Error(start, v) + } + + time.Sleep(2 * time.Second) + ticker.Reset(2 * time.Second) + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 28*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 28*time.Second { + t.Error(start, v) + } + + time.Sleep(7 * time.Second) + ticker.Reset(5 * time.Second) + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 35*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 30*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 40*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 40*time.Second { + t.Error(start, v) + } +} + +func TestTimeTickerStopReset(t *testing.T) { + start := time.Now() + + ticker := time.NewTicker(5 * time.Second) + + v := <-ticker.C + if now := time.Now(); now.Sub(start) != 5*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 5*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 10*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 10*time.Second { + t.Error(start, v) + } + + ticker.Stop() + time.Sleep(10 * time.Second) + select { + case v := <-ticker.C: + t.Error(v, start) + default: + } + + ticker.Reset(3 * time.Second) + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 23*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 23*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 26*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 26*time.Second { + t.Error(start, v) + } +} + +func TestTimeTickerNonBlocking(t *testing.T) { + start := time.Now() + + ticker := time.NewTicker(5 * time.Second) + + v := <-ticker.C + if now := time.Now(); now.Sub(start) != 5*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 5*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 10*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 10*time.Second { + t.Error(start, v) + } + + time.Sleep(8 * time.Second) + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 18*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 15*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 20*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 20*time.Second { + t.Error(start, v) + } + + time.Sleep(31 * time.Second) + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 51*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 25*time.Second { + t.Error(start, v) + } + + v = <-ticker.C + if now := time.Now(); now.Sub(start) != 55*time.Second { + t.Error(start, now) + } + if v.Sub(start) != 55*time.Second { + t.Error(start, v) + } +} diff --git a/internal/tests/race/race_test.go b/internal/tests/race/race_test.go new file mode 100644 index 0000000..ede761a --- /dev/null +++ b/internal/tests/race/race_test.go @@ -0,0 +1,223 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "bufio" + "bytes" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/race" +) + +// copied and modified from go: +// XXX: use -json? +// TODO: get rid of globals please? + +var ( + passedTests = 0 + totalTests = 0 + falsePos = 0 + falseNeg = 0 + // failingPos = 0 + // failingNeg = 0 + failed = false +) + +const ( + visibleLen = 40 + testPrefix = "=== RUN Test" +) + +// TODO: this test really ought to run things under a couple of seeds +func TestRaceSim(t *testing.T) { + if !race.Enabled { + t.Skip("race.Enabled is false") + } + + path := gosimtool.GetPathForPrecompiledTestBinary(t, gosimtool.Module+"/internal/tests/race/testdata") + checkTests(t, exec.Command(path, "-test.v")) +} + +func TestRaceOriginal(t *testing.T) { + if !race.Enabled { + t.Skip("race.Enabled is false") + } + + modDir, err := gosimtool.FindGoModDir() + if err != nil { + t.Fatal(err) + } + + // TODO: get this from some shared package instead + path := filepath.Join(modDir, gosimtool.OutputDirectory, "racetest", "testdata.test") + + checkTests(t, exec.Command(path, "-test.v")) +} + +func checkTests(t *testing.T, cmd *exec.Cmd) { + files, err := filepath.Glob("./testdata/*.go") + if err != nil { + t.Fatal(err) + } + _ = files + + passedTests = 0 + totalTests = 0 + falsePos = 0 + falseNeg = 0 + // failingPos = 0 + // failingNeg = 0 + failed = false + + testOutput, err := runTests(cmd) + if err != nil { + t.Fatalf("Failed to run tests: %v\n%v", err, string(testOutput)) + } + reader := bufio.NewReader(bytes.NewReader(testOutput)) + + funcName := "" + for { + s, err := nextLine(reader) + if err != nil { + break + } + if strings.HasPrefix(s, testPrefix) { + funcName = s[len(testPrefix):] + break + } + } + + var tsanLog []string + for { + s, err := nextLine(reader) + if err != nil { + fmt.Printf("%s\n", processLog(funcName, tsanLog)) + break + } + if strings.HasPrefix(s, testPrefix) { + fmt.Printf("%s\n", processLog(funcName, tsanLog)) + tsanLog = make([]string, 0, 100) + funcName = s[len(testPrefix):] + } else { + tsanLog = append(tsanLog, s) + } + } + + if totalTests == 0 { + t.Fatalf("failed to parse test output:\n%s", testOutput) + } + fmt.Printf("\nPassed %d of %d tests (%.02f%%, %d+, %d-)\n", + passedTests, totalTests, 100*float64(passedTests)/float64(totalTests), falsePos, falseNeg) + // fmt.Printf("%d expected failures (%d has not fail)\n", failingPos+failingNeg, failingNeg) + + if totalTests == 0 { + t.Errorf("expected > 0 parsed tests, got %d", totalTests) + } + + if failed { + t.Fail() + } +} + +// nextLine is a wrapper around bufio.Reader.ReadString. +// It reads a line up to the next '\n' character. Error +// is non-nil if there are no lines left, and nil +// otherwise. +func nextLine(r *bufio.Reader) (string, error) { + s, err := r.ReadString('\n') + if err != nil { + if err != io.EOF { + log.Fatalf("nextLine: expected EOF, received %v", err) + } + return s, err + } + return s[:len(s)-1], nil +} + +// processLog verifies whether the given ThreadSanitizer's log +// contains a race report, checks this information against +// the name of the testcase and returns the result of this +// comparison. +func processLog(testName string, tsanLog []string) string { + gotRace := false + for _, s := range tsanLog { + if strings.Contains(s, "DATA RACE") { + gotRace = true + break + } + } + + // failing := strings.Contains(testName, "Failing") + expRace := !strings.HasPrefix(testName, "No") + for len(testName) < visibleLen { + testName += " " + } + if expRace == gotRace { + passedTests++ + totalTests++ + // if failing { + // failed = true + // failingNeg++ + // } + return fmt.Sprintf("%s .", testName) + } + pos := "" + if expRace { + falseNeg++ + } else { + falsePos++ + pos = "+" + } + // if failing { + // failingPos++ + // } else { + failed = true + // } + totalTests++ + + return fmt.Sprintf("%s %s%s", testName, "FAILED", pos) +} + +func runTests(cmd *exec.Cmd) ([]byte, error) { + for _, env := range os.Environ() { + if strings.HasPrefix(env, "GOMAXPROCS=") || + strings.HasPrefix(env, "GODEBUG=") || + strings.HasPrefix(env, "GORACE=") { + continue + } + cmd.Env = append(cmd.Env, env) + } + // We set GOMAXPROCS=1 to prevent test flakiness. + // There are two sources of flakiness: + // 1. Some tests rely on particular execution order. + // If the order is different, race does not happen at all. + // 2. Ironically, ThreadSanitizer runtime contains a logical race condition + // that can lead to false negatives if racy accesses happen literally at the same time. + // Tests used to work reliably in the good old days of GOMAXPROCS=1. + // So let's set it for now. A more reliable solution is to explicitly annotate tests + // with required execution order by means of a special "invisible" synchronization primitive + // (that's what is done for C++ ThreadSanitizer tests). This is issue #14119. + cmd.Env = append(cmd.Env, + "GOMAXPROCS=1", + "GORACE=suppress_equal_stacks=0 suppress_equal_addresses=0", + ) + + out, _ := cmd.CombinedOutput() + log.Println(string(out)) + if bytes.Contains(out, []byte("fatal error:")) { + // But don't expect runtime to crash. + return out, fmt.Errorf("runtime fatal error") + } + return out, nil +} diff --git a/internal/tests/race/testdata/annotate_test.go b/internal/tests/race/testdata/annotate_test.go new file mode 100644 index 0000000..f826317 --- /dev/null +++ b/internal/tests/race/testdata/annotate_test.go @@ -0,0 +1,151 @@ +//go:build !sim + +package race_test + +import ( + "sync" + "testing" + "unsafe" + + "github.com/jellevandenhooff/gosim/internal/race" +) + +//go:norace +func helper1(shared *int, wg *sync.WaitGroup) { + defer wg.Done() + *shared = 0 +} + +func TestNoRaceAnnotateOutside(t *testing.T) { + var shared int + var wg sync.WaitGroup + wg.Add(2) + go helper1(&shared, &wg) + go helper1(&shared, &wg) + wg.Wait() +} + +//go:norace +func helper2(shared *int, wg *sync.WaitGroup) { + go func() { + defer wg.Done() + *shared = 0 + }() +} + +func TestRaceAnnotateGoInside(t *testing.T) { + var shared int + var wg sync.WaitGroup + wg.Add(2) + go helper2(&shared, &wg) + go helper2(&shared, &wg) + wg.Wait() +} + +//go:norace +func helper3(shared *int, wg *sync.WaitGroup) { + defer wg.Done() + func() { + *shared = 0 + }() +} + +func TestNoRaceAnnotateInsideInline(t *testing.T) { + var shared int + var wg sync.WaitGroup + wg.Add(2) + go helper3(&shared, &wg) + go helper3(&shared, &wg) + wg.Wait() +} + +//go:noinline +func wrapper(f func()) { + // prevent inlining to ensure norace does not apply + f() +} + +//go:norace +func helper4(shared *int, wg *sync.WaitGroup) { + defer wg.Done() + wrapper(func() { + *shared = 0 + }) +} + +func TestRaceAnnotateInsideWrapper(t *testing.T) { + var shared int + var wg sync.WaitGroup + wg.Add(2) + go helper4(&shared, &wg) + go helper4(&shared, &wg) + wg.Wait() +} + +func wrappernewracectx(f func()) { + done := make(chan struct{}) + race.Release(unsafe.Pointer(&done)) + + race.Disable() + go func() { + f() + race.Acquire(unsafe.Pointer(&done)) + close(done) + }() + race.Enable() + <-done +} + +func TestRaceWrappernewracectx(t *testing.T) { + x := 0 + wrappernewracectx(func() { + x = 1 + }) + _ = x +} + +func TestNoRaceWrappernewracectx(t *testing.T) { + x := 0 + wrappernewracectx(func() { + }) + _ = x +} + +//go:norace +func helper5(x, y int) int { + var z int + wrappernewracectx(func() { + z = x + y + }) + return z +} + +func TestNoRaceWrappernewracectxReturn(t *testing.T) { + if helper5(3, 2) != 5 { + t.Error("bad") + } +} + +func helper6(x, y int) { + wrappernewracectx(func() { + if x+y != 5 { + panic("help") + } + }) +} + +func TestNoRaceWrappernewracectxArgs(t *testing.T) { + helper6(3, 2) +} + +func helper7(x, y int) int { + var z int + wrappernewracectx(func() { + z = x + y + }) + return z +} + +func TestRaceWrappernewracectxReturn(t *testing.T) { + helper7(3, 2) +} diff --git a/internal/tests/race/testdata/atomic_test.go b/internal/tests/race/testdata/atomic_test.go new file mode 100644 index 0000000..2e0562d --- /dev/null +++ b/internal/tests/race/testdata/atomic_test.go @@ -0,0 +1,330 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "runtime" + "sync" + "sync/atomic" + "testing" + "unsafe" +) + +// from go/src/runtime/race/testdata/race_test.go + +func TestNoRaceAtomicAddInt64(_ *testing.T) { + var x1, x2 int8 + _ = x1 + x2 + var s int64 + ch := make(chan bool, 2) + go func() { + x1 = 1 + if atomic.AddInt64(&s, 1) == 2 { + x2 = 1 + } + ch <- true + }() + go func() { + x2 = 1 + if atomic.AddInt64(&s, 1) == 2 { + x1 = 1 + } + ch <- true + }() + <-ch + <-ch +} + +func TestRaceAtomicAddInt64(_ *testing.T) { + var x1, x2 int8 + _ = x1 + x2 + var s int64 + ch := make(chan bool, 2) + go func() { + x1 = 1 + if atomic.AddInt64(&s, 1) == 1 { + x2 = 1 + } + ch <- true + }() + go func() { + x2 = 1 + if atomic.AddInt64(&s, 1) == 1 { + x1 = 1 + } + ch <- true + }() + <-ch + <-ch +} + +func TestNoRaceAtomicAddInt32(_ *testing.T) { + var x1, x2 int8 + _ = x1 + x2 + var s int32 + ch := make(chan bool, 2) + go func() { + x1 = 1 + if atomic.AddInt32(&s, 1) == 2 { + x2 = 1 + } + ch <- true + }() + go func() { + x2 = 1 + if atomic.AddInt32(&s, 1) == 2 { + x1 = 1 + } + ch <- true + }() + <-ch + <-ch +} + +func TestNoRaceAtomicLoadAddInt32(_ *testing.T) { + var x int64 + _ = x + var s int32 + go func() { + x = 2 + atomic.AddInt32(&s, 1) + }() + for atomic.LoadInt32(&s) != 1 { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicLoadStoreInt32(_ *testing.T) { + var x int64 + _ = x + var s int32 + go func() { + x = 2 + atomic.StoreInt32(&s, 1) + }() + for atomic.LoadInt32(&s) != 1 { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicStoreCASInt32(_ *testing.T) { + var x int64 + _ = x + var s int32 + go func() { + x = 2 + atomic.StoreInt32(&s, 1) + }() + for !atomic.CompareAndSwapInt32(&s, 1, 0) { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicCASLoadInt32(_ *testing.T) { + var x int64 + _ = x + var s int32 + go func() { + x = 2 + if !atomic.CompareAndSwapInt32(&s, 0, 1) { + panic("") + } + }() + for atomic.LoadInt32(&s) != 1 { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicCASCASInt32(_ *testing.T) { + var x int64 + _ = x + var s int32 + go func() { + x = 2 + if !atomic.CompareAndSwapInt32(&s, 0, 1) { + panic("") + } + }() + for !atomic.CompareAndSwapInt32(&s, 1, 0) { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicCASCASInt32_2(_ *testing.T) { + var x1, x2 int8 + _ = x1 + x2 + var s int32 + ch := make(chan bool, 2) + go func() { + x1 = 1 + if !atomic.CompareAndSwapInt32(&s, 0, 1) { + x2 = 1 + } + ch <- true + }() + go func() { + x2 = 1 + if !atomic.CompareAndSwapInt32(&s, 0, 1) { + x1 = 1 + } + ch <- true + }() + <-ch + <-ch +} + +func TestNoRaceAtomicLoadInt64(_ *testing.T) { + var x int32 + _ = x + var s int64 + go func() { + x = 2 + atomic.AddInt64(&s, 1) + }() + for atomic.LoadInt64(&s) != 1 { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicCASCASUInt64(_ *testing.T) { + var x int64 + _ = x + var s uint64 + go func() { + x = 2 + if !atomic.CompareAndSwapUint64(&s, 0, 1) { + panic("") + } + }() + for !atomic.CompareAndSwapUint64(&s, 1, 0) { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicLoadStorePointer(_ *testing.T) { + var x int64 + _ = x + var s unsafe.Pointer + var y int = 2 + var p unsafe.Pointer = unsafe.Pointer(&y) + go func() { + x = 2 + atomic.StorePointer(&s, p) + }() + for atomic.LoadPointer(&s) != p { + runtime.Gosched() + } + x = 1 +} + +func TestNoRaceAtomicStoreCASUint64(_ *testing.T) { + var x int64 + _ = x + var s uint64 + go func() { + x = 2 + atomic.StoreUint64(&s, 1) + }() + for !atomic.CompareAndSwapUint64(&s, 1, 0) { + runtime.Gosched() + } + x = 1 +} + +func TestRaceAtomicStoreLoad(_ *testing.T) { + c := make(chan bool) + var a uint64 + go func() { + atomic.StoreUint64(&a, 1) + c <- true + }() + _ = a + <-c +} + +func TestRaceAtomicLoadStore(_ *testing.T) { + c := make(chan bool) + var a uint64 + go func() { + _ = atomic.LoadUint64(&a) + c <- true + }() + a = 1 + <-c +} + +func TestRaceAtomicAddLoad(_ *testing.T) { + c := make(chan bool) + var a uint64 + go func() { + atomic.AddUint64(&a, 1) + c <- true + }() + _ = a + <-c +} + +func TestRaceAtomicAddStore(_ *testing.T) { + c := make(chan bool) + var a uint64 + go func() { + atomic.AddUint64(&a, 1) + c <- true + }() + a = 42 + <-c +} + +// A nil pointer in an atomic operation should not deadlock +// the rest of the program. Used to hang indefinitely. +func TestNoRaceAtomicCrash(_ *testing.T) { + var mutex sync.Mutex + var nilptr *int32 + panics := 0 + defer func() { + if x := recover(); x != nil { + mutex.Lock() + panics++ + mutex.Unlock() + } else { + panic("no panic") + } + }() + atomic.AddInt32(nilptr, 1) +} + +func TestNoRaceDeferAtomicStore(t *testing.T) { + // XXX: this test causes flakes in eg TestRaceChanSendLen + t.Skip() + + // Test that when an atomic function is deferred directly, the + // GC scans it correctly. See issue 42599. + type foo struct { + bar int64 + } + + var doFork func(f *foo, depth int) + doFork = func(f *foo, depth int) { + atomic.StoreInt64(&f.bar, 1) + defer atomic.StoreInt64(&f.bar, 0) + if depth > 0 { + for i := 0; i < 2; i++ { + f2 := &foo{} + go doFork(f2, depth-1) + } + } + runtime.GC() + } + + f := &foo{} + doFork(f, 11) +} diff --git a/internal/tests/race/testdata/chan_test.go b/internal/tests/race/testdata/chan_test.go new file mode 100644 index 0000000..7bf40e5 --- /dev/null +++ b/internal/tests/race/testdata/chan_test.go @@ -0,0 +1,789 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "runtime" + "testing" + "time" +) + +// from go/src/runtime/race/testdata/chan_test.go + +func TestNoRaceChanSync(t *testing.T) { + v := 0 + _ = v + c := make(chan int) + go func() { + v = 1 + c <- 0 + }() + <-c + v = 2 +} + +func TestNoRaceChanSyncRev(t *testing.T) { + v := 0 + _ = v + c := make(chan int) + go func() { + c <- 0 + v = 2 + }() + v = 1 + <-c +} + +func TestNoRaceChanAsync(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + v = 1 + c <- 0 + }() + <-c + v = 2 +} + +func TestRaceChanAsyncRev(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + c <- 0 + v = 1 + }() + v = 2 + <-c +} + +func TestNoRaceChanAsyncCloseRecv(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + v = 1 + close(c) + }() + func() { + defer func() { + recover() + v = 2 + }() + <-c + }() +} + +func TestNoRaceChanAsyncCloseRecv2(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + v = 1 + close(c) + }() + _, _ = <-c + v = 2 +} + +func TestNoRaceChanAsyncCloseRecv3(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + v = 1 + close(c) + }() + for range c { + } + v = 2 +} + +func TestNoRaceChanSyncCloseRecv(t *testing.T) { + v := 0 + _ = v + c := make(chan int) + go func() { + v = 1 + close(c) + }() + func() { + defer func() { + recover() + v = 2 + }() + <-c + }() +} + +func TestNoRaceChanSyncCloseRecv2(t *testing.T) { + v := 0 + _ = v + c := make(chan int) + go func() { + v = 1 + close(c) + }() + _, _ = <-c + v = 2 +} + +func TestNoRaceChanSyncCloseRecv3(t *testing.T) { + v := 0 + _ = v + c := make(chan int) + go func() { + v = 1 + close(c) + }() + for range c { + } + v = 2 +} + +func TestRaceChanSyncCloseSend(t *testing.T) { + v := 0 + _ = v + c := make(chan int) + go func() { + v = 1 + close(c) + }() + func() { + defer func() { + recover() + }() + c <- 0 + }() + v = 2 +} + +func TestRaceChanAsyncCloseSend(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + v = 1 + close(c) + }() + func() { + defer func() { + recover() + }() + for { + c <- 0 + } + }() + v = 2 +} + +func TestRaceChanCloseClose(t *testing.T) { + compl := make(chan bool, 2) + v1 := 0 + v2 := 0 + _ = v1 + v2 + c := make(chan int) + go func() { + defer func() { + if recover() != nil { + v2 = 2 + } + compl <- true + }() + v1 = 1 + close(c) + }() + go func() { + defer func() { + if recover() != nil { + v1 = 2 + } + compl <- true + }() + v2 = 1 + close(c) + }() + <-compl + <-compl +} + +func TestRaceChanSendLen(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + go func() { + v = 1 + c <- 1 + }() + for len(c) == 0 { + runtime.Gosched() + } + v = 2 +} + +func TestRaceChanRecvLen(t *testing.T) { + v := 0 + _ = v + c := make(chan int, 10) + c <- 1 + go func() { + v = 1 + <-c + }() + for len(c) != 0 { + runtime.Gosched() + } + v = 2 +} + +func TestRaceChanSendSend(t *testing.T) { + compl := make(chan bool, 2) + v1 := 0 + v2 := 0 + _ = v1 + v2 + c := make(chan int, 1) + go func() { + v1 = 1 + select { + case c <- 1: + default: + v2 = 2 + } + compl <- true + }() + go func() { + v2 = 1 + select { + case c <- 1: + default: + v1 = 2 + } + compl <- true + }() + <-compl + <-compl +} + +func TestNoRaceChanPtr(t *testing.T) { + type msg struct { + x int + } + c := make(chan *msg) + go func() { + c <- &msg{1} + }() + m := <-c + m.x = 2 +} + +func TestRaceChanWrongSend(t *testing.T) { + v1 := 0 + v2 := 0 + _ = v1 + v2 + c := make(chan int, 2) + go func() { + v1 = 1 + c <- 1 + }() + go func() { + v2 = 2 + c <- 2 + }() + time.Sleep(1e7) + if <-c == 1 { + v2 = 3 + } else { + v1 = 3 + } +} + +func TestRaceChanWrongClose(t *testing.T) { + v1 := 0 + v2 := 0 + _ = v1 + v2 + c := make(chan int, 1) + done := make(chan bool) + go func() { + defer func() { + recover() + }() + v1 = 1 + c <- 1 + done <- true + }() + go func() { + time.Sleep(1e7) + v2 = 2 + close(c) + done <- true + }() + time.Sleep(2e7) + if _, who := <-c; who { + v2 = 2 + } else { + v1 = 2 + } + <-done + <-done +} + +func TestRaceChanSendClose(t *testing.T) { + compl := make(chan bool, 2) + c := make(chan int, 1) + go func() { + defer func() { + recover() + compl <- true + }() + c <- 1 + }() + go func() { + time.Sleep(10 * time.Millisecond) + close(c) + compl <- true + }() + <-compl + <-compl +} + +func TestRaceChanSendSelectClose(t *testing.T) { + compl := make(chan bool, 2) + c := make(chan int, 1) + c1 := make(chan int) + go func() { + defer func() { + recover() + compl <- true + }() + time.Sleep(10 * time.Millisecond) + select { + case c <- 1: + case <-c1: + } + }() + go func() { + close(c) + compl <- true + }() + <-compl + <-compl +} + +func TestRaceSelectReadWriteAsync(t *testing.T) { + done := make(chan bool) + x := 0 + c1 := make(chan int, 10) + c2 := make(chan int, 10) + c3 := make(chan int) + c2 <- 1 + go func() { + select { + case c1 <- x: // read of x races with... + case c3 <- 1: + } + done <- true + }() + select { + case x = <-c2: // ... write to x here + case c3 <- 1: + } + <-done +} + +func TestRaceSelectReadWriteSync(t *testing.T) { + done := make(chan bool) + x := 0 + c1 := make(chan int) + c2 := make(chan int) + c3 := make(chan int) + // make c1 and c2 ready for communication + go func() { + <-c1 + }() + go func() { + c2 <- 1 + }() + go func() { + select { + case c1 <- x: // read of x races with... + case c3 <- 1: + } + done <- true + }() + select { + case x = <-c2: // ... write to x here + case c3 <- 1: + } + <-done +} + +func TestNoRaceSelectReadWriteAsync(t *testing.T) { + done := make(chan bool) + x := 0 + c1 := make(chan int) + c2 := make(chan int) + go func() { + select { + case c1 <- x: // read of x does not race with... + case c2 <- 1: + } + done <- true + }() + select { + case x = <-c1: // ... write to x here + case c2 <- 1: + } + <-done +} + +func TestRaceChanReadWriteAsync(t *testing.T) { + done := make(chan bool) + c1 := make(chan int, 10) + c2 := make(chan int, 10) + c2 <- 10 + x := 0 + go func() { + c1 <- x // read of x races with... + done <- true + }() + x = <-c2 // ... write to x here + <-done +} + +func TestRaceChanReadWriteSync(t *testing.T) { + done := make(chan bool) + c1 := make(chan int) + c2 := make(chan int) + // make c1 and c2 ready for communication + go func() { + <-c1 + }() + go func() { + c2 <- 10 + }() + x := 0 + go func() { + c1 <- x // read of x races with... + done <- true + }() + x = <-c2 // ... write to x here + <-done +} + +func TestNoRaceChanReadWriteAsync(t *testing.T) { + done := make(chan bool) + c1 := make(chan int, 10) + x := 0 + go func() { + c1 <- x // read of x does not race with... + done <- true + }() + x = <-c1 // ... write to x here + <-done +} + +func TestNoRaceProducerConsumerUnbuffered(t *testing.T) { + type Task struct { + f func() + done chan bool + } + + queue := make(chan Task) + + go func() { + t := <-queue + t.f() + t.done <- true + }() + + doit := func(f func()) { + done := make(chan bool, 1) + queue <- Task{f, done} + <-done + } + + x := 0 + doit(func() { + x = 1 + }) + _ = x +} + +func TestRaceChanItselfSend(t *testing.T) { + compl := make(chan bool, 1) + c := make(chan int, 10) + go func() { + c <- 0 + compl <- true + }() + c = make(chan int, 20) + <-compl +} + +func TestRaceChanItselfRecv(t *testing.T) { + compl := make(chan bool, 1) + c := make(chan int, 10) + c <- 1 + go func() { + <-c + compl <- true + }() + time.Sleep(1e7) + c = make(chan int, 20) + <-compl +} + +func TestRaceChanItselfNil(t *testing.T) { + c := make(chan int, 10) + go func() { + c <- 0 + }() + time.Sleep(1e7) + c = nil + _ = c +} + +func TestRaceChanItselfClose(t *testing.T) { + compl := make(chan bool, 1) + c := make(chan int) + go func() { + close(c) + compl <- true + }() + c = make(chan int) + <-compl +} + +func TestRaceChanItselfLen(t *testing.T) { + compl := make(chan bool, 1) + c := make(chan int) + go func() { + _ = len(c) + compl <- true + }() + c = make(chan int) + <-compl +} + +func TestRaceChanItselfCap(t *testing.T) { + compl := make(chan bool, 1) + c := make(chan int) + go func() { + _ = cap(c) + compl <- true + }() + c = make(chan int) + <-compl +} + +func TestNoRaceChanCloseLen(t *testing.T) { + c := make(chan int, 10) + r := make(chan int, 10) + go func() { + r <- len(c) + }() + go func() { + close(c) + r <- 0 + }() + <-r + <-r +} + +func TestNoRaceChanCloseCap(t *testing.T) { + c := make(chan int, 10) + r := make(chan int, 10) + go func() { + r <- cap(c) + }() + go func() { + close(c) + r <- 0 + }() + <-r + <-r +} + +func TestRaceChanCloseSend(t *testing.T) { + compl := make(chan bool, 1) + c := make(chan int, 10) + go func() { + close(c) + compl <- true + }() + c <- 0 + <-compl +} + +func TestNoRaceChanMutex(t *testing.T) { + done := make(chan struct{}) + mtx := make(chan struct{}, 1) + data := 0 + _ = data + go func() { + mtx <- struct{}{} + data = 42 + <-mtx + done <- struct{}{} + }() + mtx <- struct{}{} + data = 43 + <-mtx + <-done +} + +func TestNoRaceSelectMutex(t *testing.T) { + done := make(chan struct{}) + mtx := make(chan struct{}, 1) + aux := make(chan bool) + data := 0 + _ = data + go func() { + select { + case mtx <- struct{}{}: + case <-aux: + } + data = 42 + select { + case <-mtx: + case <-aux: + } + done <- struct{}{} + }() + select { + case mtx <- struct{}{}: + case <-aux: + } + data = 43 + select { + case <-mtx: + case <-aux: + } + <-done +} + +func TestRaceChanSem(t *testing.T) { + done := make(chan struct{}) + mtx := make(chan bool, 2) + data := 0 + _ = data + go func() { + mtx <- true + data = 42 + <-mtx + done <- struct{}{} + }() + mtx <- true + data = 43 + <-mtx + <-done +} + +func TestNoRaceChanWaitGroup(t *testing.T) { + const N = 10 + chanWg := make(chan bool, N/2) + data := make([]int, N) + for i := 0; i < N; i++ { + chanWg <- true + go func(i int) { + data[i] = 42 + <-chanWg + }(i) + } + for i := 0; i < cap(chanWg); i++ { + chanWg <- true + } + for i := 0; i < N; i++ { + _ = data[i] + } +} + +// Test that sender synchronizes with receiver even if the sender was blocked. +func TestNoRaceBlockedSendSync(t *testing.T) { + c := make(chan *int, 1) + c <- nil + go func() { + i := 42 + c <- &i + }() + // Give the sender time to actually block. + // This sleep is completely optional: race report must not be printed + // regardless of whether the sender actually blocks or not. + // It cannot lead to flakiness. + time.Sleep(10 * time.Millisecond) + <-c + p := <-c + if *p != 42 { + t.Fatal() + } +} + +// The same as TestNoRaceBlockedSendSync above, but sender unblock happens in a select. +func TestNoRaceBlockedSelectSendSync(t *testing.T) { + c := make(chan *int, 1) + c <- nil + go func() { + i := 42 + c <- &i + }() + time.Sleep(10 * time.Millisecond) + <-c + select { + case p := <-c: + if *p != 42 { + t.Fatal() + } + case <-make(chan int): + } +} + +// Test that close synchronizes with a read from the empty closed channel. +// See https://golang.org/issue/36714. +func TestNoRaceCloseHappensBeforeRead(t *testing.T) { + for i := 0; i < 100; i++ { + var loc int + write := make(chan struct{}) + read := make(chan struct{}) + + go func() { + select { + case <-write: + _ = loc + default: + } + close(read) + }() + + go func() { + loc = 1 + close(write) + }() + + <-read + } +} + +// Test that we call the proper race detector function when c.elemsize==0. +// See https://github.com/golang/go/issues/42598 +func TestNoRaceElemetSize0(t *testing.T) { + var x, y int + c := make(chan struct{}, 2) + c <- struct{}{} + c <- struct{}{} + go func() { + x += 1 + <-c + }() + go func() { + y += 1 + <-c + }() + time.Sleep(10 * time.Millisecond) + c <- struct{}{} + c <- struct{}{} + x += 1 + y += 1 +} diff --git a/internal/tests/race/testdata/io_test.go b/internal/tests/race/testdata/io_test.go new file mode 100644 index 0000000..30f35ca --- /dev/null +++ b/internal/tests/race/testdata/io_test.go @@ -0,0 +1,75 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +// from go/src/runtime/race/testdata/io_test.go + +// XXX: broken +/* +func TestNoRaceIOFile(t *testing.T) { + if !gosimruntime.IsDetgo() { + setupRealDisk(s) + } + x := 0 + path := "." // t.TempDir() + fname := filepath.Join(path, "data") + go func() { + x = 42 + f, _ := os.OpenFile(fname, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666) + f.Write([]byte("done")) + f.Close() + }() + for { + f, err := os.OpenFile(fname, os.O_RDONLY, 0) + if err != nil { + time.Sleep(1e6) + continue + } + buf := make([]byte, 100) + count, err := f.Read(buf) + if count == 0 { + time.Sleep(1e6) + continue + } + break + } + _ = x +} +*/ + +// XXX: not implemented +/* +var ( + regHandler sync.Once + handlerData int +) + +func TestNoRaceIOHttp(t *testing.T) { + regHandler.Do(func() { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + handlerData++ + fmt.Fprintf(w, "test") + handlerData++ + }) + }) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("net.Listen: %v", err) + } + defer ln.Close() + go http.Serve(ln, nil) + handlerData++ + _, err = http.Get("http://" + ln.Addr().String()) + if err != nil { + t.Fatalf("http.Get: %v", err) + } + handlerData++ + _, err = http.Get("http://" + ln.Addr().String()) + if err != nil { + t.Fatalf("http.Get: %v", err) + } + handlerData++ +} +*/ diff --git a/internal/tests/race/testdata/map_test.go b/internal/tests/race/testdata/map_test.go new file mode 100644 index 0000000..c971a28 --- /dev/null +++ b/internal/tests/race/testdata/map_test.go @@ -0,0 +1,336 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import "testing" + +// from go/src/runtime/race/testdata/map_test.go + +func TestRaceMapRW(t *testing.T) { + m := make(map[int]int) + ch := make(chan bool, 1) + go func() { + _ = m[1] + ch <- true + }() + m[1] = 1 + <-ch +} + +func TestRaceMapRW2(t *testing.T) { + m := make(map[int]int) + ch := make(chan bool, 1) + go func() { + _, _ = m[1] + ch <- true + }() + m[1] = 1 + <-ch +} + +func TestRaceMapRWArray(t *testing.T) { + // Check instrumentation of unaddressable arrays (issue 4578). + m := make(map[int][2]int) + ch := make(chan bool, 1) + go func() { + _ = m[1][1] + ch <- true + }() + m[2] = [2]int{1, 2} + <-ch +} + +func TestNoRaceMapRR(t *testing.T) { + m := make(map[int]int) + ch := make(chan bool, 1) + go func() { + _, _ = m[1] + ch <- true + }() + _ = m[1] + <-ch +} + +func TestRaceMapRange(t *testing.T) { + m := make(map[int]int) + ch := make(chan bool, 1) + go func() { + for range m { + } + ch <- true + }() + m[1] = 1 + <-ch +} + +func TestRaceMapRange2(t *testing.T) { + m := make(map[int]int) + ch := make(chan bool, 1) + go func() { + for range m { + } + ch <- true + }() + m[1] = 1 + <-ch +} + +func TestNoRaceMapRangeRange(t *testing.T) { + m := make(map[int]int) + // now the map is not empty and range triggers an event + // should work without this (as in other tests) + // so it is suspicious if this test passes and others don't + m[0] = 0 + ch := make(chan bool, 1) + go func() { + for range m { + } + ch <- true + }() + for range m { + } + <-ch +} + +func TestRaceMapLen(t *testing.T) { + m := make(map[string]bool) + ch := make(chan bool, 1) + go func() { + _ = len(m) + ch <- true + }() + m[""] = true + <-ch +} + +func TestRaceMapDelete(t *testing.T) { + m := make(map[string]bool) + ch := make(chan bool, 1) + go func() { + delete(m, "") + ch <- true + }() + m[""] = true + <-ch +} + +func TestRaceMapLenDelete(t *testing.T) { + m := make(map[string]bool) + ch := make(chan bool, 1) + go func() { + delete(m, "a") + ch <- true + }() + _ = len(m) + <-ch +} + +func TestRaceMapVariable(t *testing.T) { + ch := make(chan bool, 1) + m := make(map[int]int) + _ = m + go func() { + m = make(map[int]int) + ch <- true + }() + m = make(map[int]int) + <-ch +} + +func TestRaceMapVariable2(t *testing.T) { + ch := make(chan bool, 1) + m := make(map[int]int) + go func() { + m[1] = 1 + ch <- true + }() + m = make(map[int]int) + <-ch +} + +func TestRaceMapVariable3(t *testing.T) { + ch := make(chan bool, 1) + m := make(map[int]int) + go func() { + _ = m[1] + ch <- true + }() + m = make(map[int]int) + <-ch +} + +type Big struct { + x [17]int32 +} + +func TestRaceMapLookupPartKey(t *testing.T) { + k := &Big{} + m := make(map[Big]bool) + ch := make(chan bool, 1) + go func() { + k.x[8] = 1 + ch <- true + }() + _ = m[*k] + <-ch +} + +func TestRaceMapLookupPartKey2(t *testing.T) { + k := &Big{} + m := make(map[Big]bool) + ch := make(chan bool, 1) + go func() { + k.x[8] = 1 + ch <- true + }() + _, _ = m[*k] + <-ch +} + +func TestRaceMapDeletePartKey(t *testing.T) { + k := &Big{} + m := make(map[Big]bool) + ch := make(chan bool, 1) + go func() { + k.x[8] = 1 + ch <- true + }() + delete(m, *k) + <-ch +} + +func TestRaceMapInsertPartKey(t *testing.T) { + k := &Big{} + m := make(map[Big]bool) + ch := make(chan bool, 1) + go func() { + k.x[8] = 1 + ch <- true + }() + m[*k] = true + <-ch +} + +func TestRaceMapInsertPartVal(t *testing.T) { + v := &Big{} + m := make(map[int]Big) + ch := make(chan bool, 1) + go func() { + v.x[8] = 1 + ch <- true + }() + m[1] = *v + <-ch +} + +// Test for issue 7561. +func TestRaceMapAssignMultipleReturn(t *testing.T) { + connect := func() (int, error) { return 42, nil } + conns := make(map[int][]int) + conns[1] = []int{0} + ch := make(chan bool, 1) + var err error + _ = err + go func() { + conns[1][0], err = connect() + ch <- true + }() + x := conns[1][0] + _ = x + <-ch +} + +// BigKey and BigVal must be larger than 256 bytes, +// so that compiler sets KindGCProg for them. +type BigKey [1000]*int + +type BigVal struct { + x int + y [1000]*int +} + +func TestRaceMapBigKeyAccess1(t *testing.T) { + m := make(map[BigKey]int) + var k BigKey + ch := make(chan bool, 1) + go func() { + _ = m[k] + ch <- true + }() + k[30] = new(int) + <-ch +} + +func TestRaceMapBigKeyAccess2(t *testing.T) { + m := make(map[BigKey]int) + var k BigKey + ch := make(chan bool, 1) + go func() { + _, _ = m[k] + ch <- true + }() + k[30] = new(int) + <-ch +} + +func TestRaceMapBigKeyInsert(t *testing.T) { + m := make(map[BigKey]int) + var k BigKey + ch := make(chan bool, 1) + go func() { + m[k] = 1 + ch <- true + }() + k[30] = new(int) + <-ch +} + +func TestRaceMapBigKeyDelete(t *testing.T) { + m := make(map[BigKey]int) + var k BigKey + ch := make(chan bool, 1) + go func() { + delete(m, k) + ch <- true + }() + k[30] = new(int) + <-ch +} + +func TestRaceMapBigValInsert(t *testing.T) { + m := make(map[int]BigVal) + var v BigVal + ch := make(chan bool, 1) + go func() { + m[1] = v + ch <- true + }() + v.y[30] = new(int) + <-ch +} + +func TestRaceMapBigValAccess1(t *testing.T) { + m := make(map[int]BigVal) + var v BigVal + ch := make(chan bool, 1) + go func() { + v = m[1] + ch <- true + }() + v.y[30] = new(int) + <-ch +} + +func TestRaceMapBigValAccess2(t *testing.T) { + m := make(map[int]BigVal) + var v BigVal + ch := make(chan bool, 1) + go func() { + v, _ = m[1] + ch <- true + }() + v.y[30] = new(int) + <-ch +} diff --git a/internal/tests/race/testdata/mutex_test.go b/internal/tests/race/testdata/mutex_test.go new file mode 100644 index 0000000..1203127 --- /dev/null +++ b/internal/tests/race/testdata/mutex_test.go @@ -0,0 +1,152 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "sync" + "testing" + "time" +) + +// from go/src/runtime/race/testdata/mutex_test.go + +func TestNoRaceMutex(t *testing.T) { + var mu sync.Mutex + var x int16 = 0 + _ = x + ch := make(chan bool, 2) + go func() { + mu.Lock() + defer mu.Unlock() + x = 1 + ch <- true + }() + go func() { + mu.Lock() + x = 2 + mu.Unlock() + ch <- true + }() + <-ch + <-ch +} + +func TestRaceMutex(t *testing.T) { + var mu sync.Mutex + var x int16 = 0 + _ = x + ch := make(chan bool, 2) + go func() { + x = 1 + mu.Lock() + defer mu.Unlock() + ch <- true + }() + go func() { + x = 2 + mu.Lock() + mu.Unlock() + ch <- true + }() + <-ch + <-ch +} + +func TestRaceMutex2(t *testing.T) { + var mu1 sync.Mutex + var mu2 sync.Mutex + var x int8 = 0 + _ = x + ch := make(chan bool, 2) + go func() { + mu1.Lock() + defer mu1.Unlock() + x = 1 + ch <- true + }() + go func() { + mu2.Lock() + x = 2 + mu2.Unlock() + ch <- true + }() + <-ch + <-ch +} + +func TestNoRaceMutexPureHappensBefore(t *testing.T) { + var mu sync.Mutex + var x int16 = 0 + _ = x + written := false + ch := make(chan bool, 2) + go func() { + x = 1 + mu.Lock() + written = true + mu.Unlock() + ch <- true + }() + go func() { + time.Sleep(100 * time.Microsecond) + mu.Lock() + for !written { + mu.Unlock() + time.Sleep(100 * time.Microsecond) + mu.Lock() + } + mu.Unlock() + x = 1 + ch <- true + }() + <-ch + <-ch +} + +func TestNoRaceMutexSemaphore(t *testing.T) { + var mu sync.Mutex + ch := make(chan bool, 2) + x := 0 + _ = x + mu.Lock() + go func() { + x = 1 + mu.Unlock() + ch <- true + }() + go func() { + mu.Lock() + x = 2 + mu.Unlock() + ch <- true + }() + <-ch + <-ch +} + +// from doc/go_mem.html +func TestNoRaceMutexExampleFromHtml(t *testing.T) { + var l sync.Mutex + a := "" + + l.Lock() + go func() { + a = "hello, world" + l.Unlock() + }() + l.Lock() + _ = a +} + +func TestRaceMutexOverwrite(t *testing.T) { + c := make(chan bool, 1) + var mu sync.Mutex + go func() { + mu = sync.Mutex{} + c <- true + }() + mu.Lock() + <-c +} diff --git a/internal/tests/race/testdata/os_test.go b/internal/tests/race/testdata/os_test.go new file mode 100644 index 0000000..a5469a4 --- /dev/null +++ b/internal/tests/race/testdata/os_test.go @@ -0,0 +1,276 @@ +//go:build sim && skip + +package race_test + +import ( + "fmt" + "testing" + + "github.com/jellevandenhooff/gosim/internal/simulation/bridge" +) + +// TODO: needs to be updated for syscallabi + +func TestNoRaceOS1DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + op := func(_ struct{}, req string) string { + return req + } + + done := make(chan struct{}, 2) + + go func() { + bridge.Invoke(os, backend, op, "a") + + done <- struct{}{} + }() + + go func() { + bridge.Invoke(os, backend, op, "b") + + done <- struct{}{} + }() + + <-done + <-done + + // os.Stop() +} + +// XXX: add test that uses os as a data store (and thus sync mechanism) and show +// that it triggers the race detector still + +func TestNoRaceOS2DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + op := func(_ struct{}, req string) string { + return req + } + + resp := bridge.Invoke(os, backend, op, "hello") + if resp != "hello" { + t.Error(resp) + } + + // os.Stop() +} + +/* +func TestNoRaceOS3DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewOS() + go os.Work() + + op := func(_ struct{}, req string) (string, error) { + return "", io.EOF + } + + resp, err := bridge.Invoke(os, backend, op, "hello") + if err != io.EOF { + t.Error(err) + } + if resp != "" { + t.Error(resp) + } + + os.Stop() +} +*/ + +func TestNoRaceOS4DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + type Req struct { + In, Out bridge.ByteSliceView + } + + op := func(_ struct{}, req Req) struct{} { + buffer := get(req.In) + for i := range buffer { + buffer[i] = buffer[i] + 1 + } + req.Out.Write(buffer) + return struct{}{} + } + + in := make([]byte, 4) + for i := range in { + in[i] = byte(i) + } + out := make([]byte, 4) + + bridge.Invoke(os, backend, op, Req{In: bridge.ByteSliceView{Ptr: in}, Out: bridge.ByteSliceView{Ptr: out}}) + if len(out) != 4 { + t.Error(len(out)) + } + for i := range out { + if out[i] != in[i]+1 { + t.Error(i, out[i]) + } + } + + // os.Stop() +} + +func TestNoRaceOS5DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + type Req struct { + A int + B int + } + + type Resp struct { + A string + B string + } + + op := func(_ struct{}, req Req) Resp { + return Resp{ + A: fmt.Sprint(req.A), + B: fmt.Sprint(req.B), + } + } + + resp := bridge.Invoke(os, backend, op, Req{A: 1, B: 2}) + if resp.A != "1" || resp.B != "2" { + t.Error(resp) + } + + // os.Stop() +} + +func TestRaceOS2DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + type Req struct { + In, Out bridge.ByteSliceView + } + + op := func(_ struct{}, req Req) struct{} { + buffer := make([]byte, req.In.Len()) + copy(buffer, req.In.Ptr) // instead of req.In.Read + for i := range buffer { + buffer[i] = buffer[i] + 1 + } + req.Out.Write(buffer) + return struct{}{} + } + + in := make([]byte, 4) + for i := range in { + in[i] = byte(i) + } + out := make([]byte, 4) + + bridge.Invoke(os, backend, op, Req{In: bridge.ByteSliceView{Ptr: in}, Out: bridge.ByteSliceView{Ptr: out}}) + if len(out) != 4 { + t.Error(len(out)) + } + for i := range out { + if out[i] != in[i]+1 { + t.Error(i, out[i]) + } + } + + // os.Stop() +} + +func get(b bridge.ByteSliceView) []byte { + copy := make([]byte, b.Len()) + b.Read(copy) + return copy +} + +func TestRaceOS3DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + type Req struct { + In, Out bridge.ByteSliceView + } + + op := func(_ struct{}, req Req) struct{} { + buffer := get(req.In) + for i := range buffer { + buffer[i] = buffer[i] + 1 + } + copy(req.Out.Ptr, buffer) // instead of req.Out.Write + return struct{}{} + } + + in := make([]byte, 4) + for i := range in { + in[i] = byte(i) + } + out := make([]byte, 4) + + bridge.Invoke(os, backend, op, Req{In: bridge.ByteSliceView{Ptr: in}, Out: bridge.ByteSliceView{Ptr: out}}) + if len(out) != 4 { + t.Error(len(out)) + } + for i := range out { + if out[i] != in[i]+1 { + t.Error(i, out[i]) + } + } + + // os.Stop() +} + +func TestRaceOS1DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewBridge() + go os.Run() + + var x int + _ = x + + op := func(_ struct{}, req string) string { + x = 2 + return req + } + + x = 1 + + resp := bridge.Invoke(os, backend, op, "hello") + if resp != "hello" { + t.Error(resp) + } + + // os.Stop() +} + +/* +func TestRaceOS4DetgoOnly(t *testing.T) { + var backend struct{} + os := bridge.NewOS() + go os.Work() + + op := func(_ struct{}, req string) (string, error) { + return "", errors.New(req) + } + + resp, err := bridge.InvokeNoerr(os, backend, op, "hello") + if err == nil || err.Error() != "hello" { + t.Error(err) + } + if resp != "" { + t.Error(resp) + } + + os.Stop() +} +*/ diff --git a/internal/tests/race/testdata/race_test.go b/internal/tests/race/testdata/race_test.go new file mode 100644 index 0000000..2ed4f14 --- /dev/null +++ b/internal/tests/race/testdata/race_test.go @@ -0,0 +1,91 @@ +package race_test + +import ( + "math/rand" + "testing" + + "github.com/jellevandenhooff/gosim" +) + +func TestRaceSimple(t *testing.T) { + var x int + _ = x + + done := make(chan struct{}) + go func() { + x = 1 + done <- struct{}{} + }() + + go func() { + x = 20 + done <- struct{}{} + }() + + <-done + <-done +} + +func TestNoRaceSimple(t *testing.T) { + var x int + _ = x + + wait := make(chan struct{}) + done := make(chan struct{}) + go func() { + x = 1 + wait <- struct{}{} + done <- struct{}{} + }() + + go func() { + <-wait + x = 2 + done <- struct{}{} + }() + + <-done + <-done +} + +func TestNoRaceRand(t *testing.T) { + done := make(chan struct{}) + go func() { + rand.Float64() + done <- struct{}{} + }() + + go func() { + rand.Float64() + done <- struct{}{} + }() + + <-done + <-done +} + +func TestNoRaceLotsOfGo(t *testing.T) { + done := make(chan struct{}, 100) + + for i := 0; i < 10; i++ { + go func() { + for j := 0; j < 10; j++ { + go func() { + done <- struct{}{} + }() + } + }() + } + + for i := 0; i < 100; i++ { + <-done + } +} + +// XXX: example from go race detector blog with timer and reset +// XXX: logging should not cause edges +// XXX: filesystem works and does not cause edges (maybe?) + +func TestNoRaceIsDetgo(t *testing.T) { + gosim.IsSim() +} diff --git a/internal/tests/race/testdata/rwmutex_test.go b/internal/tests/race/testdata/rwmutex_test.go new file mode 100644 index 0000000..af222af --- /dev/null +++ b/internal/tests/race/testdata/rwmutex_test.go @@ -0,0 +1,156 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "sync" + "testing" + "time" +) + +// from go/src/runtime/race/testdata/rwmutex_test.go + +func TestRaceMutexRWMutex(t *testing.T) { + var mu1 sync.Mutex + var mu2 sync.RWMutex + var x int16 = 0 + _ = x + ch := make(chan bool, 2) + go func() { + mu1.Lock() + defer mu1.Unlock() + x = 1 + ch <- true + }() + go func() { + mu2.Lock() + x = 2 + mu2.Unlock() + ch <- true + }() + <-ch + <-ch +} + +func TestNoRaceRWMutex(t *testing.T) { + var mu sync.RWMutex + var x, y int64 = 0, 1 + _ = y + ch := make(chan bool, 2) + go func() { + mu.Lock() + defer mu.Unlock() + x = 2 + ch <- true + }() + go func() { + mu.RLock() + y = x + mu.RUnlock() + ch <- true + }() + <-ch + <-ch +} + +func TestRaceRWMutexMultipleReaders(t *testing.T) { + var mu sync.RWMutex + var x, y int64 = 0, 1 + ch := make(chan bool, 4) + go func() { + mu.Lock() + defer mu.Unlock() + x = 2 + ch <- true + }() + // Use three readers so that no matter what order they're + // scheduled in, two will be on the same side of the write + // lock above. + go func() { + mu.RLock() + y = x + 1 + mu.RUnlock() + ch <- true + }() + go func() { + mu.RLock() + y = x + 2 + mu.RUnlock() + ch <- true + }() + go func() { + mu.RLock() + y = x + 3 + mu.RUnlock() + ch <- true + }() + <-ch + <-ch + <-ch + <-ch + _ = y +} + +func TestNoRaceRWMutexMultipleReaders(t *testing.T) { + var mu sync.RWMutex + x := int64(0) + ch := make(chan bool, 4) + go func() { + mu.Lock() + defer mu.Unlock() + x = 2 + ch <- true + }() + go func() { + mu.RLock() + y := x + 1 + _ = y + mu.RUnlock() + ch <- true + }() + go func() { + mu.RLock() + y := x + 2 + _ = y + mu.RUnlock() + ch <- true + }() + go func() { + mu.RLock() + y := x + 3 + _ = y + mu.RUnlock() + ch <- true + }() + <-ch + <-ch + <-ch + <-ch +} + +func TestNoRaceRWMutexTransitive(t *testing.T) { + var mu sync.RWMutex + x := int64(0) + ch := make(chan bool, 2) + go func() { + mu.RLock() + _ = x + mu.RUnlock() + ch <- true + }() + go func() { + time.Sleep(1e7) + mu.RLock() + _ = x + mu.RUnlock() + ch <- true + }() + time.Sleep(2e7) + mu.Lock() + x = 42 + mu.Unlock() + <-ch + <-ch +} diff --git a/internal/tests/race/testdata/select_test.go b/internal/tests/race/testdata/select_test.go new file mode 100644 index 0000000..8ab75c3 --- /dev/null +++ b/internal/tests/race/testdata/select_test.go @@ -0,0 +1,272 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "runtime" + "testing" +) + +// from go/src/runtime/race/testdata/select_test.go + +func TestNoRaceSelect1(t *testing.T) { + var x int + _ = x + compl := make(chan bool) + c := make(chan bool) + c1 := make(chan bool) + + go func() { + x = 1 + // At least two channels are needed because + // otherwise the compiler optimizes select out. + // See comment in runtime/select.go:^func selectgo. + select { + case c <- true: + case c1 <- true: + } + compl <- true + }() + select { + case <-c: + case c1 <- true: + } + x = 2 + <-compl +} + +func TestNoRaceSelect2(t *testing.T) { + var x int + _ = x + compl := make(chan bool) + c := make(chan bool) + c1 := make(chan bool) + go func() { + select { + case <-c: + case <-c1: + } + x = 1 + compl <- true + }() + x = 2 + close(c) + runtime.Gosched() + <-compl +} + +func TestNoRaceSelect3(t *testing.T) { + var x int + _ = x + compl := make(chan bool) + c := make(chan bool, 10) + c1 := make(chan bool) + go func() { + x = 1 + select { + case c <- true: + case <-c1: + } + compl <- true + }() + <-c + x = 2 + <-compl +} + +func TestNoRaceSelect4(t *testing.T) { + type Task struct { + f func() + done chan bool + } + + queue := make(chan Task) + dummy := make(chan bool) + + go func() { + for { + select { + case t := <-queue: + t.f() + t.done <- true + } + } + }() + + doit := func(f func()) { + done := make(chan bool, 1) + select { + case queue <- Task{f, done}: + case <-dummy: + } + select { + case <-done: + case <-dummy: + } + } + + var x int + doit(func() { + x = 1 + }) + _ = x +} + +func TestNoRaceSelect5(t *testing.T) { + test := func(sel, needSched bool) { + var x int + _ = x + ch := make(chan bool) + c1 := make(chan bool) + + done := make(chan bool, 2) + go func() { + if needSched { + runtime.Gosched() + } + // println(1) + x = 1 + if sel { + select { + case ch <- true: + case <-c1: + } + } else { + ch <- true + } + done <- true + }() + + go func() { + // println(2) + if sel { + select { + case <-ch: + case <-c1: + } + } else { + <-ch + } + x = 1 + done <- true + }() + <-done + <-done + } + + test(true, true) + test(true, false) + test(false, true) + test(false, false) +} + +/* +// TODO: Fix +func TestRaceSelect1(t *testing.T) { + var x int + _ = x + compl := make(chan bool, 2) + c := make(chan bool) + c1 := make(chan bool) + + go func() { + <-c + <-c + }() + f := func() { + select { + case c <- true: + case c1 <- true: + } + x = 1 + compl <- true + } + go f() + go f() + <-compl + <-compl +} +*/ + +func TestRaceSelect2(t *testing.T) { + var x int + _ = x + compl := make(chan bool) + c := make(chan bool) + c1 := make(chan bool) + go func() { + x = 1 + select { + case <-c: + case <-c1: + } + compl <- true + }() + close(c) + x = 2 + <-compl +} + +func TestRaceSelect3(t *testing.T) { + var x int + _ = x + compl := make(chan bool) + c := make(chan bool) + c1 := make(chan bool) + go func() { + x = 1 + select { + case c <- true: + case c1 <- true: + } + compl <- true + }() + x = 2 + select { + case <-c: + } + <-compl +} + +func TestRaceSelect4(t *testing.T) { + done := make(chan bool, 1) + var x int + go func() { + select { + default: + x = 2 + } + done <- true + }() + _ = x + <-done +} + +// The idea behind this test: +// there are two variables, access to one +// of them is synchronized, access to the other +// is not. +// Select must (unconditionally) choose the non-synchronized variable +// thus causing exactly one race. +// Currently this test doesn't look like it accomplishes +// this goal. +func TestRaceSelect5(t *testing.T) { + done := make(chan bool, 1) + c1 := make(chan bool, 1) + c2 := make(chan bool) + var x, y int + go func() { + select { + case c1 <- true: + x = 1 + case c2 <- true: + y = 1 + } + done <- true + }() + _ = x + _ = y + <-done +} diff --git a/internal/tests/race/testdata/sync_test.go b/internal/tests/race/testdata/sync_test.go new file mode 100644 index 0000000..c54ad99 --- /dev/null +++ b/internal/tests/race/testdata/sync_test.go @@ -0,0 +1,207 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "sync" + "testing" + "time" +) + +// from go/src/runtime/race/testdata/sync_test.go + +func TestNoRaceCond(t *testing.T) { + x := 0 + _ = x + condition := 0 + var mu sync.Mutex + cond := sync.NewCond(&mu) + go func() { + x = 1 + mu.Lock() + condition = 1 + cond.Signal() + mu.Unlock() + }() + mu.Lock() + for condition != 1 { + cond.Wait() + } + mu.Unlock() + x = 2 +} + +func TestRaceCond(t *testing.T) { + done := make(chan bool) + var mu sync.Mutex + cond := sync.NewCond(&mu) + x := 0 + _ = x + condition := 0 + go func() { + time.Sleep(10 * time.Millisecond) // Enter cond.Wait loop + x = 1 + mu.Lock() + condition = 1 + cond.Signal() + mu.Unlock() + time.Sleep(10 * time.Millisecond) // Exit cond.Wait loop + mu.Lock() + x = 3 + mu.Unlock() + done <- true + }() + mu.Lock() + for condition != 1 { + cond.Wait() + } + mu.Unlock() + x = 2 + <-done +} + +// We do not currently automatically +// parse this test. It is intended that the creation +// stack is observed manually not to contain +// off-by-one errors +func TestRaceAnnounceThreads(t *testing.T) { + const N = 7 + allDone := make(chan bool, N) + + var x int + _ = x + + var f, g, h func() + f = func() { + x = 1 + go g() + go func() { + x = 1 + allDone <- true + }() + x = 2 + allDone <- true + } + + g = func() { + for i := 0; i < 2; i++ { + go func() { + x = 1 + allDone <- true + }() + allDone <- true + } + } + + h = func() { + x = 1 + x = 2 + go f() + allDone <- true + } + + go h() + + for i := 0; i < N; i++ { + <-allDone + } +} + +func TestNoRaceAfterFunc1(t *testing.T) { + i := 2 + c := make(chan bool) + var f func() + f = func() { + i-- + if i >= 0 { + time.AfterFunc(0, f) + } else { + c <- true + } + } + + time.AfterFunc(0, f) + <-c +} + +func TestNoRaceAfterFunc2(t *testing.T) { + var x int + _ = x + timer := time.AfterFunc(10, func() { + x = 1 + }) + defer timer.Stop() +} + +func TestNoRaceAfterFunc3(t *testing.T) { + c := make(chan bool, 1) + x := 0 + _ = x + time.AfterFunc(1e7, func() { + x = 1 + c <- true + }) + <-c +} + +/* +// TODO: Fix +func TestRaceAfterFunc3(t *testing.T) { + c := make(chan bool, 2) + x := 0 + _ = x + time.AfterFunc(1e7, func() { + x = 1 + c <- true + }) + time.AfterFunc(2e7, func() { + x = 2 + c <- true + }) + <-c + <-c +} +*/ + +// This test's output is intended to be +// observed manually. One should check +// that goroutine creation stack is +// comprehensible. +func TestRaceGoroutineCreationStack(t *testing.T) { + var x int + _ = x + ch := make(chan bool, 1) + + f1 := func() { + x = 1 + ch <- true + } + f2 := func() { go f1() } + f3 := func() { go f2() } + f4 := func() { go f3() } + + go f4() + x = 2 + <-ch +} + +// A nil pointer in a mutex method call should not +// corrupt the race detector state. +// Used to hang indefinitely. +func TestNoRaceNilMutexCrash(t *testing.T) { + var mutex sync.Mutex + panics := 0 + defer func() { + if x := recover(); x != nil { + mutex.Lock() + panics++ + mutex.Unlock() + } else { + panic("no panic") + } + }() + var othermutex *sync.RWMutex + othermutex.RLock() +} diff --git a/internal/tests/race/testdata/time_simonly_test.go b/internal/tests/race/testdata/time_simonly_test.go new file mode 100644 index 0000000..7c5a67c --- /dev/null +++ b/internal/tests/race/testdata/time_simonly_test.go @@ -0,0 +1,79 @@ +//go:build sim + +package race_test + +import ( + "testing" + "time" +) + +func TestNoRaceTicker1(t *testing.T) { + // little song and dance to make sure we can read timer.C without racing + timer := time.NewTicker(10 * time.Millisecond) + timer.Stop() + + select { + case <-timer.C: + default: + } + + select { + case <-timer.C: + t.Error("bad") + default: + } + + var x int + _ = x + + done := make(chan struct{}, 1) + go func() { + <-timer.C + x = 2 + done <- struct{}{} + }() + + x = 1 + timer.Reset(time.Second) + + <-done +} + +/* +// TODO: Fix +func TestRaceTimer3(t *testing.T) { + // little song and dance to make sure we can read timer.C without racing + t1 := time.NewTimer(10 * time.Millisecond) + <-t1.C + + t2 := time.NewTimer(10 * time.Millisecond) + <-t2.C + + var x int + _ = x + var y int + _ = y + + done := make(chan struct{}, 2) + go func() { + <-t1.C + x = 2 + done <- struct{}{} + }() + + go func() { + <-t2.C + y = 2 + done <- struct{}{} + }() + + // t1 will fire later but is still racing + t1.Reset(10 * time.Millisecond) + x = 1 + y = 1 + t2.Reset(5 * time.Millisecond) + + <-done + <-done +} +*/ diff --git a/internal/tests/race/testdata/time_test.go b/internal/tests/race/testdata/time_test.go new file mode 100644 index 0000000..b7db312 --- /dev/null +++ b/internal/tests/race/testdata/time_test.go @@ -0,0 +1,144 @@ +package race_test + +import ( + "testing" + "time" +) + +func TestNoRaceTimer1(t *testing.T) { + // little song and dance to make sure we can read timer.C without racing + timer := time.NewTimer(10 * time.Millisecond) + <-timer.C + + var x int + _ = x + + done := make(chan struct{}, 1) + go func() { + <-timer.C + x = 1 + done <- struct{}{} + }() + + x = 2 + timer.Reset(10 * time.Millisecond) + <-done +} + +func TestRaceTimer1(t *testing.T) { + // little song and dance to make sure we can read timer.C without racing + timer := time.NewTimer(10 * time.Millisecond) + <-timer.C + + var x int + _ = x + + done := make(chan struct{}, 1) + go func() { + <-timer.C + x = 1 + done <- struct{}{} + }() + + timer.Reset(10 * time.Millisecond) + x = 2 + <-done +} + +func TestNoRaceTimer2(t *testing.T) { + var x int + _ = x + + done := make(chan struct{}, 1) + + x = 1 + time.AfterFunc(10*time.Millisecond, func() { + x = 2 + done <- struct{}{} + }) + + <-done +} + +func TestRaceTimer2(t *testing.T) { + var x int + _ = x + + done := make(chan struct{}, 1) + + time.AfterFunc(10*time.Millisecond, func() { + x = 2 + done <- struct{}{} + }) + x = 1 + + <-done +} + +func TestRaceTicker1(t *testing.T) { + // little song and dance to make sure we can read timer.C without racing + timer := time.NewTicker(10 * time.Millisecond) + timer.Stop() + + select { + case <-timer.C: + default: + } + + select { + case <-timer.C: + t.Error("bad") + default: + } + + var x int + _ = x + + done := make(chan struct{}, 1) + go func() { + <-timer.C + x = 2 + done <- struct{}{} + }() + + timer.Reset(time.Second) + x = 1 + + <-done +} + +func TestNoRaceTimer3(t *testing.T) { + // little song and dance to make sure we can read timer.C without racing + t1 := time.NewTimer(10 * time.Millisecond) + <-t1.C + + t2 := time.NewTimer(10 * time.Millisecond) + <-t2.C + + var x int + _ = x + var y int + _ = y + + done := make(chan struct{}, 2) + go func() { + <-t1.C + x = 2 + done <- struct{}{} + }() + + go func() { + <-t2.C + y = 2 + done <- struct{}{} + }() + + // t1 will fire later but is still racing + x = 1 + y = 1 + t1.Reset(10 * time.Millisecond) + t2.Reset(5 * time.Millisecond) + + <-done + <-done +} diff --git a/internal/tests/race/testdata/waitgroup_test.go b/internal/tests/race/testdata/waitgroup_test.go new file mode 100644 index 0000000..e99c533 --- /dev/null +++ b/internal/tests/race/testdata/waitgroup_test.go @@ -0,0 +1,368 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found at https://go.googlesource.com/go/+/refs/heads/master/LICENSE. + +package race_test + +import ( + "runtime" + "sync" + "testing" + "time" +) + +// from go/src/runtime/race/testdata/waitgroup_test.go + +func TestNoRaceWaitGroup(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + n := 1 + for i := 0; i < n; i++ { + wg.Add(1) + j := i + go func() { + x = j + wg.Done() + }() + } + wg.Wait() +} + +// TODO: Fix +/* +func TestRaceWaitGroup(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + n := 2 + for i := 0; i < n; i++ { + wg.Add(1) + j := i + go func() { + x = j + wg.Done() + }() + } + wg.Wait() +} +*/ + +func TestNoRaceWaitGroup2(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + wg.Add(1) + go func() { + x = 1 + wg.Done() + }() + wg.Wait() + x = 2 +} + +// incrementing counter in Add and locking wg's mutex +func TestRaceWaitGroupAsMutex(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + c := make(chan bool, 2) + go func() { + wg.Wait() + time.Sleep(100 * time.Millisecond) + wg.Add(+1) + x = 1 + wg.Add(-1) + c <- true + }() + go func() { + wg.Wait() + time.Sleep(100 * time.Millisecond) + wg.Add(+1) + x = 2 + wg.Add(-1) + c <- true + }() + <-c + <-c +} + +// Incorrect usage: Add is too late. +func TestRaceWaitGroupWrongWait(t *testing.T) { + c := make(chan bool, 2) + var x int + _ = x + var wg sync.WaitGroup + go func() { + wg.Add(1) + runtime.Gosched() + x = 1 + wg.Done() + c <- true + }() + go func() { + wg.Add(1) + runtime.Gosched() + x = 2 + wg.Done() + c <- true + }() + wg.Wait() + <-c + <-c +} + +func TestRaceWaitGroupWrongAdd(t *testing.T) { + c := make(chan bool, 2) + var wg sync.WaitGroup + go func() { + wg.Add(1) + time.Sleep(100 * time.Millisecond) + wg.Done() + c <- true + }() + go func() { + wg.Add(1) + time.Sleep(100 * time.Millisecond) + wg.Done() + c <- true + }() + time.Sleep(50 * time.Millisecond) + wg.Wait() + <-c + <-c +} + +func TestNoRaceWaitGroupMultipleWait(t *testing.T) { + c := make(chan bool, 2) + var wg sync.WaitGroup + go func() { + wg.Wait() + c <- true + }() + go func() { + wg.Wait() + c <- true + }() + wg.Wait() + <-c + <-c +} + +func TestNoRaceWaitGroupMultipleWait2(t *testing.T) { + c := make(chan bool, 2) + var wg sync.WaitGroup + wg.Add(2) + go func() { + wg.Done() + wg.Wait() + c <- true + }() + go func() { + wg.Done() + wg.Wait() + c <- true + }() + wg.Wait() + <-c + <-c +} + +func TestNoRaceWaitGroupMultipleWait3(t *testing.T) { + const P = 3 + var data [P]int + done := make(chan bool, P) + var wg sync.WaitGroup + wg.Add(P) + for p := 0; p < P; p++ { + go func(p int) { + data[p] = 42 + wg.Done() + }(p) + } + for p := 0; p < P; p++ { + go func() { + wg.Wait() + for p1 := 0; p1 < P; p1++ { + _ = data[p1] + } + done <- true + }() + } + for p := 0; p < P; p++ { + <-done + } +} + +// TODO: Fix +/* +// Correct usage but still a race +func TestRaceWaitGroup2(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + wg.Add(2) + go func() { + x = 1 + wg.Done() + }() + go func() { + x = 2 + wg.Done() + }() + wg.Wait() +} +*/ + +func TestNoRaceWaitGroupPanicRecover(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + defer func() { + err := recover() + if err != "sync: negative WaitGroup counter" { + t.Fatal("Unexpected panic: ", err) + } + x = 2 + }() + x = 1 + wg.Add(-1) +} + +// TODO: this is actually a panic-synchronization test, not a +// WaitGroup test. Move it to another *_test file +// Is it possible to get a race by synchronization via panic? +func TestNoRaceWaitGroupPanicRecover2(t *testing.T) { + var x int + _ = x + var wg sync.WaitGroup + ch := make(chan bool, 1) + var f func() = func() { + x = 2 + ch <- true + } + go func() { + defer func() { + err := recover() + if err != "sync: negative WaitGroup counter" { + } + go f() + }() + x = 1 + wg.Add(-1) + }() + + <-ch +} + +func TestNoRaceWaitGroupTransitive(t *testing.T) { + x, y := 0, 0 + var wg sync.WaitGroup + wg.Add(2) + go func() { + x = 42 + wg.Done() + }() + go func() { + time.Sleep(1e7) + y = 42 + wg.Done() + }() + wg.Wait() + _ = x + _ = y +} + +func TestNoRaceWaitGroupReuse(t *testing.T) { + const P = 3 + var data [P]int + var wg sync.WaitGroup + for try := 0; try < 3; try++ { + wg.Add(P) + for p := 0; p < P; p++ { + go func(p int) { + data[p]++ + wg.Done() + }(p) + } + wg.Wait() + for p := 0; p < P; p++ { + data[p]++ + } + } +} + +func TestNoRaceWaitGroupReuse2(t *testing.T) { + const P = 3 + var data [P]int + var wg sync.WaitGroup + for try := 0; try < 3; try++ { + wg.Add(P) + for p := 0; p < P; p++ { + go func(p int) { + data[p]++ + wg.Done() + }(p) + } + done := make(chan bool) + go func() { + wg.Wait() + for p := 0; p < P; p++ { + data[p]++ + } + done <- true + }() + wg.Wait() + <-done + for p := 0; p < P; p++ { + data[p]++ + } + } +} + +func TestRaceWaitGroupReuse(t *testing.T) { + const P = 3 + const T = 3 + done := make(chan bool, T) + var wg sync.WaitGroup + for try := 0; try < T; try++ { + var data [P]int + wg.Add(P) + for p := 0; p < P; p++ { + go func(p int) { + time.Sleep(50 * time.Millisecond) + data[p]++ + wg.Done() + }(p) + } + go func() { + wg.Wait() + for p := 0; p < P; p++ { + data[p]++ + } + done <- true + }() + time.Sleep(100 * time.Millisecond) + wg.Wait() + } + for try := 0; try < T; try++ { + <-done + } +} + +func TestNoRaceWaitGroupConcurrentAdd(t *testing.T) { + const P = 4 + waiting := make(chan bool, P) + var wg sync.WaitGroup + for p := 0; p < P; p++ { + go func() { + wg.Add(1) + waiting <- true + wg.Done() + }() + } + for p := 0; p < P; p++ { + <-waiting + } + wg.Wait() +} diff --git a/internal/tests/script/script_test.go b/internal/tests/script/script_test.go new file mode 100644 index 0000000..19c6f4e --- /dev/null +++ b/internal/tests/script/script_test.go @@ -0,0 +1,77 @@ +package script_test + +import ( + "io/fs" + "log" + "os" + "path/filepath" + "testing" + + "github.com/rogpeppe/go-internal/gotooltest" + "github.com/rogpeppe/go-internal/testscript" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" +) + +func TestScript(t *testing.T) { + p := testscript.Params{ + Dir: "testdata", + } + gotooltest.Setup(&p) + + curModDir, err := gosimtool.FindGoModDir() + if err != nil { + t.Fatal(err) + } + extractedModDir := filepath.Join(curModDir, gosimtool.OutputDirectory, "scripttest", "mod") + + // add test dependencies to the copied gosim module + if err := filepath.Walk(extractedModDir, func(path string, info fs.FileInfo, err error) error { + return nil + }); err != nil { + log.Fatal(err) + } + + origSetup := p.Setup + p.Setup = func(e *testscript.Env) error { + // put a fake go.mod in the work directory + // this include a copy of the gosim module (not the entire module because we want to minimize test dependencies) + if err := gosimtool.MakeGoModForTest(extractedModDir, e.WorkDir, []string{ + "github.com/dave/dst", + "github.com/google/go-cmp", + "github.com/mattn/go-isatty", + "github.com/mattn/go-sqlite3", + "golang.org/x/mod", + "golang.org/x/sync", + "golang.org/x/sys", + "golang.org/x/tools", + "mvdan.cc/gofumpt", + }); err != nil { + return err + } + + // add /bin to PATH in tests + binDir := filepath.Join(curModDir, gosimtool.OutputDirectory, "scripttest", "bin") + // add dependencies on /bin + if err := filepath.Walk(binDir, func(path string, info fs.FileInfo, err error) error { + return nil + }); err != nil { + log.Fatal(err) + } + e.Setenv("PATH", binDir+string(filepath.ListSeparator)+e.Getenv("PATH")) + e.Setenv("GOSIMTOOL", filepath.Join(binDir, "gosim")) + + // allow the module to access our shared cache + e.Vars = append(e.Vars, "GOSIMCACHE="+filepath.Join(curModDir, gosimtool.OutputDirectory)) + + if origSetup != nil { + return origSetup(e) + } + return nil + } + testscript.Run(t, p) +} + +func TestMain(m *testing.M) { + os.Exit(testscript.RunMain(m, nil)) +} diff --git a/internal/tests/script/testdata/gosimpassfail.txtar b/internal/tests/script/testdata/gosimpassfail.txtar new file mode 100644 index 0000000..1fa5d2f --- /dev/null +++ b/internal/tests/script/testdata/gosimpassfail.txtar @@ -0,0 +1,22 @@ +# running gosim +! exec gosim test -run TestFail -v . +stdout 'no good' + +exec gosim test -run TestPass -v . +stdout 'quite good' + + +-- machinecrash_test.go -- +package behavior_test + +import ( + "testing" +) + +func TestFail(t *testing.T) { + t.Fatal("no good") +} + +func TestPass(t *testing.T) { + t.Log("quite good") +} \ No newline at end of file diff --git a/internal/tests/script/testdata/simmetatest.txtar b/internal/tests/script/testdata/simmetatest.txtar new file mode 100644 index 0000000..9df0fbb --- /dev/null +++ b/internal/tests/script/testdata/simmetatest.txtar @@ -0,0 +1,25 @@ +# running gosim +! exec gosim test -run=Other -v . +stdout 'metatest cannot be used from within gosim' + +! exec gosim test -run=Self -v . +stdout 'metatest cannot be used from within gosim' + +-- machinecrash_test.go -- +package behavior_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestSelf(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + metatesting.CheckDeterministic(t, mt) +} + +func TestOther(t *testing.T) { + mt := metatesting.ForOtherPackage(t, "foo") + metatesting.CheckDeterministic(t, mt) +} \ No newline at end of file diff --git a/internal/tests/script/testdata/testbuilding.txtar b/internal/tests/script/testdata/testbuilding.txtar new file mode 100644 index 0000000..64b437b --- /dev/null +++ b/internal/tests/script/testdata/testbuilding.txtar @@ -0,0 +1,75 @@ +# This tests verifies the metatest test binary building and caching logic. + +# Inner test should work +exec gosim test -v . +stdout 'inside gosim hello' + +# Cached metatest fails without binary +! exec go test -v . +stdout 'Could not find pre-built gosim test binary' + +# Uncached metatest works +exec go test -v -count=1 . +stdout 'looks like an uncached test run.*build-tests' +stdout 'inside gosim hello' + +# Build binary +exec gosim build-tests . + +# Cached metatest now works +exec go test -v . +stdout 'Using pre-built gosim test binary' +stdout 'inside gosim hello' + +# Uncached metatest still works +exec go test -v -count=1 . +stdout 'looks like an uncached test run.*build-tests' +stdout 'inside gosim hello' + +# Touch input file +exec touch test_test.go + +# Cached metatest no longer works +! exec go test -v . +stdout 'Using pre-built gosim test binary' +stdout 'changed' + +# Uncached metatest still works +exec go test -v -count=1 . +stdout 'looks like an uncached test run.*build-tests' +stdout 'inside gosim hello' + +-- metatest_test.go -- +//go:build !sim + +package behavior_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestSelf(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + _, err := mt.Run(t, &metatesting.RunConfig{ + Test: "TestHello", + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } +} + +-- test_test.go -- +//go:build sim + +package behavior_test + +import ( + "testing" +) + +func TestHello(t *testing.T) { + t.Log("inside gosim hello") +} \ No newline at end of file diff --git a/internal/tests/testpb.proto b/internal/tests/testpb.proto new file mode 100644 index 0000000..d4c4b9b --- /dev/null +++ b/internal/tests/testpb.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +option go_package = "github.com/jellevandenhooff/gosim/internal/tests/testpb"; + +package testpb; + +service EchoServer { + rpc Echo(EchoRequest) returns (EchoResponse) {} +} + +message EchoRequest { + string message = 1; +} + +message EchoResponse { + string message = 1; +} + diff --git a/internal/tests/testpb/testpb.pb.go b/internal/tests/testpb/testpb.pb.go new file mode 100644 index 0000000..925d5f6 --- /dev/null +++ b/internal/tests/testpb/testpb.pb.go @@ -0,0 +1,213 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.1 +// protoc v3.21.6 +// source: testpb.proto + +package testpb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EchoRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (x *EchoRequest) Reset() { + *x = EchoRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_testpb_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EchoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EchoRequest) ProtoMessage() {} + +func (x *EchoRequest) ProtoReflect() protoreflect.Message { + mi := &file_testpb_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EchoRequest.ProtoReflect.Descriptor instead. +func (*EchoRequest) Descriptor() ([]byte, []int) { + return file_testpb_proto_rawDescGZIP(), []int{0} +} + +func (x *EchoRequest) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +type EchoResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` +} + +func (x *EchoResponse) Reset() { + *x = EchoResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_testpb_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EchoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EchoResponse) ProtoMessage() {} + +func (x *EchoResponse) ProtoReflect() protoreflect.Message { + mi := &file_testpb_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EchoResponse.ProtoReflect.Descriptor instead. +func (*EchoResponse) Descriptor() ([]byte, []int) { + return file_testpb_proto_rawDescGZIP(), []int{1} +} + +func (x *EchoResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +var File_testpb_proto protoreflect.FileDescriptor + +var file_testpb_proto_rawDesc = []byte{ + 0x0a, 0x0c, 0x74, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, + 0x74, 0x65, 0x73, 0x74, 0x70, 0x62, 0x22, 0x27, 0x0a, 0x0b, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, + 0x28, 0x0a, 0x0c, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x32, 0x41, 0x0a, 0x0a, 0x45, 0x63, 0x68, + 0x6f, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x33, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, + 0x13, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x70, 0x62, 0x2e, 0x45, 0x63, + 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x39, 0x5a, 0x37, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6a, 0x65, 0x6c, 0x6c, 0x65, + 0x76, 0x61, 0x6e, 0x64, 0x65, 0x6e, 0x68, 0x6f, 0x6f, 0x66, 0x66, 0x2f, 0x67, 0x6f, 0x73, 0x69, + 0x6d, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2f, 0x74, 0x65, 0x73, 0x74, 0x73, + 0x2f, 0x74, 0x65, 0x73, 0x74, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_testpb_proto_rawDescOnce sync.Once + file_testpb_proto_rawDescData = file_testpb_proto_rawDesc +) + +func file_testpb_proto_rawDescGZIP() []byte { + file_testpb_proto_rawDescOnce.Do(func() { + file_testpb_proto_rawDescData = protoimpl.X.CompressGZIP(file_testpb_proto_rawDescData) + }) + return file_testpb_proto_rawDescData +} + +var file_testpb_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_testpb_proto_goTypes = []interface{}{ + (*EchoRequest)(nil), // 0: testpb.EchoRequest + (*EchoResponse)(nil), // 1: testpb.EchoResponse +} +var file_testpb_proto_depIdxs = []int32{ + 0, // 0: testpb.EchoServer.Echo:input_type -> testpb.EchoRequest + 1, // 1: testpb.EchoServer.Echo:output_type -> testpb.EchoResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_testpb_proto_init() } +func file_testpb_proto_init() { + if File_testpb_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_testpb_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EchoRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_testpb_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EchoResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_testpb_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_testpb_proto_goTypes, + DependencyIndexes: file_testpb_proto_depIdxs, + MessageInfos: file_testpb_proto_msgTypes, + }.Build() + File_testpb_proto = out.File + file_testpb_proto_rawDesc = nil + file_testpb_proto_goTypes = nil + file_testpb_proto_depIdxs = nil +} diff --git a/internal/tests/testpb/testpb_grpc.pb.go b/internal/tests/testpb/testpb_grpc.pb.go new file mode 100644 index 0000000..9472eef --- /dev/null +++ b/internal/tests/testpb/testpb_grpc.pb.go @@ -0,0 +1,103 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.21.6 +// source: testpb.proto + +package testpb + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// EchoServerClient is the client API for EchoServer service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type EchoServerClient interface { + Echo(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) +} + +type echoServerClient struct { + cc grpc.ClientConnInterface +} + +func NewEchoServerClient(cc grpc.ClientConnInterface) EchoServerClient { + return &echoServerClient{cc} +} + +func (c *echoServerClient) Echo(ctx context.Context, in *EchoRequest, opts ...grpc.CallOption) (*EchoResponse, error) { + out := new(EchoResponse) + err := c.cc.Invoke(ctx, "/testpb.EchoServer/Echo", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// EchoServerServer is the server API for EchoServer service. +// All implementations should embed UnimplementedEchoServerServer +// for forward compatibility +type EchoServerServer interface { + Echo(context.Context, *EchoRequest) (*EchoResponse, error) +} + +// UnimplementedEchoServerServer should be embedded to have forward compatible implementations. +type UnimplementedEchoServerServer struct { +} + +func (UnimplementedEchoServerServer) Echo(context.Context, *EchoRequest) (*EchoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Echo not implemented") +} + +// UnsafeEchoServerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to EchoServerServer will +// result in compilation errors. +type UnsafeEchoServerServer interface { + mustEmbedUnimplementedEchoServerServer() +} + +func RegisterEchoServerServer(s grpc.ServiceRegistrar, srv EchoServerServer) { + s.RegisterService(&EchoServer_ServiceDesc, srv) +} + +func _EchoServer_Echo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EchoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EchoServerServer).Echo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/testpb.EchoServer/Echo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EchoServerServer).Echo(ctx, req.(*EchoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// EchoServer_ServiceDesc is the grpc.ServiceDesc for EchoServer service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var EchoServer_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "testpb.EchoServer", + HandlerType: (*EchoServerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Echo", + Handler: _EchoServer_Echo_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "testpb.proto", +} diff --git a/internal/translate/cache.go b/internal/translate/cache.go new file mode 100644 index 0000000..4154541 --- /dev/null +++ b/internal/translate/cache.go @@ -0,0 +1,162 @@ +package translate + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/gob" + "encoding/hex" + "log" + "maps" + "os" + "slices" + + "golang.org/x/tools/go/packages" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/translate/cache" +) + +type Hash [32]byte + +func hashfile(path string) (Hash, error) { + // XXX: cache? + bytes, err := os.ReadFile(path) + if err != nil { + return Hash{}, err + } + return sha256.Sum256(bytes), nil +} + +type Hasher struct { + buffer *bytes.Buffer +} + +func NewHasher() *Hasher { + return &Hasher{ + buffer: bytes.NewBuffer(nil), + } +} + +func (h *Hasher) addHash(hash Hash) { + h.buffer.Write(hash[:]) +} + +func (h *Hasher) addString(s string) { + hash := sha256.Sum256([]byte(s)) + h.addHash(hash) +} + +func (h *Hasher) addFileContents(path string) error { + hash, err := hashfile(path) + if err != nil { + return err + } + h.addHash(hash) + return nil +} + +func (h *Hasher) addFile(path string) error { + h.addString(path) + return h.addFileContents(path) +} + +func (h *Hasher) Sum256() Hash { + return sha256.Sum256(h.buffer.Bytes()) +} + +func computeTranslateToolHash(cfg gosimtool.BuildConfig) Hash { + h := NewHasher() + h.addString("runner") + h.addString(cfg.AsDirname()) + binaryPath, err := os.Executable() + if err != nil { + log.Fatal(err) + } + // log.Println(binaryPath) + // binaryPath = "./foobar" + // do not hash path because it might change. also it does not matter. + if err := h.addFileContents(binaryPath); err != nil { + log.Fatal(err) + } + return h.Sum256() +} + +func computePackageHash(translateToolHash Hash, pkg *packages.Package, importHashes map[string]Hash) Hash { + cacheKeyHasher := NewHasher() + cacheKeyHasher.addHash(translateToolHash) + cacheKeyHasher.addString(pkg.ID) + for _, path := range pkg.GoFiles { + // XXX: why not compiled go files? + if err := cacheKeyHasher.addFile(path); err != nil { + log.Fatal(err) + } + } + for _, path := range pkg.OtherFiles { + if err := cacheKeyHasher.addFile(path); err != nil { + log.Fatal(err) + } + } + keys := slices.Sorted(maps.Keys(importHashes)) + for _, key := range keys { + if key == pkg.ID { + continue + } + cacheKeyHasher.addString(key) + cacheKeyHasher.addHash(importHashes[key]) + } + return cacheKeyHasher.Sum256() +} + +func marshalCachedResult(res *TranslatePackageResult) ([]byte, error) { + var buf bytes.Buffer + cacheWriter := gzip.NewWriter(&buf) + if err := gob.NewEncoder(cacheWriter).Encode(res); err != nil { + log.Fatal(err) + } + if err := cacheWriter.Close(); err != nil { + log.Fatal(err) + } + return buf.Bytes(), nil +} + +func cachePut(c *cache.Cache, h Hash, res *TranslatePackageResult) error { + b, err := marshalCachedResult(res) + if err != nil { + log.Fatal(err) + } + if err := c.Put(hex.EncodeToString(h[:]), b); err != nil { + log.Fatal(err) + } + return nil +} + +func unmarshalCachedResult(b []byte) (*TranslatePackageResult, error) { + reader, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + log.Fatal(err) + } + var result TranslatePackageResult + if err := gob.NewDecoder(reader).Decode(&result); err != nil { + log.Fatal(err) + } + return &result, nil +} + +func cacheGet(c *cache.Cache, h Hash) (*TranslatePackageResult, error) { + blob, err := c.Get(hex.EncodeToString(h[:])) + if err != nil && err != cache.ErrNoSuchKey { + log.Fatal(err) + } + + if blob == nil { + return nil, nil + } + + res, err := unmarshalCachedResult(blob) + if err != nil { + log.Fatal(err) + } + + return res, nil +} diff --git a/internal/translate/cache/cache.go b/internal/translate/cache/cache.go new file mode 100644 index 0000000..8674e68 --- /dev/null +++ b/internal/translate/cache/cache.go @@ -0,0 +1,115 @@ +package cache + +import ( + "database/sql" + "errors" + "sync" + "time" + + _ "github.com/mattn/go-sqlite3" +) + +type DB struct { + db *sql.DB +} + +func NewDB(path string) (*DB, error) { + db, err := sql.Open("sqlite3", path) + if err != nil { + return nil, err + } + + if _, err := db.Exec("CREATE TABLE IF NOT EXISTS cache (key TEXT PRIMARY KEY NOT NULL, timestamp INT, contents BLOB) STRICT"); err != nil { + db.Close() + return nil, err + } + + return &DB{ + db: db, + }, nil +} + +func (d *DB) Close() error { + return d.db.Close() +} + +var ErrNoSuchKey = errors.New("no such key") + +func (d *DB) Get(key string) ([]byte, error) { + row := d.db.QueryRow("SELECT contents FROM cache WHERE key = ?", key) + if err := row.Err(); err != nil { + return nil, err + } + + var blob []byte + if err := row.Scan(&blob); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNoSuchKey + } + return nil, err + } + + return blob, nil +} + +func (d *DB) Put(key string, timestamp time.Time, blob []byte) error { + if _, err := d.db.Exec("INSERT OR IGNORE INTO cache (key, timestamp, contents) VALUES (?, ?, ?)", key, timestamp.Unix(), blob); err != nil { + return err + } + return nil +} + +func (d *DB) Touch(key string, timestamp time.Time) error { + if _, err := d.db.Exec("UPDATE cache SET timestamp = ? WHERE key = ? AND timestamp < ?", timestamp.Unix(), key, timestamp.Unix()); err != nil { + return err + } + return nil +} + +func (d *DB) Clean(timestamp time.Time) error { + if _, err := d.db.Exec("DELETE FROM cache WHERE timestamp < ?", timestamp.Unix()); err != nil { + return err + } + return nil +} + +type Cache struct { + mu sync.Mutex + db *DB + now time.Time +} + +func NewCache(db *DB) *Cache { + return &Cache{ + db: db, + now: time.Now().Truncate(time.Hour), + } +} + +func (c *Cache) Get(key string) ([]byte, error) { + c.mu.Lock() + defer c.mu.Unlock() + + blob, err := c.db.Get(key) + if err != nil { + return nil, err + } + if err := c.db.Touch(key, c.now); err != nil { + return nil, err + } + return blob, nil +} + +func (c *Cache) Put(key string, blob []byte) error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.db.Put(key, c.now, blob) +} + +func (c *Cache) Clean() error { + c.mu.Lock() + defer c.mu.Unlock() + + return c.db.Clean(c.now.Add(-7 * 24 * time.Hour)) +} diff --git a/internal/translate/cache/cache_test.go b/internal/translate/cache/cache_test.go new file mode 100644 index 0000000..be4e622 --- /dev/null +++ b/internal/translate/cache/cache_test.go @@ -0,0 +1,79 @@ +package cache_test + +import ( + "path" + "testing" + "time" + + "github.com/jellevandenhooff/gosim/internal/translate/cache" +) + +func mustMiss(t *testing.T, db *cache.DB, key string) { + t.Helper() + if _, err := db.Get(key); err != cache.ErrNoSuchKey { + t.Errorf("get %s: unexpected error %s, expected %s", key, err, cache.ErrNoSuchKey) + } +} + +func mustGet(t *testing.T, db *cache.DB, key string, value string) { + t.Helper() + blob, err := db.Get(key) + if err != nil { + t.Errorf("get %s: unexpected error %s", key, err) + return + } + if string(blob) != value { + t.Errorf("get %s: got %s, expected %s", key, string(blob), value) + } +} + +func TestDB(t *testing.T) { + dir := t.TempDir() + + file := path.Join(dir, "cache.sqlite3") + + db, err := cache.NewDB(file) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + timestamp := time.Now() + + mustMiss(t, db, "foo") + + if err := db.Put("foo", timestamp, []byte("hello")); err != nil { + t.Errorf("put foo: unexpected error %s", err) + } + mustGet(t, db, "foo", "hello") + + if err := db.Put("bar", timestamp.Add(2*time.Hour), []byte("goodbye")); err != nil { + t.Errorf("put bar: unexpected error %s", err) + } + mustGet(t, db, "bar", "goodbye") + + if err := db.Clean(timestamp); err != nil { + t.Errorf("clean: unexpected error %s", err) + } + mustGet(t, db, "foo", "hello") + mustGet(t, db, "bar", "goodbye") + + if err := db.Clean(timestamp.Add(time.Hour)); err != nil { + t.Errorf("clean: unexpected error %s", err) + } + mustMiss(t, db, "foo") + mustGet(t, db, "bar", "goodbye") + + if err := db.Touch("bar", timestamp.Add(4*time.Hour)); err != nil { + t.Errorf("touch bar: unexpected error %s", err) + } + if err := db.Clean(timestamp.Add(3 * time.Hour)); err != nil { + t.Errorf("clean: unexpected error %s", err) + } + mustGet(t, db, "bar", "goodbye") + + if err := db.Clean(timestamp.Add(5 * time.Hour)); err != nil { + t.Errorf("clean: unexpected error %s", err) + } + mustMiss(t, db, "bar") +} diff --git a/internal/translate/chan.go b/internal/translate/chan.go new file mode 100644 index 0000000..a129bf5 --- /dev/null +++ b/internal/translate/chan.go @@ -0,0 +1,409 @@ +package translate + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "log" + "strconv" + "strings" + + "github.com/dave/dst" + "github.com/dave/dst/dstutil" +) + +func isRecvExpr(node dst.Node) (*dst.UnaryExpr, bool) { + expr, ok := node.(dst.Expr) + if !ok { + return nil, false + } + expr = dstutil.Unparen(expr) + if recvExpr, ok := expr.(*dst.UnaryExpr); ok && recvExpr.Op == token.ARROW { + return recvExpr, true + } + return nil, false +} + +type ChanType struct { + Type *types.Chan + Named types.Type +} + +func (t *packageTranslator) isChanType(expr dst.Expr) (ChanType, bool) { + if astExpr, ok := t.astMap.Nodes[expr].(ast.Expr); ok { + if convertedType, ok := t.implicitConversions[astExpr]; ok { + if chanType, ok := convertedType.Underlying().(*types.Chan); ok { + return ChanType{Type: chanType, Named: convertedType}, true + } + } + } + + if typ, ok := t.getType(expr); ok { + if chanTyp, ok := typ.Underlying().(*types.Chan); ok { + return ChanType{Type: chanTyp, Named: typ}, true + } + } + + return ChanType{}, false +} + +func (t *packageTranslator) maybeConvertChanType(x dst.Expr, typ ChanType) dst.Expr { + if typ.Named == typ.Type { + return x + } + + return &dst.CallExpr{ + Fun: t.makeTypeExpr(typ.Named), + Args: []dst.Expr{ + x, + }, + } +} + +func (t *packageTranslator) maybeExtractNamedChanType(x dst.Expr, typ ChanType) dst.Expr { + if typ.Named == typ.Type { + return x + } + return &dst.CallExpr{Fun: t.newRuntimeSelector("ExtractChan"), Args: []dst.Expr{x}} +} + +func (t *packageTranslator) rewriteChanLen(c *dstutil.Cursor) { + if call, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(call.Fun, "len") { + if typ, ok := t.isChanType(call.Args[0]); ok { + call.Fun = &dst.SelectorExpr{ + X: t.maybeExtractNamedChanType(call.Args[0], typ), + Sel: dst.NewIdent("Len"), + } + call.Args = nil + } + } +} + +func (t *packageTranslator) rewriteChanCap(c *dstutil.Cursor) { + if call, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(call.Fun, "cap") { + if typ, ok := t.isChanType(call.Args[0]); ok { + call.Fun = &dst.SelectorExpr{ + X: t.maybeExtractNamedChanType(call.Args[0], typ), + Sel: dst.NewIdent("Cap"), + } + call.Args = nil + } + } +} + +func (t *packageTranslator) rewriteChanType(c *dstutil.Cursor) { + // chan type + if chanType, ok := c.Node().(*dst.ChanType); ok { + // XXX: this ignores the arrow... + c.Replace(&dst.IndexListExpr{ + X: t.newRuntimeSelector("Chan"), + Indices: []dst.Expr{ + chanType.Value, // XXX: apply? + }, + }) + } +} + +func (t *packageTranslator) rewriteChanRange(c *dstutil.Cursor) { + if rangeStmt, ok := c.Node().(*dst.RangeStmt); ok { + if typ, ok := t.isChanType(rangeStmt.X); ok { + rangeStmt.X = &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedChanType(rangeStmt.X, typ), + Sel: dst.NewIdent("Range"), + }, + } + } + } +} + +func (t *packageTranslator) rewriteChanRecvSimpleExpr(c *dstutil.Cursor) { + // read from chan in simple form + // <-ch + if recvExpr, ok := isRecvExpr(c.Node()); ok { + // we catch (val, ok) cases earlier + typ, _ := t.isChanType(recvExpr.X) + + c.Replace(&dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedChanType(recvExpr.X, typ), + Sel: dst.NewIdent("Recv"), + }, + }) + } +} + +func (t *packageTranslator) rewriteChanRecvOk(c *dstutil.Cursor) { + // read from chan in val, ok = <-ch + + rhs, ok := isDualAssign(c) + if !ok { + return + } + + if recvExpr, ok := isRecvExpr(*rhs); ok { + *rhs = &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: recvExpr.X, + Sel: dst.NewIdent("RecvOk"), + }, + } + } +} + +func (t *packageTranslator) rewriteChanClose(c *dstutil.Cursor) { + // close ch + if callExpr, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(callExpr.Fun, "close") { + typ, _ := t.isChanType(callExpr.Args[0]) + r := &dst.CallExpr{ + Decs: callExpr.Decs, + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedChanType(callExpr.Args[0], typ), + Sel: dst.NewIdent("Close"), + }, + } + c.Replace(r) + } +} + +func (t *packageTranslator) rewriteChanSend(c *dstutil.Cursor) { + // write to chan + // ch <- + if sendStmt, ok := c.Node().(*dst.SendStmt); ok { + typ, _ := t.isChanType(sendStmt.Chan) + c.Replace(&dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedChanType(sendStmt.Chan, typ), + Sel: dst.NewIdent("Send"), + }, + Args: []dst.Expr{ + sendStmt.Value, + }, + }, + }) + } +} + +func (t *packageTranslator) rewriteChanLiteral(c *dstutil.Cursor) { + if ident, ok := c.Node().(*dst.Ident); ok && ident.Name == "nil" { + if chanType, ok := t.isChanType(ident); ok { + c.Replace(&dst.CallExpr{ + Fun: &dst.IndexListExpr{ + X: t.newRuntimeSelector("NilChan"), + Indices: []dst.Expr{ + t.makeTypeExpr(chanType.Type.Elem()), + }, + }, + }) + } + } +} + +func (t *packageTranslator) rewriteMakeChan(c *dstutil.Cursor) { + // make(chan) + if makeExpr, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(makeExpr.Fun, "make") { + if typ, ok := t.isChanType(makeExpr); ok { + var lenArg dst.Expr = &dst.BasicLit{Kind: token.INT, Value: "0"} + if len(makeExpr.Args) >= 2 { + lenArg = t.apply(makeExpr.Args[1]).(dst.Expr) + } + + c.Replace(t.maybeConvertChanType(&dst.CallExpr{ + Fun: &dst.IndexListExpr{ + X: t.newRuntimeSelector("NewChan"), + Indices: []dst.Expr{ + t.makeTypeExpr(typ.Type.Elem()), + }, + }, + Args: []dst.Expr{ + lenArg, + }, + }, typ)) + } + } +} + +func (t *packageTranslator) rewriteSelectStmt(c *dstutil.Cursor) { + // select stmt + if selectStmt, ok := c.Node().(*dst.SelectStmt); ok { + handlers := []dst.Expr{} + var clauses []dst.Stmt + + copyStmt := &dst.AssignStmt{ + Lhs: []dst.Expr{}, + Tok: token.DEFINE, + Rhs: []dst.Expr{}, + } + + suffix := t.suffix() + selectIdx := "idx" + suffix + selectVal := "val" + suffix + valUsed := false + selectOk := "ok" + suffix + okUsed := false + + hasDefault := false + + counter := 0 + + for _, clause := range selectStmt.Body.List { + clause := clause.(*dst.CommClause) + + isDefault := false + var handler dst.Expr + + if sendStmt, ok := clause.Comm.(*dst.SendStmt); ok { + handler = &dst.CallExpr{ + Fun: &dst.SelectorExpr{X: sendStmt.Chan, Sel: dst.NewIdent("SendSelector")}, + Args: []dst.Expr{sendStmt.Value}, + } + } else if exprStmt, ok := clause.Comm.(*dst.ExprStmt); ok { + recvExpr, ok := isRecvExpr(exprStmt.X) + if !ok { + log.Fatal("bad recv expr") + } + typ, _ := t.isChanType(recvExpr.X) + + handler = &dst.CallExpr{ + Fun: &dst.SelectorExpr{X: t.maybeExtractNamedChanType(recvExpr.X, typ), Sel: dst.NewIdent("RecvSelector")}, + } + } else if assignStmt, ok := clause.Comm.(*dst.AssignStmt); ok { + if len(assignStmt.Rhs) != 1 { + log.Fatal("bad recv assign stmt") + } + recvExpr, ok := isRecvExpr(assignStmt.Rhs[0]) + if !ok { + log.Fatal("bad recv expr") + } + + typ, _ := t.isChanType(recvExpr.X) + + lhs := assignStmt.Lhs + rhs := []dst.Expr{ + &dst.CallExpr{ + Fun: &dst.IndexExpr{ + X: t.newRuntimeSelector("ChanCast"), + Index: t.makeTypeExpr(typ.Type.Elem()), + }, + Args: []dst.Expr{dst.NewIdent(selectVal)}, + }, + } + valUsed = true + if len(lhs) > 1 { + rhs = append(rhs, dst.NewIdent(selectOk)) + okUsed = true + } + + clause.Body = append([]dst.Stmt{ + &dst.AssignStmt{ + Lhs: lhs, + Tok: assignStmt.Tok, + Rhs: rhs, + Decs: dst.AssignStmtDecorations{ + NodeDecs: dst.NodeDecs{ + End: clause.Decs.Comm, + }, + }, + }, + }, clause.Body...) + + handler = &dst.CallExpr{ + Fun: &dst.SelectorExpr{X: recvExpr.X, Sel: dst.NewIdent("RecvSelector")}, + } + } else if clause.Comm == nil { + hasDefault = true + isDefault = true + } else { + panic("help") + } + + var caseList []dst.Expr + if !isDefault { + caseList = []dst.Expr{&dst.BasicLit{Kind: token.INT, Value: fmt.Sprint(counter)}} + counter++ + handlers = append(handlers, handler) + } + clauses = append(clauses, &dst.CaseClause{ + List: caseList, + Body: clause.Body, + Decs: dst.CaseClauseDecorations{ + NodeDecs: clause.Decs.NodeDecs, + Case: clause.Decs.Case, + Colon: clause.Decs.Colon, + }, + }) + } + + if !hasDefault { + // output a default case to help the compiler understand we will never pass over this select + clauses = append(clauses, &dst.CaseClause{ + // stick this case at the original closing bracket to help comments find their way + List: nil, // default + Body: []dst.Stmt{ + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: dst.NewIdent("panic"), + Args: []dst.Expr{ + &dst.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote("unreachable select"), + }, + }, + }, + }, + }, + }) + } else { + handlers = append(handlers, &dst.CallExpr{ + Fun: t.newRuntimeSelector("DefaultSelector"), + }) + } + + if len(copyStmt.Lhs) > 0 { + c.InsertBefore(copyStmt) + } + + if !valUsed { + selectVal = "_" + } + if !okUsed { + selectOk = "_" + } + + var call dst.Expr + if hasDefault && counter == 1 { + // special case + call = handlers[0] + sel := call.(*dst.CallExpr).Fun.(*dst.SelectorExpr).Sel + sel.Name = "Select" + strings.TrimSuffix(sel.Name, "Selector") + "OrDefault" + } else { + call = &dst.CallExpr{ + Fun: t.newRuntimeSelector("Select"), + Args: handlers, + } + } + + replacement := &dst.SwitchStmt{ + Decs: dst.SwitchStmtDecorations{ + NodeDecs: selectStmt.Decs.NodeDecs, + Switch: selectStmt.Decs.Select, + }, + Init: &dst.AssignStmt{ + Lhs: []dst.Expr{ + dst.NewIdent(selectIdx), + dst.NewIdent(selectVal), + dst.NewIdent(selectOk), + }, + Tok: token.DEFINE, + Rhs: []dst.Expr{call}, + }, + Tag: dst.NewIdent(selectIdx), + Body: &dst.BlockStmt{ + List: clauses, + }, + } + c.Replace(replacement) + } +} diff --git a/internal/translate/globals.go b/internal/translate/globals.go new file mode 100644 index 0000000..268b65f --- /dev/null +++ b/internal/translate/globals.go @@ -0,0 +1,462 @@ +package translate + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "strconv" + + "github.com/dave/dst" + "github.com/dave/dst/dstutil" +) + +func (t *packageTranslator) rewriteJsonGlobalsHack(c *dstutil.Cursor) { + if decl, ok := c.Node().(*dst.FuncDecl); ok { + if t.pkgPath == "encoding/json" && (decl.Name.Name == "cachedTypeFields" || decl.Name.Name == "typeEncoder") { + decl.Body.List = append([]dst.Stmt{ + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: t.newRuntimeSelector("BeginControlledNondeterminism"), + }, + }, + &dst.DeferStmt{ + Call: &dst.CallExpr{ + Fun: t.newRuntimeSelector("EndControlledNondeterminism"), + }, + }, + }, decl.Body.List...) + } + } +} + +func (t *packageTranslator) rewriteInit(c *dstutil.Cursor) { + node := c.Node() + + decl, ok := node.(*dst.FuncDecl) + if !ok { + return + } + + if decl.Name.Name != "init" || decl.Recv != nil { + return + } + + astIdent, ok := t.astMap.Nodes[decl.Name].(*ast.Ident) + if !ok { + // XXX hmmm??? why does this happen?? + return + } + obj, ok := t.typesInfo.Defs[astIdent] + if !ok { + return + } + var_, ok := obj.(*types.Func) + if !ok { + return + } + pkg := var_.Pkg() + if pkg == nil { + return + } + if var_.Parent() != pkg.Scope() { + return + } + + idx := t.collect.initIdx + t.collect.initIdx++ + var forTest string + if t.forTest { + forTest = "fortest" + } + name := fmt.Sprintf("gosiminit%s%d", forTest, idx) + t.collect.inits = append(t.collect.inits, name) + + decl.Name = dst.NewIdent(name) +} + +func (t *packageTranslator) isGlobalUse(node dst.Node) (dst.Expr, bool) { + expr, ok := node.(dst.Expr) + if !ok { + return nil, false + } + expr = dstutil.Unparen(expr) + + ident, ok := expr.(*dst.Ident) + if !ok { + return nil, false + } + + astExpr, ok := t.astMap.Nodes[ident].(ast.Expr) // XXX: use expr here because a dst ident might be an ast selector; happens in other places also? + if !ok { + return nil, false + } + + astIdent, ok := astExpr.(*ast.Ident) + if !ok { + astSelector, ok := astExpr.(*ast.SelectorExpr) + if !ok { + return nil, false + } + astIdent = astSelector.Sel + } + + obj, ok := t.typesInfo.Uses[astIdent] + if !ok { + return nil, false + } + + var_, ok := obj.(*types.Var) + if !ok { + return nil, false + } + + // XXX: jank + pkg := var_.Pkg() + if pkg == nil { + return nil, false + } + + if var_.Parent() != pkg.Scope() { + return nil, false + } + + if pkgInfo, ok := t.globalInfo.ByPackage[pkg.Path()]; ok { + if container, ok := pkgInfo.Container[var_.Name()]; ok { + return &dst.SelectorExpr{ + X: &dst.CallExpr{ + Fun: &dst.Ident{Name: container, Path: ident.Path}, + }, + Sel: dst.NewIdent(ident.Name), + }, true + } + } + + return nil, false +} + +func (t *packageTranslator) rewriteGlobalRead(c *dstutil.Cursor) { + if replacement, ok := t.isGlobalUse(c.Node()); ok { + c.Replace(replacement) + } +} + +func (t *packageTranslator) rewriteGlobalDef(c *dstutil.Cursor) { + file, ok := c.Node().(*dst.File) + if !ok { + return + } + + var filtered []dst.Decl + + for _, decl := range file.Decls { + genDecl, ok := decl.(*dst.GenDecl) + if !ok { + filtered = append(filtered, decl) + continue + } + if genDecl.Tok != token.VAR { + filtered = append(filtered, decl) + continue + } + t.collectGlobalDefImpl(genDecl) + if len(genDecl.Specs) != 0 { + // XXX: for kept specs + filtered = append(filtered, genDecl) + } + } + + file.Decls = filtered +} + +func (t *packageTranslator) collectGlobalDefImpl(decl *dst.GenDecl) { + var filtered []dst.Spec + + for _, spec := range decl.Specs { + spec := spec.(*dst.ValueSpec) + + for _, name := range spec.Names { + obj, ok := t.typesInfo.Defs[t.astMap.Nodes[name].(*ast.Ident)] + if !ok { + return + } + var_, ok := obj.(*types.Var) + if !ok { + return + } + + if name.Name == "_" { + continue + } + + if _, ok := t.globalInfo.ByPackage[t.pkgPath].Container[name.Name]; !ok { + if ok := t.globalInfo.ByPackage[t.pkgPath].ShouldShare[name.Name]; !ok { + filtered = append(filtered, spec) + continue + } + } + + if !t.globalInfo.ByPackage[t.pkgPath].ShouldShare[name.Name] { + t.collect.globalFields = append(t.collect.globalFields, &dst.Field{ + Names: []*dst.Ident{name}, + Type: t.makeTypeExpr(var_.Type()), + }) + } else { + t.collect.sharedGlobalFields = append(t.collect.sharedGlobalFields, &dst.ValueSpec{ + Names: []*dst.Ident{name}, + Type: t.makeTypeExpr(var_.Type()), + }) + } + } + } + + decl.Specs = filtered +} + +func makeDetgoGlobalsFile(t *packageTranslator, pkgName string, forTest bool) (*dst.File, error) { + var inits []dst.Stmt + + // XXX: somehow keep the initializers closer to their old location in the code? + + // TODO: Consider if we can group the globals in a combined struct with + // anonymous members. That might make the source of the globals a nice + // implementation detail. + + funcName := "G" + typName := "Globals" + varName := "globals" + initFuncName := "initializers" + field := "Globals" + if forTest { + funcName = "GForTest" + typName = "GlobalsForTest" + varName = "globalsForTest" + initFuncName = "initializersForTest" + field = "ForTest" + } + + var lastIfSmt *dst.BlockStmt + + for _, initializer := range t.typesInfo.InitOrder { + var lhs []dst.Expr + + shouldShare := t.globalInfo.ByPackage[t.pkgPath].ShouldShare[initializer.Lhs[0].Name()] + + for _, var_ := range initializer.Lhs { + if var_.Name() == "_" { + lhs = append(lhs, dst.NewIdent("_")) + } else { + if !shouldShare { + lhs = append(lhs, &dst.SelectorExpr{ + X: &dst.CallExpr{Fun: dst.NewIdent(funcName)}, + Sel: dst.NewIdent(var_.Name()), + }) + } else { + lhs = append(lhs, dst.NewIdent(var_.Name())) + } + } + } + + rhs := t.apply(t.dstMap.Nodes[initializer.Rhs]).(dst.Expr) // hmm + + if shouldShare { + if lastIfSmt == nil { + lastIfSmt = &dst.BlockStmt{} + inits = append(inits, &dst.IfStmt{ + Cond: dst.NewIdent("initializeShared"), + Body: lastIfSmt, + }) + } + lastIfSmt.List = append(lastIfSmt.List, &dst.AssignStmt{ + Lhs: lhs, + Tok: token.ASSIGN, + Rhs: []dst.Expr{ + rhs, + }, + }) + if callExpr, ok := rhs.(*dst.CallExpr); ok { + if ident, ok := callExpr.Fun.(*dst.Ident); ok && ident.Name == "NewReplacer" && ident.Path == t.replacedPkgs["strings"] { + lastIfSmt.List = append(lastIfSmt.List, &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: &dst.SelectorExpr{X: dst.NewIdent(initializer.Lhs[0].Name()), Sel: dst.NewIdent("Replace")}, + Args: []dst.Expr{ + &dst.BasicLit{ + Kind: token.STRING, + Value: `""`, + }, + }, + }, + Decs: dst.ExprStmtDecorations{ + NodeDecs: dst.NodeDecs{ + End: []string{"// trigger once initialization"}, + }, + }, + }) + } + } + } else { + inits = append(inits, &dst.AssignStmt{ + Lhs: lhs, + Tok: token.ASSIGN, + Rhs: []dst.Expr{ + rhs, + }, + }) + lastIfSmt = nil + } + } + + for _, name := range t.collect.inits { + inits = append(inits, &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: dst.NewIdent(name), + }, + }) + } + var onceInits []dst.Stmt + for _, info := range t.collect.maps { + if info.onceName != "" && info.onceFn != "" { + // help how oculd one be non-empty? + onceInits = append(onceInits, &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: &dst.SelectorExpr{X: dst.NewIdent(info.onceName), Sel: dst.NewIdent("Do")}, + Args: []dst.Expr{dst.NewIdent(info.onceFn)}, + }, + }) + } + } + if len(onceInits) > 0 { + inits = append(inits, &dst.IfStmt{ + Cond: dst.NewIdent("initializeShared"), + Body: &dst.BlockStmt{ + List: onceInits, + }, + }) + } + + dstFile := &dst.File{ + Name: dst.NewIdent(pkgName), + Decls: []dst.Decl{ + &dst.GenDecl{ + Tok: token.TYPE, + Specs: []dst.Spec{ + &dst.TypeSpec{ + Name: dst.NewIdent(typName), + Type: &dst.StructType{ + Fields: &dst.FieldList{ + List: t.collect.globalFields, + }, + }, + }, + }, + }, + &dst.GenDecl{ + Tok: token.VAR, + Specs: append([]dst.Spec{ + &dst.ValueSpec{ + Names: []*dst.Ident{dst.NewIdent(varName)}, + Type: &dst.IndexListExpr{ + X: t.newRuntimeSelector("Global"), + Indices: []dst.Expr{dst.NewIdent(typName)}, + }, + }, + }, t.collect.sharedGlobalFields...), + }, + &dst.FuncDecl{ + Name: dst.NewIdent(initFuncName), + Type: &dst.FuncType{ + Params: &dst.FieldList{ + List: []*dst.Field{ + { + Names: []*dst.Ident{dst.NewIdent("initializeShared")}, + Type: dst.NewIdent("bool"), + }, + }, + }, + }, + Body: &dst.BlockStmt{ + List: inits, + }, + }, + &dst.FuncDecl{ + Name: dst.NewIdent(funcName), + Type: &dst.FuncType{ + Results: &dst.FieldList{ + List: []*dst.Field{ + { + Type: &dst.StarExpr{X: dst.NewIdent(typName)}, + }, + }, + }, + }, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ReturnStmt{ + Results: []dst.Expr{ + &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: dst.NewIdent(varName), + Sel: dst.NewIdent("Get"), + }, + }, + }, + }, + }, + }, + }, + }, + } + + globals := &dst.CompositeLit{ + Type: t.newRuntimeSelector("Globals"), + Elts: []dst.Expr{ + &dst.KeyValueExpr{ + Key: dst.NewIdent("Globals"), + Value: &dst.UnaryExpr{ + Op: token.AND, + X: dst.NewIdent(varName), + }, + }, + &dst.KeyValueExpr{ + Key: dst.NewIdent("Initializers"), + Value: dst.NewIdent(initFuncName), + }, + }, + } + + dstFile.Decls = append(dstFile.Decls, &dst.FuncDecl{ + Name: dst.NewIdent("init"), + Type: &dst.FuncType{}, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.AssignStmt{ + Lhs: []dst.Expr{ + &dst.SelectorExpr{ + X: &dst.CallExpr{ + Fun: t.newRuntimeSelector("RegisterPackage"), + Args: []dst.Expr{ + &dst.BasicLit{ + Kind: token.STRING, + Value: strconv.Quote(t.pkgPath), + }, + }, + }, + Sel: dst.NewIdent(field), + }, + }, + Tok: token.ASSIGN, + Rhs: []dst.Expr{ + &dst.UnaryExpr{ + Op: token.AND, + X: globals, + }, + }, + }, + }, + }, + }) + + dstFile.Decls = append(dstFile.Decls, makeBindFuncs(t.collect.bindspecs)...) + + return dstFile, nil +} diff --git a/internal/translate/globals_analysis.go b/internal/translate/globals_analysis.go new file mode 100644 index 0000000..f0444a4 --- /dev/null +++ b/internal/translate/globals_analysis.go @@ -0,0 +1,244 @@ +package translate + +import ( + "go/ast" + "go/token" + "go/types" + "strings" + + "github.com/dave/dst" + "github.com/dave/dst/decorator" +) + +type globalsCollector struct { + typesInfo *types.Info + astMap decorator.AstMap + pkgPath string + + globalsFields []string + shouldShare map[string]bool + tests []string + + dontTranslate map[packageSelector]bool +} + +func collectGlobalsAndTests(files []*dst.File, astMap decorator.AstMap, pkgPath string, typesInfo *types.Info, dontTranslate map[packageSelector]bool) *globalsCollector { + c := &globalsCollector{ + typesInfo: typesInfo, + astMap: astMap, + pkgPath: pkgPath, + + shouldShare: make(map[string]bool), + + dontTranslate: dontTranslate, + } + + for _, file := range files { + c.collectGlobals(file) + } + + return c +} + +var globalsDontTranslateGo123 = map[packageSelector]bool{ + {Pkg: "crypto/sha512", Selector: "_K"}: true, + // {pkg: "sync/atomic", selector: "firstStoreInProgress"}: true, // no way... universe strikes once again + {Pkg: "encoding/json", Selector: "fieldCache"}: true, + {Pkg: "encoding/json", Selector: "encoderCache"}: true, + // this would be nice but breaks determinism because sometimes the cache works....... + // XXX: register all types in our own registry, warm up the cache? + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "decomps"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "nfcValues"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "nfcIndex"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "nfcSparseValues"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "nfkcValues"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "nfkcIndex"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/norm", Selector: "nfkcSparseValues"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "decomps"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "nfcValues"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "nfcIndex"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "nfcSparseValues"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "nfkcValues"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "nfkcIndex"}: true, + {Pkg: "golang.org/x/text/unicode/norm", Selector: "nfkcSparseValues"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/bidi", Selector: "bidiValues"}: true, + {Pkg: "vendor/golang.org/x/text/unicode/bidi", Selector: "bidiIndex"}: true, + {Pkg: "golang.org/x/text/unicode/bidi", Selector: "bidiValues"}: true, + {Pkg: "golang.org/x/text/unicode/bidi", Selector: "bidiIndex"}: true, + {Pkg: "vendor/golang.org/x/net/idna", Selector: "idnaValues"}: true, + {Pkg: "vendor/golang.org/x/net/idna", Selector: "idnaIndex"}: true, + {Pkg: "vendor/golang.org/x/net/idna", Selector: "idnaSparseValues"}: true, + {Pkg: "golang.org/x/net/idna", Selector: "idnaValues"}: true, + {Pkg: "golang.org/x/net/idna", Selector: "idnaIndex"}: true, + {Pkg: "golang.org/x/net/idna", Selector: "idnaSparseValues"}: true, + {Pkg: reflectPackage, Selector: "mapInterfaceType"}: true, + {Pkg: reflectPackage, Selector: "jankHashMap"}: true, + + // xxx amd64 asm + {Pkg: "crypto/sha256", Selector: "useSHA"}: true, + {Pkg: "crypto/sha256", Selector: "useAVX2"}: true, + {Pkg: "vendor/golang.org/x/crypto/chacha20poly1305", Selector: "useAVX2"}: true, + {Pkg: "crypto/internal/bigmod", Selector: "supportADX"}: true, +} + +func (t *globalsCollector) collectGlobalsDecl(genDecl *dst.GenDecl) { + if genDecl.Tok != token.VAR { + return + } + for _, spec := range genDecl.Specs { + spec := spec.(*dst.ValueSpec) + + // XXX: handle single/multi assignment gracefully + // XXX: do initializers in TypesInfo get split? + + for _, name := range spec.Names { + obj, ok := t.typesInfo.Defs[t.astMap.Nodes[name].(*ast.Ident)] + if !ok { + continue + } + + if _, ok := obj.(*types.Var); !ok { + continue + } + + if name.Name == "_" { + continue + } + + n := name.Name + + shouldShare := false + + if t.dontTranslate[packageSelector{Pkg: t.pkgPath, Selector: name.Name}] { + shouldShare = true + } + + // XXX: reuse existing err globals? + /* + if t.pkgPath == "io" && initializer.Lhs[0].Name() == "EOF" { + rhs = &dst.Ident{ + Path: "io", + Name: "EOF", + } + } + */ + /* + // XXX: this causes problems for errors.Is(statErr, ErrNotExist) + if typ, ok := t.TypesInfo.Types[initializer.Rhs]; ok { + if named, ok := typ.Type.(*types.Named); ok && named.Obj().Name() == "error" { + if name := initializer.Lhs[0].Name(); token.IsExported(name) && !isInternal(t.pkgPath) && !isVendor(t.pkgPath) { + // XXX: can we go even deeper and rewrite all references / skip the variable altogether? + rhs = &dst.Ident{ + Name: name, + Path: t.pkgPath, + } + } + } + } + */ + + if len(spec.Values) > 0 { + if callExpr, ok := spec.Values[0].(*dst.CallExpr); ok { + if ident, ok := callExpr.Fun.(*dst.Ident); ok && ident.Name == "MustCompile" && ident.Path == "regexp" { + shouldShare = true + } + if ident, ok := callExpr.Fun.(*dst.Ident); ok && ident.Name == "NewReplacer" && ident.Path == "strings" { + shouldShare = true + } + } + if callExpr, ok := spec.Values[0].(*dst.CallExpr); ok { + if ident, ok := callExpr.Fun.(*dst.Ident); ok && ident.Name == "New" && ident.Path == "errors" { + shouldShare = true + } + } + } + + if t.pkgPath == "golang.org/x/net/http2/hpack" && n == "staticTable" { + shouldShare = true + } + if t.pkgPath == "regexp/syntax" && n == "posixGroup" { + shouldShare = true + } + if t.pkgPath == "hash/crc32" && n == "IEEETable" { + shouldShare = true + } + if t.pkgPath == "crypto/internal/edwards25519" { + shouldShare = true + } + if t.pkgPath == "vendor/golang.org/x/net/http2/hpack" && n == "staticTable" { + shouldShare = true + } + if t.pkgPath == "golang.org/x/net/http/httpguts" && n == "badTrailer" { + shouldShare = true + } + if t.pkgPath == "google.golang.org/grpc/internal/transport" && n == "http2ErrConvTab" { + shouldShare = true + } + if t.pkgPath == "google.golang.org/grpc/internal/transport" && n == "HTTPStatusConvTab" { + shouldShare = true + } + if t.pkgPath == "github.com/google/go-cmp/cmp" && n == "randBool" { + shouldShare = true + } + if t.pkgPath == "github.com/google/go-cmp/cmp/internal/diff" && n == "randBool" { + shouldShare = true + } + if t.pkgPath == "html/template" { + shouldShare = true + } + if t.pkgPath == "unicode" { + shouldShare = true + } + + t.globalsFields = append(t.globalsFields, name.Name) + if shouldShare { + t.shouldShare[name.Name] = true + } + } + } +} + +func (t *globalsCollector) collectFuncDecl(funcDecl *dst.FuncDecl) { + if !strings.HasPrefix(funcDecl.Name.Name, "Test") { + return + } + if funcDecl.Recv != nil { + return + } + if funcDecl.Type.TypeParams != nil { + return + } + if funcDecl.Type.Results != nil { + return + } + if len(funcDecl.Type.Params.List) != 1 { + return + } + param := funcDecl.Type.Params.List[0] + if len(param.Names) != 1 { + return + } + unaryExpr, ok := param.Type.(*dst.StarExpr) + if !ok { + return + } + ident, ok := unaryExpr.X.(*dst.Ident) + if !ok { + return + } + if ident.Path != "testing" || ident.Name != "T" { + return + } + t.tests = append(t.tests, funcDecl.Name.Name) +} + +func (t *globalsCollector) collectGlobals(file *dst.File) { + for _, decl := range file.Decls { + if genDecl, ok := decl.(*dst.GenDecl); ok { + t.collectGlobalsDecl(genDecl) + } + if funcDecl, ok := decl.(*dst.FuncDecl); ok { + t.collectFuncDecl(funcDecl) + } + } +} diff --git a/internal/translate/globals_analysis_ssa.go b/internal/translate/globals_analysis_ssa.go new file mode 100644 index 0000000..bd6f220 --- /dev/null +++ b/internal/translate/globals_analysis_ssa.go @@ -0,0 +1,260 @@ +package translate + +import ( + "go/ast" + "go/token" + "go/types" + "log" + "strings" + + "golang.org/x/tools/go/packages" + "golang.org/x/tools/go/ssa" + "golang.org/x/tools/go/ssa/ssautil" +) + +func collectFuncs(pkg *packages.Package, ssaPkg *ssa.Package) []*ssa.Function { + // Compute list of source functions, including literals, + // in source order. + var funcs []*ssa.Function + for _, f := range pkg.Syntax { + for _, decl := range f.Decls { + if fdecl, ok := decl.(*ast.FuncDecl); ok { + // (init functions have distinct Func + // objects named "init" and distinct + // ssa.Functions named "init#1", ...) + + fn := pkg.TypesInfo.Defs[fdecl.Name].(*types.Func) + if fn == nil { + panic(fn) + } + + f := ssaPkg.Prog.FuncValue(fn) + if f == nil { + panic(fn) + } + + var addAnons func(f *ssa.Function) + addAnons = func(f *ssa.Function) { + funcs = append(funcs, f) + for _, anon := range f.AnonFuncs { + addAnons(anon) + } + } + addAnons(f) + } + } + } + // XXX: inits? other crap? + return funcs +} + +type mapInitializer struct { + name string + onceName string + onceFn string +} + +type ssaGlobalsInfo struct { + readonlyMaps []mapInitializer +} + +func collectGlobalsSSA(pkg *packages.Package) ssaGlobalsInfo { + program, pkgs := ssautil.Packages([]*packages.Package{pkg}, ssa.BuilderMode(0)) + _ = program + + // log.Println(program, pkgs) + + if len(pkgs) != 1 { + log.Fatal("wtf") + } + + // XXX: should we check what kind of object is in the map? what if is mutable? (i mean it shouldn't be but........) + + maps := make(map[types.Object]struct{}) + mapInstrs := make(map[types.Object][]ssa.Instruction) + + funRefs := make(map[types.Object][]ssa.Instruction) + + ssaPkg := pkgs[0] + ssaPkg.Build() + + for _, member := range ssaPkg.Members { + switch member := member.(type) { + case *ssa.Global: + // log.Println(member.Name(), member.Type()) + _, ok := member.Type().(*types.Pointer).Elem().(*types.Map) + if ok { + // log.Println(member) + maps[member.Object()] = struct{}{} + } + } + } + + var rands []*ssa.Value + + for _, fun := range collectFuncs(pkg, ssaPkg) { + for _, block := range fun.Blocks { + for _, instr := range block.Instrs { + rands = instr.Operands(rands[:0]) + for _, op := range rands { + if glob, ok := (*op).(*ssa.Global); ok { + if glob.Pkg == ssaPkg { + if _, ok := maps[glob.Object()]; ok { + mapInstrs[glob.Object()] = append(mapInstrs[glob.Object()], instr) + } + } + } + if fun, ok := (*op).(*ssa.Function); ok { + if fun.Object() != nil { + funRefs[fun.Object()] = append(funRefs[fun.Object()], instr) + } + } + } + } + } + } + + var globals ssaGlobalsInfo + + for obj, instrs := range mapInstrs { + // log.Println(obj) + readonly := true + weird := false + maybeInitializers := make(map[*ssa.Function]struct{}) + + for _, instr := range instrs { + // instr.Block().Parent().WriteTo(os.Stdout) + // log.Println(instr) + + switch instr := instr.(type) { + case ssa.Value: + // log.Println("read", instr.Name(), instr.String()) + for _, next := range *instr.Referrers() { + switch next := next.(type) { + case *ssa.Lookup: + // log.Println("indexed, ok") + case *ssa.MapUpdate: + // log.Println("updated, not ok") + maybeInitializers[instr.Parent()] = struct{}{} + default: + readonly = false + weird = true + _ = next + // log.Println("unknown", next) + } + } + case *ssa.Store: + maybeInitializers[instr.Parent()] = struct{}{} + // log.Println("write", instr.String(), instr.Val) + default: + readonly = false + weird = true + // log.Println("wtf", reflect.TypeOf(instr)) + } + } + + if weird { + // log.Println("weird, can't do it") + } else if readonly && len(maybeInitializers) == 0 { + // log.Println("yay, readonly") + globals.readonlyMaps = append(globals.readonlyMaps, mapInitializer{ + name: obj.Name(), + }) + } else if readonly && len(maybeInitializers) > 0 { + // log.Println(maybeInitializers) + if len(maybeInitializers) != 1 { + // log.Println("more than one initializer, bailing...") + } else { + var fun *ssa.Function + for f := range maybeInitializers { + fun = f + } + // log.Println("checking fun", fun.Object()) + + onceName := "" + thisFunGood := fun.Parent() == nil // xxx can't handle nested funs + thisFunGood = thisFunGood && !token.IsExported(fun.Name()) && !strings.HasPrefix(fun.Name(), "init#") + for _, ref := range funRefs[fun.Object()] { + thisRefGood := false + + // log.Println("checking ref", ref) + if call, ok := ref.(*ssa.Call); ok { + if fun, ok := call.Call.Value.(*ssa.Function); ok { + if obj, ok := fun.Object().(*types.Func); ok && obj.FullName() == "(*sync.Once).Do" { // can this be faster please?? + // XXX: GREAT only initialized once. now double-check that this once doesn't do any other funny business!!! + // ie its global, and if any other calls to Do also exist for this function they must use the same once + // (i mean why not but let's just check...) + // log.Println("seems good!!") + // log.Println(call) + + onceVar := call.Call.Args[0] + // log.Println(onceVar, reflect.TypeOf(onceVar)) + if onceGlobal, ok := onceVar.(*ssa.Global); ok { + if onceName == "" { + onceName = onceGlobal.Name() + thisRefGood = true + } else { + thisRefGood = onceName == onceGlobal.Name() + } + } + + // XXX: now check that this once is only used for this func...??? + } + } + } + + thisFunGood = thisFunGood && thisRefGood + } + if thisFunGood { + // log.Println("we declare it readonly", onceName) + globals.readonlyMaps = append(globals.readonlyMaps, mapInitializer{ + name: obj.Name(), + onceName: onceName, + onceFn: fun.Name(), + }) + } + } + } + + // log.Println() + // inits: all call from ssaPkg.Members["init"] to init-named funcs + } + + // next: + // - grpc/protobuf? pretty close and would be a big win... then afterwards clean this up. + // - arrays? slices? + // - only for primitives for now? or do same sloppiness as with maps? (will get caught by shared sync.Pool...) + + // XXX: public maps bad + // XXX: registeredCodecs in google.golang.org/grpc/encoding??? + // XXX: init for go/token keywords??? + + return globals + + // XXX: can we get the init functions and initializers in the house also please? + // XXX: want to make sure they don't deref or anything + + // XXX: maybe a different helpful analysis is figuring out if a field can be + // accessed without holding a lock (or some other sync thing?) anything + // that you can get to we can mostly conclude to be immutable...??? + + /* + for _, member := range ssaPkg.Members { + switch member := member.(type) { + case *ssa.Function: + if token.IsExported(member.Name()) { + log.Println(member.Name(), member.Params) + } + + case *ssa.Type: + if token.IsExported(member.Name()) { + log.Println(member.Name()) + methods := program.MethodSets.MethodSet(member.Type()) + for i := 0; i < methods.Len(); i++ { + log.Println(methods.At(i)) + } + } + } + } + */ +} diff --git a/internal/translate/go.go b/internal/translate/go.go new file mode 100644 index 0000000..f202e32 --- /dev/null +++ b/internal/translate/go.go @@ -0,0 +1,234 @@ +package translate + +import ( + "cmp" + "fmt" + "go/types" + "slices" + + "github.com/dave/dst" + "github.com/dave/dst/dstutil" +) + +func (t *packageTranslator) rewriteGo(c *dstutil.Cursor) { + // go stmts + if goStmt, ok := c.Node().(*dst.GoStmt); ok { + var fun dst.Expr + + funcType, ok := t.getType(goStmt.Call.Fun) + simple := ok && funcType.(*types.Signature).Params().Len() == 0 && funcType.(*types.Signature).Results().Len() == 0 + + if simple { + // go func() { ... } () + fun = goStmt.Call.Fun + } else { + args := 0 + variable := false + ret := 0 + if ok { + sig := funcType.(*types.Signature) + args = sig.Params().Len() + variable = sig.Variadic() + ret = sig.Results().Len() + } + + bs := bindspec{args: args, variable: variable, ret: ret} + t.collect.bindspecs[bs] = struct{}{} + + fun = &dst.CallExpr{ + Fun: &dst.CallExpr{ + Fun: &dst.Ident{ + Name: bs.Name(), + }, + Args: []dst.Expr{goStmt.Call.Fun}, + }, + Args: goStmt.Call.Args, + } + } + + c.Replace(&dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: t.newRuntimeSelector("Go"), + Args: []dst.Expr{fun}, + }, + }) + } +} + +// bindspec describes programming-style bind functions used to rewrite go +// statements. +// +// Given a function `func foo(a int) error { ... }` with invocation `go foo(10)` +// the generated bind function and invocation are: +// +// func Bind1_1[T1, T2 any](f func(v1 T1) T2) func(v1 T1) func() { +// return func(v1 T1) func() { +// return func() { +// f(v1) +// } +// } +// } +// gosimruntime.Go(Bind1_1(foo)(10)) +// +// This trick is necessary because Go() only accepts func() arguments. Each +// converted package includes all the Bind used in that that package. bindspec +// is used for the tracking of necessary functions and their generation. +type bindspec struct { + args int // number of arguments + variable bool // is the last argumennt variable? + ret int // number of return values +} + +func (bs bindspec) Name() string { + dots := "" + if bs.variable { + dots = "var" + } + + return fmt.Sprintf("Bind%d%s_%d", bs.args, dots, bs.ret) +} + +func makeBindFunc(bs bindspec) *dst.FuncDecl { + typeargs := &dst.Field{ + Type: dst.NewIdent("any"), + } + for i := 1; i <= bs.args+bs.ret; i++ { + typeargs.Names = append(typeargs.Names, dst.NewIdent(fmt.Sprintf("T%d", i))) + } + + funcargs := func() *dst.FieldList { + var funcargs []*dst.Field + for i := 1; i <= bs.args; i++ { + var typ dst.Expr = dst.NewIdent(fmt.Sprintf("T%d", i)) + if bs.variable && i == bs.args { + typ = &dst.Ellipsis{Elt: typ} + } + funcargs = append(funcargs, &dst.Field{ + Names: []*dst.Ident{dst.NewIdent(fmt.Sprintf("v%d", i))}, + Type: typ, + }) + } + return &dst.FieldList{ + List: funcargs, + } + } + + var callargs []dst.Expr + for i := 1; i <= bs.args; i++ { + callargs = append(callargs, dst.NewIdent(fmt.Sprintf("v%d", i))) + } + + funcret := func() *dst.FieldList { + var funcret []*dst.Field + for i := 1; i <= bs.ret; i++ { + var typ dst.Expr = dst.NewIdent(fmt.Sprintf("T%d", bs.args+i)) + funcret = append(funcret, &dst.Field{ + Type: typ, + }) + } + return &dst.FieldList{ + List: funcret, + } + } + + bindret := &dst.FuncType{ + Params: funcargs(), + Results: &dst.FieldList{ + List: []*dst.Field{ + { + Type: &dst.FuncType{}, + }, + }, + }, + } + + return &dst.FuncDecl{ + Name: dst.NewIdent(bs.Name()), + Type: &dst.FuncType{ + TypeParams: &dst.FieldList{ + List: []*dst.Field{ + typeargs, + }, + }, + Params: &dst.FieldList{ + List: []*dst.Field{ + { + Names: []*dst.Ident{ + dst.NewIdent("f"), + }, + Type: &dst.FuncType{ + Params: funcargs(), + Results: funcret(), + }, + }, + }, + }, + Results: &dst.FieldList{ + List: []*dst.Field{ + { + Type: dst.Clone(bindret).(*dst.FuncType), + }, + }, + }, + }, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ReturnStmt{ + Results: []dst.Expr{ + &dst.FuncLit{ + Type: dst.Clone(bindret).(*dst.FuncType), + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ReturnStmt{ + Results: []dst.Expr{ + &dst.FuncLit{ + Type: &dst.FuncType{}, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: dst.NewIdent("f"), + Args: callargs, + Ellipsis: bs.variable, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func makeBindFuncs(bindspecs map[bindspec]struct{}) []dst.Decl { + var bss []bindspec + for bs := range bindspecs { + bss = append(bss, bs) + } + slices.SortFunc(bss, func(a, b bindspec) int { + if d := cmp.Compare(a.args, b.args); d != 0 { + return d + } + if a.variable != b.variable { + if a.variable { + return +1 + } else { + return -1 + } + } + return cmp.Compare(a.ret, b.ret) + }) + + var decls []dst.Decl + for _, bs := range bss { + decls = append(decls, makeBindFunc(bs)) + } + return decls +} diff --git a/internal/translate/hooks_go123.go b/internal/translate/hooks_go123.go new file mode 100644 index 0000000..d81d2d1 --- /dev/null +++ b/internal/translate/hooks_go123.go @@ -0,0 +1,200 @@ +package translate + +var hooksGo123 = map[packageSelector]packageSelector{ + {Pkg: "golang.org/x/sys/unix", Selector: "RawSyscall"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "RawSyscall6"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "RawSyscallNoError"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Syscall"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Syscall6"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "SyscallNoError"}: {Pkg: hooksGo123Package}, + + // amd64 only? + {Pkg: "golang.org/x/sys/unix", Selector: "gettimeofday"}: {Pkg: hooksGo123Package}, + + {Pkg: "hash/maphash", Selector: "runtime_rand"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/abi", Selector: "FuncPCABI0"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/abi", Selector: "FuncPCABIInternal"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/chacha8rand", Selector: "block"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/bytealg", Selector: "Compare"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "Count"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "CountString"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "Equal"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "Index"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "IndexByte"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "IndexByteString"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "IndexString"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "MakeNoZero"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "abigen_runtime_cmpstring"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "abigen_runtime_memequal"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/bytealg", Selector: "abigen_runtime_memequal_varlen"}: {Pkg: hooksGo123Package}, + + // arm only? + {Pkg: "internal/cpu", Selector: "getMIDR"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/cpu", Selector: "getisar0"}: {Pkg: hooksGo123Package}, + + // amd only? + {Pkg: "internal/cpu", Selector: "cpuid"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/cpu", Selector: "xgetbv"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/cpu", Selector: "getGOAMD64level"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/godebug", Selector: "registerMetric"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/godebug", Selector: "setNewIncNonDefault"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/godebug", Selector: "setUpdate"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/godebug", Selector: "write"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/poll", Selector: "runtimeNano"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_Semacquire"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_Semrelease"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_isPollServerDescriptor"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollClose"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollOpen"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollReset"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollServerInit"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollSetDeadline"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollUnblock"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollWait"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/poll", Selector: "runtime_pollWaitCanceled"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/runtime/syscall", Selector: "Syscall6"}: {Pkg: hooksGo123Package}, + + {Pkg: "internal/syscall/unix", Selector: "GetRandom"}: {Pkg: hooksGo123Package}, + {Pkg: "internal/syscall/unix", Selector: "fcntl"}: {Pkg: hooksGo123Package}, + + {Pkg: "iter", Selector: "coroswitch"}: {Pkg: hooksGo123Package}, + {Pkg: "iter", Selector: "newcoro"}: {Pkg: hooksGo123Package}, + + {Pkg: "maps", Selector: "clone"}: {Pkg: hooksGo123Package}, + + {Pkg: "math/rand", Selector: "runtime_rand"}: {Pkg: hooksGo123Package}, + + {Pkg: "math/rand/v2", Selector: "runtime_rand"}: {Pkg: hooksGo123Package}, + + {Pkg: "net", Selector: "runtime_rand"}: {Pkg: hooksGo123Package}, + + {Pkg: "os", Selector: "runtime_args"}: {Pkg: hooksGo123Package}, + {Pkg: "os", Selector: "runtime_beforeExit"}: {Pkg: hooksGo123Package}, + {Pkg: "os", Selector: "runtime_rand"}: {Pkg: hooksGo123Package}, + {Pkg: "os", Selector: "sigpipe"}: {Pkg: hooksGo123Package}, + {Pkg: "os", Selector: "checkClonePidfd"}: {Pkg: hooksGo123Package}, + {Pkg: "os", Selector: "ignoreSIGSYS"}: {Pkg: hooksGo123Package}, + {Pkg: "os", Selector: "restoreSIGSYS"}: {Pkg: hooksGo123Package}, + + {Pkg: "runtime/debug", Selector: "SetTraceback"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "WriteHeapDump"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "freeOSMemory"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "modinfo"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "readGCStats"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "setGCPercent"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "setMaxStack"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "setMaxThreads"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "setMemoryLimit"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/debug", Selector: "setPanicOnFault"}: {Pkg: hooksGo123Package}, + + {Pkg: "runtime/trace", Selector: "userLog"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/trace", Selector: "userRegion"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/trace", Selector: "userTaskCreate"}: {Pkg: hooksGo123Package}, + {Pkg: "runtime/trace", Selector: "userTaskEnd"}: {Pkg: hooksGo123Package}, + + {Pkg: "sync", Selector: "fatal"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_LoadAcquintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_Semacquire"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_SemacquireMutex"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_SemacquireRWMutex"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_SemacquireRWMutexR"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_Semrelease"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_StoreReluintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_canSpin"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_doSpin"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_nanotime"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_notifyListAdd"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_notifyListCheck"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_notifyListNotifyAll"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_notifyListNotifyOne"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_notifyListWait"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_procPin"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_procUnpin"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_randn"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "runtime_registerPoolCleanup"}: {Pkg: hooksGo123Package}, + {Pkg: "sync", Selector: "throw"}: {Pkg: hooksGo123Package}, + + {Pkg: "sync/atomic", Selector: "AddInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AddInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AddPointer"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AddUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AddUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AddUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AndInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AndInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AndUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AndUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "AndUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "CompareAndSwapInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "CompareAndSwapInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "CompareAndSwapPointer"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "CompareAndSwapUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "CompareAndSwapUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "CompareAndSwapUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "LoadInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "LoadInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "LoadPointer"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "LoadUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "LoadUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "LoadUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "OrInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "OrInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "OrUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "OrUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "OrUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "StoreInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "StoreInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "StorePointer"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "StoreUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "StoreUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "StoreUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "SwapInt32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "SwapInt64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "SwapPointer"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "SwapUint32"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "SwapUint64"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "SwapUintptr"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "runtime_procPin"}: {Pkg: hooksGo123Package}, + {Pkg: "sync/atomic", Selector: "runtime_procUnpin"}: {Pkg: hooksGo123Package}, + + {Pkg: "syscall", Selector: "Exit"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Getpagesize"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "RawSyscall6"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "cgocaller"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "hasWaitingReaders"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "rawSyscallNoError"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "rawVforkSyscall"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtimeSetenv"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtimeUnsetenv"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_AfterExec"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_AfterFork"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_AfterForkInChild"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_BeforeExec"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_BeforeFork"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_doAllThreadsSyscall"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_entersyscall"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_envs"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "runtime_exitsyscall"}: {Pkg: hooksGo123Package}, + + // amd64 only? + {Pkg: "syscall", Selector: "gettimeofday"}: {Pkg: hooksGo123Package}, + + {Pkg: "time", Selector: "Sleep"}: {Pkg: hooksGo123Package}, + {Pkg: "time", Selector: "now"}: {Pkg: hooksGo123Package}, + {Pkg: "time", Selector: "newTimer"}: {Pkg: hooksGo123Package}, + {Pkg: "time", Selector: "resetTimer"}: {Pkg: hooksGo123Package}, + {Pkg: "time", Selector: "runtimeNano"}: {Pkg: hooksGo123Package}, + {Pkg: "time", Selector: "stopTimer"}: {Pkg: hooksGo123Package}, +} + +var hooksGensyscallGo123ByArch = map[string]map[packageSelector]packageSelector{} + +var acceptedgo123Linknames = map[packageSelector]packageSelector{ + {Pkg: "hash/maphash", Selector: "runtime_memhash"}: {Pkg: "runtime", Selector: "memhash"}, +} diff --git a/internal/translate/hooks_go123_amd64_gensyscall.go b/internal/translate/hooks_go123_amd64_gensyscall.go new file mode 100644 index 0000000..7f12fea --- /dev/null +++ b/internal/translate/hooks_go123_amd64_gensyscall.go @@ -0,0 +1,57 @@ +// Code generated by gensyscall. DO NOT EDIT. +package translate + +func init() { + hooksGensyscallGo123ByArch["amd64"] = map[packageSelector]packageSelector{ + {Pkg: "golang.org/x/sys/unix", Selector: "accept4"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "bind"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Close"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "connect"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Fstat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Fstatat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Fsync"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Ftruncate"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Getdents"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "getpeername"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Getpid"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Getrandom"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "getsockname"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "getsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Listen"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "openat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "pread"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "pwrite"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "read"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Renameat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "setsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "socket"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Uname"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Unlinkat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "write"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "accept4"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "bind"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Close"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "connect"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "fcntl"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Fstat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "fstatat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Fsync"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Ftruncate"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Getdents"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "getpeername"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Getpid"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "getsockname"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "getsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Listen"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "openat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "pread"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "pwrite"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "read"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Renameat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "setsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "socket"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Uname"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "unlinkat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "write"}: {Pkg: hooksGo123Package}, + } +} diff --git a/internal/translate/hooks_go123_arm64_gensyscall.go b/internal/translate/hooks_go123_arm64_gensyscall.go new file mode 100644 index 0000000..d4c8cd5 --- /dev/null +++ b/internal/translate/hooks_go123_arm64_gensyscall.go @@ -0,0 +1,57 @@ +// Code generated by gensyscall. DO NOT EDIT. +package translate + +func init() { + hooksGensyscallGo123ByArch["arm64"] = map[packageSelector]packageSelector{ + {Pkg: "golang.org/x/sys/unix", Selector: "accept4"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "bind"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Close"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "connect"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Fstat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Fstatat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Fsync"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Ftruncate"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Getdents"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "getpeername"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Getpid"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Getrandom"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "getsockname"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "getsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Listen"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "openat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "pread"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "pwrite"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "read"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Renameat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "setsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "socket"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Uname"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "Unlinkat"}: {Pkg: hooksGo123Package}, + {Pkg: "golang.org/x/sys/unix", Selector: "write"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "accept4"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "bind"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Close"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "connect"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "fcntl"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Fstat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "fstatat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Fsync"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Ftruncate"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Getdents"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "getpeername"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Getpid"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "getsockname"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "getsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Listen"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "openat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "pread"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "pwrite"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "read"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Renameat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "setsockopt"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "socket"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "Uname"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "unlinkat"}: {Pkg: hooksGo123Package}, + {Pkg: "syscall", Selector: "write"}: {Pkg: hooksGo123Package}, + } +} diff --git a/internal/translate/main.go b/internal/translate/main.go new file mode 100644 index 0000000..5ebd809 --- /dev/null +++ b/internal/translate/main.go @@ -0,0 +1,594 @@ +package translate + +import ( + "cmp" + _ "embed" + "flag" + "go/token" + "log" + "maps" + "os" + "path" + "runtime/pprof" + "slices" + "strings" + "time" + + "golang.org/x/mod/modfile" + "golang.org/x/tools/go/packages" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/translate/cache" +) + +var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") + +var skippedPackagesGo123 = map[string]bool{ + "runtime": true, // XXX wait what + "errors": true, + "reflect": true, + // "strings": true, + "strconv": true, + // embed: true, // this will also have io.EOF problems... unless we can just use the original there please??? + // XXX: for all unconverted packages, figure out all references to converted packages, and have a plan. + "embed": true, // XXX can we link back to the original here somehow??? + "math": true, + "math/big": true, + + "unsafe": true, + + "os/signal": true, // XXX for now + "runtime/coverage": true, // XXX for now + "runtime/metrics": true, // XXX for now + "runtime/pprof": true, // XXX for now + + // XXX: rewrite internal/cpu to golang.org/x/sys/cpu? + "vendor/golang.org/x/sys/cpu": true, // XXX for now + + "unique": true, // XXX: yes + + "testing": true, + "testing/internal/testdeps": true, + "internal/reflectlite": true, + gosimruntimePackage: true, + gosimruntimePackage + "_test": true, // eh + gosimruntimePackage + ".test": true, // eh + // reflectPackage: true, +} + +var keepAsmPackagesGo123 = map[string]bool{ + "crypto/aes": true, + "crypto/internal/boring/sig": true, + "crypto/internal/nistec": true, + "crypto/md5": true, + "crypto/sha1": true, + "crypto/sha256": true, + "crypto/sha512": true, + "crypto/subtle": true, + "crypto/internal/bigmod": true, + "crypto/internal/edwards25519/field": true, + "vendor/golang.org/x/crypto/chacha20": true, + "vendor/golang.org/x/crypto/internal/poly1305": true, + "vendor/golang.org/x/crypto/chacha20poly1305": true, + "vendor/golang.org/x/crypto/sha3": true, + "hash/crc32": true, + + "net/url": true, // XXX: linkname setpath nonsense + "net/http": true, // XXX: linkname roundtrip nonsense +} + +var PublicExportHacks = map[string][]string{ + "encoding/binary": {"littleEndian"}, + "internal/poll": {"errNetClosing"}, +} + +// XXX: replace the package instead +var replacements = map[packageSelector]packageSelector{ + {Pkg: "runtime", Selector: "SetFinalizer"}: {Pkg: gosimruntimePackage, Selector: "SetFinalizer"}, + {Pkg: "runtime", Selector: "GOOS"}: {Pkg: gosimruntimePackage, Selector: "GOOS"}, + {Pkg: "runtime", Selector: "Gosched"}: {Pkg: gosimruntimePackage, Selector: "Yield"}, +} + +const gosimModPath = gosimtool.Module + +const ( + gosimruntimePackage = gosimModPath + "/gosimruntime" + hooksGo123Package = gosimModPath + "/internal/hooks/go123" + reflectPackage = gosimModPath + "/internal/reflect" + simulationPackage = gosimModPath + "/internal/simulation" + testingPackage = gosimModPath + "/internal/testing" +) + +var TranslatedRuntimePackages = []string{ + hooksGo123Package, + reflectPackage, + simulationPackage, + testingPackage, +} + +const ( + // loadDepGraph fairly quickly loads the dependency graph + loadDepGraph = packages.NeedName | packages.NeedFiles | packages.NeedModule | packages.NeedImports | packages.NeedDeps + // loadSyntaxAndTypes relatively slowly loads detailed types and syntax information + loadSyntaxAndTypes = packages.NeedSyntax | packages.NeedName | packages.NeedTypes | + packages.NeedTypesInfo | packages.NeedFiles | packages.NeedImports +) + +func loadPackages(patterns []string, b gosimtool.BuildConfig, mode packages.LoadMode) ([]*packages.Package, error) { + cfg := &packages.Config{ + Mode: mode, + Tests: true, + Fset: token.NewFileSet(), + } + + // apply build config to packages config + // TODO: build tags plan (rename files, force GOOS and GOARCH) + tags := []string{"sim"} + if b.Race { + tags = append(tags, "race") + } + // TODO: if we support varying GOARCH here, we should select the arch-specific hooks at runtime... + env := append(os.Environ(), "GOOS="+b.GOOS, "GOARCH="+b.GOARCH, "CGO_ENABLED=0") + cfg.BuildFlags = []string{"-tags", strings.Join(tags, ",")} + cfg.Env = env + + // load packages + packages, err := packages.Load(cfg, patterns...) + if err != nil { + return nil, err + } + if len(packages) == 0 { + log.Println(err) + log.Fatal("failed to load packages... missing go.mod deps?") + } + + // check for errors + for _, pkg := range packages { + if pkg.Errors != nil { + log.Println("errors for ", pkg.PkgPath) + for _, err := range pkg.Errors { + log.Println(err) + } + os.Exit(1) + } + } + + return packages, nil +} + +type packageKind string + +const ( + PackageKindTestBinary = "testbinary" + PackageKindBase = "base" + PackageKindForTest = "fortest" + PackageKindTests = "tests" +) + +func classifyPackage(pkg *packages.Package) (packageKind, string) { + switch { + case pkg.Name == "main" && strings.HasSuffix(pkg.PkgPath, ".test"): + // XXX: is this correct? + return PackageKindTestBinary, strings.TrimSuffix(pkg.PkgPath, ".test") + + case strings.HasSuffix(pkg.Name, "_test"): + return PackageKindTests, strings.TrimSuffix(pkg.PkgPath, "_test") + + case strings.HasSuffix(pkg.ID, ".test]"): + return PackageKindForTest, pkg.PkgPath + + case !strings.HasSuffix(pkg.Name, "_test") && pkg.PkgPath == pkg.ID: + return PackageKindBase, pkg.PkgPath + + default: + log.Fatal("weird package", pkg.Name, pkg.PkgPath, pkg.ID) + panic("unreachable") + } +} + +func collectImports(roots []*packages.Package, skip map[string]bool) []*packages.Package { + seen := make(map[*packages.Package]bool) + var order []*packages.Package + var visit func(pkg *packages.Package) + visit = func(pkg *packages.Package) { + if skip[pkg.PkgPath] || seen[pkg] { + return + } + seen[pkg] = true + order = append(order, pkg) + for _, dep := range pkg.Imports { + visit(dep) + } + } + for _, root := range roots { + visit(root) + } + slices.SortFunc(order, func(a, b *packages.Package) int { + return cmp.Compare(a.ID, b.ID) + }) + return order +} + +func writeGoModFile(modDir string, modFile *modfile.File, writer *outputWriter) { + isGosim := modFile.Module.Mod.Path == gosimModPath + // take the existing go.mod and make it work for a sub-directory containing + // a module translated + if err := modFile.AddModuleStmt("translated"); err != nil { + log.Fatal(err) + } + + if isGosim { + // special case running translate in the gosim module + if err := modFile.AddRequire(gosimModPath, "v0.0.0"); err != nil { + log.Fatal(err) + } + if err := modFile.AddReplace(gosimModPath, "", "../../../", ""); err != nil { + log.Fatal(err) + } + } else { + // adjust relative paths for the new module's location + for _, replace := range modFile.Replace { + // > Third, filesystem paths found in "replace" directives are + // represented by a path with an empty version. + if replace.New.Version == "" && !path.IsAbs(replace.New.Path) { + // The output module is located three directories deeper than the old module. + newPath := path.Join("../../../", replace.New.Path) + if err := modFile.AddReplace(replace.Old.Path, replace.Old.Version, newPath, ""); err != nil { + log.Fatal(err) + } + } + } + } + + bytes, err := modFile.Format() + if err != nil { + log.Fatal(err) + } + if err := writer.stage("go.mod", bytes); err != nil { + log.Fatal(err) + } + goSumBytes, err := os.ReadFile(path.Join(modDir, "go.sum")) + if err != nil { + log.Fatal(err) + } + if err := writer.stage("go.sum", goSumBytes); err != nil { + log.Fatal(err) + } +} + +type TranslateInput struct { + Packages []string + Cfg gosimtool.BuildConfig +} + +func Translate(input *TranslateInput) (*gosimtool.TranslateOutput, error) { + // XXX: dedup? + modDir, err := gosimtool.FindGoModDir() + if err != nil { + log.Fatal(err) + } + rootOutputDir := path.Join(modDir, gosimtool.OutputDirectory, "translated", input.Cfg.AsDirname()) + + cacheDir := path.Join(modDir, gosimtool.OutputDirectory) + if override := os.Getenv("GOSIMCACHE"); override != "" { + cacheDir = override + } + + cachePath := path.Join(cacheDir, "cache.sqlite3") + if err := os.MkdirAll(path.Dir(cachePath), 0o755); err != nil { + log.Fatal(err) + } + db, err := cache.NewDB(cachePath) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + cache := cache.NewCache(db) + defer cache.Clean() + // XXX: cap size at something reasonable? 100x working set? + + if err := os.MkdirAll(rootOutputDir, 0o755); err != nil { + log.Fatal(err) + } + return translatePackages(cache, input.Packages, rootOutputDir, input.Cfg) +} + +func buildReplacePackagesAndPackageNames(convertPkgs, allPkgs []*packages.Package) (replacedPkgs map[string]string, packageNames map[string]string) { + packageNames = make(map[string]string) + packageNames["golang.org/x/sys/cpu"] = "cpu" // XXX: here because we replace the vendored one with the common one + for _, pkg := range allPkgs { + packageNames[pkg.PkgPath] = pkg.Name + } + + replacedPkgs = make(map[string]string) + for _, pkg := range convertPkgs { + inputPackage := pkg.PkgPath + outputPackage := "translated/" + gosimtool.ReplaceSpecialPackages(pkg.PkgPath) + replacedPkgs[inputPackage] = outputPackage + packageNames[outputPackage] = packageNames[inputPackage] + } + + // override reflect and testing + replacedPkgs["reflect"] = replacedPkgs[reflectPackage] + replacedPkgs["internal/reflectlite"] = replacedPkgs[reflectPackage] + replacedPkgs["testing"] = replacedPkgs[testingPackage] + + // handle the linkname in the os package + replacedPkgs["net"] = "translated/" + gosimtool.ReplaceSpecialPackages("net") + + // not replaced, but need to know for rewrites + replacedPkgs[gosimruntimePackage] = gosimruntimePackage + return +} + +func checkGosimDep(modFile *modfile.File) { + // work in the gosim module + if isGosim := modFile.Module.Mod.Path == gosimModPath; isGosim { + return + } + for _, req := range modFile.Require { + // work if there is an explicit dependency + if req.Mod.Path == gosimModPath { + return + } + } + // complain otherwise + log.Fatalf("current module does not depend on %v, try running init", gosimModPath) +} + +func checkSingleModule(modPath string, pkgs []*packages.Package) { + for _, pkg := range pkgs { + // allow packages from the current module + if pkg.Module.GoMod == modPath { + continue + } + // allow packages explicitly added by translate + _, path := classifyPackage(pkg) + if slices.Contains(TranslatedRuntimePackages, path) { + continue + } + // complain about others + log.Fatal("packages from outside module modules ", pkg.PkgPath, " ", modPath) + } +} + +func translatePackages(cache *cache.Cache, listPatterns []string, rootOutputDir string, cfg gosimtool.BuildConfig) (*gosimtool.TranslateOutput, error) { + if *cpuprofile != "" { + f, err := os.Create(*cpuprofile) + if err != nil { + log.Fatal(err) + } + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + + listPatterns = append(listPatterns, TranslatedRuntimePackages...) + + listedPkgs, err := loadPackages(listPatterns, cfg, loadDepGraph) + if err != nil { + log.Fatal(err) + } + if len(listedPkgs) == 0 { + log.Fatal("no packages") + } + + modPath, modFile, err := gosimtool.FindGoMod() + if err != nil { + log.Fatal(err) + } + modDir := path.Dir(modPath) + + checkGosimDep(modFile) + checkSingleModule(modPath, listedPkgs) + + allPkgs := collectImports(listedPkgs, nil) + convertPkgs := collectImports(listedPkgs, skippedPackagesGo123) + + packageGraph := newDepGraph() + basePkgs := make(map[string]*packages.Package) + pkgById := make(map[string]*packages.Package) + for _, pkg := range convertPkgs { + packageGraph.addNode(pkg.ID) + pkgById[pkg.ID] = pkg + + if kind, path := classifyPackage(pkg); kind == PackageKindBase { + basePkgs[path] = pkg + } + } + + for _, pkg := range convertPkgs { + for _, dep := range pkg.Imports { + if _, ok := packageGraph.nodes[dep.ID]; !ok { + // XXX? + continue + } + packageGraph.addDep(pkg.ID, dep.ID) + } + + // XXX: add a package from the "for test" to the "main" package + kind, path := classifyPackage(pkg) + if kind == PackageKindForTest || kind == PackageKindTests { + if basePkgs[path] != nil { + packageGraph.addDep(pkg.ID, basePkgs[path].ID) + } else { + log.Println("huh", kind, pkg.PkgPath, path) + log.Fatal(":(") + } + } + } + + translateToolHash := computeTranslateToolHash(cfg) + + numWorkers := 32 + + packageHashes := make(map[string]Hash) + + buildInParallel(packageGraph, numWorkers, packageHashes, func(pkgId string, importHashes map[string]Hash) Hash { + return computePackageHash(translateToolHash, pkgById[pkgId], importHashes) + }) + + allResults := make(map[string]*TranslatePackageResult) + cacheHits := make(map[Hash]bool) + + uncachedPackages := make(map[string]struct{}) + for pkgId, hash := range packageHashes { + pkg := pkgById[pkgId] + + res, err := cacheGet(cache, hash) + if err != nil { + log.Fatal(err) + } + if res != nil { + allResults[pkgId] = res + cacheHits[hash] = true // record cache hits so we don't update the cache for them + } else { + uncachedPackages[strings.TrimSuffix(pkg.PkgPath, "_test")] = struct{}{} + } + } + + reloaded, err := loadPackages(slices.Collect(maps.Keys(uncachedPackages)), cfg, loadSyntaxAndTypes) + if err != nil { + log.Fatal(err) + } + pkgsWithTypesAndAst := make(map[string]*packages.Package) + for _, pkg := range reloaded { + pkgsWithTypesAndAst[pkg.ID] = pkg + } + + replacedPkgs, packageNames := buildReplacePackagesAndPackageNames(convertPkgs, allPkgs) + + buildInParallel(packageGraph, numWorkers, allResults, func(pkgId string, localResults map[string]*TranslatePackageResult) *TranslatePackageResult { + return translatePackage(&translatePackageArgs{ + cfg: cfg, + pkg: pkgById[pkgId], + replacedPkgs: replacedPkgs, + hooksPackage: hooksGo123Package, + packageNames: packageNames, + importResults: localResults, + pkgWithTypesAndAst: pkgsWithTypesAndAst[pkgId], + }) + }) + + for pkgId, res := range allResults { + hash := packageHashes[pkgId] + if !cacheHits[hash] { + if err := cachePut(cache, hash, res); err != nil { + log.Fatal(err) + } + } + } + + writer := newOutputWriter() + for _, res := range allResults { + if err := writer.merge(res.TranslatedFiles); err != nil { + log.Fatal(err) + } + } + + writeGoModFile(modDir, modFile, writer) + + if err := writer.writeFiles(rootOutputDir); err != nil { + log.Fatal(err) + } + if err := writer.maybeDeleteGeneratedFiles(rootOutputDir); err != nil { + log.Fatal(err) + } + + var out []string + for _, pkg := range listedPkgs { + kind, _ := classifyPackage(pkg) + if kind != PackageKindBase { + continue + } + if fromGosim := slices.Contains(TranslatedRuntimePackages, pkg.PkgPath); fromGosim { + continue + } + out = append(out, replacedPkgs[pkg.PkgPath]) + } + + deps := make(map[string]map[string]time.Time) + modTimeCache := make(map[string]time.Time) + for _, pkg := range listedPkgs { + kind, path := classifyPackage(pkg) + if kind != PackageKindTestBinary { + continue + } + + // TODO: also add deps from the mandatory linked packages? + files := findAllDepFiles(pkg, modDir) + times, err := loadModTimes(files, modTimeCache) + if err != nil { + log.Fatal(err) + } + deps[replacedPkgs[path]] = times + } + + return &gosimtool.TranslateOutput{ + RootOutputDir: rootOutputDir, + Packages: out, + Deps: deps, + }, nil +} + +func findAllDepFiles(pkg *packages.Package, root string) []string { + seen := make(map[*packages.Package]struct{}) + var walk func(*packages.Package) + var files []string + walk = func(pkg *packages.Package) { + if _, ok := seen[pkg]; ok { + return + } + seen[pkg] = struct{}{} + for _, dep := range pkg.Imports { + walk(dep) + } + for _, file := range pkg.GoFiles { + if strings.HasPrefix(file, root) { + files = append(files, file) + } + } + } + walk(pkg) + slices.Sort(files) + return files +} + +func loadModTimes(files []string, cache map[string]time.Time) (map[string]time.Time, error) { + result := make(map[string]time.Time) + for _, file := range files { + if t, ok := cache[file]; ok { + result[file] = t + continue + } + + info, err := os.Stat(file) + if err != nil { + return nil, err + } + t := info.ModTime() + cache[file] = t + result[file] = t + } + return result, nil +} + +// Go version? +// - GOTOOLCHAIN aware? + +// Test perf? +// - only run once for all testdata? + +// Cmd +// - nicer flags for gosim test + +// TODO: hash more singletons? +// - env vars? anything influencing go packages? +// - use go packages export file hash? + +// next: +// - verify determinism +// - deal with logging output + +// log progress? +// log.Println(results.pkgID, hex.EncodeToString(results.inputHash[:])) diff --git a/internal/translate/map.go b/internal/translate/map.go new file mode 100644 index 0000000..fbef7b3 --- /dev/null +++ b/internal/translate/map.go @@ -0,0 +1,461 @@ +package translate + +import ( + "go/ast" + "go/token" + "go/types" + "log" + + "github.com/dave/dst" + "github.com/dave/dst/dstutil" +) + +type MapType struct { + Type *types.Map + Named types.Type +} + +func isTypMapType(typ types.Type) (MapType, bool) { + if mapTyp, ok := typ.Underlying().(*types.Map); ok { + return MapType{Type: mapTyp, Named: typ}, true + } + + if param, ok := typ.(*types.TypeParam); ok { + iface := param.Constraint().Underlying().(*types.Interface) + if iface.NumEmbeddeds() == 1 { + if union, ok := iface.EmbeddedType(0).(*types.Union); ok { + if union.Len() == 1 { + if mapType, ok := union.Term(0).Type().(*types.Map); ok { + return MapType{Type: mapType, Named: typ}, true + } + } + } + } + } + + return MapType{}, false +} + +func (t *packageTranslator) isMapType(expr dst.Expr) (MapType, bool) { + if astExpr, ok := t.astMap.Nodes[expr].(ast.Expr); ok { + if convertedType, ok := t.implicitConversions[astExpr]; ok { + // check here in case we pass a map[string]string to an any field + if typ, ok := isTypMapType(convertedType); ok { + return typ, ok + } + } + } + + if typ, ok := t.getType(expr); ok { + return isTypMapType(typ) + } + + return MapType{}, false +} + +func (t *packageTranslator) isMapIndex(node dst.Node) (*dst.IndexExpr, MapType, bool) { + expr, ok := node.(dst.Expr) + if !ok { + return nil, MapType{}, false + } + expr = dstutil.Unparen(expr) + if indexExpr, ok := expr.(*dst.IndexExpr); ok { + if mapType, ok := t.isMapType(indexExpr.X); ok { + return indexExpr, mapType, true + } + } + return nil, MapType{}, false +} + +func (t *packageTranslator) maybeExtractNamedMapType(x dst.Expr, typ MapType) dst.Expr { + if typ.Type == typ.Named { + return x + } + return &dst.CallExpr{Fun: t.newRuntimeSelector("ExtractMap"), Args: []dst.Expr{x}} +} + +func (t *packageTranslator) maybeConvertMapType(x dst.Expr, typ MapType) dst.Expr { + if typ.Named == typ.Type { + return x + } + + return &dst.CallExpr{ + Fun: t.makeTypeExpr(typ.Named), + Args: []dst.Expr{ + x, + }, + } +} + +func (t *packageTranslator) rewriteMapRange(c *dstutil.Cursor) { + // map range + if rangeStmt, ok := c.Node().(*dst.RangeStmt); ok { + if typ, ok := t.isMapType(rangeStmt.X); ok { + rangeStmt.X = &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(rangeStmt.X, typ), + Sel: dst.NewIdent("Range"), + }, + } + } + } +} + +func (t *packageTranslator) rewriteMapDelete(c *dstutil.Cursor) { + // delete from map + if callExpr, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(callExpr.Fun, "delete") { + if typ, ok := t.isMapType(callExpr.Args[0]); ok { + c.Replace(&dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(t.apply(callExpr.Args[0]).(dst.Expr), typ), + Sel: dst.NewIdent("Delete"), + }, + Args: []dst.Expr{ + t.apply(callExpr.Args[1]).(dst.Expr), + }, + }) + } + } +} + +func (t *packageTranslator) rewriteMapClear(c *dstutil.Cursor) { + // delete from map + if callExpr, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(callExpr.Fun, "clear") { + if typ, ok := t.isMapType(callExpr.Args[0]); ok { + c.Replace(&dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(t.apply(callExpr.Args[0]).(dst.Expr), typ), + Sel: dst.NewIdent("Clear"), + }, + }) + } + } +} + +func (t *packageTranslator) rewriteMakeMap(c *dstutil.Cursor) { + if makeExpr, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(makeExpr.Fun, "make") { + if typ, ok := t.isMapType(makeExpr); ok { + c.Replace(t.maybeConvertMapType(&dst.CallExpr{ + Fun: &dst.IndexListExpr{ + X: t.newRuntimeSelector("NewMap"), + Indices: []dst.Expr{ + t.makeTypeExpr(typ.Type.Key()), + t.makeTypeExpr(typ.Type.Elem()), + }, + }, + Args: []dst.Expr{}, + }, typ)) + } + } +} + +func (t *packageTranslator) rewriteMapLiteral(c *dstutil.Cursor) { + // map literal + if ident, ok := c.Node().(*dst.Ident); ok && ident.Name == "nil" { + if mapType, ok := t.isMapType(ident); ok { + c.Replace(t.maybeConvertMapType(&dst.CallExpr{ + Fun: &dst.IndexListExpr{ + X: t.newRuntimeSelector("NilMap"), + Indices: []dst.Expr{ + t.makeTypeExpr(mapType.Type.Key()), + t.makeTypeExpr(mapType.Type.Elem()), + }, + }, + }, mapType)) + } + } else if compositeLit, ok := c.Node().(*dst.CompositeLit); ok { + if typ, ok := t.isMapType(compositeLit); ok { + var pairs []dst.Expr + for _, elt := range compositeLit.Elts { + kv, ok := elt.(*dst.KeyValueExpr) + if !ok { + log.Fatal("map composite lit elt is not keyvalueexpr") + } + nodeDecs := kv.Decs.NodeDecs + kv.Decs.NodeDecs = dst.NodeDecs{ + Start: nodeDecs.Start, + } + nodeDecs.Start = nil + + key := t.apply(kv.Key).(dst.Expr) + value := t.apply(kv.Value).(dst.Expr) + + if nestedLit, ok := key.(*dst.CompositeLit); ok && nestedLit.Type == nil { + nestedLit.Type = t.makeTypeExpr(typ.Type.Key()) + } + if nestedLit, ok := value.(*dst.CompositeLit); ok && nestedLit.Type == nil { + // XXX: janky hack + if ptr, ok := typ.Type.Elem().Underlying().(*types.Pointer); ok { + nestedLit.Type = t.makeTypeExpr(ptr.Elem()) + value = &dst.UnaryExpr{ + Op: token.AND, + X: value, + } + } else { + nestedLit.Type = t.makeTypeExpr(typ.Type.Elem()) + } + } + + pairs = append(pairs, &dst.CompositeLit{ + Elts: []dst.Expr{ + &dst.KeyValueExpr{ + Key: dst.NewIdent("K"), + Value: key, + }, + &dst.KeyValueExpr{ + Key: dst.NewIdent("V"), + Value: value, + }, + }, + Decs: dst.CompositeLitDecorations{ + NodeDecs: nodeDecs, + }, + }) + } + + c.Replace(t.maybeConvertMapType(&dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: &dst.CompositeLit{ + Type: &dst.IndexListExpr{ + X: t.newRuntimeSelector("MapLiteral"), + Indices: []dst.Expr{ + t.makeTypeExpr(typ.Type.Key()), + t.makeTypeExpr(typ.Type.Elem()), + }, + }, + Elts: pairs, + }, + Sel: dst.NewIdent("Build"), + }, + }, typ)) + } + } else if expr, ok := c.Node().(dst.Expr); ok { + astExpr, _ := t.astMap.Nodes[expr].(ast.Expr) + if typ, ok := t.implicitConversions[astExpr]; ok { + if _, ok := isTypMapType(typ); ok { + c.Replace(&dst.CallExpr{ + Fun: t.makeTypeExpr(typ), + Args: []dst.Expr{expr}, + }) + } + } + } +} + +func (t *packageTranslator) rewriteMapLen(c *dstutil.Cursor) { + if lenExpr, ok := c.Node().(*dst.CallExpr); ok && t.isNamedBuiltIn(lenExpr.Fun, "len") { + if typ, ok := t.isMapType(lenExpr.Args[0]); ok { + // XXX: len, cap chan? + lenExpr.Fun = &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(lenExpr.Args[0], typ), + Sel: dst.NewIdent("Len"), + } + lenExpr.Args = nil + } + } +} + +func (t *packageTranslator) rewriteMapType(c *dstutil.Cursor) { + // map type + if mapType, ok := c.Node().(*dst.MapType); ok { + c.Replace(&dst.IndexListExpr{ + X: t.newRuntimeSelector("Map"), + Indices: []dst.Expr{ + mapType.Key, + t.apply(mapType.Value).(dst.Expr), + }, + }) + } + + // for generics, eject the outer ~ on map... + if unary, ok := c.Node().(*dst.UnaryExpr); ok && unary.Op == token.TILDE { + if mapType, ok := unary.X.(*dst.MapType); ok { + c.Replace(&dst.IndexListExpr{ + X: t.newRuntimeSelector("Map"), + Indices: []dst.Expr{ + mapType.Key, + t.apply(mapType.Value).(dst.Expr), + }, + }) + } + } +} + +func (t *packageTranslator) rewriteMapIndex(c *dstutil.Cursor) { + if indexExpr, wrapped, ok := t.isMapIndex(c.Node()); ok { + // we catch (val, ok) cases earlier + c.Replace(&dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(t.apply(indexExpr.X).(dst.Expr), wrapped), + Sel: dst.NewIdent("Get"), + }, + Args: []dst.Expr{ + t.apply(indexExpr.Index).(dst.Expr), + }, + }) + } +} + +func (t *packageTranslator) rewriteMapGetOk(c *dstutil.Cursor) { + // get from map in v, ok = m[k] + + rhs, ok := isDualAssign(c) + if !ok { + return + } + + if index, wrapped, ok := t.isMapIndex(*rhs); ok { + *rhs = &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(index.X, wrapped), + Sel: dst.NewIdent("GetOk"), + }, + Args: []dst.Expr{ + index.Index, + }, + } + } +} + +func isOpAssign(tok token.Token) (token.Token, bool) { + ops := map[token.Token]token.Token{ + token.ADD_ASSIGN: token.ADD, + token.SUB_ASSIGN: token.SUB, + token.MUL_ASSIGN: token.MUL, + token.QUO_ASSIGN: token.QUO, + token.REM_ASSIGN: token.REM, + token.AND_ASSIGN: token.AND, + token.OR_ASSIGN: token.OR, + token.XOR_ASSIGN: token.XOR, + token.SHL_ASSIGN: token.SHL, + token.SHR_ASSIGN: token.SHR, + token.AND_NOT_ASSIGN: token.AND_NOT, + } + out, ok := ops[tok] + return out, ok +} + +type mapModifyInfo struct { + copyVarsStmt dst.Stmt + writeExpr dst.Expr +} + +type mapAssignInfo struct { + mapModify func(op token.Token, val dst.Expr) mapModifyInfo + mapSet func(val dst.Expr) dst.Expr +} + +func (t *packageTranslator) isMapAssign(lhs dst.Expr) (mapAssignInfo, bool) { + if index, wrapped, ok := t.isMapIndex(lhs); ok { + return mapAssignInfo{ + mapModify: func(op token.Token, val dst.Expr) mapModifyInfo { + suffix := t.suffix() + keyName := "key" + suffix + mapName := "map" + suffix + return mapModifyInfo{ + copyVarsStmt: &dst.AssignStmt{ + Lhs: []dst.Expr{dst.NewIdent(mapName), dst.NewIdent(keyName)}, + Tok: token.DEFINE, + Rhs: []dst.Expr{t.apply(index.X).(dst.Expr), t.apply(index.Index).(dst.Expr)}, + }, + writeExpr: &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: dst.NewIdent(mapName), + Sel: dst.NewIdent("Set"), + }, + Args: []dst.Expr{ + dst.NewIdent(keyName), + &dst.BinaryExpr{ + X: &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: dst.NewIdent(mapName), + Sel: dst.NewIdent("Get"), + }, + Args: []dst.Expr{ + dst.NewIdent(keyName), + }, + }, + Op: op, + Y: t.apply(val).(dst.Expr), + }, + }, + }, + } + }, + mapSet: func(val dst.Expr) dst.Expr { + return &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: t.maybeExtractNamedMapType(t.apply(index.X).(dst.Expr), wrapped), + Sel: dst.NewIdent("Set"), + }, + Args: []dst.Expr{ + t.apply(index.Index).(dst.Expr), + t.apply(val).(dst.Expr), + }, + } + }, + }, true + } + return mapAssignInfo{}, false +} + +func (t *packageTranslator) rewriteMapAssign(c *dstutil.Cursor) { + // map assignment + // "m[x] =" + if assignStmt, ok := c.Node().(*dst.AssignStmt); ok { + hasAnySpecial := false + for _, lhs := range assignStmt.Lhs { + if _, ok := t.isMapAssign(lhs); ok { + hasAnySpecial = true + } + } + if hasAnySpecial && len(assignStmt.Lhs) > 1 { + // XXX: could fix this by introducing temp values and then doing sets right after... + log.Fatal("do not support map or global assign to multiple values...") + } + if !hasAnySpecial { + return + } + + info, _ := t.isMapAssign(assignStmt.Lhs[0]) + + if op, ok := isOpAssign(assignStmt.Tok); ok { + info2 := info.mapModify(op, assignStmt.Rhs[0]) + c.InsertBefore(info2.copyVarsStmt) + c.Replace(&dst.ExprStmt{ + Decs: dst.ExprStmtDecorations{ + NodeDecs: assignStmt.Decs.NodeDecs, + }, + X: info2.writeExpr, + }) + } else { + c.Replace(&dst.ExprStmt{ + Decs: dst.ExprStmtDecorations{ + NodeDecs: assignStmt.Decs.NodeDecs, + }, + X: info.mapSet(assignStmt.Rhs[0]), + }) + } + } + + if incDecStmt, ok := c.Node().(*dst.IncDecStmt); ok { + info, ok := t.isMapAssign(incDecStmt.X) + if !ok { + return + } + + op := map[token.Token]token.Token{ + token.INC: token.ADD, + token.DEC: token.SUB, + }[incDecStmt.Tok] + + info2 := info.mapModify(op, &dst.BasicLit{Kind: token.INT, Value: "1"}) + c.InsertBefore(info2.copyVarsStmt) + c.Replace(&dst.ExprStmt{ + Decs: dst.ExprStmtDecorations{ + NodeDecs: incDecStmt.Decs.NodeDecs, + }, + X: info2.writeExpr, + }) + } +} diff --git a/internal/translate/outputwriter.go b/internal/translate/outputwriter.go new file mode 100644 index 0000000..04a2fcd --- /dev/null +++ b/internal/translate/outputwriter.go @@ -0,0 +1,273 @@ +package translate + +import ( + "bytes" + "errors" + "flag" + "fmt" + "go/ast" + "go/format" + "go/printer" + "go/token" + "io" + "io/fs" + "log" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/dave/dst" + "github.com/dave/dst/decorator" + "github.com/dave/dst/decorator/resolver" +) + +const header = "// Code generated by gosim. DO NOT EDIT.\n" + +type jankyGuessResolver struct { + known map[string]string +} + +func (r jankyGuessResolver) ResolvePackage(importPath string) (string, error) { + if known, ok := r.known[importPath]; ok { + return known, nil + } + log.Fatalf("unknown package %q", importPath) + panic(importPath) +} + +func newJankyGuessResolver(known map[string]string) resolver.RestorerResolver { + return &jankyGuessResolver{ + known: known, + } +} + +func dstFileToAstFile(f *dst.File, path string, known map[string]string) (*ast.File, *token.FileSet, error) { + res := decorator.NewRestorer() + res.Resolver = newJankyGuessResolver(known) + res.Path = path + + var buf bytes.Buffer + buf.WriteString(header) + + // lifted from res.RestoreFile but modified so we can have the alias + restorer := res.FileRestorer() + restorer.Alias["testing"] = "testing_original" + af, err := restorer.RestoreFile(f) + if err != nil { + return nil, nil, err + } + + seen := make(map[string]struct{}) + + for _, decl := range af.Decls { + genDecl, ok := decl.(*ast.GenDecl) + if !ok { + continue + } + + if genDecl.Tok != token.IMPORT { + continue + } + + var filtered []ast.Spec + for _, spec := range genDecl.Specs { + importSpec, _ := spec.(*ast.ImportSpec) + if importSpec.Name != nil { + filtered = append(filtered, importSpec) + continue + } + if _, ok := seen[importSpec.Path.Value]; ok { + continue + } + seen[importSpec.Path.Value] = struct{}{} + filtered = append(filtered, importSpec) + } + + genDecl.Specs = filtered + } + + return af, res.Fset, nil +} + +func dstFileToBytes(f *dst.File, known map[string]string, path string) ([]byte, error) { + af, fset, err := dstFileToAstFile(f, path, known) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + buf.WriteString(header) + if err := formatNode(&buf, fset, af); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func formatNode(w io.Writer, fset *token.FileSet, af any) error { + if err := format.Node(w, fset, af); err != nil { + var buf2 bytes.Buffer + if err := printer.Fprint(&buf2, fset, af); err != nil { + return fmt.Errorf("error formatting\ninner: %v\norig: %w", err, err) + } + return fmt.Errorf("error formatting\n%s\norig: %w", buf2.String(), err) + } + return nil +} + +type outputWriter struct { + mu sync.Mutex + desired map[string][]byte +} + +func newOutputWriter() *outputWriter { + return &outputWriter{ + desired: make(map[string][]byte), + } +} + +func (w *outputWriter) stage(path string, contents []byte) error { + w.mu.Lock() + defer w.mu.Unlock() + + w.desired[path] = contents + return nil +} + +func (w *outputWriter) extract() map[string][]byte { + return w.desired +} + +func (w *outputWriter) merge(m map[string][]byte) error { + for path, contents := range m { + if err := w.stage(path, contents); err != nil { + return err + } + } + return nil +} + +func maybeDeleteGeneratedFile(path string) error { + contents, err := os.ReadFile(path) + if err != nil { + return err + } + if !bytes.HasPrefix(contents, []byte(header)) { + return nil + } + if *dryrun { + log.Println("deleting", path) + } else { + if err := os.Remove(path); err != nil { + return err + } + } + return nil +} + +var dryrun = flag.Bool("dryrun", false, "dryrun") + +func (w *outputWriter) maybeDeleteGeneratedFiles(rootOutputDir string) error { + desiredFinal := make(map[string]struct{}) + for p := range w.desired { + desiredFinal[filepath.Join(rootOutputDir, p)] = struct{}{} + } + + touchedDirs := make(map[string]struct{}) + for path := range desiredFinal { + // mark this package + touchedDirs[filepath.Dir(path)] = struct{}{} + } + + var dirs []string + if err := filepath.WalkDir(rootOutputDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if path == rootOutputDir && errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + + if d.IsDir() { + dirs = append(dirs, path) + return nil + } + + if _, ok := desiredFinal[path]; !ok { + // XXX: what about .s files? why these special cases? + if !strings.HasSuffix(path, "_gosim.go") && + !strings.HasSuffix(path, "_gosim_test.go") && + filepath.Base(path) != "globals.go" && + filepath.Base(path) != "globals_for_test.go" && + filepath.Base(path) != "globals_test.go" && + filepath.Base(path) != "gosim_meta_test.go" && + filepath.Base(path) != "gosimaliashack.go" { + return nil + } + + if _, ok := touchedDirs[filepath.Dir(path)]; !ok { + // skip directories that we are not writing to + return nil + } + + if err := maybeDeleteGeneratedFile(path); err != nil { + return err + } + return nil + } + + return nil + }); err != nil { + return err + } + + if !*dryrun { + for i := len(dirs) - 1; i >= 0; i-- { + dir := dirs[i] + contents, err := os.ReadDir(dir) + if err != nil { + return err + } + if len(contents) == 0 { + if err := os.Remove(dir); err != nil { + return err + } + } + } + } + + return nil +} + +func (w *outputWriter) writeFiles(rootDir string) error { + for p, desired := range w.desired { + finalP := filepath.Join(rootDir, p) + + existing, err := os.ReadFile(finalP) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return err + } + } + if err == nil { + if bytes.Equal(desired, existing) { + continue + } + } + + if *dryrun { + log.Println("writing", finalP) + } else { + if err := os.MkdirAll(path.Dir(finalP), 0o755); err != nil { + return err + } + if err := os.WriteFile(finalP, desired, 0o644); err != nil { + return err + } + } + } + + return nil +} diff --git a/internal/translate/rewrites.go b/internal/translate/rewrites.go new file mode 100644 index 0000000..bea58cf --- /dev/null +++ b/internal/translate/rewrites.go @@ -0,0 +1,268 @@ +package translate + +import ( + "fmt" + "go/token" + "log/slog" + "os" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/dave/dst" + "github.com/dave/dst/dstutil" +) + +var rewriteSelectorRegexp = regexp.MustCompile(`[/.]`) + +// RewriteSelector rewrites internal/bytealg.Equal to InternalBytealg_Equal +func RewriteSelector(pkg, selector string) string { + pkgParts := rewriteSelectorRegexp.Split(pkg, -1) + var rewritten strings.Builder + for _, part := range pkgParts { + rewritten.WriteString(strings.ToUpper(part[0:1])) + rewritten.WriteString(part[1:]) + } + rewritten.WriteString("_") + rewritten.WriteString(selector) + return rewritten.String() +} + +func (t *packageTranslator) rewriteImport(c *dstutil.Cursor) { + node := c.Node() + + imp, ok := node.(*dst.ImportSpec) + if !ok { + return + } + + val, err := strconv.Unquote(imp.Path.Value) + if err != nil { + return + } + + if imp.Name != nil && imp.Name.Name == "." { + imp.Name = nil + } + + if replaced, ok := t.replacedPkgs[val]; ok { + imp.Path.Value = strconv.Quote(replaced) + } +} + +func (t *packageTranslator) rewriteIdentImpl(ident *dst.Ident) { + lookupPath := ident.Path + if lookupPath == "" { + lookupPath = t.pkgPath + } + + // special case new plan + /* + if info, ok := t.globalInfo.ByPackage[lookupPath]; ok { + if newName, ok := info.Rewritten[ident.Name]; ok { + if _, ok := t.astMap.Nodes[ident]; !ok { + // XXX: hack job to skip newly created idents + return + } + + ident.Name = newName + if ident.Path != "" { + ident.Path = t.replacedPkgs[ident.Path] // XXX: how do we know this? + } + return + } + } + */ + + // rewrite specific selectors + selector := packageSelector{Pkg: ident.Path, Selector: ident.Name} + replacement, ok := replacements[selector] + if !ok { + // rewrite entire packages + if replaced, ok := t.replacedPkgs[ident.Path]; ok { + if ident.Path == "testing" { + if _, ok := t.astMap.Nodes[ident]; !ok { + // XXX: hack job to skip newly created idents + return + } + } + + ident.Path = replaced + ident.Obj = nil + return + } + return + } + + pkg := replacement.Pkg + if pkg == "" { + panic(replacement.Pkg + " " + replacement.Selector) + } + // XXX + // pkg = t.replacedPkgs[pkg] + // if pkg == "" { + // panic(replacement.Pkg + " " + replacement.Selector) + // } + + ident.Name = replacement.Selector + ident.Path = pkg + ident.Obj = nil +} + +func (t *packageTranslator) rewriteIdent(c *dstutil.Cursor) { + if ident, ok := c.Node().(*dst.Ident); ok { + t.rewriteIdentImpl(ident) + } +} + +func (t *packageTranslator) rewriteStdlibEmptyAndLinkname(c *dstutil.Cursor) { + node := c.Node() + + decl, ok := node.(*dst.FuncDecl) + if !ok { + return + } + + // XXX: this misses some linkname stuff + // XXX: this wrongly assumes a linkname is going to be part of a FuncDecl's Decs; they can be anywhere in the code + + hasLinkname := false + for i, val := range decl.Decs.Start { + if val == "//go:uintptrkeepalive" { + decl.Decs.Start[i] = "//go:uintptrescapes" // uintptrkeepalive is only allowed in standard library... + } + + if strings.HasPrefix(val, "//go:linkname") { + hasLinkname = true + + if _, ok := t.hooks[packageSelector{Pkg: t.pkgPath, Selector: decl.Name.Name}]; ok { + decl.Decs.Start[i] = "//" + hasLinkname = false + continue + } + + parts := strings.Split(val, " ") + + if len(parts) == 2 && decl.Body == nil { + // TODO: make this fail the build? + slog.Error("unknown linkname with no body", "pkg", t.pkgPath, "name", decl.Name.Name) + } + + if len(parts) == 3 { + dest := parts[2] + idx1 := strings.LastIndex(dest, "/") + 1 + idx2 := strings.Index(dest[idx1:], ".") + if idx2 != -1 { + idx := idx1 + idx2 + pkg, name := dest[:idx], dest[idx+1:] + if newPkg, ok := t.replacedPkgs[pkg]; ok { + slog.Debug("rewriting linkname", "pkg", t.pkgPath, "name", decl.Name.Name) + + parts[2] = fmt.Sprintf("%s.%s", newPkg, name) + decl.Decs.Start[i] = strings.Join(parts, " ") + } else if t.acceptedLinknames[packageSelector{Pkg: t.pkgPath, Selector: decl.Name.Name}] == (packageSelector{Pkg: pkg, Selector: name}) { + slog.Debug("keeping linkname", "pkg", t.pkgPath, "name", decl.Name.Name, "targetPkg", pkg, "targetName", name) + } else if strings.HasPrefix(pkg, "gosimnotranslate/") { + parts[2] = fmt.Sprintf("%s.%s", strings.TrimPrefix(pkg, "gosimnotranslate/"), name) + decl.Decs.Start[i] = strings.Join(parts, " ") + } else { + // TODO: make this fail the build? + slog.Error("unknown linkname", "pkg", t.pkgPath, "name", decl.Name.Name, "targetPkg", pkg, "targetName", name) + } + } + } + } + } + + if decl.Body != nil { + if _, ok := t.hooks[packageSelector{Pkg: t.pkgPath, Selector: decl.Name.Name}]; ok { + // take over! + decl.Body = nil + } + } + + // XXX: hack + if decl.Body == nil && !hasLinkname { + for i, val := range decl.Decs.Start { + if val == "//go:noescape" { + decl.Decs.Start[i] = "//" + } + } + + // XXX: detect all these not implemented ones? output them? + // XXX: something similar for... other "bad" calls? + + if linkTo, ok := t.hooks[packageSelector{Pkg: t.pkgPath, Selector: decl.Name.Name}]; ok { + selector := RewriteSelector(t.pkgPath, decl.Name.Name) + + pkg := linkTo.Pkg + if replaced, ok := t.replacedPkgs[pkg]; ok { + // XXX: replace machinePackage properly + pkg = replaced + } + + decl.Decs.Start = append(decl.Decs.Start, fmt.Sprintf("//go:linkname %s %s.%s", + decl.Name.Name, pkg, selector)) + t.needsUnsafe = true + + } else if decl.Name.Name == "newUnixFile" { + // need a go:linkname that is not required in the standard library... + decl.Decs.Start = append(decl.Decs.Start, fmt.Sprintf("//go:linkname %s", + decl.Name.Name)) + t.needsUnsafe = true + + } else if t.keepAsmPkgs[t.pkgPath] { + slog.Debug("keeping asm header", "pkg", t.pkgPath, "name", decl.Name.Name) + } else { + slog.Error("missing function body", "pkg", t.pkgPath, "name", decl.Name.Name) + os.Exit(1) + } + } +} + +func removeLinknames(commentLines []string) { + for i, line := range commentLines { + if strings.HasPrefix(line, "//go:linkname") { + commentLines[i] = "// gosim removed: " + line + } + } +} + +func (t *packageTranslator) rewriteDanglingLinknames(c *dstutil.Cursor) { + node := c.Node() + + if funcDecl, ok := node.(*dst.FuncDecl); ok { + endIdx := 0 + for i, line := range slices.Backward(funcDecl.Decs.Start) { + if line == "\n" { + endIdx = i + break + } + } + removeLinknames(funcDecl.Decs.Start[:endIdx]) + removeLinknames(funcDecl.Decs.End) + } + + if genDecl, ok := node.(*dst.GenDecl); ok { + for _, spec := range genDecl.Specs { + removeLinknames(spec.Decorations().Start) + removeLinknames(spec.Decorations().End) + } + } +} + +func (t *packageTranslator) rewriteNotifyListHack(c *dstutil.Cursor) { + if decl, ok := c.Node().(*dst.GenDecl); ok { + if decl.Tok == token.TYPE { + for _, spec := range decl.Specs { + if spec, ok := spec.(*dst.TypeSpec); ok { + if t.pkgPath == "sync" && spec.Name.Name == "notifyList" { + spec.Assign = true + spec.Type = t.newRuntimeSelector("NotifyList") + } + } + } + } + } +} diff --git a/internal/translate/testdata/basic.translate.txt b/internal/translate/testdata/basic.translate.txt new file mode 100644 index 0000000..2dc98d8 --- /dev/null +++ b/internal/translate/testdata/basic.translate.txt @@ -0,0 +1,1300 @@ +-- chan.go -- +package basicgosim + +import "log" + +var chanLen = 10 + +func Test() { + structCh := make(chan struct{}) + close(structCh) + explicitZeroCh := make(chan struct{}, 0) + close(explicitZeroCh) + intChCh := make(chan chan int, 5) + close(intChCh) + + var stringCh chan string + + stringCh <- "hello" + log.Println(<-stringCh, <-stringCh) + + x, ok := <-stringCh + x = <-stringCh + x, ok = <-stringCh + log.Println(x, ok) + + var y, yOk = <-stringCh + log.Println(y, yOk) + + log.Println(len(structCh)) + log.Println(cap(structCh)) + + rewriteLenCh := make(chan int, chanLen) + close(rewriteLenCh) +} +-- translated/basic/chan_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Test() { + structCh := gosimruntime.NewChan[struct{}](0) + structCh.Close() + explicitZeroCh := gosimruntime.NewChan[struct{}](0) + explicitZeroCh.Close() + intChCh := gosimruntime.NewChan[gosimruntime.Chan[int]](5) + intChCh.Close() + + var stringCh gosimruntime.Chan[string] + + stringCh.Send("hello") + log.Println(stringCh.Recv(), stringCh.Recv()) + + x, ok := stringCh.RecvOk() + x = stringCh.Recv() + x, ok = stringCh.RecvOk() + log.Println(x, ok) + + var y, yOk = stringCh.RecvOk() + log.Println(y, yOk) + + log.Println(structCh.Len()) + log.Println(structCh.Cap()) + + rewriteLenCh := gosimruntime.NewChan[int](G().chanLen) + rewriteLenCh.Close() +} +-- chaniter.go -- +package basicgosim + +import "log" + +func Chaniter() { + var c chan string + var v string + + for v := range c { + log.Println(v) + } + + for v = range c { + log.Println(v) + } + + for range c { + log.Println() + } + + for _ = range c { + log.Println() + } + + f := func() chan string { + return c + } + + for range f() { + } +} +-- translated/basic/chaniter_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Chaniter() { + var c gosimruntime.Chan[string] + var v string + + for v := range c.Range() { + log.Println(v) + } + + for v = range c.Range() { + log.Println(v) + } + + for range c.Range() { + log.Println() + } + + for _ = range c.Range() { + log.Println() + } + + f := func() gosimruntime.Chan[string] { + return c + } + + for range f().Range() { + } +} +-- chanshadow.go -- +package basicgosim + +import ( + "log" +) + +func Chanshadow() { + len := func(ch chan int) int { + return -1 + } + log.Println(len(make(chan int, 3))) + close := func(ch chan int) {} + close(make(chan int, 2)) +} +-- translated/basic/chanshadow_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Chanshadow() { + len := func(ch gosimruntime.Chan[int]) int { + return -1 + } + log.Println(len(gosimruntime.NewChan[int](3))) + close := func(ch gosimruntime.Chan[int]) {} + close(gosimruntime.NewChan[int](2)) +} +-- context.go -- +package basicgosim + +import ( + "context" + "time" +) + +func Context() { + ctx := context.Background() + + _, cancel := context.WithCancel(ctx) + defer cancel() + + _, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, cancel = context.WithDeadline(ctx, time.Now().Add(10*time.Second)) + defer cancel() + + <-ctx.Done() +} +-- translated/basic/context_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/context" + "translated/time" +) + +func Context() { + ctx := context.Background() + + _, cancel := context.WithCancel(ctx) + defer cancel() + + _, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + _, cancel = context.WithDeadline(ctx, time.Now().Add(10*time.Second)) + defer cancel() + + ctx.Done().Recv() +} +-- funccast.go -- +package basicgosim + +type Ftype func() + +func (f Ftype) Invoke() { + f() +} + +func Funccast() { + var f = Ftype(func() {}) + f() + f.Invoke() +} +-- translated/basic/funccast_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +type Ftype func() + +func (f Ftype) Invoke() { + f() +} + +func Funccast() { + var f = Ftype(func() {}) + f() + f.Invoke() +} +-- global.go -- +package basicgosim + +import ( + "log" + "sync" + + "test/basic/global" +) + +var x int +var y = "hello" + +type Foo struct { +} + +var m = Foo{} +var n Foo + +var o *Foo +var p []Foo +var q map[Foo]Foo + +var r = [...]string{"hello", "world"} +var s [3 * 3]string + +var t <-chan string + +var mm = map[Foo]int{ + m: 3, +} + +var () + +type Yep[T any] struct { + yep T +} + +var u Yep[Yep[string]] + +var mu sync.Map + +var ( + deps1 = compute() + deps3 = "foo" +) +var deps2 = deps3 + "bar" + +func compute() string { + return deps3 + deps2 +} + +func double() (string, string) { + return "a", "b" +} + +var ( + a, b string = double() + c, d = 2, 3.0 + _, interesting = double() + _ = 3.0 + _ = compute() +) + +var _ = 10 + +func Global() { + // XXX: take address of global? + log.Println(x) + x = 3 + x++ + x *= x + + log.Println(global.Hello) +} +-- translated/basic/global/globals.go -- +// Code generated by gosim. DO NOT EDIT. +package global + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +type Globals struct { + Hello string +} + +var globals gosimruntime.Global[Globals] + +func initializers(initializeShared bool) {} +func G() *Globals { return globals.Get() } +func init() { + gosimruntime.RegisterPackage("test/basic/global").Globals = &gosimruntime.Globals{Globals: &globals, Initializers: initializers} +} +-- global/nested.go -- +package global + +var Hello string +-- translated/basic/global/nested_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package global +-- translated/basic/global_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "translated/test/basic/global" +) + +type Foo struct { +} + +type Yep[T any] struct { + yep T +} + +func compute() string { + return G().deps3 + G().deps2 +} + +func double() (string, string) { + return "a", "b" +} + +func Global() { + // XXX: take address of global? + log.Println(G().x) + G().x = 3 + G().x++ + G().x *= G().x + + log.Println(global.G().Hello) +} +-- globalmaptricky.go -- +package basicgosim + +var ( + globalmaptricky map[string]int +) + +func Globalmaptricky() { + globalmaptricky["foo"]++ +} +-- translated/basic/globalmaptricky_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +func Globalmaptricky() { + map0, key0 := G().globalmaptricky, "foo" + map0.Set(key0, map0.Get(key0)+1) +} +-- translated/basic/globals.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/sync" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +type Globals struct { + chanLen int + x int + y string + m Foo + n Foo + o *Foo + p []Foo + q gosimruntime.Map[Foo, Foo] + r [2]string + s [9]string + t gosimruntime.Chan[string] + mm gosimruntime.Map[Foo, int] + u Yep[Yep[string]] + mu sync.Map + deps1 string + deps3 string + deps2 string + a string + b string + c int + d float64 + interesting string + globalmaptricky gosimruntime.Map[string, int] +} + +var globals gosimruntime.Global[Globals] + +func initializers(initializeShared bool) { + G().chanLen = 10 + G().y = "hello" + G().m = Foo{} + G().r = [...]string{"hello", "world"} + G().mm = gosimruntime.MapLiteral[Foo, int]{ + {K: G().m, V: 3}, + }.Build() + G().deps3 = "foo" + G().deps2 = G().deps3 + "bar" + G().deps1 = compute() + G().a, G().b = double() + G().c = 2 + G().d = 3.0 + _, G().interesting = double() + _ = 3.0 + _ = compute() + _ = 10 + gosiminit0() + gosiminit1() +} +func G() *Globals { return globals.Get() } +func init() { + gosimruntime.RegisterPackage("test/basic").Globals = &gosimruntime.Globals{Globals: &globals, Initializers: initializers} +} +func Bind0_1[T1 any](f func() T1) func() func() { + return func() func() { + return func() { + f() + } + } +} +func Bind1_0[T1 any](f func(v1 T1)) func(v1 T1) func() { + return func(v1 T1) func() { + return func() { + f(v1) + } + } +} +func Bind1var_0[T1 any](f func(v1 ...T1)) func(v1 ...T1) func() { + return func(v1 ...T1) func() { + return func() { + f(v1...) + } + } +} +func Bind2_0[T1, T2 any](f func(v1 T1, v2 T2)) func(v1 T1, v2 T2) func() { + return func(v1 T1, v2 T2) func() { + return func() { + f(v1, v2) + } + } +} +-- go.go -- +package basicgosim + +import ( + "log" + "time" +) + +func Go() { + go func() { + log.Println("hello!") + }() + + x := 5 + y := "foo" + go func(x int, y string) { + log.Println(x, y) + }(x, y) + + go func() error { + return nil + }() + + go func(s ...string) { + }() + + go func(x *int, y string) { + }(nil, y) + + go func(x time.Duration) { + }(100) +} +-- translated/basic/go_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + "translated/time" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Go() { + gosimruntime.Go(func() { + log.Println("hello!") + }) + + x := 5 + y := "foo" + gosimruntime.Go(Bind2_0(func(x int, y string) { + log.Println(x, y) + })(x, y)) + gosimruntime.Go(Bind0_1(func() error { + return nil + })()) + gosimruntime.Go(Bind1var_0(func(s ...string) { + })()) + gosimruntime.Go(Bind2_0(func(x *int, y string) { + })(nil, y)) + gosimruntime.Go(Bind1_0(func(x time.Duration) { + })(100)) +} +-- init.go -- +package basicgosim + +import "log" + +func init() { + log.Println("hello") +} + +func Init() { + +} + +type S struct { +} + +func (s S) init() { +} + +func init() { + log.Println("hello") +} +-- translated/basic/init_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import "translated/log" + +func gosiminit0() { + log.Println("hello") +} + +func Init() { + +} + +type S struct { +} + +func (s S) init() { +} + +func gosiminit1() { + log.Println("hello") +} +-- translated/basic/justinit/globals.go -- +// Code generated by gosim. DO NOT EDIT. +package justinit + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +type Globals struct { +} + +var globals gosimruntime.Global[Globals] + +func initializers(initializeShared bool) { gosiminit0() } +func G() *Globals { return globals.Get() } +func init() { + gosimruntime.RegisterPackage("test/basic/justinit").Globals = &gosimruntime.Globals{Globals: &globals, Initializers: initializers} +} +-- justinit/main.go -- +package justinit + +import ( + "log" +) + +func init() { + log.Println("HELLO") +} +-- translated/basic/justinit/main_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package justinit + +import ( + "translated/log" +) + +func gosiminit0() { + log.Println("HELLO") +} +-- log.go -- +package basicgosim + +import ( + "log" +) + +func Log() { + log.Printf("hello %s!", "you") + log.Print("a", "b", "c") +} +-- translated/basic/log_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" +) + +func Log() { + log.Printf("hello %s!", "you") + log.Print("a", "b", "c") +} +-- map.go -- +package basicgosim + +import ( + "log" +) + +type StructWithMap struct { + x int + y map[int]string +} + +func Map() { + var m map[int]string + m = make(map[int]string) + + m = map[int]string{1: "ok", 2: "bar"} + + m = map[int]string{ + 1: "ok", + 2: "bar", + } + + if m == nil { + } + + _ = StructWithMap{0, nil} + + log.Println(len(m)) + var x []int + log.Println(len(x)) + + m[0] = "ok" + delete(m, 1) + m[0] = m[1] + m[2] + v, ok := m[3] + v, ok = m[3] + log.Println(v, ok) + m[0] += "lol" + + var y, yOk = m[9] + log.Println(y, yOk) + + mi := make(map[int]int) + mi[0]++ + mi[0]-- + + mi[mi[0]]++ + delete(mi, mi[0]) +} +-- translated/basic/map_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +type StructWithMap struct { + x int + y gosimruntime.Map[int, string] +} + +func Map() { + var m gosimruntime.Map[int, string] + m = gosimruntime.NewMap[int, string]() + + m = gosimruntime.MapLiteral[int, string]{{K: 1, V: "ok"}, {K: 2, V: "bar"}}.Build() + + m = gosimruntime.MapLiteral[int, string]{ + {K: 1, V: "ok"}, + {K: 2, V: "bar"}, + }.Build() + + if m == gosimruntime.NilMap[int, string]() { + } + + _ = StructWithMap{0, gosimruntime.NilMap[int, string]()} + + log.Println(m.Len()) + var x []int + log.Println(len(x)) + + m.Set(0, "ok") + m.Delete(1) + m.Set(0, m.Get(1)+m.Get(2)) + v, ok := m.GetOk(3) + v, ok = m.GetOk(3) + log.Println(v, ok) + map1, key1 := m, 0 + map1.Set(key1, map1.Get(key1)+"lol") + + var y, yOk = m.GetOk(9) + log.Println(y, yOk) + + mi := gosimruntime.NewMap[int, int]() + map2, key2 := mi, 0 + map2.Set(key2, map2.Get(key2)+1) + map3, key3 := mi, 0 + map3.Set(key3, map3.Get(key3)-1) + + map4, key4 := mi, mi.Get(0) + + map4.Set(key4, map4.Get(key4)+1) + mi.Delete(mi.Get(0)) +} +-- mapconversions.go -- +package basicgosim + +func takeany(x any) { +} + +type Named map[string]string + +func Mapconversions() { + takeany(map[string]string{"foo": "bar"}) + var x map[string]string + _ = x + x = nil + takeany(x) + + var y Named + _ = y + y = x + x = y + y = nil + takeany(y) +} +-- translated/basic/mapconversions_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func takeany(x any) { +} + +type Named gosimruntime.Map[string, string] + +func Mapconversions() { + takeany(gosimruntime.MapLiteral[string, string]{{K: "foo", V: "bar"}}.Build()) + var x gosimruntime.Map[string, string] + _ = x + x = gosimruntime.NilMap[string, string]() + takeany(x) + + var y Named + _ = y + y = Named(x) + x = gosimruntime.Map[string, string](y) + y = Named(gosimruntime.NilMap[string, string]()) + takeany(y) +} +-- mapiter.go -- +package basicgosim + +import "log" + +func Mapiter() { + var m map[int]string + var k int + var v string + + for k, v := range m { + log.Println(k, v) + } + + for k, v = range m { + log.Println(k, v) + } + + for range m { + log.Println() + } + + for _ = range m { + log.Println() + } + + for k := range m { + log.Println(k) + } + + for k, _ := range m { + log.Println(k) + } + + for k, _ = range m { + log.Println(k) + } + + for _, v := range m { + log.Println(v) + } + + for _, _ = range m { + log.Println() + } + + func() { + for _, _ = range m { + log.Println() + } + func() { + for _, _ = range m { + log.Println() + } + }() + }() +} +-- translated/basic/mapiter_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Mapiter() { + var m gosimruntime.Map[int, string] + var k int + var v string + + for k, v := range m.Range() { + log.Println(k, v) + } + + for k, v = range m.Range() { + log.Println(k, v) + } + + for range m.Range() { + log.Println() + } + + for _ = range m.Range() { + log.Println() + } + + for k := range m.Range() { + log.Println(k) + } + + for k, _ := range m.Range() { + log.Println(k) + } + + for k, _ = range m.Range() { + log.Println(k) + } + + for _, v := range m.Range() { + log.Println(v) + } + + for _, _ = range m.Range() { + log.Println() + } + + func() { + for _, _ = range m.Range() { + log.Println() + } + func() { + for _, _ = range m.Range() { + log.Println() + } + }() + }() +} +-- mapnested.go -- +package basicgosim + +import "log" + +func Mapnested() { + var m map[string]map[string]string + m = make(map[string]map[string]string) + m["foo"] = make(map[string]string) + m["foo"]["bar"] = "baz" + log.Println(m["foo"]["bar"]) + m = map[string]map[string]string{} + m = map[string]map[string]string{ + "foo": { + "bar": "baz", + }, + } + m["foo"] = map[string]string{ + "bar": "baz", + } + + triple := map[int]map[int]map[int]int{ + 0: {0: {0: 0}}, + 1: {1: {1: 1}}, + } + log.Print(triple) + + // XXX: named nested map here + + x := map[string]map[string][]byte{ + "a": {"a": {}}, + "b": map[string][]byte{"a": {0}}, + } + log.Print(x) + + for k := range x { + for _, v := range x[k] { + log.Print(v) + } + } +} +-- translated/basic/mapnested_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Mapnested() { + var m gosimruntime.Map[string, gosimruntime.Map[string, string]] + m = gosimruntime.NewMap[string, gosimruntime.Map[string, string]]() + m.Set("foo", gosimruntime.NewMap[string, string]()) + m.Get("foo").Set("bar", "baz") + log.Println(m.Get("foo").Get("bar")) + m = gosimruntime.MapLiteral[string, gosimruntime.Map[string, string]]{}.Build() + m = gosimruntime.MapLiteral[string, gosimruntime.Map[string, string]]{ + {K: "foo", V: gosimruntime.MapLiteral[string, string]{ + {K: "bar", V: "baz"}, + }.Build()}, + }.Build() + m.Set("foo", gosimruntime.MapLiteral[string, string]{ + {K: "bar", V: "baz"}, + }.Build()) + + triple := gosimruntime.MapLiteral[int, gosimruntime.Map[int, gosimruntime.Map[int, int]]]{ + {K: 0, V: gosimruntime.MapLiteral[int, gosimruntime.Map[int, int]]{{K: 0, V: gosimruntime.MapLiteral[int, int]{{K: 0, V: 0}}.Build()}}.Build()}, + {K: 1, V: gosimruntime.MapLiteral[int, gosimruntime.Map[int, int]]{{K: 1, V: gosimruntime.MapLiteral[int, int]{{K: 1, V: 1}}.Build()}}.Build()}, + }.Build() + log.Print(triple) + + // XXX: named nested map here + + x := gosimruntime.MapLiteral[string, gosimruntime.Map[string, []byte]]{ + {K: "a", V: gosimruntime.MapLiteral[string, []byte]{{K: "a", V: []byte{}}}.Build()}, + {K: "b", V: gosimruntime.MapLiteral[string, []byte]{{K: "a", V: []byte{0}}}.Build()}, + }.Build() + log.Print(x) + + for k := range x.Range() { + for _, v := range x.Get(k).Range() { + log.Print(v) + } + } +} +-- mapptr.go -- +package basicgosim + +func Mapptr() { + type S struct { + A string + } + _ = map[int]*S{ + 1: {A: "a"}, + } +} +-- translated/basic/mapptr_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func Mapptr() { + type S struct { + A string + } + _ = gosimruntime.MapLiteral[int, *S]{ + {K: 1, V: &S{A: "a"}}, + }.Build() +} +-- mapshadow.go -- +package basicgosim + +import ( + "log" +) + +func Mapshadow() { + len := func(a map[string]int) int { + return -1 + } + log.Println(len(map[string]int{})) + delete := func(a map[string]int, b string) {} + delete(map[string]int{}, "foo") +} +-- translated/basic/mapshadow_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Mapshadow() { + len := func(a gosimruntime.Map[string, int]) int { + return -1 + } + log.Println(len(gosimruntime.MapLiteral[string, int]{}.Build())) + delete := func(a gosimruntime.Map[string, int], b string) {} + delete(gosimruntime.MapLiteral[string, int]{}.Build(), "foo") +} +-- mutex.go -- +package basicgosim + +import "sync" + +func Mutex() { + var mu sync.Mutex + mu.Lock() + defer mu.Unlock() +} +-- translated/basic/mutex_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import "translated/sync" + +func Mutex() { + var mu sync.Mutex + mu.Lock() + defer mu.Unlock() +} +-- os.go -- +package basicgosim + +import ( + "log" + "os" +) + +func Os() { + f, err := os.OpenFile("foo", os.O_RDONLY, 0) + if err != nil { + log.Fatal(err) + } + f.Close() + var g *os.File + log.Println(g) + g = f + os.Remove("foo") + fi, _ := os.Stat("foo") + var s string = fi.Name() + var b bool = fi.IsDir() + log.Print(s, b) + os.Rename("a", "b") + var entries []os.DirEntry + entries, err = os.ReadDir(".") + s = entries[0].Name() + b = entries[0].IsDir() +} +-- translated/basic/os_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + "translated/os" +) + +func Os() { + f, err := os.OpenFile("foo", os.O_RDONLY, 0) + if err != nil { + log.Fatal(err) + } + f.Close() + var g *os.File + log.Println(g) + g = f + os.Remove("foo") + fi, _ := os.Stat("foo") + var s string = fi.Name() + var b bool = fi.IsDir() + log.Print(s, b) + os.Rename("a", "b") + var entries []os.DirEntry + entries, err = os.ReadDir(".") + s = entries[0].Name() + b = entries[0].IsDir() +} +-- rand.go -- +package basicgosim + +import ( + "log" + "math/rand" +) + +func Rand() { + log.Println(rand.Float64()) +} +-- translated/basic/rand_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + "translated/math/rand" +) + +func Rand() { + log.Println(rand.Float64()) +} +-- rand_timer.go -- +package basicgosim + +import ( + "log" + "math/rand" + "time" +) + +func RandTimer() { + t := time.NewTimer(10 * time.Second) + t.Reset(time.Duration(rand.Float64() * float64(10*time.Second))) + log.Println(<-t.C) +} +-- translated/basic/rand_timer_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + "translated/math/rand" + "translated/time" +) + +func RandTimer() { + t := time.NewTimer(10 * time.Second) + t.Reset(time.Duration(rand.Float64() * float64(10*time.Second))) + log.Println(t.C.Recv()) +} +-- time.go -- +package basicgosim + +import ( + "log" + "time" +) + +func Time() { + t := time.Now() + log.Println(t) + time.Sleep(5 * time.Second) + log.Println(time.Since(t)) + log.Println(time.Until(t)) + + var x *time.Timer + x = time.NewTimer(5 * time.Second) + log.Println(x) + + ch := time.After(5 * time.Second) + log.Println(ch) + x = time.AfterFunc(5*time.Second, func() {}) + + var tt *time.Ticker + tt = time.NewTicker(10 * time.Second) + var q time.Time + q = <-tt.C + log.Println(q) + tt.Stop() + tt.Reset(5 * time.Second) +} +-- translated/basic/time_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + "translated/time" +) + +func Time() { + t := time.Now() + log.Println(t) + time.Sleep(5 * time.Second) + log.Println(time.Since(t)) + log.Println(time.Until(t)) + + var x *time.Timer + x = time.NewTimer(5 * time.Second) + log.Println(x) + + ch := time.After(5 * time.Second) + log.Println(ch) + x = time.AfterFunc(5*time.Second, func() {}) + + var tt *time.Ticker + tt = time.NewTicker(10 * time.Second) + var q time.Time + q = tt.C.Recv() + log.Println(q) + tt.Stop() + tt.Reset(5 * time.Second) +} +-- waitgroup.go -- +package basicgosim + +import "sync" + +func Waitgroup() { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + }() + wg.Wait() +} +-- translated/basic/waitgroup_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/sync" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Waitgroup() { + var wg sync.WaitGroup + wg.Add(1) + gosimruntime.Go(func() { + defer wg.Done() + }) + wg.Wait() +} diff --git a/internal/translate/testdata/comment.translate.txt b/internal/translate/testdata/comment.translate.txt new file mode 100644 index 0000000..b5c3e62 --- /dev/null +++ b/internal/translate/testdata/comment.translate.txt @@ -0,0 +1,100 @@ +-- mapcomment.go -- +package gosimcomment + +func Mapcomment() { + // hello + + var m map[ /* can */ string] /* you */ string // a map + + // ok + + m["foo" /*handle*/] /* it */ = /* in */ "bar" /* here */ + + // hmm + + _ = m["bar"] // hello + + // yay + + // hurray + m = map[ /* -1 */ string]string{ + /* 0 */ "foo" /* 1 */ :/* 2 */ "ok", // ok + /* 3 */ "bar" /* 4 */ :/* 5 */ "bar", // yes + } // mm +} +-- translated/comment/mapcomment_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package gosimcomment + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func Mapcomment() { + // hello + + var m gosimruntime.Map[string, string] // a map + + // ok + + m.Set("foo", "bar") /* here */ + + // hmm + + _ = m.Get("bar") // hello + + // yay + + // hurray + m = gosimruntime.MapLiteral[string, string]{ + {K: "foo", V: "ok"}, // ok + {K: "bar", V: "bar"}, // yes + }.Build() // mm +} +-- selectcomment.go -- +package gosimcomment + +import "log" + +func Selectcomment() { + var a, b chan struct{} + + // before + select /* ok */ { + // in + case /* foo */ <-a /* bar */ : /* baz */ + // 1 + return // 2 + // 3 + case <-b: + // between + log.Println("ok") // after + default: // me + // too + } +} +-- translated/comment/selectcomment_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package gosimcomment + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Selectcomment() { + var a, b gosimruntime.Chan[struct{}] + + // before + switch /* ok */ idx0, _, _ := gosimruntime.Select(a.RecvSelector(), b.RecvSelector(), gosimruntime.DefaultSelector()); idx0 { + // in + case /* foo */ 0: /* baz */ + // 1 + return // 2 + // 3 + case 1: + // between + log.Println("ok") // after + default: // me + // too + } +} diff --git a/internal/translate/testdata/directives.translate.txt b/internal/translate/testdata/directives.translate.txt new file mode 100644 index 0000000..8ef1490 --- /dev/null +++ b/internal/translate/testdata/directives.translate.txt @@ -0,0 +1,42 @@ +-- linkname.go -- +package basicgosim + +import ( + "log" + _ "unsafe" +) + +func Test() { + log.Println(secret()) +} + +//go:linkname secret test/directives/testpkg.secretImpl +func secret() string +-- translated/directives/linkname_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package basicgosim + +import ( + "translated/log" + _ "unsafe" +) + +func Test() { + log.Println(secret()) +} + +//go:linkname secret translated/test/directives/testpkg.secretImpl +func secret() string +-- testpkg/main.go -- +package testpkg + +func secretImpl() string { + return "secret" +} +-- translated/directives/testpkg/main_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package testpkg + +func secretImpl() string { + return "secret" +} diff --git a/internal/translate/testdata/packages.translate.txt b/internal/translate/testdata/packages.translate.txt new file mode 100644 index 0000000..5ff0a79 --- /dev/null +++ b/internal/translate/testdata/packages.translate.txt @@ -0,0 +1,157 @@ +-- anonymous/anonymous.go -- +package anonymous + +import ( + _ "test/packages/testpkg" +) +-- translated/packages/anonymous/anonymous_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package anonymous + +import ( + _ "translated/test/packages/testpkg" +) +-- dot/dot.go -- +package dot + +import ( + "log" + + . "test/packages/testpkg" +) + +func Use() { + log.Println(DoSomeStuff()) +} +-- translated/packages/dot/dot_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package dot + +import ( + "translated/log" + + "translated/test/packages/testpkg" +) + +func Use() { + log.Println(testpkg.DoSomeStuff()) +} +-- nestingpkg/nestedpkg.go -- +package nestedpkg + +import ( + "test/packages/testpkg" +) + +func Hello() chan struct{} { + testpkg.DoSomeStuff() + return make(chan struct{}) +} +-- translated/packages/nestingpkg/nestedpkg_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package nestedpkg + +import ( + "translated/test/packages/testpkg" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Hello() gosimruntime.Chan[struct{}] { + testpkg.DoSomeStuff() + return gosimruntime.NewChan[struct{}](0) +} +-- translated/packages/testpkg/gosim_meta_test.go -- +// Code generated by gosim. DO NOT EDIT. +package testpkg_test + +import ( + testing_original "testing" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "translated/github.com/jellevandenhooff/gosim/internal_/hooks/go123" +) + +func init() { + gosimruntime.SetAllTests([]gosimruntime.Test{ + {Name: "TestHello", Test: ImplTestHello}, + }) +} +func TestMain(m *testing_original.M) { + gosimruntime.TestMain(go123.Runtime()) +} +-- testpkg/main.go -- +package testpkg + +func DoSomeStuff() map[string]int { + m := make(map[string]int) + m["foo"] = 5 + return m +} +-- translated/packages/testpkg/main_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package testpkg + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func DoSomeStuff() gosimruntime.Map[string, int] { + m := gosimruntime.NewMap[string, int]() + m.Set("foo", 5) + return m +} +-- translated/packages/testpkg/main_gosim_test.go -- +// Code generated by gosim. DO NOT EDIT. +package testpkg_test + +import ( + "translated/github.com/jellevandenhooff/gosim/internal_/testing" + "translated/log" + + "translated/test/packages/testpkg" +) + +func ImplTestHello(t *testing.T) { + x := testpkg.DoSomeStuff() + log.Println(x.Get("foo")) + log.Println("hello") +} +-- testpkg/main_test.go -- +package testpkg_test + +import ( + "log" + "testing" + + "test/packages/testpkg" +) + +func TestHello(t *testing.T) { + x := testpkg.DoSomeStuff() + log.Println(x["foo"]) + log.Println("hello") +} +-- testprogram/main.go -- +package main + +import ( + "log" + "time" +) + +func main() { + time.Sleep(10 * time.Second) + log.Println("hello world") +} +-- translated/packages/testprogram/main_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package main + +import ( + "translated/log" + "translated/time" +) + +func main() { + time.Sleep(10 * time.Second) + log.Println("hello world") +} diff --git a/internal/translate/testdata/select.translate.txt b/internal/translate/testdata/select.translate.txt new file mode 100644 index 0000000..d830378 --- /dev/null +++ b/internal/translate/testdata/select.translate.txt @@ -0,0 +1,219 @@ +-- select.go -- +package gosimselect + +import "log" + +func Select() { + ch := make(chan struct{}) + ch2 := make(chan int) + + var y int + + select { + case <-ch: + case y = <-ch2: + log.Println(y) + case x := <-ch: + log.Println(x) + case x, ok := <-ch: + log.Println(x, ok) + case ch <- struct{}{}: + default: + } +} +-- translated/select/select_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package gosimselect + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Select() { + ch := gosimruntime.NewChan[struct{}](0) + ch2 := gosimruntime.NewChan[int](0) + + var y int + + switch idx0, val0, ok0 := gosimruntime.Select(ch.RecvSelector(), ch2.RecvSelector(), ch.RecvSelector(), ch.RecvSelector(), ch.SendSelector(struct{}{}), gosimruntime.DefaultSelector()); idx0 { + case 0: + case 1: + y = gosimruntime.ChanCast[int](val0) + log.Println(y) + case 2: + x := gosimruntime.ChanCast[struct{}](val0) + log.Println(x) + case 3: + x, ok := gosimruntime.ChanCast[struct{}](val0), ok0 + log.Println(x, ok) + case 4: + default: + } +} +-- selectcopy.go -- +package gosimselect + +func Selectcopy() { + var x = func() chan struct{} { + ch := make(chan struct{}) + close(ch) + return ch + } + + ch := make(chan struct{}) + + select { + case <-ch: + case <-x(): + case x() <- struct{}{}: + default: + } +} +-- translated/select/selectcopy_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package gosimselect + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func Selectcopy() { + var x = func() gosimruntime.Chan[struct{}] { + ch := gosimruntime.NewChan[struct{}](0) + ch.Close() + return ch + } + + ch := gosimruntime.NewChan[struct{}](0) + + switch idx1, _, _ := gosimruntime.Select(ch.RecvSelector(), x().RecvSelector(), x().SendSelector(struct{}{}), gosimruntime.DefaultSelector()); idx1 { + case 0: + case 1: + case 2: + default: + } +} +-- selectdefault.go -- +package gosimselect + +func Selectdefault() { + a := make(chan struct{}) + + select {} + + select { + default: + } + + select { + case <-a: + } + + select { + default: + case <-a: + } + + select { + case <-a: + default: + case <-a: + } + + select { + case a <- struct{}{}: + default: + } +} +-- translated/select/selectdefault_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package gosimselect + +import "github.com/jellevandenhooff/gosim/gosimruntime" + +func Selectdefault() { + a := gosimruntime.NewChan[struct{}](0) + + switch idx2, _, _ := gosimruntime.Select(); idx2 { + default: + panic("unreachable select") + } + + switch idx3, _, _ := gosimruntime.Select(gosimruntime.DefaultSelector()); idx3 { + default: + } + + switch idx4, _, _ := gosimruntime.Select(a.RecvSelector()); idx4 { + case 0: + default: + panic("unreachable select") + } + + switch idx5, _, _ := a.SelectRecvOrDefault(); idx5 { + default: + case 0: + } + + switch idx6, _, _ := gosimruntime.Select(a.RecvSelector(), a.RecvSelector(), gosimruntime.DefaultSelector()); idx6 { + case 0: + default: + case 1: + } + + switch idx7, _, _ := a.SelectSendOrDefault(struct{}{}); idx7 { + case 0: + default: + } +} +-- selectnested.go -- +package gosimselect + +import "log" + +func Selectnested() { + ch := make(chan struct{}) + ch2 := make(chan int) + + var y int + + select { + case <-ch: + case y = <-ch2: + log.Println(y) + select { + case <-ch: + log.Println("wacky") + default: + } + default: + } +} +-- translated/select/selectnested_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package gosimselect + +import ( + "translated/log" + + "github.com/jellevandenhooff/gosim/gosimruntime" +) + +func Selectnested() { + ch := gosimruntime.NewChan[struct{}](0) + ch2 := gosimruntime.NewChan[int](0) + + var y int + + switch idx9, val9, _ := gosimruntime.Select(ch.RecvSelector(), ch2.RecvSelector(), gosimruntime.DefaultSelector()); idx9 { + case 0: + case 1: + y = gosimruntime.ChanCast[int](val9) + log.Println(y) + switch idx8, _, _ := ch.SelectRecvOrDefault(); idx8 { + case 0: + log.Println("wacky") + default: + } + default: + } +} diff --git a/internal/translate/testdata/slog.translate.txt b/internal/translate/testdata/slog.translate.txt new file mode 100644 index 0000000..88e3c7a --- /dev/null +++ b/internal/translate/testdata/slog.translate.txt @@ -0,0 +1,23 @@ +-- slog.go -- +package main + +import ( + "context" + "log/slog" +) + +func main() { + slog.Log(context.TODO(), slog.LevelInfo, "hi") +} +-- translated/slog/slog_gosim.go -- +// Code generated by gosim. DO NOT EDIT. +package main + +import ( + "translated/context" + "translated/log/slog" +) + +func main() { + slog.Log(context.TODO(), slog.LevelInfo, "hi") +} diff --git a/internal/translate/testdata/test.translate.txt b/internal/translate/testdata/test.translate.txt new file mode 100644 index 0000000..f626afa --- /dev/null +++ b/internal/translate/testdata/test.translate.txt @@ -0,0 +1,59 @@ +-- translated/test/basictest_gosim_test.go -- +// Code generated by gosim. DO NOT EDIT. +package basictest + +import ( + "translated/github.com/jellevandenhooff/gosim/internal_/testing" + "translated/time" +) + +func helper() string { + return "ok" +} + +func testOther(t *testing.T) { + helper() +} + +func ImplTestFoo(t *testing.T) { + time.Sleep(10 * time.Second) +} +-- basictest_test.go -- +package basictest + +import ( + "testing" + "time" +) + +func helper() string { + return "ok" +} + +func testOther(t *testing.T) { + helper() +} + +func TestFoo(t *testing.T) { + time.Sleep(10 * time.Second) +} +-- translated/test/gosim_meta_test.go -- +// Code generated by gosim. DO NOT EDIT. +package basictest_test + +import ( + testing_original "testing" + "translated/test/test" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "translated/github.com/jellevandenhooff/gosim/internal_/hooks/go123" +) + +func init() { + gosimruntime.SetAllTests([]gosimruntime.Test{ + {Name: "TestFoo", Test: basictest.ImplTestFoo}, + }) +} +func TestMain(m *testing_original.M) { + gosimruntime.TestMain(go123.Runtime()) +} diff --git a/internal/translate/tests.go b/internal/translate/tests.go new file mode 100644 index 0000000..7f196e9 --- /dev/null +++ b/internal/translate/tests.go @@ -0,0 +1,212 @@ +package translate + +import ( + "cmp" + "go/token" + "slices" + "strconv" + + "github.com/dave/dst" + "github.com/dave/dst/dstutil" +) + +func (t *packageTranslator) rewriteTestFunc(c *dstutil.Cursor) { + file, ok := c.Node().(*dst.File) + if !ok { + return + } + + var newDecls []dst.Decl + + for _, decl := range file.Decls { + funcDecl, ok := decl.(*dst.FuncDecl) + if !ok { + newDecls = append(newDecls, decl) + continue + } + if funcDecl.Recv != nil || !t.testFuncs[funcDecl.Name.Name] { + newDecls = append(newDecls, decl) + continue + } + + oldName := funcDecl.Name.Name + newName := t.globalInfo.ByPackage[t.pkgPath].Rewritten[oldName] // "Impl" + oldName + + funcDecl.Name.Name = newName + newDecls = append(newDecls, funcDecl) + /* + newDecls = append(newDecls, &dst.FuncDecl{ + Name: dst.NewIdent(oldName), + Type: &dst.FuncType{ + Params: &dst.FieldList{ + List: []*dst.Field{ + { + Names: []*dst.Ident{ + dst.NewIdent("t"), + }, + Type: &dst.StarExpr{ + X: &dst.Ident{ + Path: "testing", + Name: "T", + }, + }, + }, + }, + }, + }, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: &dst.Ident{Path: t.replacedPkgs[testingPackage], Name: "RunTest"}, + Args: []dst.Expr{ + dst.NewIdent("t"), + dst.NewIdent(newName), + }, + }, + }, + }, + }, + }) + */ + } + + file.Decls = newDecls +} + +func (t *packageTranslator) makeMetaTestFile(pkgName string, tests []packageSelector, runtime string) (*dst.File, error) { + var elts []dst.Expr + slices.SortFunc(tests, func(a, b packageSelector) int { + if d := cmp.Compare(a.Pkg, b.Pkg); d != 0 { + return d + } + return cmp.Compare(a.Selector, b.Selector) + }) + + for _, name := range tests { + elts = append(elts, &dst.CompositeLit{ + Elts: []dst.Expr{ + &dst.KeyValueExpr{ + Key: dst.NewIdent("Name"), // XXX: what happens if two tests have the same name in foo and foo_test? + Value: &dst.BasicLit{Kind: token.STRING, Value: strconv.Quote(name.Selector)}, + }, + &dst.KeyValueExpr{ + Key: dst.NewIdent("Test"), + // XXX: include name + // XXX: include original pkg + Value: &dst.Ident{ + Path: t.replacedPkgs[name.Pkg], + Name: "Impl" + name.Selector, + }, + }, + }, + Decs: dst.CompositeLitDecorations{ + NodeDecs: dst.NodeDecs{ + Before: dst.NewLine, + After: dst.NewLine, + }, + }, + }) + } + + dstFile := &dst.File{ + Name: dst.NewIdent(pkgName + "_test"), + Decls: []dst.Decl{ + /* + &dst.GenDecl{ + Tok: token.VAR, + Specs: specs, + }, + */ + &dst.FuncDecl{ + Name: dst.NewIdent("init"), + Type: &dst.FuncType{ + Params: &dst.FieldList{}, + }, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: &dst.Ident{Path: t.replacedPkgs[gosimruntimePackage], Name: "SetAllTests"}, + Args: []dst.Expr{ + &dst.CompositeLit{ + Type: &dst.ArrayType{ + Elt: &dst.Ident{Path: t.replacedPkgs[gosimruntimePackage], Name: "Test"}, + }, + Elts: elts, + }, + }, + }, + Decs: dst.ExprStmtDecorations{ + NodeDecs: dst.NodeDecs{ + After: dst.NewLine, + }, + }, + }, + }, + }, + }, + &dst.FuncDecl{ + Name: dst.NewIdent("TestMain"), + Type: &dst.FuncType{ + Params: &dst.FieldList{ + List: []*dst.Field{ + { + Names: []*dst.Ident{ + dst.NewIdent("m"), + }, + Type: &dst.StarExpr{ + X: &dst.Ident{Path: "testing", Name: "M"}, + }, + }, + }, + }, + }, + Body: &dst.BlockStmt{ + List: []dst.Stmt{ + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: t.newRuntimeSelector("TestMain"), + Args: []dst.Expr{ + &dst.CallExpr{ + Fun: &dst.Ident{ + Path: t.replacedPkgs[runtime], + Name: "Runtime", + }, + }, + }, + }, + Decs: dst.ExprStmtDecorations{ + NodeDecs: dst.NodeDecs{ + After: dst.NewLine, + }, + }, + }, + /* + &dst.ExprStmt{ + X: &dst.CallExpr{ + Fun: &dst.Ident{Path: "os", Name: "Exit"}, + Args: []dst.Expr{ + &dst.CallExpr{ + Fun: &dst.SelectorExpr{ + X: dst.NewIdent("m"), + Sel: dst.NewIdent("Run"), + }, + }, + }, + }, + Decs: dst.ExprStmtDecorations{ + NodeDecs: dst.NodeDecs{ + After: dst.NewLine, + }, + }, + }, + */ + }, + }, + }, + }, + } + + return dstFile, nil +} diff --git a/internal/translate/translate.go b/internal/translate/translate.go new file mode 100644 index 0000000..fd68479 --- /dev/null +++ b/internal/translate/translate.go @@ -0,0 +1,637 @@ +package translate + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "log" + "log/slog" + "maps" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/dave/dst" + "github.com/dave/dst/decorator" + "github.com/dave/dst/decorator/resolver/gotypes" + "github.com/dave/dst/dstutil" + "golang.org/x/tools/go/packages" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" +) + +type translatePackageArgs struct { + cfg gosimtool.BuildConfig + pkg *packages.Package + hooksPackage string + packageNames map[string]string + pkgWithTypesAndAst *packages.Package + replacedPkgs map[string]string + importResults map[string]*TranslatePackageResult +} + +type PackageGlobalInfo struct { + Container map[string]string + ShouldShare map[string]bool + Rewritten map[string]string +} + +type GlobalInfo struct { + ByPackage map[string]*PackageGlobalInfo +} + +func (gi *GlobalInfo) merge(pkg string, globals *PackageGlobalInfo) { + // XXX: these copies are a little jank. fixes race. handle more cleverly? + // (eg. keep both around?) happens for packages with test versions. why do + // they even show up here anyway? you can't import them. + if existing, ok := gi.ByPackage[pkg]; ok { + maps.Copy(existing.Container, globals.Container) + maps.Copy(existing.Rewritten, globals.Rewritten) + } else { + gi.ByPackage[pkg] = &PackageGlobalInfo{ + Container: maps.Clone(globals.Container), + ShouldShare: maps.Clone(globals.ShouldShare), + Rewritten: maps.Clone(globals.Rewritten), + } + } +} + +// exported for gob (?) +type TranslatePackageResult struct { + PackageName string + PackagePath string + CollectedTests []packageSelector + GlobalInfo *PackageGlobalInfo + TranslatedFiles map[string][]byte +} + +func filterTestFiles(files []*ast.File, fset *token.FileSet) []*ast.File { + var filtered []*ast.File + for _, file := range files { + fullPath := fset.File(file.Package).Name() + if strings.HasSuffix(filepath.Base(fullPath), "_test.go") { + filtered = append(filtered, file) + } + } + return filtered +} + +func renameFile(filePath string) string { + if strings.HasSuffix(filePath, "_test.go") { + return strings.TrimSuffix(filePath, "_test.go") + "_gosim_test.go" + } + if strings.HasSuffix(filePath, ".go") { + return strings.TrimSuffix(filePath, ".go") + "_gosim.go" + } + panic(filePath) +} + +func pathForFile(pkg, file string) string { + if !strings.HasPrefix(pkg, "translated/") { + panic("bad path") + } + pkg = strings.TrimPrefix(pkg, "translated/") + pkg = strings.TrimSuffix(pkg, "_test") + return filepath.Join(pkg, file) +} + +func translatePackage(args *translatePackageArgs) *TranslatePackageResult { + kind, extractedPath := classifyPackage(args.pkg) + + writer := newOutputWriter() + + // build meta-test + if kind == PackageKindTestBinary { + var collectedTests []packageSelector + var packageName string + + for _, dep := range args.pkg.Imports { + kind2, path2 := classifyPackage(dep) + if path2 == extractedPath && (kind2 == PackageKindTests || kind2 == PackageKindForTest) { + results := args.importResults[dep.ID] + if results != nil && results.CollectedTests != nil { + collectedTests = append(collectedTests, results.CollectedTests...) + } + if results != nil && results.PackageName != "" { + packageName = results.PackageName + } + } + } + + outputPackage := args.replacedPkgs[extractedPath] + + // XXX: help + t := &packageTranslator{ + replacedPkgs: args.replacedPkgs, + } + + file, err := t.makeMetaTestFile(packageName, collectedTests, args.hooksPackage) // XXX get right pkg name please.... + if err != nil { + log.Fatal(err) + } + bytes, err := dstFileToBytes(file, args.packageNames, args.replacedPkgs[extractedPath]+"_test") + if err != nil { + log.Fatal(err) + } + if err := writer.stage(pathForFile(outputPackage, "gosim_meta_test.go"), bytes); err != nil { + log.Fatal(err) + } + return &TranslatePackageResult{TranslatedFiles: writer.extract()} + } + + if kind != PackageKindBase && kind != PackageKindForTest && kind != PackageKindTests { + panic("help") + } + + allGlobals := &GlobalInfo{ + ByPackage: make(map[string]*PackageGlobalInfo), + } + for _, prev := range args.importResults { + if prev != nil && prev.GlobalInfo != nil { + allGlobals.merge(prev.PackagePath, prev.GlobalInfo) + } + } + + if args.pkgWithTypesAndAst == nil { + log.Fatalf("could not load package %s", args.pkg.ID) + } + args.pkg = args.pkgWithTypesAndAst + + pkgFiles := args.pkg.Syntax + if kind == PackageKindForTest { + // The PackageKindForTest build for package "foo" includes both _test.go + // files with "package foo" (non-test) and normal .go files with + // "package foo". The normal files are already translated with + // PackageKindBase, so skip them here. Translating again would duplicate + // the globals. + pkgFiles = filterTestFiles(pkgFiles, args.pkg.Fset) + } + + dec := decorator.NewDecorator(args.pkg.Fset) + dec.Resolver = gotypes.New(args.pkg.TypesInfo.Uses) + dec.Path = args.pkg.PkgPath + + var dstFiles []*dst.File + dstFilePaths := make(map[*dst.File]string) + + // translate package files + for _, file := range pkgFiles { + filePath := args.pkg.Fset.File(file.Package).Name() + dstFile, err := dec.DecorateFile(file) + if err != nil { + log.Fatal("decorate", filePath, err) + } + dstFiles = append(dstFiles, dstFile) + dstFilePaths[dstFile] = filePath + } + + testFuncs := make(map[string]bool) + + var results TranslatePackageResult + + globals := collectGlobalsAndTests(dstFiles, dec.Ast, args.pkg.PkgPath, args.pkg.TypesInfo, globalsDontTranslateGo123) + globalInfo := &PackageGlobalInfo{ + Container: make(map[string]string), + ShouldShare: make(map[string]bool), + Rewritten: make(map[string]string), + } + + // if args.pkg.PkgPath == "net/http" { + // log.Println(args.pkg.PkgPath, collectGlobalsSSA(args.pkg)) + ssaGlobals := collectGlobalsSSA(args.pkg) + // } + + globalField := map[packageKind]string{ + // TODO: share these constants with the global file code + PackageKindBase: "G", + PackageKindTests: "G", + PackageKindForTest: "GForTest", + }[kind] + for _, global := range globals.globalsFields { + if _, ok := globalInfo.Container[global]; ok { + log.Fatal("help", args.pkg.PkgPath, global) + } + if !globals.shouldShare[global] { + globalInfo.Container[global] = globalField + } + } + for global, val := range globals.shouldShare { + globalInfo.ShouldShare[global] = val + } + for _, info := range ssaGlobals.readonlyMaps { + globalInfo.ShouldShare[info.name] = true + delete(globalInfo.Container, info.name) + if info.onceName != "" && info.onceFn != "" { + globalInfo.ShouldShare[info.onceName] = true + delete(globalInfo.Container, info.onceName) + } + } + + if kind == PackageKindBase || kind == PackageKindForTest { + results.PackageName = args.pkg.Name + } + if kind == PackageKindTests { + results.PackageName = strings.TrimSuffix(args.pkg.Name, "_test") + } + if kind == PackageKindForTest || kind == PackageKindTests { + for _, test := range globals.tests { + testFuncs[test] = true + results.CollectedTests = append(results.CollectedTests, packageSelector{Pkg: args.pkg.PkgPath, Selector: test}) + globalInfo.Rewritten[test] = "Impl" + test + } + } + + outputPackage := args.replacedPkgs[args.pkg.PkgPath] + + // handle standard library vendored packages + localReplacedPkgs := maps.Clone(args.replacedPkgs) + for path, dep := range args.pkg.Imports { + if path != dep.PkgPath { + if args.pkg.Module != nil { + log.Fatal("pkg with vendored dep not in stdlib ", path) + } + if dep.Module != nil { + log.Fatal("vendored dep not in stdlib ", dep.PkgPath) + } + // XXX: check if this should be rewritten at all? + if _, ok := localReplacedPkgs[dep.PkgPath]; ok { + localReplacedPkgs[path] = localReplacedPkgs[dep.PkgPath] + } + } + } + + forTest := kind == PackageKindForTest + + allGlobals.merge(args.pkg.PkgPath, globalInfo) + + hooks := maps.Clone(hooksGo123) + maps.Copy(hooks, hooksGensyscallGo123ByArch[args.cfg.GOARCH]) + + translator := &packageTranslator{ + typesInfo: args.pkg.TypesInfo, + astMap: dec.Map.Ast, + dstMap: dec.Map.Dst, + replacedPkgs: localReplacedPkgs, + pkgPath: args.pkg.PkgPath, + implicitConversions: buildImplicitConversions(args.pkg), + globalInfo: allGlobals, + testFuncs: testFuncs, + forTest: forTest, + collect: translateCollect{ + bindspecs: make(map[bindspec]struct{}), + maps: ssaGlobals.readonlyMaps, + }, + hooks: hooks, + acceptedLinknames: acceptedgo123Linknames, + keepAsmPkgs: keepAsmPackagesGo123, + } + + // translate package files + for _, dstFile := range dstFiles { + filePath := dstFilePaths[dstFile] + file, err := translator.translateFile(dstFile) + if err != nil { + log.Fatal("translate", filePath, err) + } + bytes, err := dstFileToBytes(file, args.packageNames, "main") + if err != nil { + log.Fatal(err) + } + if err := writer.stage(pathForFile(outputPackage, renameFile(filepath.Base(filePath))), bytes); err != nil { + log.Fatal(err) + } + } + + if globalsName, ok := map[packageKind]string{ + PackageKindBase: "globals.go", + PackageKindForTest: "globals_for_test.go", + PackageKindTests: "globals_test.go", + }[kind]; ok { + // write a globals.go, if applicable + if len(translator.collect.globalFields) > 0 || len(translator.collect.inits) > 0 || len(translator.collect.bindspecs) > 0 || len(translator.collect.sharedGlobalFields) > 0 || len(translator.collect.maps) > 0 { + file, err := makeDetgoGlobalsFile(translator, args.pkg.Name, forTest) + if err != nil { + log.Fatal(err) + } + bytes, err := dstFileToBytes(file, args.packageNames, "main") + if err != nil { + log.Fatal(err) + } + if err := writer.stage(pathForFile(outputPackage, globalsName), bytes); err != nil { + log.Fatal(err) + } + } + } + + if kind == PackageKindBase && translator.keepAsmPkgs[args.pkg.PkgPath] { + // copy over assembly implementations for specific packages + if translator.keepAsmPkgs[args.pkg.PkgPath] { + for _, filePath := range args.pkg.OtherFiles { + slog.Debug("copying other file", "pkg", args.pkg.PkgPath, "file", filePath) + bytes, err := os.ReadFile(filePath) + if err != nil { + log.Fatal(err) + } + if err := writer.stage(pathForFile(outputPackage, filepath.Base(filePath)), bytes); err != nil { + log.Fatal(err) + } + } + } + } + + // write a gosimaliashack.go file if needed + if kind == PackageKindBase { + // XXX: compute these automatically? + if vars, ok := PublicExportHacks[args.pkg.PkgPath]; ok { + file, err := makePublicExportHackFile(path.Base(outputPackage), vars) + if err != nil { + log.Fatal(err) + } + bytes, err := dstFileToBytes(file, args.packageNames, "main") + if err != nil { + log.Fatal(err) + } + if err := writer.stage(pathForFile(outputPackage, "gosimaliashack.go"), bytes); err != nil { + log.Fatal(err) + } + } + } + + results.GlobalInfo = globalInfo + results.PackagePath = args.pkg.PkgPath + results.TranslatedFiles = writer.extract() + + return &results +} + +type translateCollect struct { + globalFields []*dst.Field + sharedGlobalFields []dst.Spec + initIdx int + inits []string + bindspecs map[bindspec]struct{} + maps []mapInitializer +} + +type packageSelector struct { + Pkg string + Selector string +} + +type packageTranslator struct { + typesInfo *types.Info + suffixCounter int + astMap decorator.AstMap + dstMap decorator.DstMap + replacedPkgs map[string]string + pkgPath string + + hooks map[packageSelector]packageSelector + acceptedLinknames map[packageSelector]packageSelector + keepAsmPkgs map[string]bool + + collect translateCollect + implicitConversions map[ast.Expr]types.Type + + globalInfo *GlobalInfo + + needsUnsafe bool + + forTest bool + + testFuncs map[string]bool +} + +func (t *packageTranslator) isBuiltIn(expr dst.Expr) bool { + return t.typesInfo.Types[t.astMap.Nodes[expr].(ast.Expr)].IsBuiltin() +} + +func (t *packageTranslator) isNamedBuiltIn(expr dst.Expr, name string) bool { + if ident, ok := expr.(*dst.Ident); ok { + return ident.Name == name && t.isBuiltIn(expr) + } + return false +} + +func isDualAssign(c *dstutil.Cursor) (rhs *dst.Expr, isDualAssign bool) { + // read from chan or map in (x, ok) form + // "x, ok =" and "x, ok :=" + if assignStmt, ok := c.Node().(*dst.AssignStmt); ok && len(assignStmt.Lhs) == 2 && len(assignStmt.Rhs) == 1 { + return &assignStmt.Rhs[0], true + } + // "var x, ok" + if varSpec, ok := c.Node().(*dst.ValueSpec); ok && len(varSpec.Names) == 2 && len(varSpec.Values) == 1 { + return &varSpec.Values[0], true + } + + return nil, false +} + +func (t *packageTranslator) suffix() string { + suffix := fmt.Sprint(t.suffixCounter) + t.suffixCounter++ + return suffix +} + +func (t *packageTranslator) getType(expr dst.Expr) (types.Type, bool) { + astExpr, ok := t.astMap.Nodes[expr].(ast.Expr) + if !ok { + // XXX hmmm??? why does this happen?? + return nil, false + } + if typ, ok := t.typesInfo.Types[astExpr]; ok { + return typ.Type, true + } + return nil, false +} + +func (t *packageTranslator) markSyncFuncsNorace(c *dstutil.Cursor) { + node := c.Node() + + decl, ok := node.(*dst.FuncDecl) + if !ok { + return + } + + if t.pkgPath == "sync" { + // XXX: jank mark all funcs norace in "sync" package + decl.Decs.Start = append(decl.Decs.Start, "//go:norace") + } +} + +func (t *packageTranslator) newRuntimeSelector(name string) *dst.Ident { + return &dst.Ident{ + Name: name, + Path: t.replacedPkgs[gosimruntimePackage], + } +} + +func (t *packageTranslator) apply(node dst.Node) dst.Node { + return dstutil.Apply(node, t.preApply, t.postApply) +} + +func (t *packageTranslator) preApply(c *dstutil.Cursor) bool { + if f, ok := c.Node().(*dst.File); ok { + for i, dec := range f.Decs.Start { + if strings.HasPrefix(dec, "//go:build") { + f.Decs.Start[i] = "//" // "//gosim filtered: " + dec + } + if strings.HasPrefix(dec, "// +build") { + f.Decs.Start[i] = "//" // "//gosim filtered: " + dec + } + } + } + + t.rewriteTestFunc(c) + t.rewriteIdent(c) + t.rewriteImport(c) + t.rewriteNotifyListHack(c) + t.rewriteJsonGlobalsHack(c) + + t.rewriteMapGetOk(c) + t.rewriteChanRecvOk(c) + t.rewriteMapLiteral(c) + t.rewriteChanLiteral(c) + t.rewriteMapLen(c) + t.rewriteMapAssign(c) + t.rewriteMapIndex(c) + t.rewriteMapDelete(c) + t.rewriteMapClear(c) + t.rewriteMakeMap(c) + t.rewriteMakeChan(c) + t.rewriteMapType(c) + t.rewriteChanLen(c) + t.rewriteChanCap(c) + t.rewriteGlobalDef(c) + t.rewriteGlobalRead(c) // XXX: after rewriteIdent + t.rewriteInit(c) + t.markSyncFuncsNorace(c) + t.rewriteStdlibEmptyAndLinkname(c) + + t.rewriteDanglingLinknames(c) + + return true +} + +func (t *packageTranslator) postApply(c *dstutil.Cursor) bool { + t.rewriteChanType(c) + t.rewriteChanRecvSimpleExpr(c) + t.rewriteChanRange(c) + t.rewriteChanClose(c) + t.rewriteChanSend(c) + + t.rewriteGo(c) + + return true +} + +func (t *packageTranslator) translateFile(dstFile *dst.File) (*dst.File, error) { + // XXX: janky + var keptPkgs []string + for _, decl := range dstFile.Decls { + genDecl, ok := decl.(*dst.GenDecl) + if !ok { + continue + } + if genDecl.Tok != token.IMPORT { + continue + } + for _, spec := range genDecl.Specs { + importSpec := spec.(*dst.ImportSpec) + decs := importSpec.Decs.End.All() + if len(decs) >= 1 && decs[0] == "//gosim:notranslate" { + unquoted, err := strconv.Unquote(importSpec.Path.Value) + if err != nil { + return nil, err + } + keptPkgs = append(keptPkgs, unquoted) + } + } + } + if len(keptPkgs) > 0 { + old := t.replacedPkgs + defer func() { + t.replacedPkgs = old + }() + t.replacedPkgs = maps.Clone(t.replacedPkgs) + for _, pkg := range keptPkgs { + delete(t.replacedPkgs, pkg) + } + } + + t.needsUnsafe = false + + // in a first pass, rewrite all select stmts. unclear if this is necessary, + // but trying to do it preApply didn't translate inner nested selects. + dstFile = dstutil.Apply(dstFile, nil, func(c *dstutil.Cursor) bool { + t.rewriteSelectStmt(c) + t.rewriteMapRange(c) + t.rewriteGlobalDef(c) // XXX: help. this is for var closedchan = make(chan struct{}) + return true + }).(*dst.File) + + dstFile = t.apply(dstFile).(*dst.File) + + if t.needsUnsafe { + var importDecl *dst.GenDecl + + for _, decl := range dstFile.Decls { + genDecl, ok := decl.(*dst.GenDecl) + if !ok { + continue + } + + if genDecl.Tok != token.IMPORT { + continue + } + + importDecl = genDecl + } + + // TODO: do this earlier, messes up comments + if t.needsUnsafe { + if importDecl == nil { + importDecl = &dst.GenDecl{ + Tok: token.IMPORT, + } + dstFile.Decls = append([]dst.Decl{importDecl}, dstFile.Decls...) + } + importDecl.Specs = append(importDecl.Specs, + &dst.ImportSpec{ + Name: dst.NewIdent("_"), + Path: &dst.BasicLit{Kind: token.STRING, Value: strconv.Quote("unsafe")}, + }, + ) + } + } + + return dstFile, nil +} + +func makePublicExportHackFile(pkgName string, names []string) (*dst.File, error) { + slices.Sort(names) + var specs []dst.Spec + for _, name := range names { + specs = append(specs, &dst.TypeSpec{ + Name: dst.NewIdent("GosimPublicExportHack" + name), + Assign: true, + Type: dst.NewIdent(name), + }) + } + + dstFile := &dst.File{ + Name: dst.NewIdent(pkgName), + Decls: []dst.Decl{ + &dst.GenDecl{ + Tok: token.TYPE, + Specs: specs, + }, + }, + } + + return dstFile, nil +} diff --git a/internal/translate/translate_test.go b/internal/translate/translate_test.go new file mode 100644 index 0000000..f506469 --- /dev/null +++ b/internal/translate/translate_test.go @@ -0,0 +1,221 @@ +package translate + +import ( + "cmp" + "flag" + "go/format" + "io/fs" + "log" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + + gocmp "github.com/google/go-cmp/cmp" + "golang.org/x/tools/txtar" + + "github.com/jellevandenhooff/gosim/internal/gosimtool" +) + +var ( + rewrite = flag.Bool("rewrite", false, "rewrite golden outputs") + useworkdir = flag.Bool("useworkdir", false, "write in testdata/workdir instead of tempdir") +) + +// TestTranslate runs gosim translate on all go files specified in testdata/. +// +// Each file in testdata that ends in .translate.txt is a directory that will be +// translated. All files in all directories are put in one module and +// translated at the same time because otherwise this test gets very slow. +// +// The expected output of the translation is stored with the testdata/ (next to +// each input file) and checked or rewritten depending on the -rewrite flag. +// +// TODO: this test relies on a compiled and up-to-date gosim binary from the +// script test... this is brittle and will be a pain at some point... +func TestTranslate(t *testing.T) { + entries, err := os.ReadDir("./testdata") + if err != nil { + t.Fatal(err) + } + + // prepare work directory + var workDir string + if *useworkdir { + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + workDir = path.Join(wd, "testdata/workdir") + if err := os.RemoveAll(workDir); err != nil { + // XXX: only empty files inside workdir? + t.Fatal(err) + } + if err := os.MkdirAll(workDir, 0o755); err != nil { + t.Fatal(err) + } + } else { + workDir = t.TempDir() + } + + curModDir, err := gosimtool.FindGoModDir() + if err != nil { + t.Fatal(err) + } + + extractedModDir := filepath.Join(curModDir, gosimtool.OutputDirectory, "scripttest", "mod") + + // add test dependencies to the copied gosim module + if err := filepath.Walk(extractedModDir, func(path string, info fs.FileInfo, err error) error { + return nil + }); err != nil { + log.Fatal(err) + } + + // put a fake go.mod in the work directory + if err := gosimtool.MakeGoModForTest(extractedModDir, workDir, []string{ + "github.com/dave/dst", + "github.com/google/go-cmp", + "github.com/mattn/go-isatty", + "github.com/mattn/go-sqlite3", + "golang.org/x/mod", + "golang.org/x/sync", + "golang.org/x/sys", + "golang.org/x/tools", + "mvdan.cc/gofumpt", + }); err != nil { + t.Fatal(err) + } + + // TODO: dedup this code with scripttest, somehow + binPath := filepath.Join(curModDir, gosimtool.OutputDirectory, "scripttest", "bin", "gosim") + envVars := append(os.Environ(), "GOSIMCACHE="+filepath.Join(curModDir, gosimtool.OutputDirectory)) + + // parse all inputs and write them to work directory + archiveByPackage := make(map[string]*txtar.Archive) + desiredFilesByPackage := make(map[string][]txtar.File) + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".translate.txt") { + continue + } + pkgName := strings.TrimSuffix(entry.Name(), ".translate.txt") + + archive, err := txtar.ParseFile(path.Join("./testdata", entry.Name())) + if err != nil { + t.Error(err) + } + + // store archive to check against later + archiveByPackage[pkgName] = archive + + for _, file := range archive.Files { + // don't write output files + if strings.HasPrefix(file.Name, "translated/") { + continue + } + + p := path.Join(workDir, pkgName, file.Name) + if err := os.MkdirAll(path.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, file.Data, 0o644); err != nil { + t.Fatal(err) + } + + // keep a formatted version of this file to check against later + formatted, err := format.Source(file.Data) + if err != nil { + t.Fatal(err) + } + desiredFilesByPackage[pkgName] = append(desiredFilesByPackage[pkgName], txtar.File{ + Name: file.Name, + Data: formatted, + }) + } + } + + // run the tool in the work directory + // TODO: use a race version, optionally + cmd := exec.Command(binPath, "translate", "./...") + cmd.Dir = workDir + cmd.Env = envVars + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + os.Exit(exitErr.ExitCode()) + } + log.Fatal(err) + } + + cfg := gosimtool.BuildConfig{ + GOOS: "linux", + // TODO: varying the architecture for snapshots is fine as long as we don't translate arch-specific stuff; cross-arch tests should catch problems... + GOARCH: runtime.GOARCH, + Race: false, + } + outputPath := filepath.Join(workDir, gosimtool.OutputDirectory, "translated", cfg.AsDirname(), "test") + + // read all output files + if err := filepath.WalkDir(outputPath, func(path string, d fs.DirEntry, _ error) error { + if !d.Type().IsRegular() { + return nil + } + + trimmedPath := strings.TrimPrefix(path, outputPath+"/") + + outBytes, err := os.ReadFile(path) + if err != nil { + return err + } + + pkgName := trimmedPath[:strings.Index(trimmedPath, "/")] + if _, ok := desiredFilesByPackage[pkgName]; !ok { + t.Errorf("help unknown package %q for path %q", pkgName, path) + } + desiredFilesByPackage[pkgName] = append(desiredFilesByPackage[pkgName], txtar.File{ + Name: "translated/" + trimmedPath, + Data: outBytes, + }) + return nil + }); err != nil { + t.Fatal(err) + } + + // check each input file one-by-one + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".translate.txt") { + continue + } + pkgName := strings.TrimSuffix(entry.Name(), ".translate.txt") + + desiredFiles := desiredFilesByPackage[pkgName] + + slices.SortFunc(desiredFiles, func(a, b txtar.File) int { + plainA := strings.TrimPrefix(a.Name, "translated/"+pkgName+"/") + plainB := strings.TrimPrefix(b.Name, "translated/"+pkgName+"/") + return cmp.Compare(plainA, plainB) + }) + + archive := archiveByPackage[pkgName] + + if *rewrite { + newArchive := &txtar.Archive{ + Comment: archive.Comment, + Files: desiredFiles, + } + if err := os.WriteFile(path.Join("./testdata", entry.Name()), txtar.Format(newArchive), 0o644); err != nil { + t.Error(err) + } + } else { + if diff := gocmp.Diff(archive.Files, desiredFiles); diff != "" { + t.Error(diff) + } + } + } +} diff --git a/internal/translate/types.go b/internal/translate/types.go new file mode 100644 index 0000000..61d7496 --- /dev/null +++ b/internal/translate/types.go @@ -0,0 +1,462 @@ +package translate + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "log" + "slices" + "strconv" + + "github.com/dave/dst" + "golang.org/x/tools/go/ast/astutil" + "golang.org/x/tools/go/packages" +) + +// makeTypeExpr converts a type-checker types.Type into an AST expression +// describing that type. For example, it converts a *types.Slice of a +// *types.Named "foo.Id" into a *dst.ArrayType of a *dst.Ident "foo.Id". +// +// In most cases its preferable to reuse AST expressions instead of building new +// ones from types, but sometimes translated codes needs a type when no +// convenient AST expression exists. See the callers of makeTypeExpr. +func (t *packageTranslator) makeTypeExpr(typ types.Type) dst.Expr { + if typ == nil { + return nil + } + + // XXX: + // - more type params + // - anonymous structs + // - anonymous interfaces + // - ??? + + switch typ := typ.(type) { + case *types.Alias: + // helpp, copy pasted from Named below + + var path string + if pkg := typ.Obj().Pkg(); pkg != nil { + // eg. builtin error has no path + path = pkg.Path() + if path == t.pkgPath { + path = "" + } + } + + id := &dst.Ident{Name: typ.Obj().Name(), Path: path} + t.rewriteIdentImpl(id) + + if id.Path != "" && id.Path != t.pkgPath && !token.IsExported(id.Name) { + // bad news bears here... + if slices.Index(PublicExportHacks[path], id.Name) == -1 { + log.Fatalf("need public export hack for %s.%s", path, id.Name) + } + id.Name = "GosimPublicExportHack" + id.Name + } + + res := dst.Expr(id) + if typ.TypeArgs() != nil { + var args []dst.Expr + for i := 0; i < typ.TypeArgs().Len(); i++ { + args = append(args, t.makeTypeExpr(typ.TypeArgs().At(i))) + } + res = &dst.IndexListExpr{ + X: res, + Indices: args, + } + } + + return res + + case *types.Named: + var path string + if pkg := typ.Obj().Pkg(); pkg != nil { + // eg. builtin error has no path + path = pkg.Path() + if path == t.pkgPath { + path = "" + } + } + + id := &dst.Ident{Name: typ.Obj().Name(), Path: path} + t.rewriteIdentImpl(id) + + if id.Path != "" && id.Path != t.pkgPath && !token.IsExported(id.Name) { + // bad news bears here... + if slices.Index(PublicExportHacks[path], id.Name) == -1 { + log.Fatalf("need public export hack for %s.%s", path, id.Name) + } + id.Name = "GosimPublicExportHack" + id.Name + } + + res := dst.Expr(id) + if typ.TypeArgs() != nil { + var args []dst.Expr + for i := 0; i < typ.TypeArgs().Len(); i++ { + args = append(args, t.makeTypeExpr(typ.TypeArgs().At(i))) + } + res = &dst.IndexListExpr{ + X: res, + Indices: args, + } + } + + return res + + case *types.Basic: + path := "" + if token.IsExported(typ.Name()) { + // XXX: hack for unsafe.Pointer + path = "unsafe" + } + return &dst.Ident{Name: typ.Name(), Path: path} + case *types.Pointer: + return &dst.StarExpr{X: t.makeTypeExpr(typ.Elem())} + case *types.Slice: + return &dst.ArrayType{Elt: t.makeTypeExpr(typ.Elem())} + case *types.Map: + return &dst.IndexListExpr{ + X: t.newRuntimeSelector("Map"), + Indices: []dst.Expr{ + t.makeTypeExpr(typ.Key()), + t.makeTypeExpr(typ.Elem()), + }, + } + // return &dst.MapType{Key: t.makeType(typ.Key()), Value: t.makeType(typ.Elem())} + case *types.Chan: + return &dst.IndexListExpr{ + X: t.newRuntimeSelector("Chan"), + Indices: []dst.Expr{ + t.makeTypeExpr(typ.Elem()), + }, + } + /* + dir := map[types.ChanDir]dst.ChanDir{ + types.SendRecv: dst.SEND | dst.RECV, + types.SendOnly: dst.SEND, + types.RecvOnly: dst.RECV, + }[typ.Dir()] + // XXX: use netDetgoSelector instead?? + return &dst.ChanType{Dir: dir, Value: t.makeType(typ.Elem())} + */ + case *types.Array: + return &dst.ArrayType{Len: &dst.BasicLit{Kind: token.INT, Value: fmt.Sprint(typ.Len())}, Elt: t.makeTypeExpr(typ.Elem())} + case *types.Interface: + if typ.Empty() { + return &dst.InterfaceType{Methods: &dst.FieldList{ + Opening: true, + Closing: true, + }} + // XXX: any might overlap HELP + // return &dst.Ident{Name: "any"} + } + var methods []*dst.Field + for i := 0; i < typ.NumExplicitMethods(); i++ { + m := typ.ExplicitMethod(i) + methods = append(methods, &dst.Field{ + Names: []*dst.Ident{dst.NewIdent(m.Name())}, // XXX: wrong? + Type: t.makeTypeExpr(m.Type()), + }) + } + return &dst.InterfaceType{ + Methods: &dst.FieldList{ + List: methods, + }, + } + + case *types.Struct: + /* + if typ.NumFields() != 0 { + panic(typ) + } + */ + // XXX: jank return empty structs for lols + if typ.NumFields() == 0 { + return &dst.StructType{ + Fields: &dst.FieldList{ + Opening: true, + Closing: true, + }, + } + } + var fields []*dst.Field + for i := 0; i < typ.NumFields(); i++ { + field := typ.Field(i) + + var names []*dst.Ident + if !field.Anonymous() { + names = []*dst.Ident{dst.NewIdent(field.Name())} + } + fields = append(fields, &dst.Field{ + Names: names, + Type: t.makeTypeExpr(field.Type()), + Tag: &dst.BasicLit{Kind: token.STRING, Value: strconv.Quote(typ.Tag(i))}, // XXX: wrong? + }) + } + return &dst.StructType{ + Fields: &dst.FieldList{ + List: fields, + }, + } + case *types.Signature: + /* + // XXX: skip; this lets us output interfaces hopefully + if typ.Recv() != nil { + panic(typ) + } + */ + if typ.RecvTypeParams() != nil { + panic(typ) + } + if typ.TypeParams() != nil { + panic(typ) + } + if typ.Variadic() { + panic(typ) + } + + var params, results []*dst.Field + + for i := 0; i < typ.Params().Len(); i++ { + param := typ.Params().At(i) + + params = append(params, &dst.Field{ + Names: []*dst.Ident{dst.NewIdent(param.Name())}, + Type: t.makeTypeExpr(param.Type()), + }) + } + + for i := 0; i < typ.Results().Len(); i++ { + param := typ.Results().At(i) + results = append(results, &dst.Field{ + Names: []*dst.Ident{dst.NewIdent(param.Name())}, + Type: t.makeTypeExpr(param.Type()), + }) + } + + return &dst.FuncType{ + Func: true, + Params: &dst.FieldList{ + List: params, + }, + Results: &dst.FieldList{ + List: results, + }, + } + case *types.TypeParam: + // XXX help + return dst.NewIdent(typ.Obj().Name()) + + default: + panic(typ) + } +} + +// buildImplicitConversions computes the type of expressions that are converted +// according to the assignability (https://go.dev/ref/spec#Assignability) rule +// in the go spec. +// +// An implicit conversion happens on assignments like: +// +// types Values map[string]string +// var foo Values /* (the type) */ = map[string]string{} /* (the value) */ +// +// Here, the spec allows assigning a value of unnamed type map[string]string{} +// to the variable foo of type Values. This is an implicit conversion of a +// map[string]string into a Values. For gosim this causes a problem because the +// translated type of map[string]string is a named type Map[string, string] +// which cannot be converted to a Values without an explicit conversion. +// +// buildImplicitConversions returns a map from ast.Expr of values that are +// implicitly converted to the type they are converted to. In this example, it +// would map the expression of the literal map[string]string{} to the Values +// type. +// +// Issue https://github.com/golang/go/issues/47151 tracks adding this +// functionality to the go/types package. +func buildImplicitConversions(pkg *packages.Package) map[ast.Expr]types.Type { + builder := &implicitConversionsBuilder{ + conversions: make(map[ast.Expr]types.Type), + typesInfo: pkg.TypesInfo, + sigstack: nil, + cursig: nil, + } + for _, file := range pkg.Syntax { + astutil.Apply(file, builder.before, builder.after) + } + return builder.conversions +} + +type implicitConversionsBuilder struct { + // conversions is the computed conversions map + conversions map[ast.Expr]types.Type + + typesInfo *types.Info + + // cursig and sigstack track the signature of the functions the astutil + // is visiting, used to compute the expected types for return statements. + // before pushes onto the stack, after pops. + cursig *types.Signature + sigstack []*types.Signature +} + +func (b *implicitConversionsBuilder) before(c *astutil.Cursor) bool { + switch node := c.Node().(type) { + case *ast.FuncDecl: + if b.typesInfo.Defs[node.Name] == nil { + log.Println(node.Name) + } + sig := b.typesInfo.Defs[node.Name].Type().(*types.Signature) + b.sigstack = append(b.sigstack, sig) + b.cursig = sig + + case *ast.FuncLit: + sig := b.typesInfo.Types[node].Type.(*types.Signature) + b.sigstack = append(b.sigstack, sig) + b.cursig = sig + + case *ast.CompositeLit: + struc, ok := b.typesInfo.Types[node].Type.Underlying().(*types.Struct) + if !ok { + break + } + + for idx, elt := range node.Elts { + if node, ok := elt.(*ast.KeyValueExpr); ok { + y := b.typesInfo.Types[node.Value] + + name := node.Key.(*ast.Ident).Name + + for i := 0; i < struc.NumFields(); i++ { + field := struc.Field(i) + if name == field.Name() { + x := field.Type() + if !types.Identical(y.Type, x) { + b.conversions[node.Value] = x + } + } + } + } else if idx < struc.NumFields() { + x := struc.Field(idx).Type() + y := b.typesInfo.Types[elt] + if !types.Identical(y.Type, x) { + b.conversions[elt] = x + } + } + } + + case *ast.CallExpr: + tv := b.typesInfo.Types[node.Fun] + + if tv.IsType() { + // cast + x := b.typesInfo.Types[node.Fun].Type + y := b.typesInfo.Types[node.Args[0]] + if y.IsNil() { + if x == nil { + panic("help") + } + b.conversions[node.Args[0]] = x + } + } else if sig, ok := tv.Type.Underlying().(*types.Signature); ok { + // this breaks for a type Foo func() type-cast?!?!!??! + for i, y := range node.Args { + var x types.Type + if i >= sig.Params().Len()-1 && sig.Variadic() { + slice, ok := sig.Params().At(sig.Params().Len() - 1).Type().(*types.Slice) + if !ok { + continue + // XXX: this happens with a string and a splat maybe??? + } + if node.Ellipsis != token.NoPos { + x = slice + } else { + x = slice.Elem() + } + } else { + // log.Println(sig, i, sig.Params(), node.Args) + x = sig.Params().At(i).Type() + } + + yType := b.typesInfo.Types[y] + if !types.Identical(yType.Type, x) { + if x == nil { + panic("help") + } + b.conversions[y] = x + } + } + } else { + // XXX? + } + case *ast.AssignStmt: + // xxx: this can happen if we have a call + if len(node.Lhs) != len(node.Rhs) { + break + } + + for i := range node.Lhs { + x := b.typesInfo.Types[node.Lhs[i]] + y := b.typesInfo.Types[node.Rhs[i]] + if !types.Identical(x.Type, y.Type) { + if x.Type == nil { + continue + // happens for :=, var ... = + log.Println(node.Lhs, node.Rhs[i]) + panic("help") + } + b.conversions[node.Rhs[i]] = x.Type + } + } + + case *ast.BinaryExpr: + x := b.typesInfo.Types[node.X] + y := b.typesInfo.Types[node.Y] + if !types.Identical(x.Type, y.Type) { + if x.IsNil() { + if y.Type == nil { + panic("help") + } + b.conversions[node.X] = y.Type + } else { + if x.Type == nil { + panic("help") + } + b.conversions[node.Y] = x.Type + } + } + + case *ast.ReturnStmt: + // XXX: gotta check len because of returning multiple + if len(node.Results) > 0 && len(node.Results) == b.cursig.Results().Len() { + for i, expr := range node.Results { + x := b.typesInfo.Types[expr] + y := b.cursig.Results().At(i).Type() + if !types.Identical(x.Type, y) { + if y == nil { + panic("help") + } + b.conversions[expr] = y + } + } + } + } + return true +} + +func (b *implicitConversionsBuilder) after(c *astutil.Cursor) bool { + switch c.Node().(type) { + case *ast.FuncDecl: + b.sigstack = b.sigstack[:len(b.sigstack)-1] + case *ast.FuncLit: + b.sigstack = b.sigstack[:len(b.sigstack)-1] + } + if len(b.sigstack) == 0 { + b.cursig = nil + } else { + b.cursig = b.sigstack[len(b.sigstack)-1] + } + return true +} diff --git a/internal/translate/workqueue.go b/internal/translate/workqueue.go new file mode 100644 index 0000000..d6a0382 --- /dev/null +++ b/internal/translate/workqueue.go @@ -0,0 +1,153 @@ +package translate + +import ( + "sync" +) + +type depGraph struct { + nodes map[string]struct{} + deps map[string]map[string]struct{} +} + +func newDepGraph() *depGraph { + return &depGraph{ + nodes: make(map[string]struct{}), + deps: make(map[string]map[string]struct{}), + } +} + +func (d *depGraph) addNode(e string) { + d.nodes[e] = struct{}{} + d.deps[e] = make(map[string]struct{}) +} + +func (d *depGraph) addDep(from, to string) { + d.deps[from][to] = struct{}{} +} + +type workQueue struct { + afterMe map[string][]string + pending map[string]int + deps map[string][]string + queue []string +} + +func newWorkQueue(g *depGraph) *workQueue { + wq := &workQueue{ + afterMe: make(map[string][]string), + pending: make(map[string]int), + deps: make(map[string][]string), + } + for node := range g.nodes { + wq.pending[node] = 0 + } + for from, deps := range g.deps { + for to := range deps { + wq.pending[from]++ + wq.afterMe[to] = append(wq.afterMe[to], from) + wq.deps[from] = append(wq.deps[from], to) + } + } + + return wq +} + +func (wq *workQueue) next() (string, bool) { + if len(wq.queue) == 0 { + return "", false + } + e := wq.queue[0] + wq.queue = wq.queue[1:] + return e, true +} + +func (wq *workQueue) done() bool { + return len(wq.pending) == 0 && len(wq.queue) == 0 +} + +func (wq *workQueue) finish(e string) { + for _, next := range wq.afterMe[e] { + wq.pending[next]-- + if wq.pending[next] == 0 { + delete(wq.pending, next) + wq.queue = append(wq.queue, next) + } + } +} + +func (wq *workQueue) workInParallel(numWorkers int, worker func(string)) { + workCh := make(chan string, len(wq.pending)) + + var mu sync.Mutex + + submit := func() { + for { + work, ok := wq.next() + if !ok { + break + } + workCh <- work + if wq.done() { + close(workCh) + } + } + } + + for next, cnt := range wq.pending { + if cnt == 0 { + delete(wq.pending, next) + wq.queue = append(wq.queue, next) + } + } + submit() + + var wg sync.WaitGroup + for i := 0; i < numWorkers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for work := range workCh { + worker(work) + mu.Lock() + wq.finish(work) + submit() + mu.Unlock() + } + }() + } + wg.Wait() +} + +func collectPreviousResults[V any](g *depGraph, cur string, prev map[string]V) map[string]V { + results := make(map[string]V) + var collect func(string) + collect = func(e string) { + if _, ok := results[e]; ok { + return + } + results[e] = prev[e] + for dep := range g.deps[e] { + collect(dep) + } + } + collect(cur) + return results +} + +func buildInParallel[V any](g *depGraph, numWorkers int, results map[string]V, work func(e string, inputs map[string]V) V) { + var mu sync.Mutex + newWorkQueue(g).workInParallel(numWorkers, func(e string) { + mu.Lock() + // short-circuit + if _, ok := results[e]; ok { + mu.Unlock() + return + } + prev := collectPreviousResults(g, e, results) + mu.Unlock() + here := work(e, prev) + mu.Lock() + results[e] = here + mu.Unlock() + }) +} diff --git a/machine.go b/machine.go new file mode 100644 index 0000000..7189709 --- /dev/null +++ b/machine.go @@ -0,0 +1,202 @@ +package gosim + +import ( + "iter" + "net/netip" + + "github.com/jellevandenhooff/gosim/internal/simulation" + "github.com/jellevandenhooff/gosim/internal/simulation/fs" +) + +// A Machine represents a simulated machine. Each machine has its own disk, +// network stack, and set of global variables. Machines can only communicate +// over the network. +// +// A Machine can be stopped and restarted. After a restart, a Machine has +// a new, freshly-initialized set of global variables, but the same disk +// and network address as before. +// +// Once a machine's main function returns, the machine stops as +// if called [Machine.Stop] with all writes flushed to disk and +// all network connections closed gracefully. +// +// When a machine stops all code running on the machine is +// stopped without running any pending `defer`ed calls. +// All timers are stopped and will not fire. +type Machine struct { + // underlying + id int +} + +// CurrentMachine returns the Machine the caller is running on. +func CurrentMachine() Machine { + return Machine{id: simulation.CurrentMachineID()} +} + +// NewSimpleMachine creates a new Machine that will run the given main function. The +// Machine will start immediately. +// +// See [Machine] for a detailed description of machines and their behavior. +func NewSimpleMachine(mainFunc func()) Machine { + machineID := simulation.SyscallMachineNew("", "", mainFunc) + // TODO: make a New machine default to a Stopped state? + return Machine{id: machineID} +} + +// MachineConfig configures a Machine. +type MachineConfig struct { + // Label is the displayed name of a machine in the logs and the hostname + // returned by os.Hostname. Optional, will be a numbered machine if + // empty. + Label string + // Addr is the network address of the machine. Only IPv4 addresses are + // currently supported. Optional, will be allocated if empty. + Addr netip.Addr + // MainFunc is the starting function of the machine. When MainFunc returns + // the machine will be stopped. + MainFunc func() +} + +// NewMachine creates a new Machine with the given configuration. +// The Machine will start immediately. +// +// See [Machine] for a detailed description of machines and their behavior. +func NewMachine(opts MachineConfig) Machine { + machineID := simulation.SyscallMachineNew(opts.Label, opts.Addr.String(), opts.MainFunc) + return Machine{id: machineID} +} + +// Crash stops the machine harshly. A crash does not flush any non-fsynced +// writes to disk, and does not properly close open network connections. +func (m Machine) Crash() { + // TODO: add more tests around stop/crash/restart, for current machine/others. + // TODO: document what Crash does if the machine is already stopped? + + // if m == CurrentMachine() { + // panic("help can't crash self (yet)") + // } + simulation.SyscallMachineStop(m.id, false) +} + +// Stop stops the machine gracefully. Pending writes will be fsynced to disk, +// and open network connections will be closed. +func (m Machine) Stop() { + // TODO: document and consider difference between Crash and Stop? + simulation.SyscallMachineStop(m.id, true) +} + +// SetMainFunc changes the main function of the machine for future restarts. +func (m Machine) SetMainFunc(mainFunc func()) { + if err := simulation.SyscallMachineSetBootProgram(m.id, mainFunc); err != nil { + panic(err) + } +} + +// Restart restarts the machine. The machine must be stopped +// before Restart can be called. +func (m Machine) Restart() { + if m == CurrentMachine() { + panic("help can't crash self (yet)") + } + if err := simulation.SyscallMachineRestart(m.id, false); err != nil { + panic(err) + } +} + +// RestartWithPartialDisk restarts the machine. A random subset of non-fsynced +// writes will be flushed to disk. +// TODO: Move this partial flushing to crash instead. +func (m Machine) RestartWithPartialDisk() { + if m == CurrentMachine() { + panic("help can't crash self (yet)") + } + if err := simulation.SyscallMachineRestart(m.id, true); err != nil { + panic(err) + } +} + +// InodeInfo holds low-level information of the requested +// inode. +// +// TODO: make internal? +// TODO: copy pasted from fs +type InodeInfo struct { + MemLinks int + MemExists bool + DiskLinks int + DiskExists bool + Handles int + Ops int +} + +// GetInodeInfo inspects the disk of the machine, returning +// low-level information of the requested inode. +func (m Machine) GetInodeInfo(inode int) InodeInfo { + var info fs.InodeInfo + simulation.SyscallMachineInodeInfo(m.id, inode, &info) + return InodeInfo(info) +} + +/* +func (m *Machine) Recover() { + // XXX: require a crash beforehand? + // XXX: go through OS somehow? + simCrash(m.os.filesystem) +} +*/ + +// XXX: add some tests that simCrash returns a subset of (or all?) traces found by Recover + +// Wait waits for the machine to finish to stop. A machine stops when its main +// function returns or when it is explicitly stopped with [Machine.Stop] or +// [Machine.Crash]. For details, see the documentation of [Machine]. +func (m Machine) Wait() { + simulation.SyscallMachineWait(m.id) +} + +// IterDiskCrashStates iterates over all possible subsets of partially +// fsync-ed writes on the disk of the crashed machine. +// +// TODO: Make this iterate over disks instead? +// TODO: Make internal? +func (m Machine) IterDiskCrashStates(program func()) iter.Seq[Machine] { + iter := simulation.SyscallMachineRecoverInit(m.id, program) + called := false + return func(yield func(Machine) bool) { + if called { + panic("iterator only works once") + } + called = true + defer simulation.SyscallMachineRecoverRelease(iter) + for { + machineID, ok := simulation.SyscallMachineRecoverNext(iter) + if !ok { + return + } + m := Machine{id: machineID} + if !yield(m) { + break + } + } + } +} + +// SetSometimesCrashOnSync configures the machine to sometimes crash +// before or after the fsync system call. +// +// TODO: What is the probability? +// TODO: Have a generic mechanism that covers all system calls instead? +func (m Machine) SetSometimesCrashOnSync(crash bool) { + if err := simulation.SyscallMachineSetSometimesCrashOnSync(m.id, crash); err != nil { + panic(err) + } +} + +// Label returns the label of the machine. +func (m Machine) Label() string { + label, err := simulation.SyscallMachineGetLabel(m.id) + if err != nil { + panic(err) + } + return label +} diff --git a/metatesting/doc.go b/metatesting/doc.go new file mode 100644 index 0000000..3d85100 --- /dev/null +++ b/metatesting/doc.go @@ -0,0 +1,46 @@ +/* +Package metatesting is a package for writing normal go tests that invoke gosim +tests. Such metatests can run gosim tests with different seeds, assert that some +scenarios happen often or never, etc. With metatesting, gosim tests can be +integrated in a normal go test run, so that 'go test ./...' for a module can +cover both simulated and non-simulated tests. + +# Using build constraints to combine gosim tests and metatests + +The metatesting API does not work inside gosim. To combine gosim tests and metatests +in a single package, use the '//go:build sim' and '//go:build !sim' contraints +at the top of test files. + +# Working with the go test cache + +Metatesting has some unfortunate interactions with go test cache. To run +metatests using 'go test', the run needs either be non-cached or the metatests +need to be precompiled. + +To run metatests with a non-cached 'go test' invocation, run 'go test -count=1 +./package/with/metatests'. Then metatesting will build the gosim tests when +needed. This is convenient, but this metatest will not be cached. + +To cache tests, the gosim tests need to be built before running the metatest +with 'gosim build-tests'. If metatesting is used incorrectly it tries to give a +helpful error message. A simple way to run tests is to use a script that invoke +'gosim build-tests' before running 'go test': + + # build gosim test binaries + go run github.com/jellevandenhooff/gosim/cmd/gosim build-tests pkgA pkgA/pkgB + # then run tests + go test ./... + +Tests need to be precompiled when using the test cache because the test cache +should be invalidated when the gosim test changes. Normal go tests get +invalidated when their files change because the go tool knows which files are +compiled into the test. However, for a metatest that invokes 'gosim test' the go +tool does not know what files 'gosim test' accesses, and so does not invalidate +the cache if those files change. To sidestep those issues, run 'gosim +build-tests' before running 'go test'. + +TODO: Support fuzzing. + +TODO: Support parallelism. +*/ +package metatesting diff --git a/metatesting/metatest.go b/metatesting/metatest.go new file mode 100644 index 0000000..845cbd5 --- /dev/null +++ b/metatesting/metatest.go @@ -0,0 +1,426 @@ +package metatesting + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/gosimtool" + "github.com/jellevandenhooff/gosim/internal/prettylog" +) + +// The structs below are copied from gosimruntime.runConfig and +// gosimruntime.runResult. Keep in sync. +// They are copied to give a nice documentation view without refering to +// internal types. + +// A RunConfig configures a test invocation. +type RunConfig struct { + Test string + Seed int64 + ExtraEnv []string +} + +// A RunResult contains the result fo a test run. +type RunResult struct { + Seed int64 + Trace []byte + Failed bool + LogOutput []byte + Err string // TODO: reconsider this type? +} + +// A MetaT runs gosim tests for a given package. It can run tests with various +// flags and returns their results. +// +// Behind the scenes a MetaT executes and controls a gosim test binary. +type MetaT struct { + cmd *exec.Cmd + + w *json.Encoder + r *json.Decoder +} + +/* +type indenter struct{} + +func (w indenter) Write(b []byte) (n int, err error) { + n = len(b) + for len(b) > 0 { + end := bytes.IndexByte(b, '\n') + if end == -1 { + end = len(b) + } else { + end++ + } + // An indent of 4 spaces will neatly align the dashes with the status + // indicator of the parent. + line := b[:end] + // if line[0] == marker { + // w.c.output = append(w.c.output, marker) + // line = line[1:] + // } + const indent = " " + fmt.Fprintf(os.Stderr, "%s%s", indent, line) + // w.c.output = append(w.c.output, indent...) + // w.c.output = append(w.c.output, line...) + b = b[end:] + } + return +} +*/ + +func newRunner(path string) (*MetaT, error) { + cmd := exec.Command(path, "-metatest") + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + + // TODO: Do not discard, and disable the formatted output print in gosimruntime instead + // TODO: Prefix output to help identify stray logging? + cmd.Stderr = io.Discard // os.Stderr + + if err := cmd.Start(); err != nil { + return nil, err + } + + w := json.NewEncoder(stdin) + r := json.NewDecoder(stdout) + + return &MetaT{ + cmd: cmd, + + w: w, + r: r, + }, nil +} + +// TODO: Support match expressions? +func (r *MetaT) ListTests() ([]string, error) { + if err := r.w.Encode(&RunConfig{ + Test: "listtests", + }); err != nil { + return nil, err + } + + var tests []string + if err := r.r.Decode(&tests); err != nil { + return nil, err + } + + return tests, nil +} + +/* +func (r *Runner) Close() { + r.cmd.Process.Kill() + r.cmd.Wait() +} +*/ + +func (r *MetaT) RunAllTests(t *testing.T) { + // TODO: Parallel tests for multiple seeds/tests? + + tests, err := r.ListTests() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + t.Run(test, func(t *testing.T) { + run, err := r.Run(t, &RunConfig{ + Test: test, + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } + if run.Failed { + t.Error("failed") + } + }) + } +} + +func (r *MetaT) Run(t *testing.T, config *RunConfig) (*RunResult, error) { + t.Helper() + + // TODO: Handle concurrent runs (lock?) + // TODO: Stop or restart process on error? + + b := new(bytes.Buffer) + + // TODO: include "gosim test ..." line? + fmt.Fprintf(b, "\n> running %s [seed=%d]\n", config.Test, config.Seed) + + if err := r.w.Encode(config); err != nil { + return nil, err + } + + var result RunResult + if err := r.r.Decode(&result); err != nil { + // TODO: Print results that fail to decode + return nil, err + } + + w := prettylog.NewWriter(b) + + out := result.LogOutput + for len(out) > 0 { + idx := bytes.IndexByte(out, '\n') + if idx == -1 { + idx = len(out) - 1 + } + line := out[:idx+1] + out = out[idx+1:] + w.Write([]byte(line)) + } + t.Log(b.String()) + + return &result, nil +} + +var ( + runnerMapMu sync.Mutex + runnerMap = make(map[string]*MetaT) +) + +// ForOtherPackage retursn a *MetaT for the specified package. +func ForOtherPackage(t *testing.T, pkg string) *MetaT { + if gosimruntime.IsSim() { + t.Fatalf("metatest cannot be used from within gosim") + } + + runnerMapMu.Lock() + defer runnerMapMu.Unlock() + + runner, ok := runnerMap[pkg] + if ok { + return runner + } + + path := gosimtool.GetPathForPrecompiledTestBinary(t, pkg) + runner, err := newRunner(path) + if err != nil { + t.Fatalf("failed to run test binary: %s", err) + } + runnerMap[pkg] = runner + return runner +} + +// getCurrentPackageFromWorkingdir guesses the package +// that `go test` is testing from the working directory. +func getCurrentPackageFromWorkingdir() string { + // go test runs all tests with the working directory set to + // the current package's source code. Walk upwards to find + // the go.mod file, and then combine the module path + // with the relative path to the working directory. + goModPath, modFile, err := gosimtool.FindGoMod() + if err != nil { + log.Fatal(err) + } + goModDir := filepath.Dir(goModPath) + baseDir, err := os.Getwd() + if err != nil { + log.Fatal(err) + } + relativePkg := strings.TrimPrefix(baseDir, goModDir) + finalPkg := modFile.Module.Mod.Path + relativePkg + return finalPkg +} + +// ForCurrentPackage returns a *MetaT for the package currently +// being tested by `go test`. +// +// ForCurrentPackage relies on the working directory to identify the current +// package, so if a test changes directories ForCurrentPackage will not work +// correctly. +func ForCurrentPackage(t *testing.T) *MetaT { + if gosimruntime.IsSim() { + t.Fatalf("metatest cannot be used from within gosim") + } + + return ForOtherPackage(t, getCurrentPackageFromWorkingdir()) +} + +func CheckDeterministic(t *testing.T, mt *MetaT) { + tests, err := mt.ListTests() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + t.Run(test, func(t *testing.T) { + run1, err := mt.Run(t, &RunConfig{ + Test: test, + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } + run2, err := mt.Run(t, &RunConfig{ + Test: test, + Seed: 1, + }) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(run1.Trace, run2.Trace) { + slog.Error("traces differ: non-determinism found") + t.Fail() + // XXX debug view? + } + if !bytes.Equal(run1.LogOutput, run2.LogOutput) { + slog.Error("logs differ: non-determinism found", "diff", cmp.Diff(run1.LogOutput, run2.LogOutput)) + t.Fail() + } + }) + } +} + +func CheckSeeds(t *testing.T, mt *MetaT, numSeeds int) { + tests, err := mt.ListTests() + if err != nil { + t.Fatal(err) + } + + for _, test := range tests { + t.Run(test, func(t *testing.T) { + for seed := int64(0); seed < 5; seed++ { + run, err := mt.Run(t, &RunConfig{ + Test: test, + Seed: seed, + }) + if err != nil { + t.Fatal(err) + } + if run.Failed { + t.Error("failed") + } + // XXX: how do we assert run success? + } + }) + } +} + +func ParseLog(logs []byte) []map[string]any { + var out []map[string]any + + for _, line := range bytes.Split(logs, []byte("\n")) { + var log map[string]any + if err := json.Unmarshal(line, &log); err != nil { + // TODO: Is this ok? + continue + } + out = append(out, log) + } + + return out +} + +func SimplifyParsedLog(logs []map[string]any) []string { + var out []string + for _, log := range logs { + out = append(out, fmt.Sprintf("%s %s", log["level"], strings.Trim(log["msg"].(string), "\n"))) + } + return out +} + +func MustFindLogValue(logs []map[string]any, msg string, key string) any { + for _, log := range logs { + if log["msg"] == msg { + val, ok := log[key] + if !ok { + panic("no matching key") + } + return val + } + } + panic("no matching msg") +} + +/* +func maybeRunInSubtest(t *testing.T, runInSubtest bool, name string, fun func(t *testing.T)) { + if runInSubtest { + t.Run(name, fun) + } else { + fun(t) + } +} + +type SimConfig struct { + Seeds []int64 + EnableTracer bool + CheckDeterminismRuns int // XXX: ensure this also checks logs + SimSubtest bool + SeedSubtests bool + RunSubtests bool + DontReportFail bool // rename? + + CaptureLog bool + LogLevelOverride string + // xxx: time limit here? steps limit? +} + +func RunNestedTest(t *testing.T, config SimConfig, test Test) []gosimruntime.RunResult { + if config.CheckDeterminismRuns > 0 && !config.EnableTracer { + // XXX: automatically set flag? + panic("must enable tracer when checking determinism") + } + if config.CheckDeterminismRuns < 0 { + panic("check determinism runs must be non-negative") + } + + // XXX + // dontReportFail := config.DontReportFail || s.dontReportFail + dontReportFail := config.DontReportFail + + var results []gosimruntime.RunResult + maybeRunInSubtest(t, config.SimSubtest, test.Name, func(t *testing.T) { + for _, seed := range config.Seeds { + maybeRunInSubtest(t, config.SeedSubtests, fmt.Sprintf("seed-%d", seed), func(t *testing.T) { + for run := 0; run < 1+config.CheckDeterminismRuns; run++ { + maybeRunInSubtest(t, config.RunSubtests, fmt.Sprintf("run-%d", run), func(t *testing.T) { + result := RunInner(test.Test, seed, config.EnableTracer, config.CaptureLog, config.LogLevelOverride, dontReportFail, t) + + if run == 0 { + results = append(results, result) + } else { + prevResult := results[len(results)-1] + if !bytes.Equal(result.Trace, prevResult.Trace) { + slog.Error("traces differ: non-determinism found") + t.Fail() + // XXX debug view? + } + if !bytes.Equal(result.LogOutput, prevResult.LogOutput) { + slog.Error("logs differ: non-determinism found", "diff", cmp.Diff(result.LogOutput, prevResult.LogOutput)) + t.Fail() + } + results = append(results, result) + } + }) + } + }) + } + }) + return results +} +*/ diff --git a/metatesting/metatest_test.go b/metatesting/metatest_test.go new file mode 100644 index 0000000..28ac3ad --- /dev/null +++ b/metatesting/metatest_test.go @@ -0,0 +1,28 @@ +//go:build !sim + +package metatesting_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestMetatest(t *testing.T) { + runner := metatesting.ForOtherPackage(t, "github.com/jellevandenhooff/gosim/internal/tests/behavior") + tests, err := runner.ListTests() + if err != nil { + t.Fatal(err) + } + t.Log(tests) + + run, err := runner.Run(t, &metatesting.RunConfig{ + Test: "TestHello", + Seed: 1, + ExtraEnv: []string{"HELLO=goodbye"}, + }) + if err != nil { + t.Fatal(err) + } + t.Log(string(run.LogOutput)) +} diff --git a/nemesis/meta_test.go b/nemesis/meta_test.go new file mode 100644 index 0000000..920dcc9 --- /dev/null +++ b/nemesis/meta_test.go @@ -0,0 +1,24 @@ +//go:build !sim + +package nemesis_test + +import ( + "testing" + + "github.com/jellevandenhooff/gosim/metatesting" +) + +func TestMetaDeterministic(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + metatesting.CheckDeterministic(t, mt) +} + +func TestMetaSeeds(t *testing.T) { + mt := metatesting.ForCurrentPackage(t) + metatesting.CheckSeeds(t, mt, 5) +} + +func TestGosim(t *testing.T) { + runner := metatesting.ForCurrentPackage(t) + runner.RunAllTests(t) +} diff --git a/nemesis/nemesis.go b/nemesis/nemesis.go new file mode 100644 index 0000000..0c50e92 --- /dev/null +++ b/nemesis/nemesis.go @@ -0,0 +1,158 @@ +/* +Package nemesis contains pre-built scenarios that introduce problems into +distributed systems to try and trigger rare bugs. +*/ +package nemesis + +import ( + "log" + "math/rand" + "time" + + "github.com/jellevandenhooff/gosim" +) + +/* +// Nemsis sleeps a random goroutine for 1 simulated second. It starts after +// calling Yield n times to give goroutines some time to get started. +// +// TODO: Wait some random number of simulated seconds instead of yielding? +// TODO: Do not export this function. It is too specific? +func Nemesis(n int) { + // figure out how many steps we are running (external argument?) + for i := 0; i < n; i++ { + gosim.Yield() + } + other := gosimruntime.PickRandomOtherGoroutine() // XXX: make this pick a runnable goroutine instead? + gosimruntime.PauseGoroutine(other) + // XXX: what if other has stopped here? make sure it doesn't break + // slog.Info("sleeping goroutine", "id", g.ID, "selected", g.selected) + time.Sleep(time.Second) + gosimruntime.ResumeGoroutine(other) + + // sleep for rand(steps)... or, call Yield N times + // pause random goroutine gosimruntime.PickRandomOtherGoroutine() + // sleep (clock) for 1s gosimruntime.PauseGoroutine + // unpause that goroutine gosimruntime.ContinueGoroutine +} + +func Restarter(n int, machines []gosim.Machine) { + // XXX: figure out which machines exist and are reasonable candidates? + for i := 0; i < n; i++ { + gosim.Yield() + } + i := rand.Intn(len(machines)) + m := machines[i] + m.Crash() + time.Sleep(5 * time.Second) + m.Restart() +} +*/ + +// A Scenario is a potentially challenging scenario that can be run and +// introduce chaos to running machines or a network to see if they keep behaving +// as expected. +type Scenario interface { + Run() +} + +// Sleep is a scenario that simply sleeps. +type Sleep struct { + Duration time.Duration +} + +// Run implements Scenario. +func (s Sleep) Run() { + time.Sleep(s.Duration) +} + +// PartitionMachines is a scenario that groups the given addresses in two +// different sets and disables the network between the groups. +type PartitionMachines struct { + Addresses []string + Duration time.Duration +} + +// Run implements Scenario. +func (p PartitionMachines) Run() { + var a, b []string + + for { + for _, addr := range p.Addresses { + if rand.Intn(2) == 0 { + a = append(a, addr) + } else { + b = append(b, addr) + } + } + if len(p.Addresses) <= 1 || (len(a) > 0 && len(b) > 0) { + break + } + } + + log.Printf("partition machines: partitioning network into %v and %v", a, b) + + for _, a := range a { + for _, b := range b { + gosim.SetConnected(a, b, false) + } + } + + time.Sleep(p.Duration) + + log.Printf("partition machines: rejoining network between %v and %v", a, b) + + for _, a := range a { + for _, b := range b { + gosim.SetConnected(a, b, true) + } + } +} + +// RestartRandomly restarts one randomly chosen machine. +type RestartRandomly struct { + // Machines to choose from for restarting + Machines []gosim.Machine + // Time to wait between crashing machine and starting again + Downtime time.Duration +} + +// Run implements Scenario. +func (r RestartRandomly) Run() { + if len(r.Machines) == 0 { + panic("restart randomly: need at least one machine") + } + m := r.Machines[rand.Intn(len(r.Machines))] + + log.Printf("restart randomly: crashing machine %s", m.Label()) + + m.Crash() + time.Sleep(r.Downtime) + + log.Printf("restart randomly: restarting machine %s", m.Label()) + m.RestartWithPartialDisk() +} + +type funcScenario func() + +func (f funcScenario) Run() { + f() +} + +// Repeat repeats the given scenario a number of times. +func Repeat(scenario Scenario, times int) Scenario { + return funcScenario(func() { + for range times { + scenario.Run() + } + }) +} + +// Sequence runs the given scenearios in sequence. +func Sequence(scenarios ...Scenario) Scenario { + return funcScenario(func() { + for _, s := range scenarios { + s.Run() + } + }) +} diff --git a/nemesis/nemesis_test.go b/nemesis/nemesis_test.go new file mode 100644 index 0000000..ee57682 --- /dev/null +++ b/nemesis/nemesis_test.go @@ -0,0 +1,122 @@ +//go:build sim + +package nemesis_test + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "net/netip" + "testing" + "time" + + "golang.org/x/sync/errgroup" + + "github.com/jellevandenhooff/gosim" + "github.com/jellevandenhooff/gosim/nemesis" +) + +func makeAddresses() []string { + var addresses []string + for i := 0; i < 3; i++ { + addresses = append(addresses, fmt.Sprintf("10.0.0.%d", i+1)) + } + return addresses +} + +// request makes a HTTP GET request with a 1 second timeout to +// http:///ping and prints the result +func request(addr string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://%s/ping", addr), nil) + if err != nil { + log.Fatal(err) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + log.Printf("pinging %s: got error: %v", addr, err) + return + } + defer resp.Body.Close() + bytes, err := io.ReadAll(resp.Body) + if err != nil { + log.Printf("pinging %s: got error: %v", addr, err) + return + } + log.Printf("pinging %s: got response %q", addr, string(bytes)) +} + +func pingerMain() { + log.Printf("starting pinger") + + // figure out our own address + // TODO: make another API work + addr := gosim.CurrentMachine().Label() + + // run http server + // TODO: make ":80" without addr work + go http.ListenAndServe(addr+":80", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "hello there") + })) + + addrs := makeAddresses() + + // ping others every 5 seconds + t := time.NewTicker(5 * time.Second) + for range t.C { + // ping in parallel + var g errgroup.Group + for _, other := range addrs { + if addr != other { + g.Go(func() error { + request(other) + return nil + }) + } + } + g.Wait() + } +} + +func TestPartition(t *testing.T) { + addrs := makeAddresses() + + // run ping on several machines + var machines []gosim.Machine + for _, addr := range addrs { + m := gosim.NewMachine(gosim.MachineConfig{ + Label: addr, // fmt.Sprintf("server-%d", i), + Addr: netip.MustParseAddr(addr), + MainFunc: pingerMain, + }) + machines = append(machines, m) + } + + // let machines communicate for a while, then randomly partition, then + // repair, then crash one machine, and repair again + scenario := nemesis.Sequence( + nemesis.Sleep{ + Duration: 10 * time.Second, + }, + nemesis.PartitionMachines{ + // TODO: make this API based of machines? + Addresses: addrs, + Duration: 10 * time.Second, + }, + nemesis.Sleep{ + Duration: 10 * time.Second, + }, + nemesis.RestartRandomly{ + Machines: machines, + Downtime: 10 * time.Second, + }, + nemesis.Sleep{ + Duration: 10 * time.Second, + }, + ) + + scenario.Run() +} diff --git a/simulation.go b/simulation.go new file mode 100644 index 0000000..1a6eb4c --- /dev/null +++ b/simulation.go @@ -0,0 +1,84 @@ +package gosim + +import ( + "time" + + "github.com/jellevandenhooff/gosim/gosimruntime" + "github.com/jellevandenhooff/gosim/internal/simulation" +) + +// IsSim returns if the program is running inside of a gosim simulation. +// +// To include or exclude code or tests only if running inside of a gosim the +// gosim tool also supports a build tag. Use the `//go:build sim` build +// constraint at the top of a go file to only include it in simulated builds or +// `//go:build !sim` to exclude it from simulated from builds. +func IsSim() bool { + return gosimruntime.IsSim() +} + +// Yield yields the processor, allowing other goroutines to run. Similar to +// [runtime.Gosched] but for gosim's scheduler. +// +// TODO: Replace with runtime.Gosched() instead? +func Yield() { + gosimruntime.Yield() +} + +/* +type Simulation struct{} + +func CurrentSimulation() Simulation { + return Simulation{} +} + +// SetConnected configures the network connectivity between addresses a and b. +// +// TODO: How will this fail? Unrouteable or rejected +// TODO: Support asymmetry +func (s Simulation) SetConnected(a, b string, connected bool) error { + return simulation.SyscallSetConnected(a, b, connected) +} + +// SetDelay configures the network latency between addresses a and b. +// +// TODO: Support asymmetry +func (s Simulation) SetDelay(a, b string, delay time.Duration) error { + return simulation.SyscallSetDelay(a, b, delay) +} + +// SetSimulationTimeout resets the simulation timeout to the given duration. +// After the simulated duration the simululation will fail with a test timeout +// error. +func (s Simulation) SetSimulationTimeout(timeout time.Duration) { + err := simulation.SyscallSetSimulationTimeout(timeout) + if err != nil { + panic(err) + } +} +*/ + +// SetConnected configures the network connectivity between addresses a and b. +// +// TODO: How will this fail? Unrouteable or rejected +// TODO: Support asymmetry +func SetConnected(a, b string, connected bool) error { + return simulation.SyscallSetConnected(a, b, connected) +} + +// SetDelay configures the network latency between addresses a and b. +// +// TODO: Support asymmetry +func SetDelay(a, b string, delay time.Duration) error { + return simulation.SyscallSetDelay(a, b, delay) +} + +// SetSimulationTimeout resets the simulation timeout to the given duration. +// After the simulated duration the simululation will fail with a test timeout +// error. +func SetSimulationTimeout(d time.Duration) { + err := simulation.SyscallSetSimulationTimeout(d) + if err != nil { + panic(err) + } +} diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..e4fce44 --- /dev/null +++ b/test.sh @@ -0,0 +1,5 @@ +#!/bin/bash +cd "${0%/*}" +set -e + +go run github.com/go-task/task/v3/cmd/task test "$@"