Panic at loom-0.6.0/src/rt/scheduler.rs:134:48

[sporksmith]

Panic at loom-0.6.0/src/rt/scheduler.rs:134:48:

thread 'atomic_tls_map_tests::test_get_and_remove_threaded' panicked at 'Box<dyn Any>', /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/yield_.rs:58:9
stack backtrace:
   0:     0x560d42552d21 - std::backtrace_rs::backtrace::libunwind::trace::hc317256c0daecb22
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/../../backtrace/src/backtrace/libunwind.rs:93:5
   1:     0x560d42552d21 - std::backtrace_rs::backtrace::trace_unsynchronized::h85008105ccd95ba6
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/../../backtrace/src/backtrace/mod.rs:66:5
   2:     0x560d42552d21 - std::sys_common::backtrace::_print_fmt::hb6e7cbe31fecc91c
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/sys_common/backtrace.rs:65:5
   3:     0x560d42552d21 - <std::sys_common::backtrace::_print::DisplayBacktrace as core::fmt::Display>::fmt::h53097d3751045778
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/sys_common/backtrace.rs:44:22
   4:     0x560d4257a2df - core::fmt::rt::Argument::fmt::hdbe0b5dfb2c09e86
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/core/src/fmt/rt.rs:138:9
   5:     0x560d4257a2df - core::fmt::write::h4ae17c68778671a0
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/core/src/fmt/mod.rs:1094:21
   6:     0x560d42550591 - std::io::Write::write_fmt::hfd7c55e775c6dc60
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/io/mod.rs:1713:15
   7:     0x560d42552b35 - std::sys_common::backtrace::_print::h81eabb0036b692a0
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/sys_common/backtrace.rs:47:5
   8:     0x560d42552b35 - std::sys_common::backtrace::print::h1b1f4d1a23fac3fd
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/sys_common/backtrace.rs:34:9
   9:     0x560d425546a7 - std::panicking::default_hook::{{closure}}::hc17d2bd8f3da442c
  10:     0x560d42554494 - std::panicking::default_hook::h3a32a9336113ba96
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panicking.rs:288:9
  11:     0x560d42554b5c - std::panicking::rust_panic_with_hook::h90e4d15277396259
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panicking.rs:705:13
  12:     0x560d4229de54 - std::panicking::begin_panic::{{closure}}::h37c03febea5cf1de
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panicking.rs:626:9
  13:     0x560d4229442c - std::sys_common::backtrace::__rust_end_short_backtrace::h5b4341804d34695b
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/sys_common/backtrace.rs:151:18
  14:     0x560d4229dde3 - std::panicking::begin_panic::h2244459bce473ae2
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panicking.rs:625:12
  15:     0x560d4226e684 - std::panic::panic_any::h7288c5f69105ea63
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panic.rs:63:5
  16:     0x560d42299b40 - generator::yield_::raw_yield::hbb9741924ab715a9
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/yield_.rs:58:9
  17:     0x560d422999dd - generator::yield_::yield_::hca56ea75864421e4
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/yield_.rs:102:5
  18:     0x560d4228aba1 - loom::rt::scheduler::spawn_thread::{{closure}}::h2884670aa130c9ac
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/loom-0.6.0/src/rt/scheduler.rs:134:48
  19:     0x560d4229e98e - core::ops::function::FnOnce::call_once::h72eae66edd093d2b
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/core/src/ops/function.rs:250:5
  20:     0x560d422a257e - generator::gen_impl::GeneratorImpl<A,T>::init_code::{{closure}}::h756087e3cd3f236d
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:336:21
  21:     0x560d42298ad5 - generator::stack::StackBox<F>::call_once::h3fc2e4fc2660a310
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:139:13
  22:     0x560d4250ec10 - generator::stack::Func::call_once::h5ff7baf698cbcb3d
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/stack/mod.rs:121:9
  23:     0x560d42510955 - generator::gen_impl::gen_init::{{closure}}::h5f50a24ebb2d84c3
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:552:9
  24:     0x560d4250dd83 - core::ops::function::FnOnce::call_once::hcb886d17c0f293b0
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/core/src/ops/function.rs:250:5
  25:     0x560d4250e9fa - std::panicking::try::do_call::h3d4cff7f20857cf3
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panicking.rs:500:40
  26:     0x560d4250eb6b - __rust_try
  27:     0x560d4250e98a - std::panicking::try::h6114cb4f790b6a44
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panicking.rs:464:19
  28:     0x560d42510cbe - std::panic::catch_unwind::hf8711f47d38c9a04
                               at /rustc/f0411ffcebcd7f75ac02ed45feb53ffd07b75398/library/std/src/panic.rs:142:14
  29:     0x560d4251068d - generator::gen_impl::catch_unwind_filter::hff393a5c10e919a6
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:543:5
  30:     0x560d42510827 - generator::gen_impl::gen_init::he6684bd1e199fb85
                               at /home/jnewsome/.cargo/registry/src/index.crates.io-6f17d22bba15001f/generator-0.7.4/src/gen_impl.rs:570:25
  31:                0x0 - <unknown>

I get this crash at this commit in my own code, with NTHREADS at this line set to 3 or more: https://github.com/shadow/shadow/pull/3063/files#diff-fc1070fc17824d4f886ddf84ded2d1cdcf42dd2b26552cdb0d5dd1d38ad98025R166 .

I've found that --test-threads 1, or disabling one of the tests test_reuse_keys_after_thread_exit or test_get_and_remove_threaded also suppresses the crash; it seems like some concurrency or resource issue inside loom? I don't think these tests share any global state in my own code that would cause them to interact.