Skip to content

ppc64le fails on RUST_OPT_LEVEL_{0,1} #155

Closed
#192
Closed
ppc64le fails on RUST_OPT_LEVEL_{0,1}#155
#192
Labels
• archRelated to a particular arch, `arch/` support in general...

Description

@ojeda
ojeda
opened on Mar 31, 2021
Large array has length: 514
Kernel panic - not syncing: corrupted stack end detected inside scheduler
CPU: 0 PID: 1 Comm: swapper/0 Not tainted 5.12.0-rc3+ #1

Removing the large array does not fix it -- something else is going on.

When this is fixed, switch the CI debug config back to RUST_OPT_LEVEL_0.

Activity

  1. ojeda
    added
    • archRelated to a particular arch, `arch/` support in general...
    prio: normal
    on Mar 31, 2021

  2. mpe commented on Apr 10, 2021

    @mpe
    mpe
    on Apr 10, 2021

    This is caused by stack overflow. We have 16K stacks, but there are functions generated with ~20K frames!

    The stack trace for that panic is:

    (gdb) bt
#0  schedule_debug (prev=0xc0000000073c0080, preempt=true) at kernel/sched/core.c:4822
#1  0xc000000000609da8 in __schedule (preempt=<error reading variable: value has been optimised out>) at kernel/sched/core.c:4967
#2  0xc00000000060a6c4 in preempt_schedule_common () at kernel/sched/core.c:5233
#3  __cond_resched () at kernel/sched/core.c:6982
#4  0xc00000000029b9dc in might_alloc (gfp_mask=0) at /home/michael/src/rust/linux/./include/linux/sched/mm.h:197
#5  slab_pre_alloc_hook (s=0xc000000007011c80, objcgp=<optimised out>, size=<optimised out>, flags=0) at /home/michael/src/rust/linux/mm/slab.h:497
#6  slab_alloc_node (s=0xc000000007011c80, gfpflags=117513344, node=-1, addr=13835058055286880704, orig_size=<optimised out>) at mm/slub.c:2826
#7  slab_alloc (s=0xc000000007011c80, gfpflags=117513344, addr=13835058055286880704, orig_size=<optimised out>) at mm/slub.c:2915
#8  __kmalloc_track_caller (size=152, gfpflags=3264, caller=13835058055286880704) at mm/slub.c:4561
#9  0xc00000000024a050 in __do_krealloc (p=0x0, new_size=152, flags=11398400) at mm/slab_common.c:1153
#10 krealloc (p=0x0, new_size=152, flags=3264) at mm/slab_common.c:1186
#11 0xc00000000047f9c0 in kernel::allocator::__rust_alloc (size=152, _align=8) at /home/michael/src/rust/linux/rust/kernel/allocator.rs:35
#12 0xc0000000005bc2c8 in alloc::alloc::alloc (layout=...) at /home/michael/.rustup/toolchains/nightly-2021-02-20-powerpc64le-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/alloc.rs:86
#13 0xc0000000005b6f04 in <alloc::alloc::Global>::alloc_impl (self=0xc0000000074024c0, layout=..., zeroed=false)
    at /home/michael/.rustup/toolchains/nightly-2021-02-20-powerpc64le-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/alloc.rs:166
#14 0xc0000000005bdf38 in <alloc::alloc::Global as core::alloc::Allocator>::allocate (self=0xc0000000074024c0, layout=...)
    at /home/michael/.rustup/toolchains/nightly-2021-02-20-powerpc64le-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/alloc.rs:226
#15 0xc0000000005bb1c8 in <alloc::boxed::Box<kernel::sync::mutex::Mutex<i32>>>::try_new_uninit_in (alloc=...)
    at /home/michael/.rustup/toolchains/nightly-2021-02-20-powerpc64le-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:425
#16 0xc0000000005bb09c in <alloc::boxed::Box<kernel::sync::mutex::Mutex<i32>>>::try_new_in (x=..., alloc=...)
    at /home/michael/.rustup/toolchains/nightly-2021-02-20-powerpc64le-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:360
#17 0xc0000000005b71c0 in <alloc::boxed::Box<kernel::sync::mutex::Mutex<i32>>>::try_new (x=...)
    at /home/michael/.rustup/toolchains/nightly-2021-02-20-powerpc64le-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:261
#18 0xc0000000005c08b8 in <rust_example::RustExample as kernel::KernelModule>::init () at /home/michael/src/rust/linux/drivers/char/rust_example.rs:192
#19 0xc0000000005c196c in rust_example::__init () at /home/michael/src/rust/linux/drivers/char/rust_example.rs:21
#20 0xc0000000005c18d8 in rust_example::__rust_example_init () at /home/michael/src/rust/linux/drivers/char/rust_example.rs:21
#21 0xc00000000001112c in do_one_initcall (fn=0xc0000000005c18c0 <rust_example::__rust_example_init>) at init/main.c:1226
#22 0xc0000000009c5be8 in do_initcall_level (level=<optimised out>, command_line=<optimised out>) at init/main.c:1299
#23 0xc0000000009c5ab4 in do_initcalls () at init/main.c:1315
#24 0xc0000000009c5a20 in do_basic_setup () at init/main.c:1335
#25 0xc0000000009c5904 in kernel_init_freeable () at init/main.c:1537
#26 0xc000000000011f7c in kernel_init (unused=<optimised out>) at init/main.c:1424
#27 0xc00000000000cb78 in ret_from_kernel_thread () at arch/powerpc/kernel/entry_64.S:416

    Looking up the stack I see this function:

    (gdb) up
#18 0xc0000000005c08b8 in _RNvXs3_CsbDqzXfLQacH_12rust_exampleNtB5_11RustExampleNtCsbDqzXfLQacH_6kernel12KernelModule4init () at /home/michael/src/rust/linux/drivers/char/rust_example.rs:192
                          <rust_example::RustExample as kernel::KernelModule>::init
192                 let data = Pin::from(Box::try_new(unsafe { Mutex::new(0) })?);

    The disassembly is:

    c0000000005c0200 <_RNvXs3_CsbDqzXfLQacH_12rust_exampleNtB5_11RustExampleNtCsbDqzXfLQacH_6kernel12KernelModule4init>:
                 <rust_example::RustExample as kernel::KernelModule>::init
c0000000005c0200:       5e 00 4c 3c     addis   r2,r12,94
c0000000005c0204:       00 eb 42 38     addi    r2,r2,-5376
c0000000005c0208:       a6 02 08 7c     mflr    r0
c0000000005c020c:       10 00 01 f8     std     r0,16(r1)
c0000000005c0210:       d1 af 21 f8     stdu    r1,-20528(r1)

    That stdu is creating a 20528 byte stack frame.

    Building with -Z emit-stack-sizes I see ~30 Rust functions that have large (> 2KB) frames.

  3. mpe commented on Apr 10, 2021

    @mpe
    mpe
    on Apr 10, 2021

    I can fix the CI failure at opt level 0 by increasing the stack size, do we want to do that for now?

    See mpe/linux@27a1886

  4. ojeda commented on Apr 10, 2021

    @ojeda
    ojeda
    on Apr 10, 2021
    MemberAuthor

    It would be nice to do it, yeah. We should also add a comment in CONFIG_RUST_OPT_LEVEL_0 and 1 explaining that the stack size might need a substantial increase in some cases.

    Also please update Doc/rust/arch-support.rst to remove the restriction! :-)

  5. mpe commented on Apr 13, 2021

    @mpe
    mpe
    on Apr 13, 2021

    For the record here are the functions I see with frames > 2KB:

    0xc000000000363040      2064    core[8787f43e282added]::num::flt2dec::to_exact_fixed_str::<f32, core[8787f43e282added]::num::flt2dec::strategy::grisu::format_exact>
0xc000000000362250      2080    core[8787f43e282added]::num::flt2dec::to_exact_fixed_str::<f64, core[8787f43e282added]::num::flt2dec::strategy::grisu::format_exact>
0xc00000000048ef20      2112    <kernel[8787f43e282added]::bindings::bindings_raw::srcu_struct as core[8787f43e282added]::default::Default>::default
0xc000000000494150      2160    <kernel[8787f43e282added]::bindings::bindings_raw::device as core[8787f43e282added]::default::Default>::default
0xc00000000035c840      2208    core[8787f43e282added]::num::flt2dec::digits_to_dec_str
0xc0000000004941e0      2336    <kernel[8787f43e282added]::bindings::bindings_raw::device_link as core[8787f43e282added]::default::Default>::default
0xc0000000003cf610      2368    <core[8787f43e282added]::core_arch::simd::i8x64 as core[8787f43e282added]::fmt::Debug>::fmt
0xc0000000003d06b0      2368    <core[8787f43e282added]::core_arch::simd::u8x64 as core[8787f43e282added]::fmt::Debug>::fmt
0xc00000000048f1b0      2416    <kernel[8787f43e282added]::bindings::bindings_raw::srcu_notifier_head as core[8787f43e282added]::default::Default>::default
0xc000000000495e00      2416    <kernel[8787f43e282added]::bindings::bindings_raw::signal_struct as core[8787f43e282added]::default::Default>::default
0xc0000000005b21c0      2416    <kernel[8787f43e282added]::chrdev::Registration<2: usize>>::register::<rust_example[8787f43e282added]::RustFile>
0xc0000000004db810      2528    <kernel[8787f43e282added]::bindings::bindings_raw::machdep_calls as core[8787f43e282added]::fmt::Debug>::fmt
0xc00000000048e9c0      2560    <kernel[8787f43e282added]::bindings::bindings_raw::zone as core[8787f43e282added]::default::Default>::default
0xc000000000491230      2560    <kernel[8787f43e282added]::bindings::bindings_raw::hrtimer_cpu_base as core[8787f43e282added]::default::Default>::default
0xc00000000048bbe0      2752    <kernel[8787f43e282added]::bindings::bindings_raw::thread_struct as core[8787f43e282added]::default::Default>::default
0xc000000000488e10      2816    <kernel[8787f43e282added]::bindings::bindings_raw::module as core[8787f43e282added]::default::Default>::default
0xc00000000048e490      2944    <kernel[8787f43e282added]::bindings::bindings_raw::mm_struct__bindgen_ty_1 as core[8787f43e282added]::default::Default>::default
0xc00000000048e520      2944    <kernel[8787f43e282added]::bindings::bindings_raw::mm_struct as core[8787f43e282added]::default::Default>::default
0xc000000000382770      3552    core[8787f43e282added]::fmt::float::float_to_decimal_common_exact::<f64>
0xc0000000003828c0      3552    core[8787f43e282added]::fmt::float::float_to_decimal_common_exact::<f32>
0xc000000000382f70      3792    core[8787f43e282added]::fmt::float::float_to_exponential_common_exact::<f64>
0xc0000000003830e0      3792    core[8787f43e282added]::fmt::float::float_to_exponential_common_exact::<f32>
0xc00000000048ae70      4432    <kernel[8787f43e282added]::bindings::bindings_raw::mce_info as core[8787f43e282added]::default::Default>::default
0xc000000000495cf0      4464    <kernel[8787f43e282added]::bindings::bindings_raw::sighand_struct as core[8787f43e282added]::default::Default>::default
0xc0000000004902a0      5248    <kernel[8787f43e282added]::bindings::bindings_raw::kobj_uevent_env as core[8787f43e282added]::default::Default>::default
0xc000000000497bc0      5376    <kernel[8787f43e282added]::bindings::bindings_raw::super_block as core[8787f43e282added]::default::Default>::default
0xc00000000048af00      6144    <kernel[8787f43e282added]::bindings::bindings_raw::paca_struct as core[8787f43e282added]::default::Default>::default
0xc00000000048a4a0      6288    <kernel[8787f43e282added]::bindings::bindings_raw::hash_mm_context as core[8787f43e282added]::default::Default>::default
0xc00000000048ec20      7168    <kernel[8787f43e282added]::bindings::bindings_raw::pglist_data as core[8787f43e282added]::default::Default>::default
0xc00000000048a380      8256    <kernel[8787f43e282added]::bindings::bindings_raw::hv_gpci_request_buffer as core[8787f43e282added]::default::Default>::default
0xc0000000004952e0      9344    <kernel[8787f43e282added]::bindings::bindings_raw::ppc64_tlb_batch as core[8787f43e282added]::default::Default>::default
0xc000000000492c40      11776   <kernel[8787f43e282added]::bindings::bindings_raw::task_struct as core[8787f43e282added]::default::Default>::default
0xc000000000351ae0      12352   core[8787f43e282added]::num::dec2flt::algorithm::power_of_ten
0xc0000000005c0200      20528   <rust_example[8787f43e282added]::RustExample as kernel[8787f43e282added]::KernelModule>::init
0xc0000000005c4a40      24864   <rust_example_2[8787f43e282added]::RustExample2 as kernel[8787f43e282added]::KernelModule>::init
0xc000000000492cf0      33024   <kernel[8787f43e282added]::bindings::bindings_raw::thread_union as core[8787f43e282added]::default::Default>::default
  6. ojeda
    closed this as completedin #192on Apr 21, 2021
  7. ojeda
    added a commit that references this issue on Jul 11, 2022

    netfilter: nft_set_pipapo: release elements in clone from abort path

    9827a0e
  8. fbq
    added 2 commits that reference this issue on Sep 25, 2023

    drm/drm_exec: Work around a WW mutex lockdep oddity

    d20b484

    drm/tests: helpers: Avoid a driver uaf

    139a278
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    • archRelated to a particular arch, `arch/` support in general...

    Type

    No type

    Projects

    No projects

      Milestone

      No milestone

      Relationships

      None yet

      Development

      No branches or pull requests

      Issue actions