WARNING: kernel stack frame pointer at ffff880156a5fea0 in bash:2103 has bad value 00007ffec7d87e50
8 messages,
4 authors,
2017-09-30 · open the first message on its own page
Hi!
While playing with bcc's opensnoop tool on Linux 4.14-rc2 I managed to trigger
this splat:
[ 297.629773] WARNING: kernel stack frame pointer at ffff880156a5fea0 in
bash:2103 has bad value 00007ffec7d87e50
[ 297.629777] unwind stack type:0 next_sp: (null) mask:0x6
graph_idx:0
[ 297.629783] ffff88015b207ae0: ffff88015b207b68 (0xffff88015b207b68)
[ 297.629790] ffff88015b207ae8: ffffffffb163c00e (__save_stack_trace+0x6e/
0xd0)
[ 297.629792] ffff88015b207af0: 0000000000000000 ...
[ 297.629795] ffff88015b207af8: ffff880156a58000 (0xffff880156a58000)
[ 297.629799] ffff88015b207b00: ffff880156a60000 (0xffff880156a60000)
[ 297.629800] ffff88015b207b08: 0000000000000000 ...
[ 297.629803] ffff88015b207b10: 0000000000000006 (0x6)
[ 297.629806] ffff88015b207b18: ffff880151b02700 (0xffff880151b02700)
[ 297.629809] ffff88015b207b20: 0000010100000000 (0x10100000000)
[ 297.629812] ffff88015b207b28: ffff880156a5fea0 (0xffff880156a5fea0)
[ 297.629815] ffff88015b207b30: ffff88015b207ae0 (0xffff88015b207ae0)
[ 297.629818] ffff88015b207b38: ffffffffc0050282 (0xffffffffc0050282)
[ 297.629819] ffff88015b207b40: 0000000000000000 ...
[ 297.629822] ffff88015b207b48: 0000000001000000 (0x1000000)
[ 297.629825] ffff88015b207b50: ffff880157b98280 (0xffff880157b98280)
[ 297.629828] ffff88015b207b58: ffff880157b98380 (0xffff880157b98380)
[ 297.629831] ffff88015b207b60: ffff88015ad2b500 (0xffff88015ad2b500)
[ 297.629834] ffff88015b207b68: ffff88015b207b78 (0xffff88015b207b78)
[ 297.629838] ffff88015b207b70: ffffffffb163c086 (save_stack_trace+0x16/0x20)
[ 297.629841] ffff88015b207b78: ffff88015b207da8 (0xffff88015b207da8)
[ 297.629847] ffff88015b207b80: ffffffffb18a8ed6 (save_stack+0x46/0xd0)
[ 297.629850] ffff88015b207b88: 000000400000000c (0x400000000c)
[ 297.629852] ffff88015b207b90: ffff88015b207ba0 (0xffff88015b207ba0)
[ 297.629855] ffff88015b207b98: ffff880100000000 (0xffff880100000000)
[ 297.629859] ffff88015b207ba0: ffffffffb163c086 (save_stack_trace+0x16/0x20)
[ 297.629864] ffff88015b207ba8: ffffffffb18a8ed6 (save_stack+0x46/0xd0)
[ 297.629868] ffff88015b207bb0: ffffffffb18a9752 (kasan_slab_free+0x72/0xc0)
[ 297.629873] ffff88015b207bb8: ffffffffb18a5e90 (kmem_cache_free+0x70/0x190)
[ 297.629879] ffff88015b207bc0: ffffffffb18b7e94 (file_free_rcu+0x34/0x40)
[ 297.629886] ffff88015b207bc8: ffffffffb172580c (rcu_process_callbacks
+0x2dc/0xcd0)
[ 297.629892] ffff88015b207bd0: ffffffffb2646cbc (__do_softirq+0x12c/0x343)
[ 297.629897] ffff88015b207bd8: ffffffffb1692304 (irq_exit+0xe4/0xf0)
[ 297.629902] ffff88015b207be0: ffffffffb2646446 (smp_apic_timer_interrupt
+0x86/0x1a0)
[ 297.629907] ffff88015b207be8: ffffffffb26452f3 (apic_timer_interrupt
+0x93/0xa0)
[ 297.629913] ffff88015b207bf0: ffffffffb1667417 (optimized_callback
+0x67/0x100)
[ 297.629916] ffff88015b207bf8: ffffffffc0050282 (0xffffffffc0050282)
[ 297.629918] ffff88015b207c00: 0000000000000000 ...
[ 297.629921] ffff88015b207c08: ffff88015a77e24c (0xffff88015a77e24c)
[ 297.629924] ffff88015b207c10: ffff88015b207c38 (0xffff88015b207c38)
[ 297.629927] ffff88015b207c18: ffff88015b207c38 (0xffff88015b207c38)
[ 297.629929] ffff88015b207c20: 0000000000000086 (0x86)
[ 297.629932] ffff88015b207c28: ffff88015a77db00 (0xffff88015a77db00)
[ 297.629935] ffff88015b207c30: 1ffff1002b640f91 (0x1ffff1002b640f91)
[ 297.629938] ffff88015b207c38: ffff88015b207d10 (0xffff88015b207d10)
[ 297.629945] ffff88015b207c40: ffffffffb16c9f60 (try_to_wake_up+0xb0/0x710)
[ 297.629947] ffff88015b207c48: 0000000000000000 ...
[ 297.629952] ffff88015b207c50: ffffffffb2dfd3c0 (machine_ops+0x40/0x40)
[ 297.629954] ffff88015b207c58: ffff88015a77df94 (0xffff88015a77df94)
[ 297.629957] ffff88015b207c60: 0000000000023540 (0x23540)
[ 297.629960] ffff88015b207c68: ffff88015b215c38 (0xffff88015b215c38)
[ 297.629963] ffff88015b207c70: ffff88015b200000 (0xffff88015b200000)
[ 297.629965] ffff88015b207c78: 0000000000000086 (0x86)
[ 297.629968] ffff88015b207c80: 0000000100000000 (0x100000000)
[ 297.629971] ffff88015b207c88: 0000000041b58ab3 (0x41b58ab3)
[ 297.629975] ffff88015b207c90: ffffffffb2d919f2 (.LC2+0x6e0e/0x83b5)
[ 297.629981] ffff88015b207c98: ffffffffb16c9eb0 (migrate_swap_stop
+0x2e0/0x2e0)
[ 297.629986] ffff88015b207ca0: ffffffffb16d0f73 (account_entity_dequeue
+0x73/0x110)
[ 297.629989] ffff88015b207ca8: 0000000000100000 (0x100000)
[ 297.629992] ffff88015b207cb0: ffff88015b2235a0 (0xffff88015b2235a0)
[ 297.629994] ffff88015b207cb8: ffff88015061e280 (0xffff88015061e280)
[ 297.629997] ffff88015b207cc0: ffff88015b207ce8 (0xffff88015b207ce8)
[ 297.630003] ffff88015b207cc8: ffffffffb16c87ed (sched_avg_update+0x2d/0x90)
[ 297.630005] ffff88015b207cd0: 0000000000000005 (0x5)
[ 297.630008] ffff88015b207cd8: ffff88015b223570 (0xffff88015b223570)
[ 297.630010] ffff88015b207ce0: 00000000000000dd (0xdd)
[ 297.630013] ffff88015b207ce8: ffff88015a017ea0 (0xffff88015a017ea0)
[ 297.630021] ffff88015b207cf0: ffffffffb30b7128 (rcu_sched_state
+0x928/0xaa0)
[ 297.630024] ffff88015b207cf8: ffff880151b02700 (0xffff880151b02700)
[ 297.630026] ffff88015b207d00: 0000000000000001 (0x1)
[ 297.630031] ffff88015b207d08: ffffffffb30b6800 (rcu_bh_varname+0x60/0x60)
[ 297.630034] ffff88015b207d10: ffff88015b207d20 (0xffff88015b207d20)
[ 297.630040] ffff88015b207d18: ffffffffb16ca5d0 (wake_up_process+0x10/0x20)
[ 297.630043] ffff88015b207d20: ffff88015b207d48 (0xffff88015b207d48)
[ 297.630045] ffff88015b207d28: ffff88015b207d48 (0xffff88015b207d48)
[ 297.630048] ffff88015b207d30: 0000000000000202 (0x202)
[ 297.630053] ffff88015b207d38: ffffffffb30b7120 (rcu_sched_state
+0x920/0xaa0)
[ 297.630056] ffff88015b207d40: 0000000000000202 (0x202)
[ 297.630059] ffff88015b207d48: ffff88015b207d68 (0xffff88015b207d68)
[ 297.630063] ffff88015b207d50: ffffffffb16ee225 (swake_up+0x25/0x30)
[ 297.630069] ffff88015b207d58: ffffffffb30b6800 (rcu_bh_varname+0x60/0x60)
[ 297.630072] ffff88015b207d60: ffff88015a77db00 (0xffff88015a77db00)
[ 297.630074] ffff88015b207d68: ffff88015b207d90 (0xffff88015b207d90)
[ 297.630079] ffff88015b207d70: ffffffffb1720016 (rcu_gp_kthread_wake
+0x56/0x60)
[ 297.630082] ffff88015b207d78: 0000000000000002 (0x2)
[ 297.630087] ffff88015b207d80: ffffffffb30b7138 (rcu_sched_state
+0x938/0xaa0)
[ 297.630092] ffff88015b207d88: ffffffffb30b6800 (rcu_bh_varname+0x60/0x60)
[ 297.630095] ffff88015b207d90: ffff88015b207e18 (0xffff88015b207e18)
[ 297.630099] ffff88015b207d98: ffffffffb1720521 (rcu_report_qs_rnp
+0x2f1/0x310)
[ 297.630102] ffff88015b207da0: ffff88015ad2b500 (0xffff88015ad2b500)
[ 297.630105] ffff88015b207da8: ffff88015b207dd0 (0xffff88015b207dd0)
[ 297.630110] ffff88015b207db0: ffffffffb18a9752 (kasan_slab_free+0x72/0xc0)
[ 297.630113] ffff88015b207db8: ffff880157b98280 (0xffff880157b98280)
[ 297.630116] ffff88015b207dc0: ffffea00055ee600 (0xffffea00055ee600)
[ 297.630121] ffff88015b207dc8: ffffffffb18b7e94 (file_free_rcu+0x34/0x40)
[ 297.630124] ffff88015b207dd0: ffff88015b207e00 (0xffff88015b207e00)
[ 297.630128] ffff88015b207dd8: ffffffffb18a5e90 (kmem_cache_free+0x70/0x190)
[ 297.630131] ffff88015b207de0: ffff880157b98280 (0xffff880157b98280)
[ 297.630135] ffff88015b207de8: ffffffffb18b7e60 (get_max_files+0x10/0x10)
[ 297.630141] ffff88015b207df0: ffffffffb30b72a0 (rcu_sched_state
+0xaa0/0xaa0)
[ 297.630143] ffff88015b207df8: 000000000000000f (0xf)
[ 297.630146] ffff88015b207e00: ffff88015b207e18 (0xffff88015b207e18)
[ 297.630150] ffff88015b207e08: ffffffffb18b7e94 (file_free_rcu+0x34/0x40)
[ 297.630153] ffff88015b207e10: ffff880157b98280 (0xffff880157b98280)
[ 297.630156] ffff88015b207e18: ffff88015b207f30 (0xffff88015b207f30)
[ 297.630161] ffff88015b207e20: ffffffffb172580c (rcu_process_callbacks
+0x2dc/0xcd0)
[ 297.630164] ffff88015b207e28: ffff88015b21b000 (0xffff88015b21b000)
[ 297.630167] ffff88015b207e30: ffff88015b21b070 (0xffff88015b21b070)
[ 297.630170] ffff88015b207e38: 1ffff1002b640fd5 (0x1ffff1002b640fd5)
[ 297.630173] ffff88015b207e40: ffff880151b02700 (0xffff880151b02700)
[ 297.630176] ffff88015b207e48: ffff88015b224200 (0xffff88015b224200)
[ 297.630178] ffff88015b207e50: ffff88015b224280 (0xffff88015b224280)
[ 297.630181] ffff88015b207e58: ffff88015b2242b0 (0xffff88015b2242b0)
[ 297.630184] ffff88015b207e60: ffff88015b207f08 (0xffff88015b207f08)
[ 297.630187] ffff88015b207e68: ffff880151b0274c (0xffff880151b0274c)
[ 297.630190] ffff88015b207e70: ffff880151b02700 (0xffff880151b02700)
[ 297.630195] ffff88015b207e78: ffffffffb30b7258 (rcu_sched_state
+0xa58/0xaa0)
[ 297.630198] ffff88015b207e80: ffff880157b98288 (0xffff880157b98288)
[ 297.630203] ffff88015b207e88: ffffffffb30b6800 (rcu_bh_varname+0x60/0x60)
[ 297.630206] ffff88015b207e90: ffff88015b224238 (0xffff88015b224238)
[ 297.630209] ffff88015b207e98: ffff88015b207ec8 (0xffff88015b207ec8)
[ 297.630211] ffff88015b207ea0: 000000000000000a (0xa)
[ 297.630214] ffff88015b207ea8: 0000000041b58ab3 (0x41b58ab3)
[ 297.630218] ffff88015b207eb0: ffffffffb2d944f5 (.LC0+0x155c/0xa3a6)
[ 297.630223] ffff88015b207eb8: ffffffffb1725530 (note_gp_changes+0xe0/0xe0)
[ 297.630226] ffff88015b207ec0: ffff88015b215740 (0xffff88015b215740)
[ 297.630229] ffff88015b207ec8: ffff880157b983c0 (0xffff880157b983c0)
[ 297.630231] ffff88015b207ed0: ffff88014ac19eb0 (0xffff88014ac19eb0)
[ 297.630234] ffff88015b207ed8: ffffffffffffffff (0xffffffffffffffff)
[ 297.630236] ffff88015b207ee0: 0000000000000000 ...
[ 297.630239] ffff88015b207ee8: 0000004552dda1c0 (0x4552dda1c0)
[ 297.630240] ffff88015b207ef0: 0000000000000000 ...
[ 297.630243] ffff88015b207ef8: ffff88015b207f20 (0xffff88015b207f20)
[ 297.630249] ffff88015b207f00: ffffffffb174a0a8 (tick_program_event
+0x48/0x80)
[ 297.630252] ffff88015b207f08: 0000000000000009 (0x9)
[ 297.630259] ffff88015b207f10: ffffffffb3009148 (softirq_vec+0x48/0x80)
[ 297.630261] ffff88015b207f18: 0000000000000009 (0x9)
[ 297.630263] ffff88015b207f20: 0000000000000008 (0x8)
[ 297.630265] ffff88015b207f28: 0000000000000009 (0x9)
[ 297.630268] ffff88015b207f30: ffff88015b207fa8 (0xffff88015b207fa8)
[ 297.630273] ffff88015b207f38: ffffffffb2646cbc (__do_softirq+0x12c/0x343)
[ 297.630276] ffff88015b207f40: 0000000a00404100 (0xa00404100)
[ 297.630279] ffff88015b207f48: ffff880151b02700 (0xffff880151b02700)
[ 297.630282] ffff88015b207f50: 00000000fffff730 (0xfffff730)
[ 297.630284] ffff88015b207f58: 0000000000000009 (0x9)
[ 297.630286] ffff88015b207f60: 0000000000000040 (0x40)
[ 297.630289] ffff88015b207f68: 000001005b21c294 (0x1005b21c294)
[ 297.630294] ffff88015b207f70: ffffffffb3009110 (softirq_vec+0x10/0x80)
[ 297.630297] ffff88015b207f78: 0000008000000008 (0x8000000008)
[ 297.630300] ffff88015b207f80: ffff88015a77ce00 (0xffff88015a77ce00)
[ 297.630303] ffff88015b207f88: ffff88015b215840 (0xffff88015b215840)
[ 297.630304] ffff88015b207f90: 0000000000000000 ...
[ 297.630307] ffff88015b207f98: ffff880156a5feb0 (0xffff880156a5feb0)
[ 297.630311] ffff88015b207fa0: ffffffffb18b23b1 (SyS_open+0x1/0x20)
[ 297.630314] ffff88015b207fa8: ffff88015b207fc0 (0xffff88015b207fc0)
[ 297.630318] ffff88015b207fb0: ffffffffb1692304 (irq_exit+0xe4/0xf0)
[ 297.630321] ffff88015b207fb8: ffff88015b215740 (0xffff88015b215740)
[ 297.630324] ffff88015b207fc0: ffff88015b207fe8 (0xffff88015b207fe8)
[ 297.630329] ffff88015b207fc8: ffffffffb2646446 (smp_apic_timer_interrupt
+0x86/0x1a0)
[ 297.630332] ffff88015b207fd0: ffff88015104d500 (0xffff88015104d500)
[ 297.630335] ffff88015b207fd8: ffff88015b215840 (0xffff88015b215840)
[ 297.630338] ffff88015b207fe0: 0000000000000246 (0x246)
[ 297.630341] ffff88015b207fe8: ffff880156a5fdc9 (0xffff880156a5fdc9)
[ 297.630345] ffff88015b207ff0: ffffffffb26452f3 (apic_timer_interrupt
+0x93/0xa0)
[ 297.630348] ffff88015b207ff8: ffff880156a5fdc8 (0xffff880156a5fdc8)
[ 297.630352] ffff880156a5fdc8: ffffffffb18b23b1 (SyS_open+0x1/0x20)
[ 297.630355] ffff880156a5fdd0: ffff880156a5feb0 (0xffff880156a5feb0)
[ 297.630357] ffff880156a5fdd8: 0000000000000246 (0x246)
[ 297.630360] ffff880156a5fde0: ffff88015b215840 (0xffff88015b215840)
[ 297.630363] ffff880156a5fde8: ffff880156a5fea0 (0xffff880156a5fea0)
[ 297.630366] ffff880156a5fdf0: ffff88015104d500 (0xffff88015104d500)
[ 297.630369] ffff880156a5fdf8: fffff52000140c08 (0xfffff52000140c08)
[ 297.630372] ffff880156a5fe00: ffffc90000a0603f (0xffffc90000a0603f)
[ 297.630375] ffff880156a5fe08: fffff52000140c07 (0xfffff52000140c07)
[ 297.630378] ffff880156a5fe10: fffff52000140c08 (0xfffff52000140c08)
[ 297.630379] ffff880156a5fe18: 0000000000000000 ...
[ 297.630385] ffff880156a5fe20: ffffffffb178d9eb (opt_pre_handler+0x6b/0x80)
[ 297.630388] ffff880156a5fe28: dffffc0000000000 (0xdffffc0000000000)
[ 297.630391] ffff880156a5fe30: dffffc0000000000 (0xdffffc0000000000)
[ 297.630393] ffff880156a5fe38: 0000000000000246 (0x246)
[ 297.630396] ffff880156a5fe40: ffffffffffffff10 (0xffffffffffffff10)
[ 297.630401] ffff880156a5fe48: ffffffffb1667417 (optimized_callback
+0x67/0x100)
[ 297.630404] ffff880156a5fe50: 0000000000000010 (0x10)
[ 297.630406] ffff880156a5fe58: 0000000000000246 (0x246)
[ 297.630409] ffff880156a5fe60: ffff880156a5fe78 (0xffff880156a5fe78)
[ 297.630412] ffff880156a5fe68: 0000000000000018 (0x18)
[ 297.630414] ffff880156a5fe70: 0000000000000246 (0x246)
[ 297.630417] ffff880156a5fe78: 00000000026aed08 (0x26aed08)
[ 297.630419] ffff880156a5fe80: 0000000000000005 (0x5)
[ 297.630421] ffff880156a5fe88: 0000000000000003 (0x3)
[ 297.630423] ffff880156a5fe90: 0000000000000000 ...
[ 297.630425] ffff880156a5fe98: 00000000025d1568 (0x25d1568)
[ 297.630428] ffff880156a5fea0: 00007ffec7d87e50 (0x7ffec7d87e50)
[ 297.630431] ffff880156a5fea8: ffffffffc0050282 (0xffffffffc0050282)
[ 297.630433] ffff880156a5feb0: 00000000025d1568 (0x25d1568)
[ 297.630435] ffff880156a5feb8: 0000000000000000 ...
[ 297.630437] ffff880156a5fec0: 0000000000000003 (0x3)
[ 297.630440] ffff880156a5fec8: 0000000000000005 (0x5)
[ 297.630442] ffff880156a5fed0: 00007ffec7d87e50 (0x7ffec7d87e50)
[ 297.630445] ffff880156a5fed8: 00000000026aed08 (0x26aed08)
[ 297.630448] ffff880156a5fee0: ffff880151b02700 (0xffff880151b02700)
[ 297.630450] ffff880156a5fee8: 0000000002675e00 (0x2675e00)
[ 297.630453] ffff880156a5fef0: 0000000000000001 (0x1)
[ 297.630455] ffff880156a5fef8: 0000000000000002 (0x2)
[ 297.630457] ffff880156a5ff00: 0000000000000002 (0x2)
[ 297.630460] ffff880156a5ff08: 0000000002675e00 (0x2675e00)
[ 297.630462] ffff880156a5ff10: 0000000000000180 (0x180)
[ 297.630464] ffff880156a5ff18: 0000000000000000 ...
[ 297.630466] ffff880156a5ff20: 000000000272a008 (0x272a008)
[ 297.630469] ffff880156a5ff28: ffffffffffffffff (0xffffffffffffffff)
[ 297.630473] ffff880156a5ff30: ffffffffb18b23b1 (SyS_open+0x1/0x20)
[ 297.630475] ffff880156a5ff38: 0000000000000010 (0x10)
[ 297.630478] ffff880156a5ff40: 0000000000000293 (0x293)
[ 297.630481] ffff880156a5ff48: ffff880156a5ff50 (0xffff880156a5ff50)
[ 297.630485] ffff880156a5ff50: ffffffffb1665770 (copy_oldmem_page+0x90/0x90)
[ 297.630488] ffff880156a5ff58: 00000000025d1b28 (0x25d1b28)
[ 297.630489] ffff880156a5ff60: 0000000000000000 ...
[ 297.630492] ffff880156a5ff68: 0000000000000003 (0x3)
[ 297.630494] ffff880156a5ff70: 0000000000000005 (0x5)
[ 297.630497] ffff880156a5ff78: 00007ffec7d87e50 (0x7ffec7d87e50)
[ 297.630499] ffff880156a5ff80: 00000000026aed08 (0x26aed08)
[ 297.630502] ffff880156a5ff88: 0000000000000246 (0x246)
[ 297.630504] ffff880156a5ff90: 0000000002675e00 (0x2675e00)
[ 297.630506] ffff880156a5ff98: 0000000000000001 (0x1)
[ 297.630509] ffff880156a5ffa0: 0000000000000002 (0x2)
[ 297.630511] ffff880156a5ffa8: ffffffffffffffda (0xffffffffffffffda)
[ 297.630514] ffff880156a5ffb0: 00007f3d3f7be4e0 (0x7f3d3f7be4e0)
[ 297.630517] ffff880156a5ffb8: 0000000000000180 (0x180)
[ 297.630518] ffff880156a5ffc0: 0000000000000000 ...
[ 297.630521] ffff880156a5ffc8: 000000000272a008 (0x272a008)
[ 297.630523] ffff880156a5ffd0: 0000000000000002 (0x2)
[ 297.630526] ffff880156a5ffd8: 00007f3d3f7be4e0 (0x7f3d3f7be4e0)
[ 297.630528] ffff880156a5ffe0: 0000000000000033 (0x33)
[ 297.630530] ffff880156a5ffe8: 0000000000000246 (0x246)
[ 297.630533] ffff880156a5fff0: 00007ffec7d87db8 (0x7ffec7d87db8)
[ 297.630535] ffff880156a5fff8: 000000000000002b (0x2b)
opensnoop(pythong) itself blocks too:
root@test:~# cat /proc/2075/stack
[<ffffffffb79a0a07>] ring_buffer_wait+0x167/0x2e0
[<ffffffffb79a34e7>] wait_on_pipe+0x77/0x80
[<ffffffffb79aa7a1>] tracing_wait_pipe.isra.69+0x51/0xf0
[<ffffffffb79abdf9>] tracing_read_pipe+0x1c9/0x500
[<ffffffffb7ab5e62>] __vfs_read+0xd2/0x370
[<ffffffffb7ab61b7>] vfs_read+0xb7/0x1a0
[<ffffffffb7ab6bd0>] SyS_read+0xa0/0x120
[<ffffffffb8843c37>] entry_SYSCALL_64_fastpath+0x1a/0xa5
[<ffffffffffffffff>] 0xffffffffffffffff
Thanks,
//richard
--
sigma star gmbh - Eduard-Bodem-Gasse 6 - 6020 Innsbruck - Austria
ATU66964118 - FN 374287y
On Mon, Sep 25, 2017 at 11:23:31PM +0200, Richard Weinberger wrote: Hi!
While playing with bcc's opensnoop tool on Linux 4.14-rc2 I managed to trigger
this splat:
[ 297.629773] WARNING: kernel stack frame pointer at ffff880156a5fea0 in
bash:2103 has bad value 00007ffec7d87e50
[ 297.629777] unwind stack type:0 next_sp: (null) mask:0x6
graph_idx:0
[ 297.629783] ffff88015b207ae0: ffff88015b207b68 (0xffff88015b207b68)
[ 297.629790] ffff88015b207ae8: ffffffffb163c00e (__save_stack_trace+0x6e/
0xd0)
[ 297.629792] ffff88015b207af0: 0000000000000000 ...
[ 297.629795] ffff88015b207af8: ffff880156a58000 (0xffff880156a58000)
[ 297.629799] ffff88015b207b00: ffff880156a60000 (0xffff880156a60000)
[ 297.629800] ffff88015b207b08: 0000000000000000 ...
[ 297.629803] ffff88015b207b10: 0000000000000006 (0x6)
[ 297.629806] ffff88015b207b18: ffff880151b02700 (0xffff880151b02700)
[ 297.629809] ffff88015b207b20: 0000010100000000 (0x10100000000)
[ 297.629812] ffff88015b207b28: ffff880156a5fea0 (0xffff880156a5fea0)
[ 297.629815] ffff88015b207b30: ffff88015b207ae0 (0xffff88015b207ae0)
[ 297.629818] ffff88015b207b38: ffffffffc0050282 (0xffffffffc0050282)
[ 297.629819] ffff88015b207b40: 0000000000000000 ...
[ 297.629822] ffff88015b207b48: 0000000001000000 (0x1000000)
[ 297.629825] ffff88015b207b50: ffff880157b98280 (0xffff880157b98280)
[ 297.629828] ffff88015b207b58: ffff880157b98380 (0xffff880157b98380)
[ 297.629831] ffff88015b207b60: ffff88015ad2b500 (0xffff88015ad2b500)
[ 297.629834] ffff88015b207b68: ffff88015b207b78 (0xffff88015b207b78)
[ 297.629838] ffff88015b207b70: ffffffffb163c086 (save_stack_trace+0x16/0x20)
[ 297.629841] ffff88015b207b78: ffff88015b207da8 (0xffff88015b207da8)
[ 297.629847] ffff88015b207b80: ffffffffb18a8ed6 (save_stack+0x46/0xd0)
[ 297.629850] ffff88015b207b88: 000000400000000c (0x400000000c)
[ 297.629852] ffff88015b207b90: ffff88015b207ba0 (0xffff88015b207ba0)
[ 297.629855] ffff88015b207b98: ffff880100000000 (0xffff880100000000)
[ 297.629859] ffff88015b207ba0: ffffffffb163c086 (save_stack_trace+0x16/0x20)
[ 297.629864] ffff88015b207ba8: ffffffffb18a8ed6 (save_stack+0x46/0xd0)
[ 297.629868] ffff88015b207bb0: ffffffffb18a9752 (kasan_slab_free+0x72/0xc0)
Thanks for the report!
I'm not sure I understand what's going on here.
It seems you have kasan enabled and it's trying to do save_stack()
and something crashing?
I don't see any bpf related helpers in the stack trace.
Which architecture is this? and .config ?
Is bpf jit enabled? If so, make sure that net.core.bpf_jit_kallsyms=1
Alexei,
CC'ing Josh and Ingo.
Am Dienstag, 26. September 2017, 06:09:02 CEST schrieb Alexei Starovoitov: On Mon, Sep 25, 2017 at 11:23:31PM +0200, Richard Weinberger wrote: quoted Hi!
While playing with bcc's opensnoop tool on Linux 4.14-rc2 I managed to
trigger this splat:
[ 297.629773] WARNING: kernel stack frame pointer at ffff880156a5fea0 in
bash:2103 has bad value 00007ffec7d87e50
[ 297.629777] unwind stack type:0 next_sp: (null) mask:0x6
graph_idx:0
[ 297.629783] ffff88015b207ae0: ffff88015b207b68 (0xffff88015b207b68)
[ 297.629790] ffff88015b207ae8: ffffffffb163c00e
(__save_stack_trace+0x6e/
0xd0)
[ 297.629792] ffff88015b207af0: 0000000000000000 ...
[ 297.629795] ffff88015b207af8: ffff880156a58000 (0xffff880156a58000)
[ 297.629799] ffff88015b207b00: ffff880156a60000 (0xffff880156a60000)
[ 297.629800] ffff88015b207b08: 0000000000000000 ...
[ 297.629803] ffff88015b207b10: 0000000000000006 (0x6)
[ 297.629806] ffff88015b207b18: ffff880151b02700 (0xffff880151b02700)
[ 297.629809] ffff88015b207b20: 0000010100000000 (0x10100000000)
[ 297.629812] ffff88015b207b28: ffff880156a5fea0 (0xffff880156a5fea0)
[ 297.629815] ffff88015b207b30: ffff88015b207ae0 (0xffff88015b207ae0)
[ 297.629818] ffff88015b207b38: ffffffffc0050282 (0xffffffffc0050282)
[ 297.629819] ffff88015b207b40: 0000000000000000 ...
[ 297.629822] ffff88015b207b48: 0000000001000000 (0x1000000)
[ 297.629825] ffff88015b207b50: ffff880157b98280 (0xffff880157b98280)
[ 297.629828] ffff88015b207b58: ffff880157b98380 (0xffff880157b98380)
[ 297.629831] ffff88015b207b60: ffff88015ad2b500 (0xffff88015ad2b500)
[ 297.629834] ffff88015b207b68: ffff88015b207b78 (0xffff88015b207b78)
[ 297.629838] ffff88015b207b70: ffffffffb163c086
(save_stack_trace+0x16/0x20) [ 297.629841] ffff88015b207b78:
ffff88015b207da8 (0xffff88015b207da8) [ 297.629847] ffff88015b207b80:
ffffffffb18a8ed6 (save_stack+0x46/0xd0) [ 297.629850] ffff88015b207b88:
000000400000000c (0x400000000c)
[ 297.629852] ffff88015b207b90: ffff88015b207ba0 (0xffff88015b207ba0)
[ 297.629855] ffff88015b207b98: ffff880100000000 (0xffff880100000000)
[ 297.629859] ffff88015b207ba0: ffffffffb163c086
(save_stack_trace+0x16/0x20) [ 297.629864] ffff88015b207ba8:
ffffffffb18a8ed6 (save_stack+0x46/0xd0) [ 297.629868] ffff88015b207bb0:
ffffffffb18a9752 (kasan_slab_free+0x72/0xc0) Thanks for the report!
I'm not sure I understand what's going on here.
It seems you have kasan enabled and it's trying to do save_stack()
and something crashing?
I don't see any bpf related helpers in the stack trace.
Which architecture is this? and .config ?
Is bpf jit enabled? If so, make sure that net.core.bpf_jit_kallsyms=1
I found some time to dig a little further.
It seems to happen only when CONFIG_DEBUG_SPINLOCK is enabled, please see the
attached .config. The JIT is off.
KAsan is also not involved at all, the regular stack saving machinery from the
trace framework initiates the stack unwinder.
The issue arises as soon as in pre_handler_kretprobe() raw_spin_lock_irqsave()
is being called.
It happens on all releases that have commit c32c47c68a0a ("x86/unwind: Warn on
bad frame pointer").
Interestingly it does not happen when I run
samples/kprobes/kretprobe_example.ko. So, BPF must be involved somehow.
Here is another variant of the warning, it matches the attached .config:
[ 42.729039] WARNING: kernel stack frame pointer at ffff99ef4076bea0 in
opensnoop:2008 has bad value 0000000000000008
[ 42.729041] unwind stack type:0 next_sp: (null) mask:0x2
graph_idx:0
[ 42.729042] ffff99ef4076bcb0: ffff99ef4076bd38 (0xffff99ef4076bd38)
[ 42.729044] ffff99ef4076bcb8: ffffffffac42781e (__save_stack_trace+0x6e/
0xd0)
[ 42.729044] ffff99ef4076bcc0: 0000000000000000 ...
[ 42.729045] ffff99ef4076bcc8: ffff99ef40768000 (0xffff99ef40768000)
[ 42.729045] ffff99ef4076bcd0: ffff99ef4076c000 (0xffff99ef4076c000)
[ 42.729045] ffff99ef4076bcd8: 0000000000000000 ...
[ 42.729046] ffff99ef4076bce0: 0000000000000002 (0x2)
[ 42.729046] ffff99ef4076bce8: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729047] ffff99ef4076bcf0: 0000000100000000 (0x100000000)
[ 42.729047] ffff99ef4076bcf8: ffff99ef4076bea0 (0xffff99ef4076bea0)
[ 42.729048] ffff99ef4076bd00: ffff99ef4076bcb0 (0xffff99ef4076bcb0)
[ 42.729048] ffff99ef4076bd08: ffffffffc00b302f (0xffffffffc00b302f)
[ 42.729048] ffff99ef4076bd10: 0000000000000000 ...
[ 42.729049] ffff99ef4076bd18: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729049] ffff99ef4076bd20: 0000000000000000 ...
[ 42.729052] ffff99ef4076bd28: ffffffffadb9ccc0 (lock_classes
+0x55500/0x29fec0)
[ 42.729052] ffff99ef4076bd30: 0000000000000000 ...
[ 42.729052] ffff99ef4076bd38: ffff99ef4076bd48 (0xffff99ef4076bd48)
[ 42.729053] ffff99ef4076bd40: ffffffffac427896 (save_stack_trace+0x16/0x20)
[ 42.729054] ffff99ef4076bd48: ffff99ef4076bd98 (0xffff99ef4076bd98)
[ 42.729055] ffff99ef4076bd50: ffffffffac4a18d5 (__lock_acquire.isra.
34+0x525/0x700)
[ 42.729055] ffff99ef4076bd58: 0000000000000000 ...
[ 42.729055] ffff99ef4076bd68: ffff99ef00000411 (0xffff99ef00000411)
[ 42.729056] ffff99ef4076bd70: 0000000000000046 (0x46)
[ 42.729056] ffff99ef4076bd78: 0000000000000000 ...
[ 42.729057] ffff99ef4076bd98: ffff99ef4076be00 (0xffff99ef4076be00)
[ 42.729057] ffff99ef4076bda0: ffffffffac4a224a (lock_acquire+0xca/0x170)
[ 42.729059] ffff99ef4076bda8: ffffffffac50a2cd (pre_handler_kretprobe+0x3d/
0x1b0)
[ 42.729059] ffff99ef4076bdb0: 0000000100000000 (0x100000000)
[ 42.729060] ffff99ef4076bdb8: ffff8a1c00000000 (0xffff8a1c00000000)
[ 42.729063] ffff99ef4076bdc0: 0000000000000046 (0x46)
[ 42.729063] ffff99ef4076bdc8: 00000001ac47ee61 (0x1ac47ee61)
[ 42.729064] ffff99ef4076bdd0: ffff8a1c37b0e0d0 (0xffff8a1c37b0e0d0)
[ 42.729064] ffff99ef4076bdd8: ffff8a1c37b0e0b8 (0xffff8a1c37b0e0b8)
[ 42.729067] ffff99ef4076bde0: 0000000000000082 (0x82)
[ 42.729067] ffff99ef4076bde8: ffff8a1c37b0e0b8 (0xffff8a1c37b0e0b8)
[ 42.729067] ffff99ef4076bdf0: ffff99ef4076beb0 (0xffff99ef4076beb0)
[ 42.729068] ffff99ef4076bdf8: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729068] ffff99ef4076be00: ffff99ef4076be28 (0xffff99ef4076be28)
[ 42.729070] ffff99ef4076be08: fffffffface13e56 (_raw_spin_lock_irqsave
+0x46/0x60)
[ 42.729071] ffff99ef4076be10: ffffffffac50a2cd (pre_handler_kretprobe+0x3d/
0x1b0)
[ 42.729072] ffff99ef4076be18: ffff8a1c37b0e010 (0xffff8a1c37b0e010)
[ 42.729072] ffff99ef4076be20: ffff8a1c37b0e010 (0xffff8a1c37b0e010)
[ 42.729073] ffff99ef4076be28: ffff99ef4076be60 (0xffff99ef4076be60)
[ 42.729074] ffff99ef4076be30: ffffffffac50a2cd (pre_handler_kretprobe+0x3d/
0x1b0)
[ 42.729074] ffff99ef4076be38: ffff8a1c37b0e010 (0xffff8a1c37b0e010)
[ 42.729074] ffff99ef4076be40: ffff8a1c38cc1780 (0xffff8a1c38cc1780)
[ 42.729075] ffff99ef4076be48: ffff99ef4076beb0 (0xffff99ef4076beb0)
[ 42.729075] ffff99ef4076be50: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729076] ffff99ef4076be58: 000055b4ee9920a0 (0x55b4ee9920a0)
[ 42.729076] ffff99ef4076be60: ffff99ef4076be88 (0xffff99ef4076be88)
[ 42.729077] ffff99ef4076be68: ffffffffac509f6a (opt_pre_handler+0x3a/0x60)
[ 42.729078] ffff99ef4076be70: 0000000000000246 (0x246)
[ 42.729078] ffff99ef4076be78: 000055b4ef12cd70 (0x55b4ef12cd70)
[ 42.729079] ffff99ef4076be80: 0000000000000001 (0x1)
[ 42.729079] ffff99ef4076be88: ffff99ef4076bea0 (0xffff99ef4076bea0)
[ 42.729080] ffff99ef4076be90: ffffffffac442721 (optimized_callback
+0x81/0x90)
[ 42.729081] ffff99ef4076be98: 000055b4ef134d50 (0x55b4ef134d50)
[ 42.729081] ffff99ef4076bea0: 0000000000000008 (0x8)
[ 42.729082] ffff99ef4076bea8: ffffffffc00b302f (0xffffffffc00b302f)
[ 42.729082] ffff99ef4076beb0: 000055b4ee9920a0 (0x55b4ee9920a0)
[ 42.729083] ffff99ef4076beb8: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729083] ffff99ef4076bec0: 0000000000000001 (0x1)
[ 42.729084] ffff99ef4076bec8: 000055b4ef12cd70 (0x55b4ef12cd70)
[ 42.729084] ffff99ef4076bed0: 0000000000000008 (0x8)
[ 42.729084] ffff99ef4076bed8: 000055b4ef134d50 (0x55b4ef134d50)
[ 42.729085] ffff99ef4076bee0: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729085] ffff99ef4076bee8: 0000000000000000 ...
[ 42.729086] ffff99ef4076bef0: 0000000000000001 (0x1)
[ 42.729086] ffff99ef4076bef8: 0000000000000008 (0x8)
[ 42.729086] ffff99ef4076bf00: 0000000000000002 (0x2)
[ 42.729087] ffff99ef4076bf08: 0000000000000000 ...
[ 42.729087] ffff99ef4076bf10: 00000000000001b6 (0x1b6)
[ 42.729087] ffff99ef4076bf18: 0000000000000000 ...
[ 42.729088] ffff99ef4076bf20: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729088] ffff99ef4076bf28: ffffffffffffffff (0xffffffffffffffff)
[ 42.729090] ffff99ef4076bf30: ffffffffac5c5031 (SyS_open+0x1/0x20)
[ 42.729090] ffff99ef4076bf38: 0000000000000010 (0x10)
[ 42.729090] ffff99ef4076bf40: 0000000000000293 (0x293)
[ 42.729091] ffff99ef4076bf48: ffff99ef4076bf50 (0xffff99ef4076bf50)
[ 42.729092] ffff99ef4076bf50: fffffffface13f77 (entry_SYSCALL_64_fastpath
+0x1a/0xaa)
[ 42.729092] ffff99ef4076bf58: 0000000000000026 (0x26)
[ 42.729093] ffff99ef4076bf60: 00007f276f5e2600 (0x7f276f5e2600)
[ 42.729093] ffff99ef4076bf68: 0000000000000001 (0x1)
[ 42.729094] ffff99ef4076bf70: 0000000000000026 (0x26)
[ 42.729094] ffff99ef4076bf78: 000055b4ef1035d0 (0x55b4ef1035d0)
[ 42.729094] ffff99ef4076bf80: 0000000000000026 (0x26)
[ 42.729095] ffff99ef4076bf88: 0000000000000246 (0x246)
[ 42.729095] ffff99ef4076bf90: 0000000000000000 ...
[ 42.729095] ffff99ef4076bf98: 0000000000000001 (0x1)
[ 42.729096] ffff99ef4076bfa0: 0000000000000008 (0x8)
[ 42.729096] ffff99ef4076bfa8: ffffffffffffffda (0xffffffffffffffda)
[ 42.729097] ffff99ef4076bfb0: 00007f276f3234e0 (0x7f276f3234e0)
[ 42.729097] ffff99ef4076bfb8: 00000000000001b6 (0x1b6)
[ 42.729097] ffff99ef4076bfc0: 0000000000000000 ...
[ 42.729098] ffff99ef4076bfc8: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729098] ffff99ef4076bfd0: 0000000000000002 (0x2)
[ 42.729099] ffff99ef4076bfd8: 00007f276f3234e0 (0x7f276f3234e0)
[ 42.729099] ffff99ef4076bfe0: 0000000000000033 (0x33)
[ 42.729100] ffff99ef4076bfe8: 0000000000000246 (0x246)
[ 42.729100] ffff99ef4076bff0: 00007ffd98082448 (0x7ffd98082448)
[ 42.729100] ffff99ef4076bff8: 000000000000002b (0x2b)
Thanks,
//richard
--
sigma star gmbh - Eduard-Bodem-Gasse 6 - 6020 Innsbruck - Austria
ATU66964118 - FN 374287y
On 09/26/2017 11:51 PM, Richard Weinberger wrote: Alexei,
CC'ing Josh and Ingo.
Am Dienstag, 26. September 2017, 06:09:02 CEST schrieb Alexei Starovoitov: quoted On Mon, Sep 25, 2017 at 11:23:31PM +0200, Richard Weinberger wrote: quoted Hi!
While playing with bcc's opensnoop tool on Linux 4.14-rc2 I managed to
trigger this splat:
[ 297.629773] WARNING: kernel stack frame pointer at ffff880156a5fea0 in
bash:2103 has bad value 00007ffec7d87e50
[ 297.629777] unwind stack type:0 next_sp: (null) mask:0x6
graph_idx:0
[ 297.629783] ffff88015b207ae0: ffff88015b207b68 (0xffff88015b207b68)
[ 297.629790] ffff88015b207ae8: ffffffffb163c00e
(__save_stack_trace+0x6e/
0xd0)
[ 297.629792] ffff88015b207af0: 0000000000000000 ...
[ 297.629795] ffff88015b207af8: ffff880156a58000 (0xffff880156a58000)
[ 297.629799] ffff88015b207b00: ffff880156a60000 (0xffff880156a60000)
[ 297.629800] ffff88015b207b08: 0000000000000000 ...
[ 297.629803] ffff88015b207b10: 0000000000000006 (0x6)
[ 297.629806] ffff88015b207b18: ffff880151b02700 (0xffff880151b02700)
[ 297.629809] ffff88015b207b20: 0000010100000000 (0x10100000000)
[ 297.629812] ffff88015b207b28: ffff880156a5fea0 (0xffff880156a5fea0)
[ 297.629815] ffff88015b207b30: ffff88015b207ae0 (0xffff88015b207ae0)
[ 297.629818] ffff88015b207b38: ffffffffc0050282 (0xffffffffc0050282)
[ 297.629819] ffff88015b207b40: 0000000000000000 ...
[ 297.629822] ffff88015b207b48: 0000000001000000 (0x1000000)
[ 297.629825] ffff88015b207b50: ffff880157b98280 (0xffff880157b98280)
[ 297.629828] ffff88015b207b58: ffff880157b98380 (0xffff880157b98380)
[ 297.629831] ffff88015b207b60: ffff88015ad2b500 (0xffff88015ad2b500)
[ 297.629834] ffff88015b207b68: ffff88015b207b78 (0xffff88015b207b78)
[ 297.629838] ffff88015b207b70: ffffffffb163c086
(save_stack_trace+0x16/0x20) [ 297.629841] ffff88015b207b78:
ffff88015b207da8 (0xffff88015b207da8) [ 297.629847] ffff88015b207b80:
ffffffffb18a8ed6 (save_stack+0x46/0xd0) [ 297.629850] ffff88015b207b88:
000000400000000c (0x400000000c)
[ 297.629852] ffff88015b207b90: ffff88015b207ba0 (0xffff88015b207ba0)
[ 297.629855] ffff88015b207b98: ffff880100000000 (0xffff880100000000)
[ 297.629859] ffff88015b207ba0: ffffffffb163c086
(save_stack_trace+0x16/0x20) [ 297.629864] ffff88015b207ba8:
ffffffffb18a8ed6 (save_stack+0x46/0xd0) [ 297.629868] ffff88015b207bb0:
ffffffffb18a9752 (kasan_slab_free+0x72/0xc0) Thanks for the report!
I'm not sure I understand what's going on here.
It seems you have kasan enabled and it's trying to do save_stack()
and something crashing?
I don't see any bpf related helpers in the stack trace.
Which architecture is this? and .config ?
Is bpf jit enabled? If so, make sure that net.core.bpf_jit_kallsyms=1
I found some time to dig a little further.
It seems to happen only when CONFIG_DEBUG_SPINLOCK is enabled, please see the
attached .config. The JIT is off.
KAsan is also not involved at all, the regular stack saving machinery from the
trace framework initiates the stack unwinder.
The issue arises as soon as in pre_handler_kretprobe() raw_spin_lock_irqsave()
is being called.
It happens on all releases that have commit c32c47c68a0a ("x86/unwind: Warn on
bad frame pointer").
Interestingly it does not happen when I run
samples/kprobes/kretprobe_example.ko. So, BPF must be involved somehow.
Some time ago, Josh fixed this one here, seems perhaps related in
some way; it was triggerable back then from one of the BPF tracing
samples if I recall correctly:
commit a8b7a92318b6d7779f6d8e9aa6ba0e3de01a8943
Author: Josh Poimboeuf [off-list ref]
Date: Wed Apr 12 13:47:12 2017 -0500
x86/unwind: Silence entry-related warnings
A few people have reported unwinder warnings like the following:
WARNING: kernel stack frame pointer at ffffc90000fe7ff0 in rsync:1157 has bad value (null)
unwind stack type:0 next_sp: (null) mask:2 graph_idx:0
ffffc90000fe7f98: ffffc90000fe7ff0 (0xffffc90000fe7ff0)
ffffc90000fe7fa0: ffffffffb7000f56 (trace_hardirqs_off_thunk+0x1a/0x1c)
ffffc90000fe7fa8: 0000000000000246 (0x246)
ffffc90000fe7fb0: 0000000000000000 ...
ffffc90000fe7fc0: 00007ffe3af639bc (0x7ffe3af639bc)
ffffc90000fe7fc8: 0000000000000006 (0x6)
ffffc90000fe7fd0: 00007f80af433fc5 (0x7f80af433fc5)
ffffc90000fe7fd8: 00007ffe3af638e0 (0x7ffe3af638e0)
ffffc90000fe7fe0: 00007ffe3af638e0 (0x7ffe3af638e0)
ffffc90000fe7fe8: 00007ffe3af63970 (0x7ffe3af63970)
ffffc90000fe7ff0: 0000000000000000 ...
ffffc90000fe7ff8: ffffffffb7b74b9a (entry_SYSCALL_64_after_swapgs+0x17/0x4f)
This warning can happen when unwinding a code path where an interrupt
occurred in x86 entry code before it set up the first stack frame.
Silently ignore any warnings for this case.
Reported-by: Daniel Borkmann [off-list ref]
Reported-by: Dave Jones [off-list ref]
Signed-off-by: Josh Poimboeuf [off-list ref]
Acked-by: Thomas Gleixner [off-list ref]
Cc: Andy Lutomirski [off-list ref]
Cc: Borislav Petkov [off-list ref]
Cc: Brian Gerst [off-list ref]
Cc: Denys Vlasenko [off-list ref]
Cc: H. Peter Anvin [off-list ref]
Cc: Linus Torvalds [off-list ref]
Cc: Peter Zijlstra [off-list ref]
Fixes: c32c47c68a0a ("x86/unwind: Warn on bad frame pointer")
Link: http://lkml.kernel.org/r/dbd6838826466a60dc23a52098185bc973ce2f1e.1492020577.git.jpoimboe@redhat.com
Signed-off-by: Ingo Molnar [off-list ref]
Here is another variant of the warning, it matches the attached .config:
[ 42.729039] WARNING: kernel stack frame pointer at ffff99ef4076bea0 in
opensnoop:2008 has bad value 0000000000000008
[ 42.729041] unwind stack type:0 next_sp: (null) mask:0x2
graph_idx:0
[ 42.729042] ffff99ef4076bcb0: ffff99ef4076bd38 (0xffff99ef4076bd38)
[ 42.729044] ffff99ef4076bcb8: ffffffffac42781e (__save_stack_trace+0x6e/
0xd0)
[ 42.729044] ffff99ef4076bcc0: 0000000000000000 ...
[ 42.729045] ffff99ef4076bcc8: ffff99ef40768000 (0xffff99ef40768000)
[ 42.729045] ffff99ef4076bcd0: ffff99ef4076c000 (0xffff99ef4076c000)
[ 42.729045] ffff99ef4076bcd8: 0000000000000000 ...
[ 42.729046] ffff99ef4076bce0: 0000000000000002 (0x2)
[ 42.729046] ffff99ef4076bce8: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729047] ffff99ef4076bcf0: 0000000100000000 (0x100000000)
[ 42.729047] ffff99ef4076bcf8: ffff99ef4076bea0 (0xffff99ef4076bea0)
[ 42.729048] ffff99ef4076bd00: ffff99ef4076bcb0 (0xffff99ef4076bcb0)
[ 42.729048] ffff99ef4076bd08: ffffffffc00b302f (0xffffffffc00b302f)
[ 42.729048] ffff99ef4076bd10: 0000000000000000 ...
[ 42.729049] ffff99ef4076bd18: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729049] ffff99ef4076bd20: 0000000000000000 ...
[ 42.729052] ffff99ef4076bd28: ffffffffadb9ccc0 (lock_classes
+0x55500/0x29fec0)
[ 42.729052] ffff99ef4076bd30: 0000000000000000 ...
[ 42.729052] ffff99ef4076bd38: ffff99ef4076bd48 (0xffff99ef4076bd48)
[ 42.729053] ffff99ef4076bd40: ffffffffac427896 (save_stack_trace+0x16/0x20)
[ 42.729054] ffff99ef4076bd48: ffff99ef4076bd98 (0xffff99ef4076bd98)
[ 42.729055] ffff99ef4076bd50: ffffffffac4a18d5 (__lock_acquire.isra.
34+0x525/0x700)
[ 42.729055] ffff99ef4076bd58: 0000000000000000 ...
[ 42.729055] ffff99ef4076bd68: ffff99ef00000411 (0xffff99ef00000411)
[ 42.729056] ffff99ef4076bd70: 0000000000000046 (0x46)
[ 42.729056] ffff99ef4076bd78: 0000000000000000 ...
[ 42.729057] ffff99ef4076bd98: ffff99ef4076be00 (0xffff99ef4076be00)
[ 42.729057] ffff99ef4076bda0: ffffffffac4a224a (lock_acquire+0xca/0x170)
[ 42.729059] ffff99ef4076bda8: ffffffffac50a2cd (pre_handler_kretprobe+0x3d/
0x1b0)
[ 42.729059] ffff99ef4076bdb0: 0000000100000000 (0x100000000)
[ 42.729060] ffff99ef4076bdb8: ffff8a1c00000000 (0xffff8a1c00000000)
[ 42.729063] ffff99ef4076bdc0: 0000000000000046 (0x46)
[ 42.729063] ffff99ef4076bdc8: 00000001ac47ee61 (0x1ac47ee61)
[ 42.729064] ffff99ef4076bdd0: ffff8a1c37b0e0d0 (0xffff8a1c37b0e0d0)
[ 42.729064] ffff99ef4076bdd8: ffff8a1c37b0e0b8 (0xffff8a1c37b0e0b8)
[ 42.729067] ffff99ef4076bde0: 0000000000000082 (0x82)
[ 42.729067] ffff99ef4076bde8: ffff8a1c37b0e0b8 (0xffff8a1c37b0e0b8)
[ 42.729067] ffff99ef4076bdf0: ffff99ef4076beb0 (0xffff99ef4076beb0)
[ 42.729068] ffff99ef4076bdf8: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729068] ffff99ef4076be00: ffff99ef4076be28 (0xffff99ef4076be28)
[ 42.729070] ffff99ef4076be08: fffffffface13e56 (_raw_spin_lock_irqsave
+0x46/0x60)
[ 42.729071] ffff99ef4076be10: ffffffffac50a2cd (pre_handler_kretprobe+0x3d/
0x1b0)
[ 42.729072] ffff99ef4076be18: ffff8a1c37b0e010 (0xffff8a1c37b0e010)
[ 42.729072] ffff99ef4076be20: ffff8a1c37b0e010 (0xffff8a1c37b0e010)
[ 42.729073] ffff99ef4076be28: ffff99ef4076be60 (0xffff99ef4076be60)
[ 42.729074] ffff99ef4076be30: ffffffffac50a2cd (pre_handler_kretprobe+0x3d/
0x1b0)
[ 42.729074] ffff99ef4076be38: ffff8a1c37b0e010 (0xffff8a1c37b0e010)
[ 42.729074] ffff99ef4076be40: ffff8a1c38cc1780 (0xffff8a1c38cc1780)
[ 42.729075] ffff99ef4076be48: ffff99ef4076beb0 (0xffff99ef4076beb0)
[ 42.729075] ffff99ef4076be50: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729076] ffff99ef4076be58: 000055b4ee9920a0 (0x55b4ee9920a0)
[ 42.729076] ffff99ef4076be60: ffff99ef4076be88 (0xffff99ef4076be88)
[ 42.729077] ffff99ef4076be68: ffffffffac509f6a (opt_pre_handler+0x3a/0x60)
[ 42.729078] ffff99ef4076be70: 0000000000000246 (0x246)
[ 42.729078] ffff99ef4076be78: 000055b4ef12cd70 (0x55b4ef12cd70)
[ 42.729079] ffff99ef4076be80: 0000000000000001 (0x1)
[ 42.729079] ffff99ef4076be88: ffff99ef4076bea0 (0xffff99ef4076bea0)
[ 42.729080] ffff99ef4076be90: ffffffffac442721 (optimized_callback
+0x81/0x90)
[ 42.729081] ffff99ef4076be98: 000055b4ef134d50 (0x55b4ef134d50)
[ 42.729081] ffff99ef4076bea0: 0000000000000008 (0x8)
[ 42.729082] ffff99ef4076bea8: ffffffffc00b302f (0xffffffffc00b302f)
[ 42.729082] ffff99ef4076beb0: 000055b4ee9920a0 (0x55b4ee9920a0)
[ 42.729083] ffff99ef4076beb8: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729083] ffff99ef4076bec0: 0000000000000001 (0x1)
[ 42.729084] ffff99ef4076bec8: 000055b4ef12cd70 (0x55b4ef12cd70)
[ 42.729084] ffff99ef4076bed0: 0000000000000008 (0x8)
[ 42.729084] ffff99ef4076bed8: 000055b4ef134d50 (0x55b4ef134d50)
[ 42.729085] ffff99ef4076bee0: ffff8a1c39163fc0 (0xffff8a1c39163fc0)
[ 42.729085] ffff99ef4076bee8: 0000000000000000 ...
[ 42.729086] ffff99ef4076bef0: 0000000000000001 (0x1)
[ 42.729086] ffff99ef4076bef8: 0000000000000008 (0x8)
[ 42.729086] ffff99ef4076bf00: 0000000000000002 (0x2)
[ 42.729087] ffff99ef4076bf08: 0000000000000000 ...
[ 42.729087] ffff99ef4076bf10: 00000000000001b6 (0x1b6)
[ 42.729087] ffff99ef4076bf18: 0000000000000000 ...
[ 42.729088] ffff99ef4076bf20: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729088] ffff99ef4076bf28: ffffffffffffffff (0xffffffffffffffff)
[ 42.729090] ffff99ef4076bf30: ffffffffac5c5031 (SyS_open+0x1/0x20)
[ 42.729090] ffff99ef4076bf38: 0000000000000010 (0x10)
[ 42.729090] ffff99ef4076bf40: 0000000000000293 (0x293)
[ 42.729091] ffff99ef4076bf48: ffff99ef4076bf50 (0xffff99ef4076bf50)
[ 42.729092] ffff99ef4076bf50: fffffffface13f77 (entry_SYSCALL_64_fastpath
+0x1a/0xaa)
[ 42.729092] ffff99ef4076bf58: 0000000000000026 (0x26)
[ 42.729093] ffff99ef4076bf60: 00007f276f5e2600 (0x7f276f5e2600)
[ 42.729093] ffff99ef4076bf68: 0000000000000001 (0x1)
[ 42.729094] ffff99ef4076bf70: 0000000000000026 (0x26)
[ 42.729094] ffff99ef4076bf78: 000055b4ef1035d0 (0x55b4ef1035d0)
[ 42.729094] ffff99ef4076bf80: 0000000000000026 (0x26)
[ 42.729095] ffff99ef4076bf88: 0000000000000246 (0x246)
[ 42.729095] ffff99ef4076bf90: 0000000000000000 ...
[ 42.729095] ffff99ef4076bf98: 0000000000000001 (0x1)
[ 42.729096] ffff99ef4076bfa0: 0000000000000008 (0x8)
[ 42.729096] ffff99ef4076bfa8: ffffffffffffffda (0xffffffffffffffda)
[ 42.729097] ffff99ef4076bfb0: 00007f276f3234e0 (0x7f276f3234e0)
[ 42.729097] ffff99ef4076bfb8: 00000000000001b6 (0x1b6)
[ 42.729097] ffff99ef4076bfc0: 0000000000000000 ...
[ 42.729098] ffff99ef4076bfc8: 000055b4ef12d1b0 (0x55b4ef12d1b0)
[ 42.729098] ffff99ef4076bfd0: 0000000000000002 (0x2)
[ 42.729099] ffff99ef4076bfd8: 00007f276f3234e0 (0x7f276f3234e0)
[ 42.729099] ffff99ef4076bfe0: 0000000000000033 (0x33)
[ 42.729100] ffff99ef4076bfe8: 0000000000000246 (0x246)
[ 42.729100] ffff99ef4076bff0: 00007ffd98082448 (0x7ffd98082448)
[ 42.729100] ffff99ef4076bff8: 000000000000002b (0x2b)
Thanks,
//richard
On Tue, Sep 26, 2017 at 11:51:31PM +0200, Richard Weinberger wrote: Alexei,
CC'ing Josh and Ingo.
Am Dienstag, 26. September 2017, 06:09:02 CEST schrieb Alexei Starovoitov: quoted On Mon, Sep 25, 2017 at 11:23:31PM +0200, Richard Weinberger wrote: quoted Hi!
While playing with bcc's opensnoop tool on Linux 4.14-rc2 I managed to
trigger this splat:
[ 297.629773] WARNING: kernel stack frame pointer at ffff880156a5fea0 in
bash:2103 has bad value 00007ffec7d87e50
[ 297.629777] unwind stack type:0 next_sp: (null) mask:0x6
graph_idx:0
[ 297.629783] ffff88015b207ae0: ffff88015b207b68 (0xffff88015b207b68)
[ 297.629790] ffff88015b207ae8: ffffffffb163c00e
(__save_stack_trace+0x6e/
0xd0)
[ 297.629792] ffff88015b207af0: 0000000000000000 ...
[ 297.629795] ffff88015b207af8: ffff880156a58000 (0xffff880156a58000)
[ 297.629799] ffff88015b207b00: ffff880156a60000 (0xffff880156a60000)
[ 297.629800] ffff88015b207b08: 0000000000000000 ...
[ 297.629803] ffff88015b207b10: 0000000000000006 (0x6)
[ 297.629806] ffff88015b207b18: ffff880151b02700 (0xffff880151b02700)
[ 297.629809] ffff88015b207b20: 0000010100000000 (0x10100000000)
[ 297.629812] ffff88015b207b28: ffff880156a5fea0 (0xffff880156a5fea0)
[ 297.629815] ffff88015b207b30: ffff88015b207ae0 (0xffff88015b207ae0)
[ 297.629818] ffff88015b207b38: ffffffffc0050282 (0xffffffffc0050282)
[ 297.629819] ffff88015b207b40: 0000000000000000 ...
[ 297.629822] ffff88015b207b48: 0000000001000000 (0x1000000)
[ 297.629825] ffff88015b207b50: ffff880157b98280 (0xffff880157b98280)
[ 297.629828] ffff88015b207b58: ffff880157b98380 (0xffff880157b98380)
[ 297.629831] ffff88015b207b60: ffff88015ad2b500 (0xffff88015ad2b500)
[ 297.629834] ffff88015b207b68: ffff88015b207b78 (0xffff88015b207b78)
[ 297.629838] ffff88015b207b70: ffffffffb163c086
(save_stack_trace+0x16/0x20) [ 297.629841] ffff88015b207b78:
ffff88015b207da8 (0xffff88015b207da8) [ 297.629847] ffff88015b207b80:
ffffffffb18a8ed6 (save_stack+0x46/0xd0) [ 297.629850] ffff88015b207b88:
000000400000000c (0x400000000c)
[ 297.629852] ffff88015b207b90: ffff88015b207ba0 (0xffff88015b207ba0)
[ 297.629855] ffff88015b207b98: ffff880100000000 (0xffff880100000000)
[ 297.629859] ffff88015b207ba0: ffffffffb163c086
(save_stack_trace+0x16/0x20) [ 297.629864] ffff88015b207ba8:
ffffffffb18a8ed6 (save_stack+0x46/0xd0) [ 297.629868] ffff88015b207bb0:
ffffffffb18a9752 (kasan_slab_free+0x72/0xc0) Thanks for the report!
I'm not sure I understand what's going on here.
It seems you have kasan enabled and it's trying to do save_stack()
and something crashing?
I don't see any bpf related helpers in the stack trace.
Which architecture is this? and .config ?
Is bpf jit enabled? If so, make sure that net.core.bpf_jit_kallsyms=1
I found some time to dig a little further.
It seems to happen only when CONFIG_DEBUG_SPINLOCK is enabled, please see the
attached .config. The JIT is off.
KAsan is also not involved at all, the regular stack saving machinery from the
trace framework initiates the stack unwinder.
The issue arises as soon as in pre_handler_kretprobe() raw_spin_lock_irqsave()
is being called.
It happens on all releases that have commit c32c47c68a0a ("x86/unwind: Warn on
bad frame pointer").
Interestingly it does not happen when I run
samples/kprobes/kretprobe_example.ko. So, BPF must be involved somehow.
Here is another variant of the warning, it matches the attached .config:
I can take a look at it. Unfortunately, for these types of issues I
often need the vmlinux file to be able to make sense of the unwinder
dump. So if you happen to have somewhere to copy the vmlinux to, that
would be helpful. Or if you give me your GCC version I can try to
rebuild it locally.
--
Josh
Am Mittwoch, 27. September 2017, 00:42:46 CEST schrieb Josh Poimboeuf: quoted Here is another variant of the warning, it matches the attached .config: I can take a look at it. Unfortunately, for these types of issues I
often need the vmlinux file to be able to make sense of the unwinder
dump. So if you happen to have somewhere to copy the vmlinux to, that
would be helpful. Or if you give me your GCC version I can try to
rebuild it locally.
There you go:
http://git.infradead.org/~rw/bpf_splat/vmlinux.xz
Thanks,
//richard
--
sigma star gmbh - Eduard-Bodem-Gasse 6 - 6020 Innsbruck - Austria
ATU66964118 - FN 374287y
On Wed, Sep 27, 2017 at 08:51:22AM +0200, Richard Weinberger wrote: Am Mittwoch, 27. September 2017, 00:42:46 CEST schrieb Josh Poimboeuf: quoted quoted Here is another variant of the warning, it matches the attached .config: I can take a look at it. Unfortunately, for these types of issues I
often need the vmlinux file to be able to make sense of the unwinder
dump. So if you happen to have somewhere to copy the vmlinux to, that
would be helpful. Or if you give me your GCC version I can try to
rebuild it locally.
There you go:
http://git.infradead.org/~rw/bpf_splat/vmlinux.xz
Thanks. Can you test this fix?
diff --git a/arch/x86/kernel/kprobes/common.h b/arch/x86/kernel/kprobes/common.h
index db2182d63ed0..3fc0f9a794cb 100644
--- a/arch/x86/kernel/kprobes/common.h
+++ b/arch/x86/kernel/kprobes/common.h @@ -3,6 +3,15 @@
/* Kprobes and Optprobes common header */
+ #include <asm/asm.h> + + #ifdef CONFIG_FRAME_POINTER + # define SAVE_RBP_STRING " push %" _ASM_BP "\n" \ + " mov %" _ASM_SP ", %" _ASM_BP " \n " + #else + # define SAVE_RBP_STRING " push %" _ASM_BP "\n" + #endif + #ifdef CONFIG_X86_64
#define SAVE_REGS_STRING \
/* Skip cs, ip, orig_ax. */ \ @@ -17,7 +26,7 @@
" pushq %r10 \n " \
" pushq %r11 \n " \
" pushq %rbx \n " \
- " pushq %rbp \n " \+ SAVE_RBP_STRING \ " pushq %r12 \n " \
" pushq %r13 \n " \
" pushq %r14 \n " \ @@ -48,7 +57,7 @@
" pushl %es \n " \
" pushl %ds \n " \
" pushl %eax \n " \
- " pushl %ebp \n " \+ SAVE_RBP_STRING \ " pushl %edi \n " \
" pushl %esi \n " \
" pushl %edx \n " \
Josh,
Am Mittwoch, 27. September 2017, 16:14:30 CEST schrieb Josh Poimboeuf: On Wed, Sep 27, 2017 at 08:51:22AM +0200, Richard Weinberger wrote: quoted Am Mittwoch, 27. September 2017, 00:42:46 CEST schrieb Josh Poimboeuf: quoted quoted Here is another variant of the warning, it matches the attached .config: quoted hunk quoted quoted I can take a look at it. Unfortunately, for these types of issues I
often need the vmlinux file to be able to make sense of the unwinder
dump. So if you happen to have somewhere to copy the vmlinux to, that
would be helpful. Or if you give me your GCC version I can try to
rebuild it locally.
There you go:
http://git.infradead.org/~rw/bpf_splat/vmlinux.xz
Thanks. Can you test this fix?
diff --git a/arch/x86/kernel/kprobes/common.h b/arch/x86/kernel/kprobes/common.h index db2182d63ed0..3fc0f9a794cb 100644 --- a/arch/x86/kernel/kprobes/common.h
+++ b/arch/x86/kernel/kprobes/common.h @@ -3,6 +3,15 @@
/* Kprobes and Optprobes common header */
+ #include <asm/asm.h> + + #ifdef CONFIG_FRAME_POINTER + # define SAVE_RBP_STRING " push %" _ASM_BP "\n" \ + " mov %" _ASM_SP ", %" _ASM_BP " \n " + #else + # define SAVE_RBP_STRING " push %" _ASM_BP "\n" + #endif + #ifdef CONFIG_X86_64
#define SAVE_REGS_STRING \
/* Skip cs, ip, orig_ax. */ \ @@ -17,7 +26,7 @@
" pushq %r10 \n " \
" pushq %r11 \n " \
" pushq %rbx \n " \
- " pushq %rbp \n " \+ SAVE_RBP_STRING \ " pushq %r12 \n " \
" pushq %r13 \n " \
" pushq %r14 \n " \ @@ -48,7 +57,7 @@
" pushl %es \n " \
" pushl %ds \n " \
" pushl %eax \n " \
- " pushl %ebp \n " \+ SAVE_RBP_STRING \ " pushl %edi \n " \
" pushl %esi \n " \
" pushl %edx \n " \
This fixes the issue for me!
Thanks,
//richard
--
sigma star gmbh - Eduard-Bodem-Gasse 6 - 6020 Innsbruck - Austria
ATU66964118 - FN 374287y