vmcore分析锁问题实例（x86-64）

问题描述：系统出现panic，dmesg有如下打印：

[122061.197311] task:irq/181-ice-enp state:D stack:0     pid:3134  ppid:2      flags:0x00004000
[122061.197315] Call Trace:
[122061.197317]  <TASK>
[122061.197318]  __schedule+0x34e/0xb00
[122061.197325]  schedule_rtlock+0x1f/0x40
[122061.197328]  rtlock_slowlock_locked+0x232/0xd40
[122061.197332]  rt_read_lock+0x54/0x130
[122061.197333]  ep_poll_callback+0x35/0x2a0
[122061.197337]  ? ktime_get+0x39/0xa0
[122061.197340]  __wake_up_common+0x7d/0x190
[122061.197343]  __wake_up_common_lock+0x7c/0xc0
[122061.197345]  sock_def_readable+0x42/0xc0
[122061.197349]  tcp_child_process+0x199/0x1f0
[122061.197354]  tcp_v4_rcv+0xaf5/0xe80
[122061.197356]  ? raw_local_deliver+0xc7/0x230
[122061.197359]  ip_protocol_deliver_rcu+0x32/0x160
[122061.197361]  ip_local_deliver_finish+0x77/0xa0
[122061.197363]  ip_sublist_rcv_finish+0x80/0x90
[122061.197364]  ip_sublist_rcv+0x1a4/0x240
[122061.197365]  ? __pfx_ip_rcv_finish+0x10/0x10
[122061.197367]  ip_list_rcv+0x136/0x160
[122061.197368]  __netif_receive_skb_list_core+0x2b1/0x2e0
[122061.197371]  netif_receive_skb_list_internal+0x1d5/0x310
[122061.197373]  napi_complete_done+0x73/0x1b0
[122061.197377]  ice_napi_poll+0xa1f/0xd80 [ice]
[122061.197444]  __napi_poll+0x29/0x1b0
[122061.197446]  net_rx_action+0x29f/0x370
[122061.197447]  ? plist_del+0x63/0xc0
[122061.197450]  handle_softirqs.constprop.0+0xb0/0x250
[122061.197453]  ? __pfx_irq_forced_thread_fn+0x10/0x10
[122061.197456]  __local_bh_enable_ip+0x6f/0xa0
[122061.197457]  irq_forced_thread_fn+0x77/0x90
[122061.197460]  irq_thread+0xed/0x1a0
[122061.197463]  ? __pfx_irq_thread_dtor+0x10/0x10
[122061.197466]  ? __pfx_irq_thread+0x10/0x10
[122061.197468]  kthread+0xdd/0x110
[122061.197470]  ? __pfx_kthread+0x10/0x10 
[122061.197471]  ret_from_fork+0x31/0x50
[122061.197474]  ? __pfx_kthread+0x10/0x10 
[122061.197476]  ret_from_fork_asm+0x1b/0x30
[122061.197479]  </TASK>

进程3134在尝试获取lock，而且处于state:D，中断无法执行，导致系统出现异常。
内核代码在rtlock_slowlock_locked中尝试获取lock，代码如下：

1802 /**                 
1803  * rtlock_slowlock_locked - Slow path lock acquisition for RT locks
1804  * @lock:   The underlying RT mutex
1805  */                 
1806 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock)
1807 {                                                                                                                                                                                                                                    
1808     struct rt_mutex_waiter waiter;
1809     struct task_struct *owner;
1810    
1811     lockdep_assert_held(&lock->wait_lock);
1812    
1813     if (try_to_take_rt_mutex(lock, current, NULL))
1814         return;     
1815    
1816     rt_mutex_init_rtlock_waiter(&waiter);
1817    
1818     /* Save current state and set state to TASK_RTLOCK_WAIT */
1819     current_save_and_set_rtlock_wait_state();
1820    
1821     trace_contention_begin(lock, LCB_F_RT);
1822    
1823     task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK);
1824    
1825     for (;;) {      
1826         /* Try to acquire the lock again */
1827         if (try_to_take_rt_mutex(lock, current, &waiter))
1828             break;
1829    
1830         if (&waiter == rt_mutex_top_waiter(lock))
1831             owner = rt_mutex_owner(lock);
1832         else        
1833             owner = NULL;
1834         raw_spin_unlock_irq(&lock->wait_lock);
1835    
1836         if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner))
1837             schedule_rtlock();
1838    
1839         raw_spin_lock_irq(&lock->wait_lock);
1840         set_current_state(TASK_RTLOCK_WAIT);
1841     }               
1842    
1843     /* Restore the task state */
1844     current_restore_rtlock_saved_state();
1845    
1846     /*              
1847      * try_to_take_rt_mutex() sets the waiter bit unconditionally.
1848      * We might have to fix that up:
1849      */             
1850     fixup_rt_mutex_waiters(lock, true);
1851     debug_rt_mutex_free_waiter(&waiter);
1852  
1853     trace_contention_end(lock, 0);
1854 }

lock结构体如下：

struct rt_mutex_base {raw_spinlock_t      wait_lock;struct rb_root_cached   waiters;struct task_struct  *owner;
};

解决思路：只要找到lock，就可以通过lock找到owner
使用crash打开vmcore分析：

crash> bt -f 3134
PID: 3134     TASK: ff27411db0af97c0  CPU: 0    COMMAND: "irq/181-ice-enp"#0 [ff468632c619f748] __schedule at ffffffff99f3c3ceff468632c619f750: 0000100000000000 ff27415b3e5f23c0 ff468632c619f760: ffffffff00000004 ffffffff9a2100a0 ff468632c619f770: 0000000000000000 0000000000000002 ff468632c619f780: da58a4eb0f1c7100 ff27411db0af97c0 ff468632c619f790: ff27411db0af97c0 ff27411f3bd0df00 ff468632c619f7a0: ff27411db0afa098 ff27411db0af97c0 ff468632c619f7b0: ff27411f89f8da98 ffffffff99f3d04f #1 [ff468632c619f7b8] schedule_rtlock at ffffffff99f3d04fff468632c619f7c0: ff468632c619f800 ffffffff99f43042 #2 [ff468632c619f7c8] rtlock_slowlock_locked at ffffffff99f43042ff468632c619f7d0: 0000000000000000 ff27411f3bd0e7d8 ff468632c619f7e0: ff27411f3bd0e7d8 0000000000000000 ff468632c619f7f0: ff27411db0af9ba8 0000000000000000 ff468632c619f800: ff468632cc40f3f0 0000000000000000 ff468632c619f810: 0000000000000000 ffffffff0000001d ff468632c619f820: 0000000000000000 0000000000000001 ff468632c619f830: 0000000000000000 0000000000000000 ff468632c619f840: ff27411d0000001d 0000000000000000 ff468632c619f850: ff27411db0af97c0 ff27411f89f8da98 ff468632c619f860: 27183d1600001000 0000000000000000 ff468632c619f870: da58a4eb0f1c7100 00000000000000c3 ff468632c619f880: ff27411f89f8da90 ff27411f89f8da98 ff468632c619f890: ff27411fbcd3d8d0 ff27411f89f8da90 ff468632c619f8a0: 00000000000000c3 ffffffff99f43c74 
…………

需要找到lock的寄存器，进而找到在何时被压栈到何处
rt_mutex_base中wait_lock是第一个成员，也就是说函数当中许多对 lock->wait_lock 的调用可以直接用lock的值，对函数rtlock_slowlock_locked进行反汇编：

crash> dis -lr ffffffff99f43042
kernel/locking/rtmutex.c: 1807
0xffffffff99f42e10 <rtlock_slowlock_locked>:	nopl   0x0(%rax,%rax,1) [FTRACE NOP]
0xffffffff99f42e15 <rtlock_slowlock_locked+5>:	push   %r15
kernel/locking/rtmutex.c: 1813
0xffffffff99f42e17 <rtlock_slowlock_locked+7>:	xor    %edx,%edx
kernel/locking/rtmutex.c: 1807
0xffffffff99f42e19 <rtlock_slowlock_locked+9>:	push   %r14
0xffffffff99f42e1b <rtlock_slowlock_locked+11>:	push   %r13
0xffffffff99f42e1d <rtlock_slowlock_locked+13>:	push   %r12
0xffffffff99f42e1f <rtlock_slowlock_locked+15>:	push   %rbp
0xffffffff99f42e20 <rtlock_slowlock_locked+16>:	mov    %rdi,%rbp
0xffffffff99f42e23 <rtlock_slowlock_locked+19>:	push   %rbx
arch/x86/include/asm/current.h: 41
0xffffffff99f42e24 <rtlock_slowlock_locked+20>:	mov    %gs:0x31b40,%r12
kernel/locking/rtmutex.c: 1813
0xffffffff99f42e2d <rtlock_slowlock_locked+29>:	mov    %r12,%rsi
kernel/locking/rtmutex.c: 1807
0xffffffff99f42e30 <rtlock_slowlock_locked+32>:	sub    $0xa8,%rsp
0xffffffff99f42e37 <rtlock_slowlock_locked+39>:	mov    %gs:0x28,%rax
0xffffffff99f42e40 <rtlock_slowlock_locked+48>:	mov    %rax,0xa0(%rsp)
0xffffffff99f42e48 <rtlock_slowlock_locked+56>:	xor    %eax,%eax
arch/x86/include/asm/current.h: 41
0xffffffff99f42e4a <rtlock_slowlock_locked+58>:	call   0xffffffff99f42430 <try_to_take_rt_mutex>
kernel/locking/rtmutex.c: 1813
0xffffffff99f42e4f <rtlock_slowlock_locked+63>:	mov    %eax,0x4(%rsp)
0xffffffff99f42e53 <rtlock_slowlock_locked+67>:	test   %eax,%eax
0xffffffff99f42e55 <rtlock_slowlock_locked+69>:	je     0xffffffff99f42e84 <rtlock_slowlock_locked+116>
kernel/locking/rtmutex.c: 1854
0xffffffff99f42e57 <rtlock_slowlock_locked+71>:	mov    0xa0(%rsp),%rax
0xffffffff99f42e5f <rtlock_slowlock_locked+79>:	sub    %gs:0x28,%rax
0xffffffff99f42e68 <rtlock_slowlock_locked+88>:	jne    0xffffffff99f43b44 <rtlock_slowlock_locked+3380>
0xffffffff99f42e6e <rtlock_slowlock_locked+94>:	add    $0xa8,%rsp
0xffffffff99f42e75 <rtlock_slowlock_locked+101>:	pop    %rbx
0xffffffff99f42e76 <rtlock_slowlock_locked+102>:	pop    %rbp
0xffffffff99f42e77 <rtlock_slowlock_locked+103>:	pop    %r12
0xffffffff99f42e79 <rtlock_slowlock_locked+105>:	pop    %r13
0xffffffff99f42e7b <rtlock_slowlock_locked+107>:	pop    %r14
0xffffffff99f42e7d <rtlock_slowlock_locked+109>:	pop    %r15
0xffffffff99f42e7f <rtlock_slowlock_locked+111>:	ret    
0xffffffff99f42e80 <rtlock_slowlock_locked+112>:	int3   
0xffffffff99f42e81 <rtlock_slowlock_locked+113>:	int3   
0xffffffff99f42e82 <rtlock_slowlock_locked+114>:	int3   
0xffffffff99f42e83 <rtlock_slowlock_locked+115>:	int3   
kernel/locking/rtmutex_common.h: 214
0xffffffff99f42e84 <rtlock_slowlock_locked+116>:	lea    0x8d8(%r12),%r13
0xffffffff99f42e8c <rtlock_slowlock_locked+124>:	lea    0x30(%rsp),%rbx
kernel/locking/rtmutex_common.h: 217
0xffffffff99f42e91 <rtlock_slowlock_locked+129>:	movq   $0x0,0x80(%rsp)
kernel/locking/rtmutex_common.h: 214
0xffffffff99f42e9d <rtlock_slowlock_locked+141>:	lea    0x58(%rsp),%rax
kernel/locking/rtmutex.c: 1819
0xffffffff99f42ea2 <rtlock_slowlock_locked+146>:	mov    %r13,%rdi
kernel/locking/rtmutex_common.h: 215
0xffffffff99f42ea5 <rtlock_slowlock_locked+149>:	mov    %rbx,0x30(%rsp)
kernel/locking/rtmutex_common.h: 214
0xffffffff99f42eaa <rtlock_slowlock_locked+154>:	mov    %rax,0x58(%rsp)
kernel/locking/rtmutex_common.h: 223
0xffffffff99f42eaf <rtlock_slowlock_locked+159>:	movl   $0x1000,0x90(%rsp)
arch/x86/include/asm/current.h: 41
0xffffffff99f42eba <rtlock_slowlock_locked+170>:	call   0xffffffff99f45c60 <_raw_spin_lock>
0xffffffff99f42ebf <rtlock_slowlock_locked+175>:	mov    0x18(%r12),%eax
0xffffffff99f42ec4 <rtlock_slowlock_locked+180>:	mov    %r13,%rdi
0xffffffff99f42ec7 <rtlock_slowlock_locked+183>:	movl   $0x1000,0x18(%r12)
0xffffffff99f42ed0 <rtlock_slowlock_locked+192>:	mov    %eax,0x1c(%r12)
0xffffffff99f42ed5 <rtlock_slowlock_locked+197>:	call   0xffffffff99f45d60 <_raw_spin_unlock>
arch/x86/include/asm/jump_label.h: 27
0xffffffff99f42eda <rtlock_slowlock_locked+202>:	nopl   0x0(%rax,%rax,1)
arch/x86/include/asm/current.h: 41
0xffffffff99f42edf <rtlock_slowlock_locked+207>:	mov    0x18(%rbp),%r13
0xffffffff99f42ee3 <rtlock_slowlock_locked+211>:	mov    %gs:0x31b40,%r15
kernel/locking/rtmutex_common.h: 161
0xffffffff99f42eec <rtlock_slowlock_locked+220>:	and    $0xfffffffffffffffe,%r13
0xffffffff99f42ef0 <rtlock_slowlock_locked+224>:	mov    %r13,%r12
kernel/locking/rtmutex.c: 1224
0xffffffff99f42ef3 <rtlock_slowlock_locked+227>:	cmp    %r13,%r15
0xffffffff99f42ef6 <rtlock_slowlock_locked+230>:	je     0xffffffff99f4302a <rtlock_slowlock_locked+538>
kernel/locking/rtmutex.c: 1227
0xffffffff99f42efc <rtlock_slowlock_locked+236>:	lea    0x8d8(%r15),%r14
0xffffffff99f42f03 <rtlock_slowlock_locked+243>:	mov    %r14,%rdi
0xffffffff99f42f06 <rtlock_slowlock_locked+246>:	call   0xffffffff99f45c60 <_raw_spin_lock>
kernel/locking/rtmutex.c: 1228
0xffffffff99f42f0b <rtlock_slowlock_locked+251>:	mov    0x6c(%r15),%ecx
kernel/locking/rtmutex.c: 350
0xffffffff99f42f0f <rtlock_slowlock_locked+255>:	mov    $0x78,%eax
kernel/locking/rtmutex.c: 365
0xffffffff99f42f14 <rtlock_slowlock_locked+260>:	mov    0x1f8(%r15),%r9
kernel/locking/rtmutex.c: 1228
0xffffffff99f42f1b <rtlock_slowlock_locked+267>:	mov    %r15,0x80(%rsp)
kernel/locking/rtmutex.c: 1229
0xffffffff99f42f23 <rtlock_slowlock_locked+275>:	cmp    $0x64,%ecx
0xffffffff99f42f26 <rtlock_slowlock_locked+278>:	mov    %rbp,0x88(%rsp)
include/linux/sched/rt.h: 11
0xffffffff99f42f2e <rtlock_slowlock_locked+286>:	cmovge %eax,%ecx
kernel/locking/rtmutex_common.h: 112
0xffffffff99f42f31 <rtlock_slowlock_locked+289>:	mov    0x8(%rbp),%rax
kernel/locking/rtmutex.c: 365
0xffffffff99f42f35 <rtlock_slowlock_locked+293>:	mov    %r9,0x50(%rsp)
kernel/locking/rtmutex.c: 379
0xffffffff99f42f3a <rtlock_slowlock_locked+298>:	mov    %r9,0x78(%rsp)
kernel/locking/rtmutex.c: 364
0xffffffff99f42f3f <rtlock_slowlock_locked+303>:	mov    %ecx,0x48(%rsp)
kernel/locking/rtmutex.c: 378
0xffffffff99f42f43 <rtlock_slowlock_locked+307>:	mov    %ecx,0x70(%rsp)
kernel/locking/rtmutex_common.h: 112
0xffffffff99f42f47 <rtlock_slowlock_locked+311>:	test   %rax,%rax
0xffffffff99f42f4a <rtlock_slowlock_locked+314>:	je     0xffffffff99f4321e <rtlock_slowlock_locked+1038>
kernel/locking/rtmutex_common.h: 130
0xffffffff99f42f50 <rtlock_slowlock_locked+320>:	mov    0x10(%rbp),%rax
0xffffffff99f42f54 <rtlock_slowlock_locked+324>:	mov    %rax,0x8(%rsp)
kernel/locking/rtmutex_common.h: 135
0xffffffff99f42f59 <rtlock_slowlock_locked+329>:	test   %rax,%rax
0xffffffff99f42f5c <rtlock_slowlock_locked+332>:	je     0xffffffff99f42f68 <rtlock_slowlock_locked+344>
kernel/locking/rtmutex_common.h: 137
0xffffffff99f42f5e <rtlock_slowlock_locked+334>:	cmp    0x58(%rax),%rbp
0xffffffff99f42f62 <rtlock_slowlock_locked+338>:	jne    0xffffffff99f43527 <rtlock_slowlock_locked+1815>
include/linux/rbtree.h: 168
0xffffffff99f42f68 <rtlock_slowlock_locked+344>:	mov    0x8(%rbp),%rax
0xffffffff99f42f6c <rtlock_slowlock_locked+348>:	lea    0x8(%rbp),%r10
include/linux/rbtree.h: 172
0xffffffff99f42f70 <rtlock_slowlock_locked+352>:	test   %rax,%rax
0xffffffff99f42f73 <rtlock_slowlock_locked+355>:	je     0xffffffff99f4315c <rtlock_slowlock_locked+844>
include/linux/rbtree.h: 170
0xffffffff99f42f79 <rtlock_slowlock_locked+361>:	mov    $0x1,%edi
0xffffffff99f42f7e <rtlock_slowlock_locked+366>:	jmp    0xffffffff99f42f9a <rtlock_slowlock_locked+394>
include/linux/sched/deadline.h: 15
0xffffffff99f42f80 <rtlock_slowlock_locked+368>:	test   %ecx,%ecx
0xffffffff99f42f82 <rtlock_slowlock_locked+370>:	js     0xffffffff99f430b8 <rtlock_slowlock_locked+680>
include/linux/rbtree.h: 177
0xffffffff99f42f88 <rtlock_slowlock_locked+376>:	mov    0x8(%rax),%rdx
0xffffffff99f42f8c <rtlock_slowlock_locked+380>:	lea    0x8(%rax),%rsi
include/linux/rbtree.h: 178
0xffffffff99f42f90 <rtlock_slowlock_locked+384>:	xor    %edi,%edi
include/linux/rbtree.h: 172
0xffffffff99f42f92 <rtlock_slowlock_locked+386>:	test   %rdx,%rdx
0xffffffff99f42f95 <rtlock_slowlock_locked+389>:	je     0xffffffff99f42fab <rtlock_slowlock_locked+411>
0xffffffff99f42f97 <rtlock_slowlock_locked+391>:	mov    %rdx,%rax
kernel/locking/rtmutex.c: 393
0xffffffff99f42f9a <rtlock_slowlock_locked+394>:	cmp    0x18(%rax),%ecx
0xffffffff99f42f9d <rtlock_slowlock_locked+397>:	jge    0xffffffff99f42f80 <rtlock_slowlock_locked+368>
include/linux/rbtree.h: 175
0xffffffff99f42f9f <rtlock_slowlock_locked+399>:	lea    0x10(%rax),%rsi
include/linux/rbtree.h: 172
0xffffffff99f42fa3 <rtlock_slowlock_locked+403>:	mov    (%rsi),%rdx
0xffffffff99f42fa6 <rtlock_slowlock_locked+406>:	test   %rdx,%rdx
0xffffffff99f42fa9 <rtlock_slowlock_locked+409>:	jne    0xffffffff99f42f97 <rtlock_slowlock_locked+391>
include/linux/rbtree.h: 62
0xffffffff99f42fab <rtlock_slowlock_locked+411>:	mov    %rax,0x30(%rsp)
include/linux/rbtree.h: 63
0xffffffff99f42fb0 <rtlock_slowlock_locked+416>:	movq   $0x0,0x38(%rsp)
0xffffffff99f42fb9 <rtlock_slowlock_locked+425>:	movq   $0x0,0x40(%rsp)
include/linux/rbtree.h: 65
0xffffffff99f42fc2 <rtlock_slowlock_locked+434>:	mov    %rbx,(%rsi)
include/linux/rbtree.h: 112
0xffffffff99f42fc5 <rtlock_slowlock_locked+437>:	test   %dil,%dil
0xffffffff99f42fc8 <rtlock_slowlock_locked+440>:	jne    0xffffffff99f4317b <rtlock_slowlock_locked+875>
include/linux/rbtree.h: 114
0xffffffff99f42fce <rtlock_slowlock_locked+446>:	mov    %rbx,%rdi
0xffffffff99f42fd1 <rtlock_slowlock_locked+449>:	mov    %r10,%rsi
0xffffffff99f42fd4 <rtlock_slowlock_locked+452>:	call   0xffffffff99f20d40 <rb_insert_color>
kernel/locking/rtmutex.c: 1238
0xffffffff99f42fd9 <rtlock_slowlock_locked+457>:	mov    %rbx,0x900(%r15)
kernel/locking/rtmutex.c: 1240
0xffffffff99f42fe0 <rtlock_slowlock_locked+464>:	mov    %r14,%rdi
0xffffffff99f42fe3 <rtlock_slowlock_locked+467>:	call   0xffffffff99f45d60 <_raw_spin_unlock>
kernel/locking/rtmutex.c: 1257
0xffffffff99f42fe8 <rtlock_slowlock_locked+472>:	test   %r13,%r13
0xffffffff99f42feb <rtlock_slowlock_locked+475>:	je     0xffffffff99f4302a <rtlock_slowlock_locked+538>
kernel/locking/rtmutex.c: 1260
0xffffffff99f42fed <rtlock_slowlock_locked+477>:	lea    0x8d8(%r13),%r8
0xffffffff99f42ff4 <rtlock_slowlock_locked+484>:	mov    %r8,%rdi
0xffffffff99f42ff7 <rtlock_slowlock_locked+487>:	mov    %r8,0x10(%rsp)
0xffffffff99f42ffc <rtlock_slowlock_locked+492>:	call   0xffffffff99f45c60 <_raw_spin_lock>
kernel/locking/rtmutex_common.h: 130
0xffffffff99f43001 <rtlock_slowlock_locked+497>:	mov    0x10(%rbp),%rax
kernel/locking/rtmutex_common.h: 135
0xffffffff99f43005 <rtlock_slowlock_locked+501>:	mov    0x10(%rsp),%r8
0xffffffff99f4300a <rtlock_slowlock_locked+506>:	test   %rax,%rax
0xffffffff99f4300d <rtlock_slowlock_locked+509>:	je     0xffffffff99f43022 <rtlock_slowlock_locked+530>
kernel/locking/rtmutex_common.h: 137
0xffffffff99f4300f <rtlock_slowlock_locked+511>:	cmp    0x58(%rax),%rbp
0xffffffff99f43013 <rtlock_slowlock_locked+515>:	jne    0xffffffff99f435fe <rtlock_slowlock_locked+2030>
kernel/locking/rtmutex_common.h: 139
0xffffffff99f43019 <rtlock_slowlock_locked+521>:	cmp    %rbx,%rax
0xffffffff99f4301c <rtlock_slowlock_locked+524>:	je     0xffffffff99f43228 <rtlock_slowlock_locked+1048>
kernel/locking/rtmutex.c: 1275
0xffffffff99f43022 <rtlock_slowlock_locked+530>:	mov    %r8,%rdi
0xffffffff99f43025 <rtlock_slowlock_locked+533>:	call   0xffffffff99f45d60 <_raw_spin_unlock>
kernel/locking/rtmutex.c: 1281
0xffffffff99f4302a <rtlock_slowlock_locked+538>:	mov    %gs:0x31b40,%r12
arch/x86/include/asm/current.h: 41
0xffffffff99f43033 <rtlock_slowlock_locked+547>:	jmp    0xffffffff99f43054 <rtlock_slowlock_locked+580>
kernel/locking/rtmutex.c: 1834
0xffffffff99f43035 <rtlock_slowlock_locked+549>:	mov    %rbp,%rdi
0xffffffff99f43038 <rtlock_slowlock_locked+552>:	call   0xffffffff99f45da0 <_raw_spin_unlock_irq>
kernel/locking/rtmutex.c: 1837
0xffffffff99f4303d <rtlock_slowlock_locked+557>:	call   0xffffffff99f3d030 <schedule_rtlock>
kernel/locking/rtmutex.c: 1839
0xffffffff99f43042 <rtlock_slowlock_locked+562>:	mov    %rbp,%rdi

在x86上面，当参数少于7时， 参数会依次放入寄存器: rdi, rsi, rdx, rcx, r8, r9，

mov    %rdi,%rbp

这一行代码已经把参数的值放到rbp寄存器当中了，而且后续可以看到一些类似“mov %r13,%rdi”的操作，说明虽然schedule_rtlock没有参数，但是rdi寄存器已经不再存放lock的值了。
从以下反汇编的内容可以得知，lock值应该就存在rbq当中

0xffffffff99f43035 <rtlock_slowlock_locked+549>:        mov    %rbp,%rdi
0xffffffff99f43038 <rtlock_slowlock_locked+552>:        call   0xffffffff99f45da0 <_raw_spin_unlock_irq>

因为原函数当中是这样进行调用的：

1839         raw_spin_lock_irq(&lock->wait_lock);

lock->wait_lock就是lock的值，在调用_raw_spin_unlock_irq的时候把rbp传给参数寄存器，也就是说rbq当中存的就是lock的值。

接下来对schedule_rtlock进行反汇编：

crash> dis -lr ffffffff99f3d04f
kernel/sched/core.c: 6874
0xffffffff99f3d030 <schedule_rtlock>:	nopw   (%rax)
kernel/sched/core.c: 6792
0xffffffff99f3d034 <schedule_rtlock+4>:	push   %rbx
arch/x86/include/asm/current.h: 41
0xffffffff99f3d035 <schedule_rtlock+5>:	mov    %gs:0x31b40,%rbx
arch/x86/include/asm/preempt.h: 80
0xffffffff99f3d03e <schedule_rtlock+14>:	incl   %gs:0x660f4b03(%rip)        # 0x31b48
kernel/sched/core.c: 6796
0xffffffff99f3d045 <schedule_rtlock+21>:	mov    $0x2,%edi
0xffffffff99f3d04a <schedule_rtlock+26>:	call   0xffffffff99f3c080 <__schedule>
arch/x86/include/asm/preempt.h: 85
0xffffffff99f3d04f <schedule_rtlock+31>:	decl   %gs:0x660f4af2(%rip)        # 0x31b48

并未发现对%rbp的操作，可能是编译器进行了优化，因为schedule_rtlock并未改变寄存器%rbq的值，所以并未进行压栈。
继续对上一层函数__schedule进行反汇编

crash> dis -lr ffffffff99f3c3ce
kernel/sched/core.c: 6598
0xffffffff99f3c080 <__schedule>:	push   %rbp
0xffffffff99f3c081 <__schedule+1>:	mov    %rsp,%rbp
0xffffffff99f3c084 <__schedule+4>:	push   %r15
0xffffffff99f3c086 <__schedule+6>:	push   %r14
0xffffffff99f3c088 <__schedule+8>:	mov    %edi,%r14d
0xffffffff99f3c08b <__schedule+11>:	push   %r13
……

可以看到在最开始的时候就对寄存器%rbp进行压栈，对应的栈地址为ff468632c619f7b0，内容为ff27411f89f8da98
在crash里面查看这个地址的内容：

crash> rt_mutex_base ff27411f89f8da98
struct rt_mutex_base {wait_lock = {raw_lock = {{val = {counter = 0},{locked = 0 '\000',pending = 0 '\000'},{locked_pending = 0,tail = 0}}}},waiters = {rb_root = {rb_node = 0xff468632cc40f3f0},rb_leftmost = 0xff468632c619f800},owner = crash> rt_mutex_base ff27411f89f8da98
}

可以看到owner为0xff27411f3bd0df01，最后的1不是task_struct的地址，是在获得锁的时候或的(#define RT_MUTEX_HAS_WAITERS 1UL)，实际的task_struct地址是0xff27411f3bd0df00

crash> task_struct.pid 0xff27411f3bd0df00 pid = 23645,

得到锁是由23645进程执有的