From c630feb44f61ad258979701937829b9c1b6fa6db Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 30 Apr 2008 16:44:51 -0700 Subject: [PATCH] kclient: drop s_mutex in __ceph_mdsc_send_cap we want to avoid ever touching the delayed work whie holding s_mutex: [10124.715227] ======================================================= [10124.718705] [ INFO: possible circular locking dependency detected ] [10124.718705] 2.6.25 #18 [10124.718705] ------------------------------------------------------- [10124.718705] events/1/10 is trying to acquire lock: [10124.718705] (&s->s_mutex){--..}, at: [] ceph_check_caps+0x458/0x4f8 [ceph] [10124.718705] [10124.718705] but task is already holding lock: [10124.718705] (&(&ci->i_cap_dwork)->work){--..}, at: [] run_workqueue+0x9f/0x1f6 [10124.718705] [10124.718705] which lock already depends on the new lock. [10124.718705] [10124.718705] [10124.718705] the existing dependency chain (in reverse order) is: [10124.718705] [10124.718705] -> #1 (&(&ci->i_cap_dwork)->work){--..}: [10124.718706] [] __lock_acquire+0xa8b/0xc8a [10124.718706] [] lock_acquire+0x8e/0xb2 [10124.718706] [] __cancel_work_timer+0xe8/0x1ea [10124.718706] [] cancel_delayed_work_sync+0xd/0xf [10124.718706] [] __ceph_mdsc_send_cap+0x247/0x264 [ceph] [10124.718706] [] ceph_check_caps+0x47e/0x4f8 [ceph] [10124.718706] [] ceph_put_wrbuffer_cap_refs+0xc9/0xd2 [ceph] [10124.718706] [] ceph_writepages+0x782/0x97a [ceph] [10124.718706] [] do_writepages+0x2b/0x3a [10124.718706] [] __writeback_single_inode+0x151/0x282 [10124.718706] [] sync_sb_inodes+0x1ab/0x26f [10124.718706] [] writeback_inodes+0x85/0xe9 [10124.718706] [] wb_kupdate+0x9f/0x10d [10124.718706] [] pdflush+0x134/0x1df [10124.718706] [] kthread+0x49/0x79 [10124.718706] [] child_rip+0xa/0x12 [10124.718706] [] 0xffffffffffffffff [10124.718706] [10124.718706] -> #0 (&s->s_mutex){--..}: [10124.718706] [] __lock_acquire+0x98d/0xc8a [10124.718706] [] lock_acquire+0x8e/0xb2 [10124.718706] [] mutex_lock_nested+0xed/0x273 [10124.718706] [] ceph_check_caps+0x458/0x4f8 [ceph] [10124.718706] [] ceph_cap_delayed_work+0x101/0x14a [ceph] [10124.718706] [] run_workqueue+0xee/0x1f6 [10124.718706] [] worker_thread+0xdb/0xe8 [10124.718706] [] kthread+0x49/0x79 [10124.718706] [] child_rip+0xa/0x12 [10124.718706] [] 0xffffffffffffffff [10124.718706] [10124.718706] other info that might help us debug this: [10124.718706] [10124.718706] 2 locks held by events/1/10: [10124.718706] #0: (events){--..}, at: [] run_workqueue+0x9f/0x1f6 [10124.718706] #1: (&(&ci->i_cap_dwork)->work){--..}, at: [] run_workqueue+0x9f/0x1f6 [10124.718706] [10124.718706] stack backtrace: [10124.718706] Pid: 10, comm: events/1 Not tainted 2.6.25 #18 [10124.718706] [10124.718706] Call Trace: [10124.718706] [] print_circular_bug_tail+0x70/0x7b [10124.718706] [] __lock_acquire+0x98d/0xc8a [10124.718706] [] ? trace_hardirqs_on_thunk+0x35/0x3a [10124.718706] [] ? native_sched_clock+0x4a/0x66 [10124.718706] [] ? :ceph:ceph_check_caps+0x458/0x4f8 [10124.718706] [] lock_acquire+0x8e/0xb2 [10124.718706] [] ? :ceph:ceph_check_caps+0x458/0x4f8 [10124.718706] [] mutex_lock_nested+0xed/0x273 [10124.718706] [] ? :ceph:ceph_check_caps+0x458/0x4f8 [10124.718706] [] :ceph:ceph_check_caps+0x458/0x4f8 [10124.718706] [] ? :ceph:ceph_cap_delayed_work+0x0/0x14a [10124.718706] [] :ceph:ceph_cap_delayed_work+0x101/0x14a [10124.718706] [] run_workqueue+0xee/0x1f6 [10124.718706] [] worker_thread+0xdb/0xe8 [10124.718706] [] ? autoremove_wake_function+0x0/0x38 [10124.718706] [] ? worker_thread+0x0/0xe8 [10124.718706] [] kthread+0x49/0x79 [10124.718706] [] child_rip+0xa/0x12 [10124.718706] [] ? restore_args+0x0/0x30 [10124.718706] [] ? kthread+0x0/0x79 [10124.718706] [] ? child_rip+0x0/0x12 [10124.718706] [10175.822587] ceph_super: kill_sb ffff81010da60000 --- src/kernel/inode.c | 3 ++- src/kernel/mds_client.c | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/kernel/inode.c b/src/kernel/inode.c index a4f3d4345a67b..d620c044dfaee 100644 --- a/src/kernel/inode.c +++ b/src/kernel/inode.c @@ -865,9 +865,10 @@ ack: } } - /* send_cap drops i_lock */ + /* send_cap drops i_lock AND s_mutex */ removed_last = __ceph_mdsc_send_cap(mdsc, session, cap, used, wanted, !is_delayed); + session = 0; if (removed_last) goto out; goto retry; diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c index cddd1ee3f0f38..4aa398bda7081 100644 --- a/src/kernel/mds_client.c +++ b/src/kernel/mds_client.c @@ -1524,6 +1524,8 @@ int __ceph_mdsc_send_cap(struct ceph_mds_client *mdsc, keep, wanted, seq, size, max_size, &mtime, &atime, session->s_mds); + mutex_unlock(&session->s_mutex); + if (wanted == 0) { if (removed_last && cancel_work) cancel_delayed_work_sync(&ci->i_cap_dwork); @@ -1551,7 +1553,7 @@ static void flush_write_caps(struct ceph_mds_client *mdsc, } used = __ceph_caps_used(cap->ci); wanted = __ceph_caps_wanted(cap->ci); - __ceph_mdsc_send_cap(mdsc, session, cap, used, wanted, 1); + __ceph_mdsc_send_cap(mdsc, session, cap, used, wanted, 0); } } -- 2.39.5