libceph: add spinlock around osd->o_requests

In a later patch, we're going to need to search for a request in
the rbtree, but taking the o_mutex is inconvenient as we already
hold the con mutex at the point where we need it.

Add a new spinlock that we take when inserting and erasing entries from
the o_requests tree. Search of the rbtree can be done with either the
mutex or the spinlock, but insertion and removal requires both.

Reviewed-by: Xiubo Li <xiubli@redhat.com>
Signed-off-by: Jeff Layton <jlayton@kernel.org>

diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
index cba8a6ffc3290d068d51f5d734c3547e4de23c4f..0fe7ab827dcc96bd5408bad2683381e82605e671 100644 (file)
--- a/include/linux/ceph/osd_client.h
+++ b/include/linux/ceph/osd_client.h
@@ -29,7 +29,12 @@ typedef void (*ceph_osdc_callback_t)(struct ceph_osd_request *);
 
 #define CEPH_HOMELESS_OSD      -1
 
-/* a given osd we're communicating with */
+/*
+ * A given osd we're communicating with.
+ *
+ * Note that the o_requests tree can be searched while holding the "lock" mutex
+ * or the "o_requests_lock" spinlock. Insertion or removal requires both!
+ */
 struct ceph_osd {
        refcount_t o_ref;
        struct ceph_osd_client *o_osdc;
@@ -37,6 +42,7 @@ struct ceph_osd {
        int o_incarnation;
        struct rb_node o_node;
        struct ceph_connection o_con;
+       spinlock_t o_requests_lock;
        struct rb_root o_requests;
        struct rb_root o_linger_requests;
        struct rb_root o_backoff_mappings;

diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 9d82bb42e958f4709b1e734377984a940d8016bb..884193a9f370b20a2e74970bf3ef44310e73426f 100644 (file)
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -1177,6 +1177,7 @@ static void osd_init(struct ceph_osd *osd)
 {
        refcount_set(&osd->o_ref, 1);
        RB_CLEAR_NODE(&osd->o_node);
+       spin_lock_init(&osd->o_requests_lock);
        osd->o_requests = RB_ROOT;
        osd->o_linger_requests = RB_ROOT;
        osd->o_backoff_mappings = RB_ROOT;
@@ -1406,7 +1407,9 @@ static void link_request(struct ceph_osd *osd, struct ceph_osd_request *req)
                atomic_inc(&osd->o_osdc->num_homeless);
 
        get_osd(osd);
+       spin_lock(&osd->o_requests_lock);
        insert_request(&osd->o_requests, req);
+       spin_unlock(&osd->o_requests_lock);
        req->r_osd = osd;
 }
 
@@ -1418,7 +1421,9 @@ static void unlink_request(struct ceph_osd *osd, struct ceph_osd_request *req)
             req, req->r_tid);
 
        req->r_osd = NULL;
+       spin_lock(&osd->o_requests_lock);
        erase_request(&osd->o_requests, req);
+       spin_unlock(&osd->o_requests_lock);
        put_osd(osd);
 
        if (!osd_homeless(osd))