From a71a2a5db5fc70c6ea9e15fdd355e368d2848220 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 13 Aug 2008 14:00:55 -0700 Subject: [PATCH] kclient: add snap_mutex --- src/kernel/inode.c | 3 +- src/kernel/mds_client.c | 80 ++++++++++++++++++++++++----------------- src/kernel/mds_client.h | 11 ++++-- src/kernel/snap.c | 26 +++++++------- src/kernel/super.c | 1 - src/kernel/super.h | 11 +++--- 6 files changed, 73 insertions(+), 59 deletions(-) diff --git a/src/kernel/inode.c b/src/kernel/inode.c index d568228b1923f..6d74a4bf3ff0b 100644 --- a/src/kernel/inode.c +++ b/src/kernel/inode.c @@ -1140,9 +1140,10 @@ int ceph_add_cap(struct inode *inode, int i; int is_new = 0; struct ceph_snaprealm *realm = 0; + struct ceph_mds_client *mdsc = &ceph_inode_to_client(inode)->mdsc; if (snapblob_len) - realm = ceph_update_snap_trace(ceph_inode_to_client(inode), + realm = ceph_update_snap_trace(mdsc, snapblob, snapblob+snapblob_len, 0); diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c index 1f12f11e1a3ca..dc90f86ed3bd9 100644 --- a/src/kernel/mds_client.c +++ b/src/kernel/mds_client.c @@ -253,17 +253,6 @@ static struct ceph_mds_session *__get_session(struct ceph_mds_client *mdsc, return session; } -static struct ceph_mds_session *get_session(struct ceph_mds_client *mdsc, - int mds) -{ - struct ceph_mds_session *session; - mutex_lock(&mdsc->mutex); - session = __get_session(mdsc, mds); - mutex_unlock(&mdsc->mutex); - return session; -} - - static void put_session(struct ceph_mds_session *s) { BUG_ON(s == NULL); @@ -380,16 +369,13 @@ void ceph_mdsc_put_request(struct ceph_mds_request *req) } } -static struct ceph_mds_request * -find_request_and_lock(struct ceph_mds_client *mdsc, __u64 tid) +static struct ceph_mds_request *__get_request(struct ceph_mds_client *mdsc, + __u64 tid) { struct ceph_mds_request *req; - mutex_lock(&mdsc->mutex); req = radix_tree_lookup(&mdsc->request_tree, tid); if (req) get_request(req); - else - mutex_unlock(&mdsc->mutex); return req; } @@ -1131,9 +1117,11 @@ void ceph_mdsc_handle_reply(struct ceph_mds_client *mdsc, struct ceph_msg *msg) tid = le64_to_cpu(head->tid); /* get request, session */ - req = find_request_and_lock(mdsc, tid); + mutex_lock(&mdsc->mutex); + req = __get_request(mdsc, tid); if (!req) { dout(1, "handle_reply on unknown tid %llu\n", tid); + mutex_unlock(&mdsc->mutex); return; } dout(10, "handle_reply %p r_expects_cap=%d\n", req, req->r_expects_cap); @@ -1149,6 +1137,8 @@ void ceph_mdsc_handle_reply(struct ceph_mds_client *mdsc, struct ceph_msg *msg) return; } BUG_ON(req->r_reply); + if (req->r_expects_cap) + mutex_lock(&mdsc->snap_mutex); mutex_unlock(&mdsc->mutex); mutex_lock(&req->r_session->s_mutex); @@ -1199,6 +1189,8 @@ void ceph_mdsc_handle_reply(struct ceph_mds_client *mdsc, struct ceph_msg *msg) } done: + if (req->r_expects_cap) + mutex_unlock(&mdsc->snap_mutex); mutex_unlock(&req->r_session->s_mutex); mutex_lock(&mdsc->mutex); if (err) { @@ -1241,18 +1233,21 @@ void ceph_mdsc_handle_forward(struct ceph_mds_client *mdsc, ceph_decode_8(&p, must_resend); /* handle */ - req = find_request_and_lock(mdsc, tid); - if (!req) - return; /* dup reply? */ + mutex_lock(&mdsc->mutex); + req = __get_request(mdsc, tid); + if (!req) { + dout(10, "forward %llu dne\n", tid); + goto out; /* dup reply? */ + } /* do we have a session with the dest mds? */ - /* yes. adjust mds set, but mds will do the forward. */ if (fwd_seq <= req->r_num_fwd) { dout(10, "forward %llu to mds%d - old seq %d <= %d\n", tid, next_mds, req->r_num_fwd, fwd_seq); } else if (!must_resend && have_session(mdsc, next_mds) && mdsc->sessions[next_mds]->s_state == CEPH_MDS_SESSION_OPEN) { + /* yes. adjust mds set, but mds will do the forward. */ dout(10, "forward %llu to mds%d (mds fwded)\n", tid, next_mds); req->r_num_fwd = fwd_seq; req->r_resend_mds = next_mds; @@ -1268,9 +1263,10 @@ void ceph_mdsc_handle_forward(struct ceph_mds_client *mdsc, req->r_resend_mds = next_mds; complete(&req->r_completion); } - mutex_unlock(&mdsc->mutex); ceph_mdsc_put_request(req); +out: + mutex_unlock(&mdsc->mutex); return; bad: @@ -1515,7 +1511,6 @@ void ceph_mdsc_handle_caps(struct ceph_mds_client *mdsc, struct ceph_msg *msg) { struct super_block *sb = mdsc->client->sb; - struct ceph_client *client = ceph_sb_to_client(sb); struct ceph_mds_session *session; struct inode *inode; struct ceph_mds_caps *h; @@ -1539,9 +1534,13 @@ void ceph_mdsc_handle_caps(struct ceph_mds_client *mdsc, max_size = le64_to_cpu(h->max_size); /* find session */ - session = get_session(&client->mdsc, mds); + mutex_lock(&mdsc->mutex); + session = __get_session(mdsc, mds); + mutex_lock(&mdsc->snap_mutex); + mutex_unlock(&mdsc->mutex); if (!session) { dout(10, "WTF, got cap but no session for mds%d\n", mds); + mutex_unlock(&mdsc->snap_mutex); return; } @@ -1561,6 +1560,7 @@ void ceph_mdsc_handle_caps(struct ceph_mds_client *mdsc, switch (op) { case CEPH_CAP_OP_GRANT: + mutex_unlock(&mdsc->snap_mutex); if (ceph_handle_cap_grant(inode, h, session) == 1) { dout(10, "sending reply back to mds%d\n", mds); ceph_msg_get(msg); @@ -1569,10 +1569,12 @@ void ceph_mdsc_handle_caps(struct ceph_mds_client *mdsc, break; case CEPH_CAP_OP_TRUNC: + mutex_unlock(&mdsc->snap_mutex); ceph_handle_cap_trunc(inode, h, session); break; case CEPH_CAP_OP_EXPORT: + mutex_unlock(&mdsc->snap_mutex); ceph_handle_cap_export(inode, h, session); break; @@ -1580,7 +1582,12 @@ void ceph_mdsc_handle_caps(struct ceph_mds_client *mdsc, ceph_handle_cap_import(inode, h, session, msg->front.iov_base + sizeof(*h), le32_to_cpu(h->snap_trace_len)); + mutex_unlock(&mdsc->snap_mutex); break; + + default: + mutex_unlock(&mdsc->snap_mutex); + derr(10, "unknown cap op %d %s\n", op, ceph_cap_op_name(op)); } iput(inode); @@ -1750,7 +1757,6 @@ void ceph_mdsc_handle_snap(struct ceph_mds_client *mdsc, struct ceph_msg *msg) { struct super_block *sb = mdsc->client->sb; - struct ceph_client *client = ceph_sb_to_client(sb); struct ceph_mds_session *session; int mds = le32_to_cpu(msg->hdr.src.name.num); u64 split; @@ -1779,7 +1785,11 @@ void ceph_mdsc_handle_snap(struct ceph_mds_client *mdsc, ceph_snap_op_name(op), split, trace_len); /* find session */ - session = get_session(&client->mdsc, mds); + mutex_lock(&mdsc->mutex); + session = __get_session(mdsc, mds); + if (session) + mutex_lock(&mdsc->snap_mutex); + mutex_unlock(&mdsc->mutex); if (!session) { dout(10, "WTF, got snap but no session for mds%d\n", mds); return; @@ -1799,7 +1809,7 @@ void ceph_mdsc_handle_snap(struct ceph_mds_client *mdsc, ceph_decode_need(&p, e, sizeof(*ri), bad); ri = p; - realm = ceph_get_snaprealm(client, split); + realm = ceph_get_snaprealm(mdsc, split); if (IS_ERR(realm)) goto out; dout(10, "splitting snaprealm %llx %p\n", realm->ino, realm); @@ -1845,18 +1855,18 @@ void ceph_mdsc_handle_snap(struct ceph_mds_client *mdsc, for (i = 0; i < num_split_realms; i++) { struct ceph_snaprealm *child = - ceph_get_snaprealm(client, + ceph_get_snaprealm(mdsc, le64_to_cpu(split_realms[i])); if (!child) continue; - ceph_adjust_snaprealm_parent(client, child, realm->ino); + ceph_adjust_snaprealm_parent(mdsc, child, realm->ino); ceph_put_snaprealm(child); } ceph_put_snaprealm(realm); } - realm = ceph_update_snap_trace(client, p, e, + realm = ceph_update_snap_trace(mdsc, p, e, op != CEPH_SNAP_OP_DESTROY); if (IS_ERR(realm)) goto bad; @@ -1884,6 +1894,7 @@ void ceph_mdsc_handle_snap(struct ceph_mds_client *mdsc, } ceph_put_snaprealm(realm); + mutex_unlock(&mdsc->snap_mutex); return; bad: @@ -1954,14 +1965,16 @@ void ceph_mdsc_handle_lease(struct ceph_mds_client *mdsc, struct ceph_msg *msg) goto bad; /* find session */ - session = get_session(mdsc, mds); + mutex_lock(&mdsc->mutex); + session = __get_session(mdsc, mds); + mutex_unlock(&mdsc->mutex); if (!session) { dout(10, "WTF, got lease but no session for mds%d\n", mds); return; } - session->s_seq++; mutex_lock(&session->s_mutex); + session->s_seq++; /* lookup inode */ inode = ceph_find_inode(sb, vino); @@ -2160,7 +2173,8 @@ void ceph_mdsc_init(struct ceph_mds_client *mdsc, struct ceph_client *client) mdsc->sessions = 0; mdsc->max_sessions = 0; mdsc->last_tid = 0; - INIT_RADIX_TREE(&mdsc->request_tree, GFP_ATOMIC); + INIT_RADIX_TREE(&mdsc->snaprealms, GFP_NOFS); + INIT_RADIX_TREE(&mdsc->request_tree, GFP_NOFS); init_completion(&mdsc->map_waiters); init_completion(&mdsc->session_close_waiters); INIT_DELAYED_WORK(&mdsc->delayed_work, delayed_work); diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h index aa1289641e969..281d24053c8a6 100644 --- a/src/kernel/mds_client.h +++ b/src/kernel/mds_client.h @@ -120,15 +120,20 @@ struct ceph_mds_request { * mds client state */ struct ceph_mds_client { - struct mutex mutex; /* all nested structures */ - struct mutex snap_mutex; /* all snaprealms */ struct ceph_client *client; + struct mutex mutex; /* all nested structures */ + struct ceph_mdsmap *mdsmap; + struct completion map_waiters, session_close_waiters; + struct ceph_mds_session **sessions; /* NULL if no session */ int max_sessions; /* len of s_mds_sessions */ + + struct mutex snap_mutex; /* all snaprealms */ + struct radix_tree_root snaprealms; + __u64 last_tid; /* most recent mds request */ struct radix_tree_root request_tree; /* pending mds requests */ - struct completion map_waiters, session_close_waiters; struct delayed_work delayed_work; /* delayed work */ unsigned long last_renew_caps; struct list_head cap_delay_list; diff --git a/src/kernel/snap.c b/src/kernel/snap.c index 9de4950c338c9..049c5126d6bad 100644 --- a/src/kernel/snap.c +++ b/src/kernel/snap.c @@ -1,4 +1,5 @@ +#include #include int ceph_debug_snap = -1; @@ -9,14 +10,14 @@ int ceph_debug_snap = -1; #include "decode.h" -struct ceph_snaprealm *ceph_get_snaprealm(struct ceph_client *client, u64 ino) +struct ceph_snaprealm *ceph_get_snaprealm(struct ceph_mds_client *mdsc, u64 ino) { struct ceph_snaprealm *realm; - realm = radix_tree_lookup(&client->snaprealms, ino); + realm = radix_tree_lookup(&mdsc->snaprealms, ino); if (!realm) { realm = kzalloc(sizeof(*realm), GFP_NOFS); - radix_tree_insert(&client->snaprealms, ino, realm); + radix_tree_insert(&mdsc->snaprealms, ino, realm); realm->nref = 1; /* in tree */ realm->ino = ino; INIT_LIST_HEAD(&realm->children); @@ -30,13 +31,10 @@ struct ceph_snaprealm *ceph_get_snaprealm(struct ceph_client *client, u64 ino) return realm; } -struct ceph_snaprealm *ceph_find_snaprealm(struct ceph_client *client, u64 ino) +struct ceph_snaprealm *ceph_find_snaprealm(struct ceph_mds_client *mdsc, + u64 ino) { - struct ceph_snaprealm *realm; - - realm = radix_tree_lookup(&client->snaprealms, ino); - - return realm; + return radix_tree_lookup(&mdsc->snaprealms, ino); } void ceph_put_snaprealm(struct ceph_snaprealm *realm) @@ -52,7 +50,7 @@ void ceph_put_snaprealm(struct ceph_snaprealm *realm) } } -int ceph_adjust_snaprealm_parent(struct ceph_client *client, +int ceph_adjust_snaprealm_parent(struct ceph_mds_client *mdsc, struct ceph_snaprealm *realm, u64 parentino) { struct ceph_snaprealm *parent; @@ -60,7 +58,7 @@ int ceph_adjust_snaprealm_parent(struct ceph_client *client, if (realm->parent_ino == parentino) return 0; - parent = ceph_get_snaprealm(client, parentino); + parent = ceph_get_snaprealm(mdsc, parentino); if (!parent) return -ENOMEM; dout(20, "adjust_snaprealm_parent %llx %p: %llx %p -> %llx %p\n", @@ -193,7 +191,7 @@ static int dup_array(u64 **dst, u64 *src, int num) return 0; } -struct ceph_snaprealm *ceph_update_snap_trace(struct ceph_client *client, +struct ceph_snaprealm *ceph_update_snap_trace(struct ceph_mds_client *mdsc, void *p, void *e, int must_flush) { struct ceph_mds_snap_realm *ri; @@ -215,7 +213,7 @@ more: prior_parent_snaps = p; p += sizeof(u64) * le32_to_cpu(ri->num_prior_parent_snaps); - realm = ceph_get_snaprealm(client, le64_to_cpu(ri->ino)); + realm = ceph_get_snaprealm(mdsc, le64_to_cpu(ri->ino)); if (!realm) goto fail; if (!first) { @@ -240,7 +238,7 @@ more: dout(10, "update_snap_trace %llx %p seq %lld unchanged\n", realm->ino, realm, realm->seq); - invalidate += ceph_adjust_snaprealm_parent(client, realm, + invalidate += ceph_adjust_snaprealm_parent(mdsc, realm, le64_to_cpu(ri->parent)); if (le64_to_cpu(ri->seq) > realm->seq) { diff --git a/src/kernel/super.c b/src/kernel/super.c index 178fb510e1610..2c1de389dfa1d 100644 --- a/src/kernel/super.c +++ b/src/kernel/super.c @@ -582,7 +582,6 @@ struct ceph_client *ceph_create_client(void) init_waitqueue_head(&client->mount_wq); spin_lock_init(&client->sb_lock); - INIT_RADIX_TREE(&client->snaprealms, GFP_NOFS); client->sb = 0; client->mount_state = CEPH_MOUNT_MOUNTING; diff --git a/src/kernel/super.h b/src/kernel/super.h index a4e974000b6ff..9b4a430bd5b35 100644 --- a/src/kernel/super.h +++ b/src/kernel/super.h @@ -149,9 +149,6 @@ struct ceph_client { struct kobject *client_kobj; - struct mutex snaprealm_mutex; - struct radix_tree_root snaprealms; - /* lets ignore all this until later */ spinlock_t sb_lock; int num_sb; /* ref count (for each sb_info that points to me) */ @@ -494,14 +491,14 @@ struct ceph_snaprealm { }; /* snap.c */ -extern struct ceph_snaprealm *ceph_get_snaprealm(struct ceph_client *client, +extern struct ceph_snaprealm *ceph_get_snaprealm(struct ceph_mds_client *mdsc, u64 ino); -extern struct ceph_snaprealm *ceph_find_snaprealm(struct ceph_client *client, +extern struct ceph_snaprealm *ceph_find_snaprealm(struct ceph_mds_client *mdsc, u64 ino); extern void ceph_put_snaprealm(struct ceph_snaprealm *realm); -extern int ceph_adjust_snaprealm_parent(struct ceph_client *client, +extern int ceph_adjust_snaprealm_parent(struct ceph_mds_client *mdsc, struct ceph_snaprealm *realm, u64 p); -extern struct ceph_snaprealm *ceph_update_snap_trace(struct ceph_client *client, +extern struct ceph_snaprealm *ceph_update_snap_trace(struct ceph_mds_client *mc, void *p, void *e, int must_flush); extern int ceph_build_snap_context(struct ceph_snaprealm *realm); -- 2.39.5