From: Sage Weil Date: Fri, 17 Apr 2009 15:44:31 +0000 (-0700) Subject: mds, kclient: fix issue_seq vs seq, limit dup cap messages X-Git-Tag: v0.7.3~138 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=1ef1316c9b946bb2544b74debce2e7e02cb4f259;p=ceph.git mds, kclient: fix issue_seq vs seq, limit dup cap messages --- diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index 0fd2de5beaaff..3e8bf65a9310d 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -1052,7 +1052,7 @@ struct ceph_mds_caps { __le32 op; __le64 ino, realm; __le64 cap_id; - __le32 seq; + __le32 seq, issue_seq; __le32 caps, wanted, dirty; __le32 migrate_seq; __le64 snap_follows; diff --git a/src/kernel/caps.c b/src/kernel/caps.c index b2bd630ee50b4..92d848ec69203 100644 --- a/src/kernel/caps.c +++ b/src/kernel/caps.c @@ -694,7 +694,8 @@ void ceph_remove_cap(struct ceph_cap *cap) * Caller should be holding s_mutex. */ static void send_cap_msg(struct ceph_mds_client *mdsc, u64 ino, u64 cid, int op, - int caps, int wanted, int dirty, u64 seq, u64 mseq, + int caps, int wanted, int dirty, + u32 seq, u32 issue_seq, u32 mseq, u64 size, u64 max_size, struct timespec *mtime, struct timespec *atime, u64 time_warp_seq, @@ -707,14 +708,15 @@ static void send_cap_msg(struct ceph_mds_client *mdsc, u64 ino, u64 cid, int op, struct ceph_msg *msg; dout(10, "send_cap_msg %s %llx %llx caps %s wanted %s dirty %s" - " seq %llu/%llu follows %lld size %llu" + " seq %u/%u mseq %u follows %lld size %llu" " xattr_ver %llu xattr_len %d\n", ceph_cap_op_name(op), cid, ino, ceph_cap_string(caps), ceph_cap_string(wanted), ceph_cap_string(dirty), - seq, mseq, follows, size, + seq, issue_seq, mseq, follows, size, xattr_version, xattrs_blob_size); - msg = ceph_msg_new(CEPH_MSG_CLIENT_CAPS, sizeof(*fc) + xattrs_blob_size, 0, 0, NULL); + msg = ceph_msg_new(CEPH_MSG_CLIENT_CAPS, sizeof(*fc) + xattrs_blob_size, + 0, 0, NULL); if (IS_ERR(msg)) return; @@ -725,6 +727,7 @@ static void send_cap_msg(struct ceph_mds_client *mdsc, u64 ino, u64 cid, int op, fc->cap_id = cpu_to_le64(cid); fc->op = cpu_to_le32(op); fc->seq = cpu_to_le32(seq); + fc->issue_seq = cpu_to_le32(issue_seq); fc->migrate_seq = cpu_to_le32(mseq); fc->caps = cpu_to_le32(caps); fc->wanted = cpu_to_le32(wanted); @@ -832,7 +835,7 @@ static void __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, int revoking = cap->implemented & ~cap->issued; int dropping = cap->issued & ~retain; int keep; - u64 seq, mseq, time_warp_seq, follows; + u64 seq, issue_seq, mseq, time_warp_seq, follows; u64 size, max_size; struct timespec mtime, atime; int wake = 0; @@ -877,6 +880,7 @@ static void __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, keep = cap->implemented; seq = cap->seq; + issue_seq = cap->issue_seq; mseq = cap->mseq; size = inode->i_size; ci->i_reported_size = size; @@ -906,7 +910,7 @@ static void __send_cap(struct ceph_mds_client *mdsc, struct ceph_cap *cap, } send_cap_msg(mdsc, ceph_vino(inode).ino, cap_id, - op, keep, want, flushing, seq, mseq, + op, keep, want, flushing, seq, issue_seq, mseq, size, max_size, &mtime, &atime, time_warp_seq, uid, gid, mode, xattr_version, @@ -996,7 +1000,7 @@ retry: inode, capsnap, next_follows, capsnap->size); send_cap_msg(mdsc, ceph_vino(inode).ino, 0, CEPH_CAP_OP_FLUSHSNAP, capsnap->issued, 0, - capsnap->dirty, 0, mseq, + capsnap->dirty, 0, 0, mseq, capsnap->size, 0, &capsnap->mtime, &capsnap->atime, capsnap->time_warp_seq, diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index 4dfeba34039e8..076fd6ffbeabd 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -545,6 +545,8 @@ void Locker::eval_gather(SimpleLock *lock, bool first, bool *need_issue) bool Locker::eval_caps(CInode *in) { bool need_issue = false; + + dout(10) << "eval_caps " << *in << dendl; if (!in->filelock.is_stable()) eval_gather(&in->filelock, false, &need_issue); @@ -565,6 +567,8 @@ bool Locker::eval_caps(CInode *in) if (need_issue) issue_caps(in); + + dout(10) << "eval_caps done" << dendl; return need_issue; } @@ -975,11 +979,16 @@ void Locker::file_update_finish(CInode *in, Mutation *mut, bool share, int clien mut->cleanup(); delete mut; - if (cap && cap->wanted() & ~cap->pending()) + bool sup = false; // avoid sending two caps msgs, one for cap expansion, one for file_max change. + if (cap && (cap->wanted() & ~cap->pending())) { issue_caps(in, cap); - + cap->inc_suppress(); + sup = true; + } if (share && in->is_auth() && in->filelock.is_stable()) share_inode_max_size(in); + if (sup) + cap->dec_suppress(); } Capability* Locker::issue_new_caps(CInode *in, @@ -1452,6 +1461,8 @@ void Locker::share_inode_max_size(CInode *in) it++) { const int client = it->first; Capability *cap = it->second; + if (cap->is_suppress()) + continue; if (cap->pending() & (CEPH_CAP_GWR<filelock.is_stable()) mds->mdlog->flush(); } - if (ceph_seq_cmp(m->get_seq(), cap->get_last_issue()) == 0) { + if (ceph_seq_cmp(m->get_issue_seq(), cap->get_last_issue()) == 0) { dout(10) << " wanted " << ccap_string(cap->wanted()) << " -> " << ccap_string(wanted) << dendl; cap->set_wanted(wanted); @@ -1599,22 +1610,30 @@ void Locker::handle_client_caps(MClientCaps *m) } else { dout(10) << " NOT changing wanted " << ccap_string(cap->wanted()) << " -> " << ccap_string(wanted) - << " (seq " << m->get_seq() << " != last_issue " << cap->get_last_issue() << ")" << dendl; + << " (issue_seq " << m->get_issue_seq() << " != last_issue " + << cap->get_last_issue() << ")" << dendl; } } if (m->get_op() == CEPH_CAP_OP_DROP) can_issue = false; - if (!_do_cap_update(in, cap, m->get_dirty(), m->get_wanted(), follows, m, ack)) { + if (_do_cap_update(in, cap, m->get_dirty(), m->get_wanted(), follows, m, ack)) { + // updated, cap msg is delayed + cap->inc_suppress(); + eval_caps(in); + cap->dec_suppress(); + + if (cap->wanted() & ~cap->pending()) + mds->mdlog->flush(); + } else { // no update, ack now. if (ack) mds->send_message_client(ack, client); - } - bool did_issue = eval_caps(in); - if (!did_issue && - cap->wanted() & ~cap->pending()) - issue_caps(in, cap); + bool did_issue = eval_caps(in); + if (!did_issue && (cap->wanted() & ~cap->pending())) + issue_caps(in, cap); + } } // done? @@ -1715,10 +1734,16 @@ bool Locker::_do_cap_update(CInode *in, Capability *cap, dout(10) << " i want to change file_max, but lock won't allow it (yet)" << dendl; in->state_set(CInode::STATE_NO_SIZE_CHECK); if (in->filelock.is_stable()) { - if (in->get_loner() >= 0) - file_excl(&in->filelock); - else - simple_lock(&in->filelock); + bool need_issue = false; + cap->inc_suppress(); + if (in->get_loner() >= 0 || in->try_choose_loner()) { + file_excl(&in->filelock, &need_issue); + need_issue = false; // loner! + } else + simple_lock(&in->filelock, &need_issue); + if (need_issue) + issue_caps(in); + cap->dec_suppress(); } in->state_clear(CInode::STATE_NO_SIZE_CHECK); if (!in->filelock.can_wrlock(client)) { diff --git a/src/messages/MClientCaps.h b/src/messages/MClientCaps.h index 35250d21eae56..6f28d9a397bdd 100644 --- a/src/messages/MClientCaps.h +++ b/src/messages/MClientCaps.h @@ -28,6 +28,7 @@ class MClientCaps : public Message { int get_wanted() { return head.wanted; } int get_dirty() { return head.dirty; } ceph_seq_t get_seq() { return head.seq; } + ceph_seq_t get_issue_seq() { return head.issue_seq; } ceph_seq_t get_mseq() { return head.migrate_seq; } inodeno_t get_ino() { return inodeno_t(head.ino); }