From: Yan, Zheng Date: Fri, 5 Jun 2015 11:43:45 +0000 (+0800) Subject: client: track flush TIDs for all pending flushing caps X-Git-Tag: v9.1.0~57^2~11 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=c8a83cd1ff03646f079669e7f2615a92ba2fb3d5;p=ceph.git client: track flush TIDs for all pending flushing caps Signed-off-by: Yan, Zheng --- diff --git a/src/client/Client.cc b/src/client/Client.cc index 4259c6ecdf60..73e4ebdb7dde 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -2924,10 +2924,7 @@ void Client::send_cap(Inode *in, MetaSession *session, Cap *cap, if (flush) { flush_tid = ++in->last_flush_tid; - for (int i = 0; i < CEPH_CAP_BITS; ++i) { - if (flush & (1<flushing_cap_tid[i] = flush_tid; - } + in->flushing_cap_tids[flush_tid] = flush; follows = in->snaprealm->get_snap_context().seq; } @@ -3585,8 +3582,10 @@ void Client::remove_session_caps(MetaSession *s) signal_cond_list(in->waitfor_caps); if (dirty_caps) { lderr(cct) << "remove_session_caps still has dirty|flushing caps on " << *in << dendl; - if (in->flushing_caps) + if (in->flushing_caps) { num_flushing_caps--; + in->flushing_cap_tids.clear(); + } in->flushing_caps = 0; in->dirty_caps = 0; put_inode(in); @@ -3763,20 +3762,17 @@ void Client::flush_caps(Inode *in, MetaSession *session) (cap->issued | cap->implemented), in->flushing_caps); } -void Client::wait_sync_caps(Inode *in, uint16_t flush_tid[]) +void Client::wait_sync_caps(Inode *in, ceph_tid_t want) { -retry: - for (int i = 0; i < CEPH_CAP_BITS; ++i) { - if (!(in->flushing_caps & (1 << i))) - continue; - // handle uint16_t wrapping - if ((int16_t)(in->flushing_cap_tid[i] - flush_tid[i]) <= 0) { - ldout(cct, 10) << "wait_sync_caps on " << *in << " flushing " - << ccap_string(1 << i) << " want " << flush_tid[i] - << " last " << in->flushing_cap_tid[i] << dendl; - wait_on_list(in->waitfor_caps); - goto retry; - } + while (in->flushing_caps) { + map::iterator it = in->flushing_cap_tids.begin(); + assert(it != in->flushing_cap_tids.end()); + if (it->first > want) + break; + ldout(cct, 10) << "wait_sync_caps on " << *in << " flushing " + << ccap_string(it->second) << " want " << want + << " last " << it->first << dendl; + wait_on_list(in->waitfor_caps); } } @@ -4310,20 +4306,32 @@ void Client::handle_cap_trunc(MetaSession *session, Inode *in, MClientCaps *m) void Client::handle_cap_flush_ack(MetaSession *session, Inode *in, Cap *cap, MClientCaps *m) { - mds_rank_t mds = session->mds_num; + ceph_tid_t flush_ack_tid = m->get_client_tid(); int dirty = m->get_dirty(); int cleaned = 0; - uint16_t flush_ack_tid = static_cast(m->get_client_tid()); - for (int i = 0; i < CEPH_CAP_BITS; ++i) { - if ((dirty & (1 << i)) && - (flush_ack_tid == in->flushing_cap_tid[i])) - cleaned |= 1 << i; + int flushed = 0; + + for (map::iterator it = in->flushing_cap_tids.begin(); + it != in->flushing_cap_tids.end(); ) { + if (it->first == flush_ack_tid) + cleaned = it->second; + if (it->first <= flush_ack_tid) { + in->flushing_cap_tids.erase(it++); + ++flushed; + continue; + } + cleaned &= ~it->second; + if (!cleaned) + break; + ++it; } - ldout(cct, 5) << "handle_cap_flush_ack mds." << mds + ldout(cct, 5) << "handle_cap_flush_ack mds." << session->mds_num << " cleaned " << ccap_string(cleaned) << " on " << *in << " with " << ccap_string(dirty) << dendl; + if (flushed) + signal_cond_list(in->waitfor_caps); if (!cleaned) { ldout(cct, 10) << " tid " << m->get_client_tid() << " != any cap bit tids" << dendl; @@ -4338,7 +4346,6 @@ void Client::handle_cap_flush_ack(MetaSession *session, Inode *in, Cap *cap, MCl num_flushing_caps--; sync_cond.Signal(); } - signal_cond_list(in->waitfor_caps); if (!in->caps_dirty()) put_inode(in); } @@ -7804,7 +7811,6 @@ int Client::fsync(int fd, bool syncdataonly) int Client::_fsync(Inode *in, bool syncdataonly) { int r = 0; - uint16_t wait_on_flush[CEPH_CAP_BITS]; bool flushed_metadata = false; Mutex lock("Client::_fsync::lock"); Cond cond; @@ -7823,10 +7829,8 @@ int Client::_fsync(Inode *in, bool syncdataonly) if (!syncdataonly && (in->dirty_caps & ~CEPH_CAP_ANY_FILE_WR)) { check_caps(in, true); - if (in->flushing_caps) { + if (in->flushing_caps) flushed_metadata = true; - memcpy(wait_on_flush, in->flushing_cap_tid, sizeof(wait_on_flush)); - } } else ldout(cct, 10) << "no metadata needs to commit" << dendl; if (object_cacher_completion) { // wait on a real reply instead of guessing @@ -7864,7 +7868,7 @@ int Client::_fsync(Inode *in, bool syncdataonly) if (!r) { if (flushed_metadata) - wait_sync_caps(in, wait_on_flush); + wait_sync_caps(in, in->last_flush_tid); ldout(cct, 10) << "ino " << in->ino << " has no uncommitted writes" << dendl; } else { diff --git a/src/client/Client.h b/src/client/Client.h index 78a2da7ee04a..f560e662b336 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -561,7 +561,7 @@ protected: void get_cap_ref(Inode *in, int cap); void put_cap_ref(Inode *in, int cap); void flush_snaps(Inode *in, bool all_again=false, CapSnap *again=0); - void wait_sync_caps(Inode *in, uint16_t flush_tid[]); + void wait_sync_caps(Inode *in, ceph_tid_t want); void wait_sync_caps(uint64_t want); void queue_cap_snap(Inode *in, SnapContext &old_snapc); void finish_cap_snap(Inode *in, CapSnap *capsnap, int used); diff --git a/src/client/Inode.cc b/src/client/Inode.cc index 16eee7a7aa49..994c2971ca17 100644 --- a/src/client/Inode.cc +++ b/src/client/Inode.cc @@ -396,11 +396,11 @@ void Inode::dump(Formatter *f) const f->dump_stream("flushings_caps") << ccap_string(flushing_caps); f->dump_unsigned("flushing_cap_seq", flushing_cap_seq); f->open_object_section("flushing_cap_tid"); - for (unsigned bit = 0; bit < CEPH_CAP_BITS; bit++) { - if (flushing_caps & (1 << bit)) { - string n(ccap_string(1 << bit)); - f->dump_unsigned(n.c_str(), flushing_cap_tid[bit]); - } + for (map::const_iterator p = flushing_cap_tids.begin(); + p != flushing_cap_tids.end(); + ++p) { + string n(ccap_string(p->second)); + f->dump_unsigned(n.c_str(), p->first); } f->close_section(); } diff --git a/src/client/Inode.h b/src/client/Inode.h index f18f65272d93..975a48b00638 100644 --- a/src/client/Inode.h +++ b/src/client/Inode.h @@ -234,7 +234,7 @@ struct Inode { Cap *auth_cap; unsigned dirty_caps, flushing_caps; uint64_t flushing_cap_seq; - __u16 flushing_cap_tid[CEPH_CAP_BITS]; + std::map flushing_cap_tids; int shared_gen, cache_gen; int snap_caps, snap_cap_refs; utime_t hold_caps_until; @@ -316,7 +316,6 @@ struct Inode { { memset(&dir_layout, 0, sizeof(dir_layout)); memset(&layout, 0, sizeof(layout)); - memset(&flushing_cap_tid, 0, sizeof(__u16)*CEPH_CAP_BITS); memset("a, 0, sizeof(quota)); } ~Inode() { }