From e00bb20eaf2798678a5eca3f17730f5f61378039 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 4 Jan 2008 17:09:56 -0800 Subject: [PATCH] per-state session xlists; kill caps on close --- src/client/Client.cc | 6 ++++- src/config.cc | 2 ++ src/config.h | 2 ++ src/mds/CInode.h | 3 ++- src/mds/Locker.cc | 5 +++++ src/mds/MDS.cc | 2 +- src/mds/Server.cc | 42 ++++++++++++++++++++--------------- src/mds/SessionMap.h | 27 +++++++++++----------- src/mds/journal.cc | 2 +- src/messages/MClientSession.h | 2 ++ 10 files changed, 58 insertions(+), 35 deletions(-) diff --git a/src/client/Client.cc b/src/client/Client.cc index ed022126acd96..0e7eefd46ba1c 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -788,9 +788,13 @@ void Client::handle_client_session(MClientSession *m) break; case CEPH_SESSION_STALE: - // hmm, verify caps have been revoked? messenger->send_message(new MClientSession(CEPH_SESSION_REQUEST_RESUME, g_clock.now()), m->get_source_inst()); + // hmm, verify caps have been revoked? + break; + + case CEPH_SESSION_RESUME: + // ?? break; default: diff --git a/src/config.cc b/src/config.cc index aaf1c85fcf2aa..ae8c8eb610cdd 100644 --- a/src/config.cc +++ b/src/config.cc @@ -227,6 +227,8 @@ md_config_t g_conf = { mds_cap_timeout: 10, // cap bits time out if client idle mds_session_autoclose: 30, // autoclose idle session + mds_tick_interval: 5, + mds_log: true, mds_log_max_events: -1, //MDS_CACHE_SIZE / 3, mds_log_max_segments: 100, diff --git a/src/config.h b/src/config.h index 165b9baef20a7..329021c954df8 100644 --- a/src/config.h +++ b/src/config.h @@ -182,6 +182,8 @@ struct md_config_t { float mds_cap_timeout; float mds_session_autoclose; + float mds_tick_interval; + bool mds_log; int mds_log_max_events; int mds_log_max_segments; diff --git a/src/mds/CInode.h b/src/mds/CInode.h index a88a5db1f6007..b5c67140ae5cd 100644 --- a/src/mds/CInode.h +++ b/src/mds/CInode.h @@ -403,7 +403,8 @@ public: for (map::iterator it = client_caps.begin(); it != client_caps.end(); it++) { - w |= it->second->wanted(); + if (!it->second->is_stale()) + w |= it->second->wanted(); //cout << " get_caps_wanted client " << it->first << " " << cap_string(it->second.wanted()) << endl; } if (is_auth()) diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index 02ba34a56352c..99e646004a2a7 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -573,6 +573,8 @@ void Locker::revoke_stale_caps(Session *session) dout(10) << " revoking " << cap_string(issued) << " on " << *in << dendl; cap->revoke(); file_eval(&in->filelock); + if (!in->is_auth()) + request_inode_file_caps(in); } else { dout(10) << " nothing issued on " << *in << dendl; } @@ -591,6 +593,7 @@ void Locker::resume_stale_caps(Session *session) dout(10) << " clearing stale flag on " << *in << dendl; cap->set_stale(false); file_eval(&in->filelock); + issue_caps(in); } } } @@ -609,6 +612,8 @@ public: void Locker::request_inode_file_caps(CInode *in) { + assert(!in->is_auth()); + int wanted = in->get_caps_wanted(); if (wanted != in->replica_caps_wanted) { diff --git a/src/mds/MDS.cc b/src/mds/MDS.cc index 854d50a114c17..9474ffcc9149c 100644 --- a/src/mds/MDS.cc +++ b/src/mds/MDS.cc @@ -318,7 +318,7 @@ void MDS::reset_tick() // schedule tick_event = new C_MDS_Tick(this); - timer.add_event_after(g_conf.mon_tick_interval, tick_event); + timer.add_event_after(g_conf.mds_tick_interval, tick_event); } void MDS::tick() diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 133d31b81d00c..2fc5006f60217 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -152,7 +152,7 @@ void Server::handle_client_session(MClientSession *m) } assert(!session); // ? session = mds->sessionmap.get_or_add_session(m->get_source_inst()); - session->state = Session::STATE_OPENING; + mds->sessionmap.set_state(session, Session::STATE_OPENING); mds->sessionmap.touch_session(session); pv = ++mds->sessionmap.projected; mdlog->submit_entry(new ESession(m->get_source_inst(), true, pv), @@ -179,7 +179,7 @@ void Server::handle_client_session(MClientSession *m) assert(0); return; } - session->state = Session::STATE_OPEN; + mds->sessionmap.set_state(session, Session::STATE_OPEN); mds->sessionmap.touch_session(session); mds->locker->resume_stale_caps(session); mds->messenger->send_message(new MClientSession(CEPH_SESSION_RESUME, m->stamp), session->inst); @@ -196,7 +196,7 @@ void Server::handle_client_session(MClientSession *m) return; } assert(m->seq == session->get_push_seq()); - session->state = Session::STATE_CLOSING; + mds->sessionmap.set_state(session, Session::STATE_CLOSING); pv = ++mds->sessionmap.projected; mdlog->submit_entry(new ESession(m->get_source_inst(), false, pv), new C_MDS_session_finish(mds, session, false, pv)); @@ -215,11 +215,19 @@ void Server::_session_logged(Session *session, bool open, version_t pv) // apply if (open) { assert(session->is_opening()); - session->state = Session::STATE_OPEN; + mds->sessionmap.set_state(session, Session::STATE_OPEN); mds->sessionmap.version++; } else if (session->is_closing()) { mds->sessionmap.remove_session(session); mds->sessionmap.version++; + + // kill any lingering capabilities + for (xlist::iterator p = session->caps.begin(); !p.end(); ++p) { + CInode *in = (*p)->get_inode(); + dout(10) << " killing capability on " << *in << dendl; + in->remove_client_cap(session->inst.name.num()); + } + } else { // close must have been canceled (by an import?) ... assert(!open); @@ -240,9 +248,8 @@ void Server::prepare_force_open_sessions(map& cm) << dendl; for (map::iterator p = cm.begin(); p != cm.end(); ++p) { Session *session = mds->sessionmap.get_or_add_session(p->second); - if (session->state == Session::STATE_UNDEF || - session->state == Session::STATE_CLOSING) - session->state = Session::STATE_OPENING; + if (session->is_undef() || session->is_closing()) + mds->sessionmap.set_state(session, Session::STATE_OPENING); } } @@ -260,7 +267,7 @@ void Server::finish_force_open_sessions(map& cm) assert(session); if (session->is_opening()) { dout(10) << "force_open_sessions opening " << session->inst << dendl; - session->state = Session::STATE_OPEN; + mds->sessionmap.set_state(session, Session::STATE_OPEN); mds->messenger->send_message(new MClientSession(CEPH_SESSION_OPEN), session->inst); } } @@ -280,7 +287,7 @@ void Server::terminate_sessions() ++p) { Session *session = *p; if (session->is_closing()) continue; - session->state = Session::STATE_CLOSING; + mds->sessionmap.set_state(session, Session::STATE_CLOSING); version_t pv = ++mds->sessionmap.projected; mdlog->submit_entry(new ESession(session->inst, false, pv), new C_MDS_session_finish(mds, session, false, pv)); @@ -297,7 +304,7 @@ void Server::find_idle_sessions() utime_t cutoff = now; cutoff -= g_conf.mds_cap_timeout; while (1) { - Session *session = mds->sessionmap.get_oldest_active_session(); + Session *session = mds->sessionmap.get_oldest_session(Session::STATE_OPEN); if (!session) break; dout(20) << "laggiest active session is " << session->inst << dendl; if (session->last_cap_renew >= cutoff) { @@ -307,7 +314,7 @@ void Server::find_idle_sessions() } dout(10) << "new stale session " << session->inst << " last " << session->last_cap_renew << dendl; - mds->sessionmap.mark_session_stale(session); + mds->sessionmap.set_state(session, Session::STATE_STALE); mds->locker->revoke_stale_caps(session); mds->messenger->send_message(new MClientSession(CEPH_SESSION_STALE, g_clock.now()), session->inst); @@ -317,8 +324,9 @@ void Server::find_idle_sessions() cutoff = now; cutoff -= g_conf.mds_session_autoclose; while (1) { - Session *session = mds->sessionmap.get_oldest_stale_session(); + Session *session = mds->sessionmap.get_oldest_session(Session::STATE_STALE); if (!session) break; + assert(session->is_stale()); dout(20) << "oldest stale session is " << session->inst << dendl; if (session->last_cap_renew >= cutoff) { dout(20) << "oldest stale session is " << session->inst << " and sufficiently new (" @@ -327,14 +335,12 @@ void Server::find_idle_sessions() } dout(10) << "autoclosing stale session " << session->inst << " last " << session->last_cap_renew << dendl; - - mds->sessionmap.mark_session_stale(session); - mds->locker->revoke_stale_caps(session); - mds->messenger->send_message(new MClientSession(CEPH_SESSION_CLOSE), - session->inst); + mds->sessionmap.set_state(session, Session::STATE_CLOSING); + version_t pv = ++mds->sessionmap.projected; + mdlog->submit_entry(new ESession(session->inst, false, pv), + new C_MDS_session_finish(mds, session, false, pv)); } - } diff --git a/src/mds/SessionMap.h b/src/mds/SessionMap.h index bc9a2402dd9dc..8a3ebf0eaa9ca 100644 --- a/src/mds/SessionMap.h +++ b/src/mds/SessionMap.h @@ -42,10 +42,14 @@ public: static const int STATE_STALE = 4; static const int STATE_RECONNECTING = 5; // ? +private: int state; + friend class SessionMap; +public: entity_inst_t inst; xlist::item session_list_item; + bool is_undef() { return state == STATE_UNDEF; } bool is_opening() { return state == STATE_OPENING; } bool is_open() { return state == STATE_OPEN; } bool is_closing() { return state == STATE_CLOSING; } @@ -123,8 +127,7 @@ private: MDS *mds; hash_map session_map; public: - xlist active_sessions; - xlist stale_sessions; + map > by_state; public: // i am lazy version_t version, projected, committing, committed; @@ -156,18 +159,16 @@ public: } void touch_session(Session *s) { s->last_cap_renew = g_clock.now(); - active_sessions.push_back(&s->session_list_item); } - Session *get_oldest_active_session() { - if (active_sessions.empty()) return 0; - return active_sessions.front(); + Session *get_oldest_session(int state) { + if (by_state[state].empty()) return 0; + return by_state[state].front(); } - void mark_session_stale(Session *s) { - stale_sessions.push_back(&s->session_list_item); - } - Session *get_oldest_stale_session() { - if (stale_sessions.empty()) return 0; - return stale_sessions.front(); + void set_state(Session *session, int s) { + if (session->state != s) { + session->state = s; + by_state[s].push_back(&session->session_list_item); + } } void get_client_set(set& s) { @@ -191,7 +192,7 @@ public: ++p) { Session *session = get_or_add_session(p->second); session->inst = p->second; - session->state = Session::STATE_OPEN; + set_state(session, Session::STATE_OPEN); } version++; } diff --git a/src/mds/journal.cc b/src/mds/journal.cc index 3f25262fdff6b..d743196d79cc5 100644 --- a/src/mds/journal.cc +++ b/src/mds/journal.cc @@ -480,7 +480,7 @@ void ESession::replay(MDS *mds) assert(mds->sessionmap.version == cmapv); if (open) { Session *session = mds->sessionmap.get_or_add_session(client_inst); - session->state = Session::STATE_OPEN; + mds->sessionmap.set_state(session, Session::STATE_OPEN); } else { Session *session = mds->sessionmap.get_session(client_inst.name); if (session) diff --git a/src/messages/MClientSession.h b/src/messages/MClientSession.h index 15b27ac3efdf3..7c709c9639f63 100644 --- a/src/messages/MClientSession.h +++ b/src/messages/MClientSession.h @@ -28,6 +28,8 @@ public: case CEPH_SESSION_REQUEST_RENEWCAPS: return "request_renewcaps"; case CEPH_SESSION_RENEWCAPS: return "renewcaps"; case CEPH_SESSION_STALE: return "stale"; + case CEPH_SESSION_REQUEST_RESUME: return "request_resume"; + case CEPH_SESSION_RESUME: return "resume"; default: assert(0); return 0; } } -- 2.39.5