From 82107d208cbcac67ea4698049b2b84876ebad80a Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Wed, 26 Mar 2008 12:25:19 -0700 Subject: [PATCH] improve dirlock vs filelock interaction, filelock lease callbacks --- src/TODO | 5 +- src/client/Client.cc | 5 +- src/include/ceph_fs.h | 12 ++-- src/mds/CacheObject.cc | 5 +- src/mds/FileLock.h | 2 + src/mds/Locker.cc | 136 ++++++++++++++++++++++++----------------- src/mds/Locker.h | 5 +- src/mds/MDCache.cc | 2 +- src/mds/Server.cc | 9 ++- src/mds/SimpleLock.h | 3 + src/mds/mdstypes.h | 2 +- 11 files changed, 114 insertions(+), 72 deletions(-) diff --git a/src/TODO b/src/TODO index d2214602ba98f..0fd3b94d50351 100644 --- a/src/TODO +++ b/src/TODO @@ -5,12 +5,15 @@ code cleanup - addr=? client leases +- FIX max_size!!!!!!!!! +- IFILE lease vs filelock and caps... +- IFILE lease vs dirlock... + - do i need to unify filelock and scatterlock? - lease length heuristics - mds lock last_change stamp? - client: preemptively release lease on dentry we are unlinking, renaming from/to, etc. - or inode fields we are chmodding, etc. - lease renewals (to avoid blocking client ops?) -- IFILE vs caps... userspace client - clean up ll_ interface, now that we have leases! diff --git a/src/client/Client.cc b/src/client/Client.cc index fd2b5cbe73728..df6b2f2c5e1b1 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -2038,8 +2038,11 @@ int Client::_do_lstat(filepath &fpath, int mask, Inode **in) } if (dn->inode->file_caps() & CEPH_CAP_EXCL) { dout(10) << "_lstat has inode " << fpath << " with CAP_EXCL, yay" << dendl; - havemask |= CEPH_LOCK_IFILE; + havemask |= CEPH_LOCK_ICONTENT; } + if (havemask & CEPH_LOCK_ICONTENT) + havemask |= CEPH_LOCK_ICONTENT; // hack: if we have one, we have both, for the purposes of below + dout(10) << "_lstat has inode " << fpath << " with mask " << havemask << ", want " << mask << dendl; } else { dout(10) << "_lstat has no dn for path " << fpath << dendl; diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index bb794e48ef337..e3d63e52a33eb 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -315,13 +315,15 @@ struct ceph_statfs { */ #define CEPH_LOCK_DN 1 #define CEPH_LOCK_IVERSION 2 /* mds internal */ -#define CEPH_LOCK_IFILE 4 +#define CEPH_LOCK_IFILE 4 /* mds internal */ #define CEPH_LOCK_IAUTH 8 #define CEPH_LOCK_ILINK 16 #define CEPH_LOCK_IDFT 32 /* dir frag tree */ #define CEPH_LOCK_IDIR 64 /* mds internal */ #define CEPH_LOCK_INO 128 /* immutable inode bits; not actually a lock */ +#define CEPH_LOCK_ICONTENT (CEPH_LOCK_IFILE|CEPH_LOCK_IDIR) /* alias for either filelock or dirlock */ + /* * stat masks are defined in terms of the locks that cover inode fields. */ @@ -333,10 +335,10 @@ struct ceph_statfs { #define CEPH_STAT_MASK_GID CEPH_LOCK_IAUTH #define CEPH_STAT_MASK_MODE CEPH_LOCK_IAUTH #define CEPH_STAT_MASK_NLINK CEPH_LOCK_ILINK -#define CEPH_STAT_MASK_MTIME CEPH_LOCK_IFILE -#define CEPH_STAT_MASK_SIZE CEPH_LOCK_IFILE -#define CEPH_STAT_MASK_ATIME CEPH_LOCK_IFILE /* fixme */ -#define CEPH_STAT_MASK_INODE_ALL (CEPH_LOCK_IFILE|CEPH_LOCK_IAUTH|CEPH_LOCK_ILINK|CEPH_LOCK_INO) +#define CEPH_STAT_MASK_MTIME CEPH_LOCK_ICONTENT +#define CEPH_STAT_MASK_SIZE CEPH_LOCK_ICONTENT +#define CEPH_STAT_MASK_ATIME CEPH_LOCK_ICONTENT /* fixme */ +#define CEPH_STAT_MASK_INODE_ALL (CEPH_LOCK_ICONTENT|CEPH_LOCK_IAUTH|CEPH_LOCK_ILINK|CEPH_LOCK_INO) /* client_session */ diff --git a/src/mds/CacheObject.cc b/src/mds/CacheObject.cc index 01e7e23d53cf3..dee13a7f618ac 100644 --- a/src/mds/CacheObject.cc +++ b/src/mds/CacheObject.cc @@ -15,6 +15,7 @@ #include "mdstypes.h" #include "SimpleLock.h" +#include "Locker.h" #include "config.h" #define dout(l) if (l<=g_conf.debug || l <= g_conf.debug_mds) *_dout << dbeginl << g_clock.now() << " " << this << " " @@ -50,7 +51,7 @@ ClientLease *MDSCacheObject::add_client_lease(int c, int mask) return l; } -int MDSCacheObject::remove_client_lease(ClientLease *l, int mask) +int MDSCacheObject::remove_client_lease(ClientLease *l, int mask, Locker *locker) { assert(l->parent == this); @@ -63,6 +64,8 @@ int MDSCacheObject::remove_client_lease(ClientLease *l, int mask) if (lock) { lock->put_client_lease(); dout(10) << "put_client_lease on " << (1 << b) << " " << *lock << dendl; + if (lock->get_num_client_lease() == 0 && !lock->is_stable()) + locker->eval_gather(lock); } } b++; diff --git a/src/mds/FileLock.h b/src/mds/FileLock.h index ef7941088c305..e3f9da5fbd129 100644 --- a/src/mds/FileLock.h +++ b/src/mds/FileLock.h @@ -239,6 +239,8 @@ class FileLock : public SimpleLock { out << get_lock_type_name(get_type()) << " "; out << get_filelock_state_name(get_state()); if (!get_gather_set().empty()) out << " g=" << get_gather_set(); + if (get_num_client_lease()) + out << " c=" << get_num_client_lease(); if (is_rdlocked()) out << " r=" << get_num_rdlocks(); if (is_xlocked()) diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index 7479d112c2d06..843a13eb7b7b6 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -118,9 +118,6 @@ void Locker::send_lock_message(SimpleLock *lock, int msg, const bufferlist &data - - - bool Locker::acquire_locks(MDRequest *mdr, set &rdlocks, set &wrlocks, @@ -384,6 +381,8 @@ bool Locker::wrlock_start(SimpleLock *lock, MDRequest *mdr) return scatter_wrlock_start((ScatterLock*)lock, mdr); case CEPH_LOCK_IVERSION: return local_wrlock_start((LocalLock*)lock, mdr); + //case CEPH_LOCK_IFILE: + //return file_wrlock_start((ScatterLock*)lock, mdr); default: assert(0); return false; @@ -534,6 +533,7 @@ Capability* Locker::issue_new_caps(CInode *in, cap->set_last_open(); // increase max_size? + /* FIXME inode_t *latest = in->get_projected_inode(); if (latest->max_size == 0 && !in->is_base() && @@ -553,6 +553,7 @@ Capability* Locker::issue_new_caps(CInode *in, mds->mdlog->submit_entry(le, new C_Locker_FileUpdate_finish(this, in, ls, true)); file_wrlock_start(&in->filelock); // wrlock for duration of journal } + */ // twiddle file_data_version? int now = cap->pending(); @@ -986,8 +987,6 @@ void Locker::handle_client_lease(MClientLease *m) dout(10) << " on " << *p << dendl; // replica and lock - SimpleLock *lock = p->get_lock(m->lock); - assert(lock); ClientLease *l = p->get_client_lease(client); if (!l) { dout(7) << "handle_client_lease didn't have lease for client" << client << " of " << *p << dendl; @@ -1001,7 +1000,7 @@ void Locker::handle_client_lease(MClientLease *m) dout(7) << "handle_client_lease client" << client << " release mask " << m->mask << " on " << *p << dendl; - int left = p->remove_client_lease(l, l->mask); + int left = p->remove_client_lease(l, l->mask, this); dout(10) << " remaining mask is " << left << " on " << *p << dendl; } break; @@ -1011,10 +1010,6 @@ void Locker::handle_client_lease(MClientLease *m) break; } - // eval/waiters - if (!lock->is_stable()) - eval_gather(lock); - delete m; } @@ -1044,9 +1039,13 @@ int Locker::issue_client_lease(CInode *in, int client, { int mask = CEPH_LOCK_INO; int pool = 1; // fixme.. do something smart! - if (in->authlock.can_rdlock(0)) mask |= CEPH_LOCK_IAUTH; - if (in->linklock.can_rdlock(0)) mask |= CEPH_LOCK_ILINK; - if (in->filelock.can_rdlock(0)) mask |= CEPH_LOCK_IFILE; + if (in->authlock.can_lease()) mask |= CEPH_LOCK_IAUTH; + if (in->linklock.can_lease()) mask |= CEPH_LOCK_ILINK; + if (in->is_dir()) { + if (in->dirlock.can_lease()) mask |= CEPH_LOCK_IDIR; + } else { + if (in->filelock.can_lease()) mask |= CEPH_LOCK_IFILE; + } _issue_client_lease(in, mask, pool, client, bl, now, session); return mask; @@ -1057,13 +1056,46 @@ int Locker::issue_client_lease(CDentry *dn, int client, { int pool = 1; // fixme.. do something smart! int mask = 0; - if (dn->lock.can_rdlock(0)) mask |= CEPH_LOCK_DN; + if (dn->lock.can_lease()) mask |= CEPH_LOCK_DN; _issue_client_lease(dn, mask, pool, client, bl, now, session); return mask; } +void Locker::revoke_client_leases(SimpleLock *lock) +{ + int n = 0; + for (hash_map::iterator p = lock->get_parent()->client_lease_map.begin(); + p != lock->get_parent()->client_lease_map.end(); + p++) { + ClientLease *l = p->second; + + if (l->mask & lock->get_type() == 0) + continue; + + n++; + if (lock->get_type() == CEPH_LOCK_DN) { + CDentry *dn = (CDentry*)lock->get_parent(); + mds->send_message_client(new MClientLease(lock->get_type(), + CEPH_MDS_LEASE_REVOKE, + lock->get_type(), + dn->get_dir()->ino(), + dn->get_name()), + l->client); + } else { + CInode *in = (CInode*)lock->get_parent(); + mds->send_message_client(new MClientLease(lock->get_type(), + CEPH_MDS_LEASE_REVOKE, + lock->get_type(), + in->ino()), + l->client); + } + } + assert(n == lock->get_num_client_lease()); +} + + @@ -1376,34 +1408,7 @@ void Locker::simple_lock(SimpleLock *lock) send_lock_message(lock, LOCK_AC_LOCK); // bcast to client replicas - int n = 0; - for (hash_map::iterator p = lock->get_parent()->client_lease_map.begin(); - p != lock->get_parent()->client_lease_map.end(); - p++) { - ClientLease *l = p->second; - - if (l->mask & lock->get_type() == 0) - continue; - - n++; - if (lock->get_type() == CEPH_LOCK_DN) { - CDentry *dn = (CDentry*)lock->get_parent(); - mds->send_message_client(new MClientLease(lock->get_type(), - CEPH_MDS_LEASE_REVOKE, - lock->get_type(), - dn->get_dir()->ino(), - dn->get_name()), - l->client); - } else { - CInode *in = (CInode*)lock->get_parent(); - mds->send_message_client(new MClientLease(lock->get_type(), - CEPH_MDS_LEASE_REVOKE, - lock->get_type(), - in->ino()), - l->client); - } - } - assert(n == lock->get_num_client_lease()); + revoke_client_leases(lock); // change lock lock->set_state(LOCK_GLOCKR); @@ -2478,14 +2483,22 @@ void Locker::file_rdlock_finish(FileLock *lock, MDRequest *mdr) bool Locker::file_wrlock_start(FileLock *lock) { - if (!lock->can_wrlock()) { - dout(7) << "wrlock_start can't on " << *lock << " on " << *lock->get_parent() << dendl; - assert(0); // ! - return false; - } - dout(7) << "wrlock_start on " << *lock << " on " << *lock->get_parent() << dendl; + dout(7) << "file_wrlock_start on " << *lock + << " on " << *lock->get_parent() << dendl; + assert(lock->can_wrlock()); lock->get_wrlock(); return true; + + /* + if (lock->can_wrlock()) { + lock->get_wrlock(); + mdr->wrlocks.insert(lock); + mdr->locks.insert(lock); + return true; + } else { + lock->add_waiter(SimpleLock::WAIT_WR|SimpleLock::WAIT_STABLE, new C_MDS_RetryRequest(mdcache, mdr)); + return false; + }*/ } void Locker::file_wrlock_finish(FileLock *lock) @@ -2629,6 +2642,7 @@ void Locker::file_eval_gather(FileLock *lock) if (in->is_auth() && !lock->is_gathering() && !lock->is_wrlocked() && + lock->get_num_client_lease() == 0 && ((issued & ~lock->caps_allowed()) == 0)) { dout(7) << "file_eval_gather finished gather" << dendl; @@ -2714,6 +2728,7 @@ void Locker::file_eval_gather(FileLock *lock) // [replica] finished caps gather? if (!in->is_auth() && + lock->get_num_client_lease() == 0 && ((issued & ~lock->caps_allowed()) == 0)) { switch (lock->get_state()) { case LOCK_GMIXEDR: @@ -2784,20 +2799,25 @@ void Locker::file_eval(FileLock *lock) // * -> sync? else if (!in->filelock.is_waiter_for(SimpleLock::WAIT_WR) && !(wanted & (CEPH_CAP_WR|CEPH_CAP_WRBUFFER)) && - ((wanted & CEPH_CAP_RD) || - in->is_replicated() || - (!loner && lock->get_state() == LOCK_LONER)) && + //((wanted & CEPH_CAP_RD) || + //in->is_replicated() || + //lock->get_num_client_lease() || + //(!loner && lock->get_state() == LOCK_LONER)) && + !(loner && lock->get_state() == LOCK_LONER) && // leave loner in loner state lock->get_state() != LOCK_SYNC) { dout(7) << "file_eval stable, bump to sync " << *lock << " on " << *lock->get_parent() << dendl; file_sync(lock); } // * -> lock? (if not replicated or open) +/* else if (!in->is_replicated() && wanted == 0 && + lock->get_num_client_lease() == 0 && lock->get_state() != LOCK_LOCK) { file_lock(lock); } +*/ } @@ -2891,10 +2911,12 @@ void Locker::file_lock(FileLock *lock) int issued = in->get_caps_issued(); if (lock->get_state() == LOCK_SYNC) { - if (in->is_replicated()) { + if (in->is_replicated() || + lock->get_num_client_lease()) { // bcast to replicas send_lock_message(lock, LOCK_AC_LOCK); lock->init_gather(); + revoke_client_leases(lock); // change lock lock->set_state(LOCK_GLOCKR); @@ -2970,9 +2992,11 @@ void Locker::file_mixed(FileLock *lock) int issued = in->get_caps_issued(); if (lock->get_state() == LOCK_SYNC) { - if (in->is_replicated()) { + if (in->is_replicated() || + lock->get_num_client_lease()) { // bcast to replicas send_lock_message(lock, LOCK_AC_MIXED); + revoke_client_leases(lock); lock->init_gather(); lock->set_state(LOCK_GMIXEDR); @@ -3031,7 +3055,7 @@ void Locker::file_mixed(FileLock *lock) void Locker::file_loner(FileLock *lock) { CInode *in = (CInode*)lock->get_parent(); - dout(7) << "inode_file_loner " << *lock << " on " << *lock->get_parent() << dendl; + dout(7) << "file_loner " << *lock << " on " << *lock->get_parent() << dendl; assert(in->is_auth()); assert(lock->is_stable()); @@ -3039,9 +3063,11 @@ void Locker::file_loner(FileLock *lock) assert((in->client_caps.size() == 1) && in->mds_caps_wanted.empty()); if (lock->get_state() == LOCK_SYNC) { - if (in->is_replicated()) { + if (in->is_replicated() || + lock->get_num_client_lease()) { // bcast to replicas send_lock_message(lock, LOCK_AC_LOCK); + revoke_client_leases(lock); lock->init_gather(); // change lock diff --git a/src/mds/Locker.h b/src/mds/Locker.h index 996c5712a5f5c..0d32e9883065d 100644 --- a/src/mds/Locker.h +++ b/src/mds/Locker.h @@ -49,6 +49,7 @@ class SimpleLock; class FileLock; class ScatterLock; class LocalLock; +class MDCache; class Locker { private: @@ -77,8 +78,8 @@ public: void drop_locks(MDRequest *mdr); -protected: void eval_gather(SimpleLock *lock); +protected: bool rdlock_start(SimpleLock *lock, MDRequest *mdr); void rdlock_finish(SimpleLock *lock, MDRequest *mdr); bool xlock_start(SimpleLock *lock, MDRequest *mdr); @@ -208,7 +209,7 @@ public: void _issue_client_lease(MDSCacheObject *p, int mask, int pool, int client, bufferlist &bl, utime_t now, Session *session); int issue_client_lease(CInode *in, int client, bufferlist &bl, utime_t now, Session *session); int issue_client_lease(CDentry *dn, int client, bufferlist &bl, utime_t now, Session *session); - + void revoke_client_leases(SimpleLock *lock); }; diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index 668a78a6ce83a..fc2b3fd8419b8 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -3511,7 +3511,7 @@ void MDCache::trim_client_leases() if (r->ttl > now) break; MDSCacheObject *p = r->parent; dout(10) << " expiring client" << r->client << " lease of " << *p << dendl; - p->remove_client_lease(r, r->mask); + p->remove_client_lease(r, r->mask, mds->locker); } dout(10) << "trim_client_leases pool " << pool << " finish - " << client_leases[pool].size() << " leases" << dendl; diff --git a/src/mds/Server.cc b/src/mds/Server.cc index b4fe81a54df28..73ce06634e80c 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -229,7 +229,7 @@ void Server::_session_logged(Session *session, bool open, version_t pv) ClientLease *r = session->leases.front(); MDSCacheObject *p = r->parent; dout(10) << " killing client lease of " << *p << dendl; - p->remove_client_lease(r, r->mask); + p->remove_client_lease(r, r->mask, mds->locker); } if (session->is_closing()) @@ -1521,9 +1521,9 @@ void Server::handle_client_stat(MDRequest *mdr) if (mask & CEPH_LOCK_ILINK) rdlocks.insert(&ref->linklock); if (mask & CEPH_LOCK_IAUTH) rdlocks.insert(&ref->authlock); if (ref->is_file() && - mask & CEPH_LOCK_IFILE) rdlocks.insert(&ref->filelock); + mask & CEPH_LOCK_ICONTENT) rdlocks.insert(&ref->filelock); if (ref->is_dir() && - mask & CEPH_STAT_MASK_MTIME) rdlocks.insert(&ref->dirlock); + mask & CEPH_LOCK_ICONTENT) rdlocks.insert(&ref->dirlock); if (!mds->locker->acquire_locks(mdr, rdlocks, wrlocks, xlocks)) return; @@ -3914,7 +3914,6 @@ void Server::handle_client_open(MDRequest *mdr) // hmm, check permissions or something. - // O_TRUNC if (flags & O_TRUNC) { assert(cur->is_auth()); @@ -3931,7 +3930,7 @@ void Server::handle_client_open(MDRequest *mdr) handle_client_opent(mdr); return; } - } + } // do it _do_open(mdr, cur); diff --git a/src/mds/SimpleLock.h b/src/mds/SimpleLock.h index 8d2dd743c039c..5734b20793897 100644 --- a/src/mds/SimpleLock.h +++ b/src/mds/SimpleLock.h @@ -281,6 +281,9 @@ public: return false; } + bool can_lease() { + return state == LOCK_SYNC; + } bool can_rdlock(MDRequest *mdr) { //if (state == LOCK_LOCK && mdr && xlock_by == mdr) return true; // xlocked by me. (actually, is this right?) //if (state == LOCK_LOCK && !xlock_by && parent->is_auth()) return true; diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h index 4eeb763021b47..9f5002a97c70e 100644 --- a/src/mds/mdstypes.h +++ b/src/mds/mdstypes.h @@ -624,7 +624,7 @@ protected: } ClientLease *add_client_lease(int c, int mask); - int remove_client_lease(ClientLease *r, int mask); // returns remaining mask (if any) + int remove_client_lease(ClientLease *r, int mask, class Locker *locker); // returns remaining mask (if any), and kicks locker eval_gathers // --------------------------------------------- -- 2.39.5