From: Sage Weil Date: Mon, 20 Apr 2009 16:31:39 +0000 (-0700) Subject: mds: move leases to dentry only; use map instead of hash_map X-Git-Tag: v0.7.3~104 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=cfb35839f6fba98b632afd28dc9498d0819203de;p=ceph.git mds: move leases to dentry only; use map instead of hash_map This lowers mds memory usage by about 20% --- diff --git a/src/Makefile.am b/src/Makefile.am index 43c3937e461..a7d4a0cf1b1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -243,7 +243,6 @@ libmds_a_SOURCES = \ mds/journal.cc \ mds/Server.cc \ mds/MDCache.cc \ - mds/CacheObject.cc \ mds/Locker.cc \ mds/Migrator.cc \ mds/MDBalancer.cc \ diff --git a/src/mds/CDentry.cc b/src/mds/CDentry.cc index 24d908071e1..60d85002fd9 100644 --- a/src/mds/CDentry.cc +++ b/src/mds/CDentry.cc @@ -21,6 +21,7 @@ #include "MDS.h" #include "MDCache.h" +#include "Locker.h" #include "LogSegment.h" #include "messages/MLock.h" @@ -496,3 +497,85 @@ void CDentry::decode_lock_state(int type, bufferlist& bl) assert(0); } } + + +ClientLease *CDentry::add_client_lease(int c, int mask) +{ + ClientLease *l; + if (client_lease_map.count(c)) + l = client_lease_map[c]; + else { + if (client_lease_map.empty()) + get(PIN_CLIENTLEASE); + l = client_lease_map[c] = new ClientLease(c, this); + } + + int adding = ~l->mask & mask; + dout(20) << " had " << l->mask << " adding " << mask + << " -> " << adding + << " ... now " << (l->mask | mask) + << dendl; + int b = 0; + while (adding) { + if (adding & 1) { + SimpleLock *lock = get_lock(1 << b); + if (lock) { + lock->get_client_lease(); + dout(20) << "get_client_lease on " << (1 << b) << " " << *lock << dendl; + } + } + b++; + adding = adding >> 1; + } + l->mask |= mask; + + return l; +} + +int CDentry::remove_client_lease(ClientLease *l, int mask, Locker *locker) +{ + assert(l->parent == this); + + list to_gather; + + int removing = l->mask & mask; + dout(20) << "had " << l->mask << " removing " << mask << " -> " << removing + << " ... now " << (l->mask & ~mask) << dendl; + int b = 0; + while (removing) { + if (removing & 1) { + SimpleLock *lock = get_lock(1 << b); + if (lock) { + lock->put_client_lease(); + dout(20) << "put_client_lease on " << (1 << b) << " " << *lock << dendl; + if (lock->get_num_client_lease() == 0 && !lock->is_stable()) + to_gather.push_back(lock); + } + } + b++; + removing = removing >> 1; + } + + l->mask &= ~mask; + int rc = l->mask; + + if (rc == 0) { + dout(20) << "removing lease for client" << l->client << dendl; + client_lease_map.erase(l->client); + l->lease_item.remove_myself(); + l->session_lease_item.remove_myself(); + delete l; + if (client_lease_map.empty()) + put(PIN_CLIENTLEASE); + } + + // do pending gathers. + while (!to_gather.empty()) { + locker->eval_gather(to_gather.front()); + to_gather.pop_front(); + } + + return rc; +} + + diff --git a/src/mds/CDentry.h b/src/mds/CDentry.h index abe1973f81b..62e62054fd7 100644 --- a/src/mds/CDentry.h +++ b/src/mds/CDentry.h @@ -42,6 +42,8 @@ class CDentry; class LogSegment; + + // define an ordering bool operator<(const CDentry& l, const CDentry& r); @@ -326,6 +328,32 @@ public: void encode_lock_state(int type, bufferlist& bl); void decode_lock_state(int type, bufferlist& bl); + + // --------------------------------------------- + // replicas (on clients) + public: + map client_lease_map; + + bool is_any_leases() { + return !client_lease_map.empty(); + } + ClientLease *get_client_lease(int c) { + if (client_lease_map.count(c)) + return client_lease_map[c]; + return 0; + } + int get_client_lease_mask(int c) { + ClientLease *l = get_client_lease(c); + if (l) + return l->mask; + else + return 0; + } + + ClientLease *add_client_lease(int c, int mask); + int remove_client_lease(ClientLease *r, int mask, class Locker *locker); // returns remaining mask (if any), and kicks locker eval_gathers + + ostream& print_db_line_prefix(ostream& out); void print(ostream& out); diff --git a/src/mds/CacheObject.cc b/src/mds/CacheObject.cc deleted file mode 100644 index aae16edd0b9..00000000000 --- a/src/mds/CacheObject.cc +++ /dev/null @@ -1,104 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab -/* - * Ceph - scalable distributed file system - * - * Copyright (C) 2004-2006 Sage Weil - * - * This is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License version 2.1, as published by the Free Software - * Foundation. See file COPYING. - * - */ - - -#include "mdstypes.h" -#include "SimpleLock.h" -#include "Locker.h" - -#include "config.h" - -#define DOUT_SUBSYS mds -#undef dout_prefix -#define dout_prefix *_dout << dbeginl << *this << " " - -ClientLease *MDSCacheObject::add_client_lease(int c, int mask) -{ - ClientLease *l; - if (client_lease_map.count(c)) - l = client_lease_map[c]; - else { - if (client_lease_map.empty()) - get(PIN_CLIENTLEASE); - l = client_lease_map[c] = new ClientLease(c, this); - } - - int adding = ~l->mask & mask; - dout(20) << " had " << l->mask << " adding " << mask - << " -> " << adding - << " ... now " << (l->mask | mask) - << dendl; - int b = 0; - while (adding) { - if (adding & 1) { - SimpleLock *lock = get_lock(1 << b); - if (lock) { - lock->get_client_lease(); - dout(20) << "get_client_lease on " << (1 << b) << " " << *lock << dendl; - } - } - b++; - adding = adding >> 1; - } - l->mask |= mask; - - return l; -} - -int MDSCacheObject::remove_client_lease(ClientLease *l, int mask, Locker *locker) -{ - assert(l->parent == this); - - list to_gather; - - int removing = l->mask & mask; - dout(20) << "had " << l->mask << " removing " << mask << " -> " << removing - << " ... now " << (l->mask & ~mask) << dendl; - int b = 0; - while (removing) { - if (removing & 1) { - SimpleLock *lock = get_lock(1 << b); - if (lock) { - lock->put_client_lease(); - dout(20) << "put_client_lease on " << (1 << b) << " " << *lock << dendl; - if (lock->get_num_client_lease() == 0 && !lock->is_stable()) - to_gather.push_back(lock); - } - } - b++; - removing = removing >> 1; - } - - l->mask &= ~mask; - int rc = l->mask; - - if (rc == 0) { - dout(20) << "removing lease for client" << l->client << dendl; - client_lease_map.erase(l->client); - l->lease_item.remove_myself(); - l->session_lease_item.remove_myself(); - delete l; - if (client_lease_map.empty()) - put(PIN_CLIENTLEASE); - } - - // do pending gathers. - while (!to_gather.empty()) { - locker->eval_gather(to_gather.front()); - to_gather.pop_front(); - } - - return rc; -} - diff --git a/src/mds/Locker.cc b/src/mds/Locker.cc index 03389d4a8d9..0584647a385 100644 --- a/src/mds/Locker.cc +++ b/src/mds/Locker.cc @@ -1231,7 +1231,7 @@ void Locker::remove_stale_leases(Session *session) dout(10) << "remove_stale_leases for " << session->inst.name << dendl; for (xlist::iterator p = session->leases.begin(); !p.end(); ++p) { ClientLease *l = *p; - MDSCacheObject *parent = l->parent; + CDentry *parent = (CDentry*)l->parent; dout(15) << " removing lease for " << l->mask << " on " << *parent << dendl; parent->remove_client_lease(l, l->mask, this); } @@ -1702,11 +1702,10 @@ void Locker::process_cap_update(int client, inodeno_t ino, __u64 cap_id, int cap CDir *dir = in->get_dirfrag(fg); if (dir) { CDentry *dn = dir->lookup(dname); - MDSCacheObject *p = dn; - ClientLease *l = p->get_client_lease(client); + ClientLease *l = dn->get_client_lease(client); if (l) { dout(10) << " removing lease on " << *dn << dendl; - p->remove_client_lease(l, l->mask, this); + dn->remove_client_lease(l, l->mask, this); } } } @@ -1951,26 +1950,23 @@ void Locker::handle_client_lease(MClientLease *m) return; } CDentry *dn = 0; - MDSCacheObject *p; - if (m->get_mask() & CEPH_LOCK_DN) { - frag_t fg = in->pick_dirfrag(m->dname); - CDir *dir = in->get_dirfrag(fg); - if (dir) - p = dn = dir->lookup(m->dname); - if (!dn) { - dout(7) << "handle_client_lease don't have dn " << m->get_ino() << " " << m->dname << dendl; - delete m; - return; - } - } else { - p = in; + assert(m->get_mask() & CEPH_LOCK_DN); + + frag_t fg = in->pick_dirfrag(m->dname); + CDir *dir = in->get_dirfrag(fg); + if (dir) + dn = dir->lookup(m->dname); + if (!dn) { + dout(7) << "handle_client_lease don't have dn " << m->get_ino() << " " << m->dname << dendl; + delete m; + return; } - dout(10) << " on " << *p << dendl; + dout(10) << " on " << *dn << dendl; // replica and lock - ClientLease *l = p->get_client_lease(client); + ClientLease *l = dn->get_client_lease(client); if (!l) { - dout(7) << "handle_client_lease didn't have lease for client" << client << " of " << *p << dendl; + dout(7) << "handle_client_lease didn't have lease for client" << client << " of " << *dn << dendl; delete m; return; } @@ -1983,9 +1979,9 @@ void Locker::handle_client_lease(MClientLease *m) } else { dout(7) << "handle_client_lease client" << client << " release mask " << m->get_mask() - << " on " << *p << dendl; - int left = p->remove_client_lease(l, l->mask, this); - dout(10) << " remaining mask is " << left << " on " << *p << dendl; + << " on " << *dn << dendl; + int left = dn->remove_client_lease(l, l->mask, this); + dout(10) << " remaining mask is " << left << " on " << *dn << dendl; } delete m; break; @@ -1994,7 +1990,7 @@ void Locker::handle_client_lease(MClientLease *m) { dout(7) << "handle_client_lease client" << client << " renew mask " << m->get_mask() - << " on " << *p << dendl; + << " on " << *dn << dendl; int pool = 1; // fixme.. do something smart! m->h.duration_ms = (int)(1000 * mdcache->client_lease_durations[pool]); m->h.seq = ++l->seq; @@ -2016,14 +2012,14 @@ void Locker::handle_client_lease(MClientLease *m) -void Locker::_issue_client_lease(MDSCacheObject *p, int mask, int pool, int client, +void Locker::_issue_client_lease(CDentry *dn, int mask, int pool, int client, bufferlist &bl, utime_t now, Session *session) { LeaseStat e; e.mask = mask; if (mask) { - ClientLease *l = p->add_client_lease(client, mask); + ClientLease *l = dn->add_client_lease(client, mask); session->touch_lease(l); e.seq = ++l->seq; @@ -2081,8 +2077,9 @@ int Locker::issue_client_lease(CDentry *dn, int client, void Locker::revoke_client_leases(SimpleLock *lock) { int n = 0; - for (hash_map::iterator p = lock->get_parent()->client_lease_map.begin(); - p != lock->get_parent()->client_lease_map.end(); + CDentry *dn = (CDentry*)lock->get_parent(); + for (map::iterator p = dn->client_lease_map.begin(); + p != dn->client_lease_map.end(); p++) { ClientLease *l = p->second; diff --git a/src/mds/Locker.h b/src/mds/Locker.h index 6998fd5494c..047698627c3 100644 --- a/src/mds/Locker.h +++ b/src/mds/Locker.h @@ -222,7 +222,7 @@ private: public: void handle_client_lease(class MClientLease *m); - void _issue_client_lease(MDSCacheObject *p, int mask, int pool, int client, bufferlist &bl, utime_t now, Session *session); + void _issue_client_lease(CDentry *dn, int mask, int pool, int client, bufferlist &bl, utime_t now, Session *session); int issue_client_lease(CInode *in, int client, bufferlist &bl, utime_t now, Session *session); int issue_client_lease(CDentry *dn, int client, bufferlist &bl, utime_t now, Session *session); void revoke_client_leases(SimpleLock *lock); diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index bab6323101b..66897ac8cf3 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -1299,21 +1299,6 @@ CInode *MDCache::cow_inode(CInode *in, snapid_t last) //oldin->inode.max_size = 0; } - // clone leases? - if (in->last != CEPH_NOSNAP) { - // only if we are not the head, and our lease may cover an - // instance in either the old or new inodes valid interval. - for (hash_map::iterator p = in->client_lease_map.begin(); - p != in->client_lease_map.end(); - p++) { - dout(10) << " cloning client" << p->first << " lease on " << p->second->mask << " to cloned inode" << dendl; - ClientLease *l = oldin->add_client_lease(p->first, p->second->mask); - l->ttl = p->second->ttl; - p->second->session_lease_item.get_xlist()->push_back(&l->session_lease_item); - p->second->lease_item.get_xlist()->push_back(&l->lease_item); - } - } - return oldin; } @@ -5088,9 +5073,9 @@ void MDCache::trim_client_leases() while (!client_leases[pool].empty()) { ClientLease *r = client_leases[pool].front(); if (r->ttl > now) break; - MDSCacheObject *p = r->parent; - dout(10) << " expiring client" << r->client << " lease of " << *p << dendl; - p->remove_client_lease(r, r->mask, mds->locker); + CDentry *dn = (CDentry*)r->parent; + dout(10) << " expiring client" << r->client << " lease of " << *dn << dendl; + dn->remove_client_lease(r, r->mask, mds->locker); } int after = client_leases[pool].size(); dout(10) << "trim_client_leases pool " << pool << " trimmed " @@ -6689,7 +6674,7 @@ void MDCache::eval_stray(CDentry *dn) } } } - if (dn->is_replicated() || in->is_any_caps() || in->is_any_leases()) return; // wait + if (dn->is_replicated() || dn->is_any_leases() || in->is_any_caps()) return; // wait if (!in->dirfrags.empty()) return; // wait for dirs to close/trim if (dn->state_test(CDentry::STATE_PURGING)) return; // already purging if (in->state_test(CInode::STATE_NEEDSRECOVER) || diff --git a/src/mds/Server.cc b/src/mds/Server.cc index 985b13382c3..4429e2b2982 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -245,9 +245,9 @@ void Server::_session_logged(Session *session, bool open, version_t pv, dequeleases.empty()) { ClientLease *r = session->leases.front(); - MDSCacheObject *p = r->parent; - dout(20) << " killing client lease of " << *p << dendl; - p->remove_client_lease(r, r->mask, mds->locker); + CDentry *dn = (CDentry*)r->parent; + dout(20) << " killing client lease of " << *dn << dendl; + dn->remove_client_lease(r, r->mask, mds->locker); } if (piv) { diff --git a/src/mds/mdstypes.h b/src/mds/mdstypes.h index f8fb71d9ccb..8e26538366f 100644 --- a/src/mds/mdstypes.h +++ b/src/mds/mdstypes.h @@ -1158,31 +1158,6 @@ protected: void set_replica_nonce(int n) { replica_nonce = n; } - // --------------------------------------------- - // replicas (on clients) - public: - hash_map client_lease_map; - - bool is_any_leases() { - return !client_lease_map.empty(); - } - ClientLease *get_client_lease(int c) { - if (client_lease_map.count(c)) - return client_lease_map[c]; - return 0; - } - int get_client_lease_mask(int c) { - ClientLease *l = get_client_lease(c); - if (l) - return l->mask; - else - return 0; - } - - ClientLease *add_client_lease(int c, int mask); - int remove_client_lease(ClientLease *r, int mask, class Locker *locker); // returns remaining mask (if any), and kicks locker eval_gathers - - // --------------------------------------------- // waiting protected: