From 711e30038a303468dbd0970a110dc130defa4c40 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Fri, 29 May 2009 11:58:22 -0700 Subject: [PATCH] kclient: trim caps on demand --- src/include/ceph_fs.h | 5 +++- src/kernel/caps.c | 17 ++++++++++++ src/kernel/mds_client.c | 51 +++++++++++++++++++++++++++++++++++ src/kernel/mds_client.h | 2 +- src/kernel/super.h | 2 ++ src/messages/MClientSession.h | 9 +++++-- 6 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index 3bae80313c710..ac37686e2d5dc 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -27,7 +27,7 @@ #define CEPH_MDS_PROTOCOL 9 /* cluster internal */ #define CEPH_MON_PROTOCOL 4 /* cluster internal */ #define CEPH_OSDC_PROTOCOL 13 /* public/client */ -#define CEPH_MDSC_PROTOCOL 20 /* public/client */ +#define CEPH_MDSC_PROTOCOL 21 /* public/client */ #define CEPH_MONC_PROTOCOL 12 /* public/client */ @@ -394,6 +394,7 @@ enum { CEPH_SESSION_REQUEST_RENEWCAPS, CEPH_SESSION_RENEWCAPS, CEPH_SESSION_STALE, + CEPH_SESSION_TRIMCAPS, }; static inline const char *ceph_session_op_name(int op) @@ -406,6 +407,7 @@ static inline const char *ceph_session_op_name(int op) case CEPH_SESSION_REQUEST_RENEWCAPS: return "request_renewcaps"; case CEPH_SESSION_RENEWCAPS: return "renewcaps"; case CEPH_SESSION_STALE: return "stale"; + case CEPH_SESSION_TRIMCAPS: return "trimcaps"; default: return "???"; } } @@ -414,6 +416,7 @@ struct ceph_mds_session_head { __le32 op; __le64 seq; struct ceph_timespec stamp; + __le32 max_caps; } __attribute__ ((packed)); /* client_request */ diff --git a/src/kernel/caps.c b/src/kernel/caps.c index 53f8faf9338f5..a755c95c5f612 100644 --- a/src/kernel/caps.c +++ b/src/kernel/caps.c @@ -605,6 +605,23 @@ int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented) return have; } +int __ceph_caps_issued_other(struct ceph_inode_info *ci, struct ceph_cap *ocap) +{ + int have = ci->i_snap_caps; + struct ceph_cap *cap; + struct rb_node *p; + + for (p = rb_first(&ci->i_caps); p; p = rb_next(p)) { + cap = rb_entry(p, struct ceph_cap, ci_node); + if (cap == ocap) + continue; + if (!__cap_is_valid(cap)) + continue; + have |= cap->issued; + } + return have; +} + static void __touch_cap(struct ceph_cap *cap) { struct ceph_mds_session *s = cap->session; diff --git a/src/kernel/mds_client.c b/src/kernel/mds_client.c index 0a047c3fd82e1..1534c1b6e0768 100644 --- a/src/kernel/mds_client.c +++ b/src/kernel/mds_client.c @@ -300,6 +300,7 @@ static struct ceph_mds_session *register_session(struct ceph_mds_client *mdsc, s->s_renew_requested = 0; INIT_LIST_HEAD(&s->s_caps); s->s_nr_caps = 0; + s->s_max_caps = 0; atomic_set(&s->s_ref, 1); INIT_LIST_HEAD(&s->s_waiting); INIT_LIST_HEAD(&s->s_unsafe); @@ -793,6 +794,52 @@ static int __close_session(struct ceph_mds_client *mdsc, return request_close_session(mdsc, session); } +/* + * Trim old(er) caps. + */ +static int trim_caps_cb(struct inode *inode, struct ceph_cap *cap, void *arg) +{ + struct ceph_mds_session *session = arg; + struct ceph_inode_info *ci = ceph_inode(inode); + int used, oissued, mine; + + if (session->s_nr_caps <= session->s_max_caps) + return -1; + + spin_lock(&inode->i_lock); + mine = cap->issued | cap->implemented; + used = __ceph_caps_used(ci); + oissued = __ceph_caps_issued_other(ci, cap); + + dout(20, "trim_caps_cb %p cap %p mine %s oissued %s used %s\n", + inode, cap, ceph_cap_string(mine), ceph_cap_string(oissued), + ceph_cap_string(used)); + if ((used & ~oissued) & mine) + goto out; /* we need these caps */ + + /* try to drop referring dentries */ + d_prune_aliases(inode); + dout(20, "trim_caps_cb %p cap %p pruned, count now %d\n", + inode, cap, atomic_read(&inode->i_count)); + +out: + spin_unlock(&inode->i_lock); + return 0; +} + +static int trim_caps(struct ceph_mds_client *mdsc, + struct ceph_mds_session *session, + int max_caps) +{ + dout(10, "trim_caps mds%d start: %d / %d caps\n", session->s_mds, + session->s_nr_caps, max_caps); + session->s_max_caps = max_caps; + iterate_session_caps(session, trim_caps_cb, session); + dout(10, "trim_caps mds%d done: %d / %d caps\n", session->s_mds, + session->s_nr_caps, max_caps); + return 0; +} + /* * Allocate cap_release messages. If there is a partially full message * in the queue, try to allocate enough to cover it's remainder, so that @@ -1724,6 +1771,10 @@ void ceph_mdsc_handle_session(struct ceph_mds_client *mdsc, send_renew_caps(mdsc, session); break; + case CEPH_SESSION_TRIMCAPS: + trim_caps(mdsc, session, le32_to_cpu(h->max_caps)); + break; + default: derr(0, "bad session op %d from mds%d\n", op, mds); WARN_ON(1); diff --git a/src/kernel/mds_client.h b/src/kernel/mds_client.h index 8a889f5855421..2bf5dc8cae81a 100644 --- a/src/kernel/mds_client.h +++ b/src/kernel/mds_client.h @@ -120,7 +120,7 @@ struct ceph_mds_session { unsigned long s_cap_ttl; /* when session caps expire */ unsigned long s_renew_requested; /* last time we sent a renew req */ struct list_head s_caps; /* all caps issued by this session */ - int s_nr_caps; + int s_nr_caps, s_max_caps; atomic_t s_ref; struct list_head s_waiting; /* waiting requests */ struct list_head s_unsafe; /* unsafe requests */ diff --git a/src/kernel/super.h b/src/kernel/super.h index 4c30fade0b1df..bb41d3cdfaf65 100644 --- a/src/kernel/super.h +++ b/src/kernel/super.h @@ -506,6 +506,8 @@ static inline bool __ceph_is_any_real_caps(struct ceph_inode_info *ci) extern int __ceph_caps_issued(struct ceph_inode_info *ci, int *implemented); extern int __ceph_caps_issued_mask(struct ceph_inode_info *ci, int mask, int t); +extern int __ceph_caps_issued_other(struct ceph_inode_info *ci, + struct ceph_cap *cap); static inline int ceph_caps_issued(struct ceph_inode_info *ci) { diff --git a/src/messages/MClientSession.h b/src/messages/MClientSession.h index f7799ac9edec7..e472870e5bab0 100644 --- a/src/messages/MClientSession.h +++ b/src/messages/MClientSession.h @@ -23,19 +23,22 @@ public: int32_t op; version_t seq; // used when requesting close, declaring stale utime_t stamp; + int32_t max_caps; MClientSession() : Message(CEPH_MSG_CLIENT_SESSION) { } MClientSession(int o, version_t s=0) : Message(CEPH_MSG_CLIENT_SESSION), - op(o), seq(s) { } + op(o), seq(s), max_caps(0) { } MClientSession(int o, utime_t st) : Message(CEPH_MSG_CLIENT_SESSION), - op(o), seq(0), stamp(st) { } + op(o), seq(0), stamp(st), max_caps(0) { } const char *get_type_name() { return "client_session"; } void print(ostream& out) { out << "client_session(" << ceph_session_op_name(op); if (seq) out << " seq " << seq; + if (op == CEPH_SESSION_TRIMCAPS) + out << " max_caps " << max_caps; out << ")"; } @@ -44,11 +47,13 @@ public: ::decode(op, p); ::decode(seq, p); ::decode(stamp, p); + ::decode(max_caps, p); } void encode_payload() { ::encode(op, payload); ::encode(seq, payload); ::encode(stamp, payload); + ::encode(max_caps, payload); } }; -- 2.39.5