From be77f3d6685fe18b1a35ed6d28a25a3bc2d56018 Mon Sep 17 00:00:00 2001 From: Sage Weil Date: Thu, 3 Jul 2008 10:07:30 -0700 Subject: [PATCH] client: snap_highwater, MClientSnap --- src/client/Client.cc | 10 +++-- src/client/Client.h | 21 ++++++---- src/include/ceph_fs.h | 13 ++++++ src/mds/MDCache.cc | 6 ++- src/mds/Migrator.cc | 7 +++- src/mds/Server.cc | 6 ++- src/mds/snap.cc | 9 ++++- src/mds/snap.h | 4 +- src/messages/MClientFileCaps.h | 11 +++-- src/messages/MClientReply.h | 2 + src/messages/MClientSnap.h | 74 ++++++++++++++++++++++++++++++++++ src/msg/Message.cc | 4 ++ 12 files changed, 146 insertions(+), 21 deletions(-) create mode 100644 src/messages/MClientSnap.h diff --git a/src/client/Client.cc b/src/client/Client.cc index 25c4c8e60a2a1..2e564b6758cc0 100644 --- a/src/client/Client.cc +++ b/src/client/Client.cc @@ -1508,7 +1508,7 @@ void Client::_flushed(Inode *in, bool checkafter) * do not block. */ void Client::add_update_inode_cap(Inode *in, int mds, - inodeno_t realm, vector &snaps, + inodeno_t realm, snapid_t snap_highwater, vector &snaps, unsigned issued, unsigned seq, unsigned mseq) { InodeCap *cap = 0; @@ -1518,7 +1518,7 @@ void Client::add_update_inode_cap(Inode *in, int mds, mds_sessions[mds].num_caps++; if (in->caps.empty()) { assert(in->snaprealm == 0); - in->snaprealm = get_snap_realm(realm, snaps); + in->snaprealm = get_snap_realm(realm); in->get(); } if (in->exporting_mds == mds) { @@ -1529,6 +1529,7 @@ void Client::add_update_inode_cap(Inode *in, int mds, } in->caps[mds] = cap = new InodeCap; } + in->snaprealm->maybe_update(snap_highwater, snaps); unsigned old_caps = cap->issued; cap->issued |= issued; @@ -1577,7 +1578,9 @@ void Client::handle_file_caps(MClientFileCaps *m) if (m->get_op() == CEPH_CAP_OP_IMPORT) { // add/update it - add_update_inode_cap(in, mds, m->get_realm(), m->get_snaps(), m->get_caps(), m->get_seq(), m->get_mseq()); + add_update_inode_cap(in, mds, + m->get_realm(), m->get_snap_highwater(), m->get_snaps(), + m->get_caps(), m->get_seq(), m->get_mseq()); if (in->exporting_mseq < m->get_mseq()) { dout(5) << "handle_file_caps ino " << m->get_ino() << " mseq " << m->get_mseq() @@ -2915,6 +2918,7 @@ int Client::_open(const filepath &path, int flags, mode_t mode, Fh **fhp, int ui int mds = reply->get_source().num(); add_update_inode_cap(in, mds, reply->get_file_caps_realm(), + reply->get_snap_highwater(), reply->get_snaps(), reply->get_file_caps(), reply->get_file_caps_seq(), diff --git a/src/client/Client.h b/src/client/Client.h index 50daad6bf9312..20042a1540dbf 100644 --- a/src/client/Client.h +++ b/src/client/Client.h @@ -129,11 +129,18 @@ struct InodeCap; struct SnapRealm { inodeno_t dirino; - vector snaps; int nref; + snapid_t snap_highwater; + vector snaps; - SnapRealm(inodeno_t i, vector &s) : dirino(i), nref(0) { - snaps.swap(s); + SnapRealm(inodeno_t i) : + dirino(i), nref(0), snap_highwater(0) { } + + void maybe_update(snapid_t sh, vector &s) { + if (sh > snap_highwater) { + snap_highwater = sh; + snaps = s; + } } }; @@ -576,12 +583,12 @@ protected: Inode* root; LRU lru; // lru list of Dentry's in our local metadata cache. - map snap_realms; + hash_map snap_realms; - SnapRealm *get_snap_realm(inodeno_t r, vector &snaps) { + SnapRealm *get_snap_realm(inodeno_t r) { SnapRealm *realm = snap_realms[r]; if (!realm) - snap_realms[r] = realm = new SnapRealm(r, snaps); + snap_realms[r] = realm = new SnapRealm(r); realm->nref++; return realm; } @@ -773,7 +780,7 @@ protected: // file caps void add_update_inode_cap(Inode *in, int mds, - inodeno_t realm, vector &snaps, + inodeno_t realm, snapid_t snap_highwater, vector &snaps, unsigned issued, unsigned seq, unsigned mseq); void remove_cap(Inode *in, int mds); void handle_file_caps(class MClientFileCaps *m); diff --git a/src/include/ceph_fs.h b/src/include/ceph_fs.h index c976023c407ec..dd05074cc6e5e 100644 --- a/src/include/ceph_fs.h +++ b/src/include/ceph_fs.h @@ -417,6 +417,7 @@ struct ceph_msg_footer { #define CEPH_MSG_CLIENT_REPLY 26 #define CEPH_MSG_CLIENT_FILECAPS 0x310 #define CEPH_MSG_CLIENT_LEASE 0x311 +#define CEPH_MSG_CLIENT_SNAP 0x312 /* osd */ #define CEPH_MSG_OSD_GETMAP 40 @@ -666,6 +667,7 @@ struct ceph_mds_reply_head { __le32 file_caps_mseq; __le32 mdsmap_epoch; __le32 num_snaps; + __le64 snap_highwater; __le64 snaps[]; } __attribute__ ((packed)); @@ -789,6 +791,7 @@ struct ceph_mds_file_caps { struct ceph_timespec mtime, atime, ctime; __le64 time_warp_seq; __le32 num_snaps; + __le64 snap_highwater; __le64 snaps[]; } __attribute__ ((packed)); @@ -814,6 +817,16 @@ struct ceph_mds_cap_reconnect { /* followed by encoded string */ +enum { + CEPH_SNAP_OP_UPDATE, + CEPH_SNAP_OP_SPLIT, +}; + +struct ceph_mds_snap { + /* ... */ +}; + + /* * osd map */ diff --git a/src/mds/MDCache.cc b/src/mds/MDCache.cc index aeb72986e2d5c..0ca59264fb6ed 100644 --- a/src/mds/MDCache.cc +++ b/src/mds/MDCache.cc @@ -2789,14 +2789,16 @@ void MDCache::rejoin_import_cap(CInode *in, int client, inode_caps_reconnect_t& session->touch_cap(cap); // send IMPORT + SnapRealm *realm = in->find_containing_snaprealm(); MClientFileCaps *reap = new MClientFileCaps(CEPH_CAP_OP_IMPORT, in->inode, - in->find_containing_snaprealm()->inode->ino(), + realm->inode->ino(), cap->get_last_seq(), cap->pending(), cap->wanted(), cap->get_mseq()); - in->find_containing_snaprealm()->get_snap_vector(reap->get_snaps()); + realm->get_snap_vector(reap->get_snaps()); + reap->set_snap_highwater(realm->snap_highwater); mds->messenger->send_message(reap, session->inst); } diff --git a/src/mds/Migrator.cc b/src/mds/Migrator.cc index 349b2469ac593..37ecc7191c3c4 100644 --- a/src/mds/Migrator.cc +++ b/src/mds/Migrator.cc @@ -2053,14 +2053,17 @@ void Migrator::finish_import_inode_caps(CInode *in, int from, } cap->merge(it->second); + SnapRealm *realm = in->find_containing_snaprealm(); MClientFileCaps *caps = new MClientFileCaps(CEPH_CAP_OP_IMPORT, in->inode, - in->find_containing_snaprealm()->inode->ino(), + realm->inode->ino(), cap->get_last_seq(), cap->pending(), cap->wanted(), cap->get_mseq()); - in->find_containing_snaprealm()->get_snap_vector(caps->get_snaps()); + realm->get_snap_vector(caps->get_snaps()); + caps->set_snap_highwater(realm->snap_highwater); + mds->send_message_client(caps, session->inst); } diff --git a/src/mds/Server.cc b/src/mds/Server.cc index cc934d28ecf91..5deb97a101381 100644 --- a/src/mds/Server.cc +++ b/src/mds/Server.cc @@ -4421,7 +4421,11 @@ void Server::_do_open(MDRequest *mdr, CInode *cur) reply->set_file_caps(cap->pending()); reply->set_file_caps_seq(cap->get_last_seq()); reply->set_file_caps_mseq(cap->get_mseq()); - cur->find_containing_snaprealm()->get_snap_vector(reply->get_snaps()); + + SnapRealm *realm = cur->find_containing_snaprealm(); + realm->get_snap_vector(reply->get_snaps()); + reply->set_snap_highwater(realm->snap_highwater); + //reply->set_file_data_version(fdv); reply_request(mdr, reply); diff --git a/src/mds/snap.cc b/src/mds/snap.cc index 11b0bf761e061..52a2ed1e2582d 100644 --- a/src/mds/snap.cc +++ b/src/mds/snap.cc @@ -68,7 +68,14 @@ void SnapRealm::get_snap_set(set &s) q->first >= p->second.first) s.insert(q->first); } - dout(10) << "build_snap_list " << s << dendl; + + if (!s.empty()) { + snapid_t t = *s.rbegin(); + if (snap_highwater < t) + snap_highwater = t; + } + + dout(10) << "build_snap_list " << s << " (highwater " << snap_highwater << ")" << dendl; } /* diff --git a/src/mds/snap.h b/src/mds/snap.h index 62310acf9c083..1dccfcd530a1f 100644 --- a/src/mds/snap.h +++ b/src/mds/snap.h @@ -101,6 +101,8 @@ struct SnapRealm { MDCache *mdcache; CInode *inode; + snapid_t snap_highwater; // largest snap this realm has exposed to clients (implicitly or explicitly) + // caches? //set cached_snaps; //set cached_active_children; // active children that are currently open @@ -108,7 +110,7 @@ struct SnapRealm { xlist inodes_with_caps; // for efficient realm splits map > client_caps; // to identify clients who need snap notifications - SnapRealm(MDCache *c, CInode *in) : mdcache(c), inode(in) {} + SnapRealm(MDCache *c, CInode *in) : mdcache(c), inode(in), snap_highwater(0) {} bool open_parents(MDRequest *mdr); void get_snap_set(set& s); diff --git a/src/messages/MClientFileCaps.h b/src/messages/MClientFileCaps.h index a52edcf0872cb..5e75298743e3f 100644 --- a/src/messages/MClientFileCaps.h +++ b/src/messages/MClientFileCaps.h @@ -37,10 +37,11 @@ class MClientFileCaps : public Message { vector snaps; public: - int get_caps() { return h.caps; } - int get_wanted() { return h.wanted; } - capseq_t get_seq() { return h.seq; } - capseq_t get_mseq() { return h.migrate_seq; } + int get_caps() { return h.caps; } + int get_wanted() { return h.wanted; } + capseq_t get_seq() { return h.seq; } + capseq_t get_mseq() { return h.migrate_seq; } + snapid_t get_snap_highwater() { return h.snap_highwater; } vector &get_snaps() { return snaps; } inodeno_t get_ino() { return inodeno_t(h.ino); } @@ -67,6 +68,8 @@ class MClientFileCaps : public Message { void set_mtime(const utime_t &t) { t.encode_timeval(&h.mtime); } void set_atime(const utime_t &t) { t.encode_timeval(&h.atime); } + void set_snap_highwater(snapid_t hw) { h.snap_highwater = hw; } + MClientFileCaps() {} MClientFileCaps(int op, inode_t& inode, diff --git a/src/messages/MClientReply.h b/src/messages/MClientReply.h index 8fdda35a5d93f..2210bb33447ec 100644 --- a/src/messages/MClientReply.h +++ b/src/messages/MClientReply.h @@ -164,6 +164,7 @@ class MClientReply : public Message { int get_result() { return (__s32)(__u32)st.result; } + snapid_t get_snap_highwater() { return st.snap_highwater; } vector &get_snaps() { return snaps; } unsigned get_file_caps() { return st.file_caps; } @@ -177,6 +178,7 @@ class MClientReply : public Message { void set_file_caps_seq(capseq_t s) { st.file_caps_seq = s; } void set_file_caps_mseq(capseq_t s) { st.file_caps_mseq = s; } //void set_file_data_version(uint64_t v) { st.file_data_version = v; } + void set_snap_highwater(snapid_t hw) { st.snap_highwater = hw; } MClientReply() {} MClientReply(MClientRequest *req, int result = 0) : diff --git a/src/messages/MClientSnap.h b/src/messages/MClientSnap.h new file mode 100644 index 0000000000000..38f3e7acb2e79 --- /dev/null +++ b/src/messages/MClientSnap.h @@ -0,0 +1,74 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2004-2006 Sage Weil + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#ifndef __MCLIENTSNAP_H +#define __MCLIENTSNAP_H + +#include "msg/Message.h" + +struct MClientSnap : public Message { + + static const char *get_opname(int o) { + switch (o) { + case CEPH_SNAP_OP_UPDATE: return "update"; + case CEPH_SNAP_OP_SPLIT: return "split"; + default: return "???"; + } + } + + __u32 op; + inodeno_t realm; + + // new snap state + snapid_t snap_highwater; + vector snaps; + + // (for split only) + inodeno_t new_realm; + list new_inodes; + + MClientSnap() : Message(CEPH_MSG_CLIENT_SNAP) {} + MClientSnap(int o, inodeno_t r) : + Message(CEPH_MSG_CLIENT_SNAP), + op(o), realm(r) {} + + const char *get_type_name() { return "Csnap"; } + void print(ostream& out) { + out << "client_snap(" << get_opname(op) << " " << realm + << " " << snaps; + out << ")"; + } + + void encode_payload() { + ::encode(op, payload); + ::encode(realm, payload); + ::encode(snap_highwater, payload); + ::encode(snaps, payload); + ::encode(new_realm, payload); + ::encode(new_inodes, payload); + } + void decode_payload() { + bufferlist::iterator p = payload.begin(); + ::decode(op, p); + ::decode(realm, p); + ::decode(snap_highwater, p); + ::decode(snaps, p); + ::decode(new_realm, p); + ::decode(new_inodes, p); + assert(p.end()); + } + +}; + +#endif diff --git a/src/msg/Message.cc b/src/msg/Message.cc index 61a37eb6882bf..026ab44cad35a 100644 --- a/src/msg/Message.cc +++ b/src/msg/Message.cc @@ -58,6 +58,7 @@ using namespace std; #include "messages/MClientReply.h" #include "messages/MClientFileCaps.h" #include "messages/MClientLease.h" +#include "messages/MClientSnap.h" #include "messages/MMDSSlaveRequest.h" @@ -252,6 +253,9 @@ decode_message(ceph_msg_header& env, bufferlist& front, bufferlist& data) case CEPH_MSG_CLIENT_LEASE: m = new MClientLease; break; + case CEPH_MSG_CLIENT_SNAP: + m = new MClientSnap; + break; // mds case MSG_MDS_SLAVE_REQUEST: -- 2.39.5