* Monitors now have a config option ``mon_osd_warn_num_repaired``, 10 by default.
If any OSD has repaired more than this many I/O errors in stored data a
``OSD_TOO_MANY_REPAIRS`` health warning is generated.
+
+* Introduce commands that manipulate required client features of a file system::
+
+ ceph fs required_client_features <fs name> add <feature>
+ ceph fs required_client_features <fs name> rm <feature>
+ ceph fs feature ls
Clients running an older version will be automatically evicted.
+Enforcing minimum version of CephFS client is achieved by setting required
+client features. Commands to manipulate required client features of a file
+system:
+
+::
+
+ fs required_client_features <fs name> add reply_encoding
+ fs required_client_features <fs name> rm reply_encoding
+
+To list all CephFS features
+
+::
+
+ fs feature ls
+
+
+CephFS features and first release they came out.
+
++------------------+--------------+-----------------+
+| Feature | Ceph release | Upstream Kernel |
++==================+==============+=================+
+| jewel | jewel | 4.5 |
++------------------+--------------+-----------------+
+| kraken | kraken | 4.13 |
++------------------+--------------+-----------------+
+| luminous | luminous | 4.13 |
++------------------+--------------+-----------------+
+| mimic | mimic | 4.19 |
++------------------+--------------+-----------------+
+| reply_encoding | nautilus | 5.1 |
++------------------+--------------+-----------------+
+| reclaim_client | nautilus | N/A |
++------------------+--------------+-----------------+
+| lazy_caps_wanted | nautilus | 5.1 |
++------------------+--------------+-----------------+
+| multi_reconnect | nautilus | 5.1 |
++------------------+--------------+-----------------+
+| deleg_ino | octopus | 5.6 |
++------------------+--------------+-----------------+
+| metric_collect | pacific | N/A |
++------------------+--------------+-----------------+
+
+CephFS Feature Descriptions
+
+
+::
+
+ reply_encoding
+
+MDS encodes request reply in extensible format if client supports this feature.
+
+
+::
+
+ reclaim_client
+
+MDS allows new client to reclaim another (dead) client's states. This feature
+is used by NFS-Ganesha.
+
+
+::
+
+ lazy_caps_wanted
+
+When a stale client resumes, if the client supports this feature, mds only needs
+to re-issue caps that are explictly wanted.
+
+
+::
+
+ multi_reconnect
+
+When mds failover, client sends reconnect messages to mds, to reestablish cache
+states. If MDS supports this feature, client can split large reconnect message
+into multiple ones.
+
+
+::
+
+ deleg_ino
+
+MDS delegate inode numbers to client if client supports this feature. Having
+delegated inode numbers is a prerequisite for client to do async file creation.
+
+
+::
+
+ metric_collect
+
+Clients can send performance metric to MDS if MDS support this feature.
+
Global settings
---------------
- exec:
mon.a:
- ceph fs dump --format=json-pretty
- - ceph fs set cephfs min_compat_client pacific
+ - ceph fs required_client_features cephfs add metric_collect
- fs.clients_evicted:
- exec:
mon.a:
- ceph fs dump --format=json-pretty
- - ceph fs set cephfs min_compat_client octopus
+ - ceph fs required_client_features cephfs add metric_collect
- fs.clients_evicted:
clients:
client.0: False
self._check_pool_application_metadata_key_value(
pool_names[i], 'cephfs', keys[i], fs_name)
+ def test_required_client_features(self):
+ """
+ That `ceph fs required_client_features` command functions.
+ """
+
+ def is_required(index):
+ out = self.fs.mon_manager.raw_cluster_cmd('fs', 'get', self.fs.name, '--format=json-pretty')
+ features = json.loads(out)['mdsmap']['required_client_features']
+ if "feature_{0}".format(index) in features:
+ return True;
+ return False;
+
+ features = json.loads(self.fs.mon_manager.raw_cluster_cmd('fs', 'feature', 'ls', '--format=json-pretty'))
+ self.assertGreater(len(features), 0);
+
+ for f in features:
+ self.fs.mon_manager.raw_cluster_cmd('fs', 'required_client_features', self.fs.name, 'rm', str(f['index']))
+
+ for f in features:
+ index = f['index']
+ feature = f['name']
+ if feature == 'reserved':
+ feature = str(index)
+
+ if index % 3 == 0:
+ continue;
+ self.fs.mon_manager.raw_cluster_cmd('fs', 'required_client_features', self.fs.name, 'add', feature)
+ self.assertTrue(is_required(index))
+
+ if index % 2 == 0:
+ continue;
+ self.fs.mon_manager.raw_cluster_cmd('fs', 'required_client_features', self.fs.name, 'rm', feature)
+ self.assertFalse(is_required(index))
+
class TestConfigCommands(CephFSTestCase):
"""
'ro_compat': JObj({}, allow_unknown=True),
'incompat': JObj({}, allow_unknown=True)
}),
- 'min_compat_client': str,
+ 'required_client_features': JObj({}, allow_unknown=True),
'data_pools': JList(int),
'info': JObj({}, allow_unknown=True),
'fs_name': str,
mds/FSMapUser.cc
mds/inode_backtrace.cc
mds/mdstypes.cc
- mds/flock.cc)
+ mds/flock.cc
+ mds/cephfs_features.cc)
add_subdirectory(json_spirit)
${PROJECT_SOURCE_DIR}/src/mgr/ServiceMap.cc
${PROJECT_SOURCE_DIR}/src/mds/inode_backtrace.cc
${PROJECT_SOURCE_DIR}/src/mds/mdstypes.cc
+ ${PROJECT_SOURCE_DIR}/src/mds/cephfs_features.cc
${PROJECT_SOURCE_DIR}/src/mds/FSMap.cc
${PROJECT_SOURCE_DIR}/src/mds/FSMapUser.cc
${PROJECT_SOURCE_DIR}/src/mds/MDSMap.cc
f->dump_int("root", root);
f->dump_int("session_timeout", session_timeout);
f->dump_int("session_autoclose", session_autoclose);
- f->dump_stream("min_compat_client") << to_integer<int>(min_compat_client) << " ("
- << min_compat_client << ")";
+ f->open_object_section("required_client_features");
+ cephfs_dump_features(f, required_client_features);
+ f->close_section();
f->dump_int("max_file_size", max_file_size);
f->dump_int("last_failure", last_failure);
f->dump_int("last_failure_osd_epoch", last_failure_osd_epoch);
out << "session_timeout\t" << session_timeout << "\n"
<< "session_autoclose\t" << session_autoclose << "\n";
out << "max_file_size\t" << max_file_size << "\n";
- out << "min_compat_client\t" << to_integer<int>(min_compat_client) << " ("
- << min_compat_client << ")\n";
+ out << "required_client_features\t" << cephfs_stringify_features(required_client_features) << "\n";
out << "last_failure\t" << last_failure << "\n"
<< "last_failure_osd_epoch\t" << last_failure_osd_epoch << "\n";
out << "compat\t" << compat << "\n";
encode(data_pools, bl);
encode(cas_pool, bl);
- __u16 ev = 15;
+ __u16 ev = 16;
encode(ev, bl);
encode(compat, bl);
encode(metadata_pool, bl);
encode(balancer, bl);
encode(standby_count_wanted, bl);
encode(old_max_mds, bl);
- encode(min_compat_client, bl);
+ {
+ ceph_release_t min_compat_client = ceph_release_t::unknown;
+ encode(min_compat_client, bl);
+ }
+ encode(required_client_features, bl);
ENCODE_FINISH(bl);
}
decode(old_max_mds, p);
}
- if (ev == 14) {
- int8_t r;
- decode(r, p);
- if (r < 0) {
- min_compat_client = ceph_release_t::unknown;
+ if (ev >= 14) {
+ ceph_release_t min_compat_client;
+ if (ev == 14) {
+ int8_t r;
+ decode(r, p);
+ if (r < 0) {
+ min_compat_client = ceph_release_t::unknown;
+ } else {
+ min_compat_client = ceph_release_t{static_cast<uint8_t>(r)};
+ }
+ } else if (ev >= 15) {
+ decode(min_compat_client, p);
+ }
+ if (ev >= 16) {
+ decode(required_client_features, p);
} else {
- min_compat_client = ceph_release_t{static_cast<uint8_t>(r)};
+ set_min_compat_client(min_compat_client);
}
- } else if (ev > 14) {
- decode(min_compat_client, p);
}
DECODE_FINISH(p);
}
return false;
}
+
+void MDSMap::set_min_compat_client(ceph_release_t version)
+{
+ vector<size_t> bits = CEPHFS_FEATURES_MDS_REQUIRED;
+
+ if (version >= ceph_release_t::octopus)
+ bits.push_back(CEPHFS_FEATURE_OCTOPUS);
+ else if (version >= ceph_release_t::nautilus)
+ bits.push_back(CEPHFS_FEATURE_NAUTILUS);
+ else if (version >= ceph_release_t::mimic)
+ bits.push_back(CEPHFS_FEATURE_MIMIC);
+ else if (version >= ceph_release_t::luminous)
+ bits.push_back(CEPHFS_FEATURE_LUMINOUS);
+ else if (version >= ceph_release_t::kraken)
+ bits.push_back(CEPHFS_FEATURE_KRAKEN);
+ else if (version >= ceph_release_t::jewel)
+ bits.push_back(CEPHFS_FEATURE_JEWEL);
+
+ std::sort(bits.begin(), bits.end());
+ required_client_features = feature_bitset_t(bits);
+}
#include "common/config.h"
#include "mds/mdstypes.h"
+#include "mds/cephfs_features.h"
#define MDS_FEATURE_INCOMPAT_BASE CompatSet::Feature(1, "base v0.20")
#define MDS_FEATURE_INCOMPAT_CLIENTRANGES CompatSet::Feature(2, "client writeable ranges")
uint64_t get_max_filesize() const { return max_file_size; }
void set_max_filesize(uint64_t m) { max_file_size = m; }
- ceph_release_t get_min_compat_client() const { return min_compat_client; }
- void set_min_compat_client(ceph_release_t version) { min_compat_client = version; }
+ void set_min_compat_client(ceph_release_t version);
+
+ void add_required_client_feature(size_t bit) {
+ required_client_features.insert(bit);
+ }
+ void remove_required_client_feature(size_t bit) {
+ required_client_features.erase(bit);
+ }
+ const auto& get_required_client_features() const {
+ return required_client_features;
+ }
int get_flags() const { return flags; }
bool test_flag(int f) const { return flags & f; }
__u32 session_autoclose = 300;
uint64_t max_file_size = 1ULL<<40; /* 1TB */
- ceph_release_t min_compat_client{ceph_release_t::unknown};
+ feature_bitset_t required_client_features;
std::vector<int64_t> data_pools; // file data pools available to clients (via an ioctl). first is the default.
int64_t cas_pool = -1; // where CAS objects go
if (objecter->get_client_incarnation() != incarnation)
objecter->set_client_incarnation(incarnation);
- if (mdsmap->get_min_compat_client() < ceph_release_t::max &&
- oldmap.get_min_compat_client() != mdsmap->get_min_compat_client())
+ if (mdsmap->get_required_client_features() != oldmap.get_required_client_features())
server->update_required_client_features();
// for debug
void Server::update_required_client_features()
{
- vector<size_t> bits = CEPHFS_FEATURES_MDS_REQUIRED;
-
- /* If this blows up on you, you added a release without adding a new release bit to cephfs_features.h */
- static_assert(CEPHFS_CURRENT_RELEASE == CEPH_RELEASE_MAX-1);
-
- ceph_release_t min_compat = mds->mdsmap->get_min_compat_client();
- if (min_compat >= ceph_release_t::octopus)
- bits.push_back(CEPHFS_FEATURE_OCTOPUS);
- else if (min_compat >= ceph_release_t::nautilus)
- bits.push_back(CEPHFS_FEATURE_NAUTILUS);
- else if (min_compat >= ceph_release_t::mimic)
- bits.push_back(CEPHFS_FEATURE_MIMIC);
- else if (min_compat >= ceph_release_t::luminous)
- bits.push_back(CEPHFS_FEATURE_LUMINOUS);
- else if (min_compat >= ceph_release_t::kraken)
- bits.push_back(CEPHFS_FEATURE_KRAKEN);
- else if (min_compat >= ceph_release_t::jewel)
- bits.push_back(CEPHFS_FEATURE_JEWEL);
-
- std::sort(bits.begin(), bits.end());
- required_client_features = feature_bitset_t(bits);
+ required_client_features = mds->mdsmap->get_required_client_features();
dout(7) << "required_client_features: " << required_client_features << dendl;
if (mds->get_state() >= MDSMap::STATE_RECONNECT) {
--- /dev/null
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include <array>
+#include <sstream>
+#include "cephfs_features.h"
+#include "mdstypes.h"
+
+static const std::array feature_names
+{
+ "reserved",
+ "reserved",
+ "reserved",
+ "reserved",
+ "reserved",
+ "jewel",
+ "kraken",
+ "luminous",
+ "mimic",
+ "reply_encoding",
+ "reclaim_client",
+ "lazy_caps_wanted",
+ "multi_reconnect",
+ "deleg_ino",
+ "metric_collect",
+};
+static_assert(feature_names.size() == CEPHFS_FEATURE_MAX + 1);
+
+std::string_view cephfs_feature_name(size_t id)
+{
+ if (id > feature_names.size())
+ return "unknown";
+ return feature_names[id];
+}
+
+int cephfs_feature_from_name(std::string_view name)
+{
+ for (size_t i = 0; i < feature_names.size(); ++i) {
+ if (name == feature_names[i])
+ return i;
+ }
+ return -1;
+}
+
+std::string cephfs_stringify_features(const feature_bitset_t& features)
+{
+ std::ostringstream ss;
+ bool first = true;
+ ss << "{";
+ for (size_t i = 0; i < feature_names.size(); ++i) {
+ if (!features.test(i))
+ continue;
+ if (!first)
+ ss << ",";
+ ss << i << "=" << cephfs_feature_name(i);
+ first = false;
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void cephfs_dump_features(ceph::Formatter *f, const feature_bitset_t& features)
+{
+ for (size_t i = 0; i < feature_names.size(); ++i) {
+ if (!features.test(i))
+ continue;
+ char s[18];
+ snprintf(s, sizeof(s), "feature_%lu", i);
+ f->dump_string(s, cephfs_feature_name(i));
+ }
+}
+
#ifndef CEPHFS_FEATURES_H
#define CEPHFS_FEATURES_H
+class feature_bitset_t;
+namespace ceph {
+ class Formatter;
+}
+
// When adding a new release, please update the "current" release below, add a
// feature bit for that release, add that feature bit to CEPHFS_FEATURES_ALL,
// and update Server::update_required_client_features(). This feature bit
#define CEPHFS_FEATURE_DELEG_INO 13
#define CEPHFS_FEATURE_OCTOPUS 13
#define CEPHFS_FEATURE_METRIC_COLLECT 14
+#define CEPHFS_FEATURE_MAX 14
#define CEPHFS_FEATURES_ALL { \
0, 1, 2, 3, 4, \
#define CEPHFS_FEATURES_CLIENT_SUPPORTED CEPHFS_FEATURES_ALL
#define CEPHFS_FEATURES_CLIENT_REQUIRED {}
+extern std::string_view cephfs_feature_name(size_t id);
+extern int cephfs_feature_from_name(std::string_view name);
+std::string cephfs_stringify_features(const feature_bitset_t& features);
+void cephfs_dump_features(ceph::Formatter *f, const feature_bitset_t& features);
+
#endif
return false;
return _vec[bit / bits_per_block] & ((block_type)1 << (bit % bits_per_block));
}
+ void insert(size_t bit) {
+ size_t n = bit / bits_per_block;
+ if (n >= _vec.size())
+ _vec.resize(n + 1);
+ _vec[n] |= ((block_type)1 << (bit % bits_per_block));
+ }
+ void erase(size_t bit) {
+ size_t n = bit / bits_per_block;
+ if (n >= _vec.size())
+ return;
+ _vec[n] &= ~((block_type)1 << (bit % bits_per_block));
+ if (n + 1 == _vec.size()) {
+ while (!_vec.empty() && _vec.back() == 0)
+ _vec.pop_back();
+ }
+ }
void clear() {
_vec.clear();
}
+ bool operator==(const feature_bitset_t& other) const {
+ return _vec == other._vec;
+ }
+ bool operator!=(const feature_bitset_t& other) const {
+ return _vec != other._vec;
+ }
void encode(ceph::buffer::list& bl) const;
void decode(ceph::buffer::list::const_iterator &p);
void dump(ceph::Formatter *f) const;
#include "FSCommands.h"
#include "MDSMonitor.h"
#include "MgrStatMonitor.h"
+#include "mds/cephfs_features.h"
using TOPNSPC::common::cmd_getval;
}
};
+class RequiredClientFeaturesHandler : public FileSystemCommandHandler
+{
+ public:
+ RequiredClientFeaturesHandler()
+ : FileSystemCommandHandler("fs required_client_features")
+ {
+ }
+
+ int handle(
+ Monitor *mon,
+ FSMap &fsmap,
+ MonOpRequestRef op,
+ const cmdmap_t& cmdmap,
+ std::stringstream &ss) override
+ {
+ std::string fs_name;
+ if (!cmd_getval(cmdmap, "fs_name", fs_name) || fs_name.empty()) {
+ ss << "Missing filesystem name";
+ return -EINVAL;
+ }
+ auto fs = fsmap.get_filesystem(fs_name);
+ if (fs == nullptr) {
+ ss << "Not found: '" << fs_name << "'";
+ return -ENOENT;
+ }
+ string subop;
+ if (!cmd_getval(cmdmap, "subop", subop) ||
+ (subop != "add" && subop != "rm")) {
+ ss << "Must either add or rm a feature; " << subop << " is not recognized";
+ return -EINVAL;
+ }
+ string val;
+ if (!cmd_getval(cmdmap, "val", val) || val.empty()) {
+ ss << "Missing feature id/name";
+ return -EINVAL;
+ }
+
+ int feature = cephfs_feature_from_name(val);
+ if (feature < 0) {
+ string err;
+ feature = strict_strtol(val.c_str(), 10, &err);
+ if (err.length()) {
+ ss << "Invalid feature name: " << val;
+ return -EINVAL;
+ }
+ if (feature < 0 || feature > CEPHFS_FEATURE_MAX) {
+ ss << "Invalid feature id: " << feature;
+ return -EINVAL;
+ }
+ }
+
+ if (subop == "add") {
+ fsmap.modify_filesystem(
+ fs->fscid,
+ [feature](std::shared_ptr<Filesystem> fs)
+ {
+ fs->mds_map.add_required_client_feature(feature);
+ });
+ ss << "added feature '" << cephfs_feature_name(feature) << "' to required_client_features";
+ } else {
+ fsmap.modify_filesystem(
+ fs->fscid,
+ [feature](std::shared_ptr<Filesystem> fs)
+ {
+ fs->mds_map.remove_required_client_feature(feature);
+ });
+ ss << "removed feature '" << cephfs_feature_name(feature) << "' to required_client_features";
+ }
+ return 0;
+ }
+};
+
+
class AddDataPoolHandler : public FileSystemCommandHandler
{
public:
handlers.push_back(std::make_shared<SetHandler>());
handlers.push_back(std::make_shared<FailHandler>());
handlers.push_back(std::make_shared<FlagSetHandler>());
+ handlers.push_back(std::make_shared<RequiredClientFeaturesHandler>());
handlers.push_back(std::make_shared<AddDataPoolHandler>(paxos));
handlers.push_back(std::make_shared<RemoveDataPoolHandler>());
handlers.push_back(std::make_shared<FsNewHandler>(paxos));
}
}
r = 0;
+ } else if (prefix == "fs feature ls") {
+ if (f) {
+ f->open_array_section("cephfs_features");
+ for (size_t i = 0; i <= CEPHFS_FEATURE_MAX; ++i) {
+ f->open_object_section("feature");
+ f->dump_int("index", i);
+ f->dump_string("name", cephfs_feature_name(i));
+ f->close_section();
+ }
+ f->close_section();
+ f->flush(ds);
+ } else {
+ for (size_t i = 0; i <= CEPHFS_FEATURE_MAX; ++i) {
+ ds << i << " " << cephfs_feature_name(i) << std::endl;
+ }
+ }
+ r = 0;
}
out:
"name=yes_i_really_mean_it,type=CephBool,req=false",
"Set a global CephFS flag",
"fs", "rw")
+
+COMMAND("fs feature ls",
+ "list available cephfs features to be set/unset",
+ "mds", "r")
+
+COMMAND("fs required_client_features "
+ "name=fs_name,type=CephString "
+ "name=subop,type=CephChoices,strings=add|rm "
+ "name=val,type=CephString ",
+ "add/remove required features of clients", "mds", "rw")
+
COMMAND("fs add_data_pool name=fs_name,type=CephString "
"name=pool,type=CephString",
"add data pool <pool>", "mds", "rw")
projects / ceph.git / commitdiff