From: Samuel Just Date: Tue, 26 Mar 2013 22:14:59 +0000 (-0700) Subject: FileStore,OSD: add mechanism for injecting EIO, truncating obj X-Git-Tag: v0.62~113^2~7 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=1940cf3e3f5100ff6a00f12ea2cd12747f6bd98d;p=ceph.git FileStore,OSD: add mechanism for injecting EIO, truncating obj This will be used in testing repair. Signed-off-by: Samuel Just --- diff --git a/src/common/config_opts.h b/src/common/config_opts.h index 76c8adaa72a..1bbb38e2c0b 100644 --- a/src/common/config_opts.h +++ b/src/common/config_opts.h @@ -446,6 +446,9 @@ OPTION(filestore, OPT_BOOL, false) // Tests index failure paths OPTION(filestore_index_retry_probability, OPT_DOUBLE, 0) +// Allow object read error injection +OPTION(filestore_debug_inject_read_err, OPT_BOOL, false) + OPTION(filestore_debug_omap_check, OPT_BOOL, 0) // Expensive debugging check on sync // Use omap for xattrs for attrs over OPTION(filestore_xattr_use_omap, OPT_BOOL, false) diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc index 5d3fd2e8e3c..8b19a6171f0 100644 --- a/src/os/FileStore.cc +++ b/src/os/FileStore.cc @@ -352,6 +352,9 @@ int FileStore::lfn_unlink(coll_t cid, const hobject_t& o, assert(!m_filestore_fail_eio || r != -EIO); return r; } + if (g_conf->filestore_debug_inject_read_err) { + debug_delete_obj(o); + } } else { /* Ensure that replay of this op doesn't result in the object_map * going away. @@ -393,6 +396,7 @@ FileStore::FileStore(const std::string &base, const std::string &jdev, const cha g_conf->filestore_op_thread_suicide_timeout, &op_tp), flusher_queue_len(0), flusher_thread(this), logger(NULL), + read_error_lock("FileStore::read_error_lock"), m_filestore_btrfs_clone_range(g_conf->filestore_btrfs_clone_range), m_filestore_btrfs_snap (g_conf->filestore_btrfs_snap ), m_filestore_commit_timeout(g_conf->filestore_commit_timeout), @@ -2723,7 +2727,12 @@ int FileStore::stat(coll_t cid, const hobject_t& oid, struct stat *st) << " = " << r << " (size " << st->st_size << ")" << dendl; } - return r; + if (g_conf->filestore_debug_inject_read_err && + debug_mdata_eio(oid)) { + return -EIO; + } else { + return r; + } } int FileStore::read(coll_t cid, const hobject_t& oid, @@ -2761,7 +2770,12 @@ int FileStore::read(coll_t cid, const hobject_t& oid, dout(10) << "FileStore::read " << cid << "/" << oid << " " << offset << "~" << got << "/" << len << dendl; - return got; + if (g_conf->filestore_debug_inject_read_err && + debug_data_eio(oid)) { + return -EIO; + } else { + return got; + } } int FileStore::fiemap(coll_t cid, const hobject_t& oid, @@ -3776,6 +3790,43 @@ int FileStore::_fsetattrs(int fd, map &aset) return 0; } +// debug EIO injection +void FileStore::inject_data_error(const hobject_t &oid) { + Mutex::Locker l(read_error_lock); + dout(10) << __func__ << ": init error on " << oid << dendl; + data_error_set.insert(oid); +} +void FileStore::inject_mdata_error(const hobject_t &oid) { + Mutex::Locker l(read_error_lock); + dout(10) << __func__ << ": init error on " << oid << dendl; + mdata_error_set.insert(oid); +} +void FileStore::debug_delete_obj(const hobject_t &oid) { + Mutex::Locker l(read_error_lock); + dout(10) << __func__ << ": clear error on " << oid << dendl; + data_error_set.erase(oid); + mdata_error_set.erase(oid); +} +bool FileStore::debug_data_eio(const hobject_t &oid) { + Mutex::Locker l(read_error_lock); + if (data_error_set.count(oid)) { + dout(10) << __func__ << ": inject error on " << oid << dendl; + return true; + } else { + return false; + } +} +bool FileStore::debug_mdata_eio(const hobject_t &oid) { + Mutex::Locker l(read_error_lock); + if (mdata_error_set.count(oid)) { + dout(10) << __func__ << ": inject error on " << oid << dendl; + return true; + } else { + return false; + } +} + + // objects int FileStore::getattr(coll_t cid, const hobject_t& oid, const char *name, bufferptr &bp) @@ -3817,7 +3868,12 @@ int FileStore::getattr(coll_t cid, const hobject_t& oid, const char *name, buffe out: dout(10) << "getattr " << cid << "/" << oid << " '" << name << "' = " << r << dendl; assert(!m_filestore_fail_eio || r != -EIO); - return r; + if (g_conf->filestore_debug_inject_read_err && + debug_mdata_eio(oid)) { + return -EIO; + } else { + return r; + } } int FileStore::getattrs(coll_t cid, const hobject_t& oid, map& aset, bool user_only) @@ -3871,7 +3927,13 @@ int FileStore::getattrs(coll_t cid, const hobject_t& oid, map& out: dout(10) << "getattrs " << cid << "/" << oid << " = " << r << dendl; assert(!m_filestore_fail_eio || r != -EIO); - return r; + + if (g_conf->filestore_debug_inject_read_err && + debug_mdata_eio(oid)) { + return -EIO; + } else { + return r; + } } int FileStore::_setattrs(coll_t cid, const hobject_t& oid, map& aset, diff --git a/src/os/FileStore.h b/src/os/FileStore.h index f1b9c1b04c9..536c98b168b 100644 --- a/src/os/FileStore.h +++ b/src/os/FileStore.h @@ -402,6 +402,16 @@ public: } uuid_d get_fsid() { return fsid; } + // DEBUG read error injection, an object is removed from both on delete() + Mutex read_error_lock; + set data_error_set; // read() will return -EIO + set mdata_error_set; // getattr(),stat() will return -EIO + void inject_data_error(const hobject_t &oid); + void inject_mdata_error(const hobject_t &oid); + void debug_delete_obj(const hobject_t &oid); + bool debug_data_eio(const hobject_t &oid); + bool debug_mdata_eio(const hobject_t &oid); + int snapshot(const string& name); // attrs diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h index ebcf058dce2..47279f76446 100644 --- a/src/os/ObjectStore.h +++ b/src/os/ObjectStore.h @@ -939,6 +939,10 @@ public: virtual void set_fsid(uuid_d u) = 0; virtual uuid_d get_fsid() = 0; + + // DEBUG + virtual void inject_data_error(const hobject_t &oid) {} + virtual void inject_mdata_error(const hobject_t &oid) {} }; diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index e72ccad3866..60add150d5b 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -1056,6 +1056,17 @@ int OSD::init() r = admin_socket->register_command("getomap", test_ops_hook, "getomap "); assert(r == 0); + r = admin_socket->register_command("truncobj", test_ops_hook, + "truncobj "); + assert(r == 0); + + r = admin_socket->register_command("injectdataerr", test_ops_hook, + "injectdataerr "); + assert(r == 0); + + r = admin_socket->register_command("injectmdataerr", test_ops_hook, + "injectmdataerr "); + assert(r == 0); service.init(); service.publish_map(osdmap); @@ -1238,6 +1249,9 @@ int OSD::shutdown() cct->get_admin_socket()->unregister_command("rmomapkey"); cct->get_admin_socket()->unregister_command("setomapheader"); cct->get_admin_socket()->unregister_command("getomap"); + cct->get_admin_socket()->unregister_command("truncobj"); + cct->get_admin_socket()->unregister_command("injectdataerr"); + cct->get_admin_socket()->unregister_command("injectmdataerr"); delete test_ops_hook; test_ops_hook = NULL; @@ -2504,6 +2518,7 @@ void OSD::check_ops_in_flight() // setomapval // rmomapkey // setomapheader
+// truncobj void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, std::string command, std::string args, ostream &ss) { @@ -2511,7 +2526,10 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, //Support changing the omap on a single osd by using the Admin Socket to //directly request the osd make a change. if (command == "setomapval" || command == "rmomapkey" || - command == "setomapheader" || command == "getomap") { + command == "setomapheader" || command == "getomap" || + command == "truncobj" || command == "injectmdataerr" || + command == "injectdataerr" + ) { std::vector argv; pg_t rawpg, pgid; int64_t pool; @@ -2607,6 +2625,23 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, } else { ss << "error=" << r; } + } else if (command == "truncobj") { + if (argc != 4) { + ss << "usage: truncobj "; + return; + } + t.truncate(coll_t(pgid), obj, atoi(argv[3].c_str())); + r = store->apply_transaction(t); + if (r < 0) + ss << "error=" << r; + else + ss << "ok"; + } else if (command == "injectdataerr") { + store->inject_data_error(obj); + ss << "ok"; + } else if (command == "injectmdataerr") { + store->inject_mdata_error(obj); + ss << "ok"; } return; }