FileStore,OSD: add mechanism for injecting EIO, truncating obj

This will be used in testing repair.

Signed-off-by: Samuel Just <sam.just@inktank.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index 76c8adaa72a28534e0bbbba57c6185746d5d6b06..1bbb38e2c0b6277eb3faa35cfa035fda44b278e7 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -446,6 +446,9 @@ OPTION(filestore, OPT_BOOL, false)
 // Tests index failure paths
 OPTION(filestore_index_retry_probability, OPT_DOUBLE, 0)
 
+// Allow object read error injection
+OPTION(filestore_debug_inject_read_err, OPT_BOOL, false)
+
 OPTION(filestore_debug_omap_check, OPT_BOOL, 0) // Expensive debugging check on sync
 // Use omap for xattrs for attrs over
 OPTION(filestore_xattr_use_omap, OPT_BOOL, false)

diff --git a/src/os/FileStore.cc b/src/os/FileStore.cc
index 5d3fd2e8e3ca31955ae675e7b6d1efd16a17a037..8b19a6171f0c55bc5094f034c22892b8c6f558da 100644 (file)
--- a/src/os/FileStore.cc
+++ b/src/os/FileStore.cc
@@ -352,6 +352,9 @@ int FileStore::lfn_unlink(coll_t cid, const hobject_t& o,
        assert(!m_filestore_fail_eio || r != -EIO);
        return r;
       }
+      if (g_conf->filestore_debug_inject_read_err) {
+       debug_delete_obj(o);
+      }
     } else {
       /* Ensure that replay of this op doesn't result in the object_map
        * going away.
@@ -393,6 +396,7 @@ FileStore::FileStore(const std::string &base, const std::string &jdev, const cha
        g_conf->filestore_op_thread_suicide_timeout, &op_tp),
   flusher_queue_len(0), flusher_thread(this),
   logger(NULL),
+  read_error_lock("FileStore::read_error_lock"),
   m_filestore_btrfs_clone_range(g_conf->filestore_btrfs_clone_range),
   m_filestore_btrfs_snap (g_conf->filestore_btrfs_snap ),
   m_filestore_commit_timeout(g_conf->filestore_commit_timeout),
@@ -2723,7 +2727,12 @@ int FileStore::stat(coll_t cid, const hobject_t& oid, struct stat *st)
             << " = " << r
             << " (size " << st->st_size << ")" << dendl;
   }
-  return r;
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_mdata_eio(oid)) {
+    return -EIO;
+  } else {
+    return r;
+  }
 }
 
 int FileStore::read(coll_t cid, const hobject_t& oid, 
@@ -2761,7 +2770,12 @@ int FileStore::read(coll_t cid, const hobject_t& oid,
 
   dout(10) << "FileStore::read " << cid << "/" << oid << " " << offset << "~"
           << got << "/" << len << dendl;
-  return got;
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_data_eio(oid)) {
+    return -EIO;
+  } else {
+    return got;
+  }
 }
 
 int FileStore::fiemap(coll_t cid, const hobject_t& oid,
@@ -3776,6 +3790,43 @@ int FileStore::_fsetattrs(int fd, map<string, bufferptr> &aset)
   return 0;
 }
 
+// debug EIO injection
+void FileStore::inject_data_error(const hobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  dout(10) << __func__ << ": init error on " << oid << dendl;
+  data_error_set.insert(oid);
+}
+void FileStore::inject_mdata_error(const hobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  dout(10) << __func__ << ": init error on " << oid << dendl;
+  mdata_error_set.insert(oid);
+}
+void FileStore::debug_delete_obj(const hobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  dout(10) << __func__ << ": clear error on " << oid << dendl;
+  data_error_set.erase(oid);
+  mdata_error_set.erase(oid);
+}
+bool FileStore::debug_data_eio(const hobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  if (data_error_set.count(oid)) {
+    dout(10) << __func__ << ": inject error on " << oid << dendl;
+    return true;
+  } else {
+    return false;
+  }
+}
+bool FileStore::debug_mdata_eio(const hobject_t &oid) {
+  Mutex::Locker l(read_error_lock);
+  if (mdata_error_set.count(oid)) {
+    dout(10) << __func__ << ": inject error on " << oid << dendl;
+    return true;
+  } else {
+    return false;
+  }
+}
+
+
 // objects
 
 int FileStore::getattr(coll_t cid, const hobject_t& oid, const char *name, bufferptr &bp)
@@ -3817,7 +3868,12 @@ int FileStore::getattr(coll_t cid, const hobject_t& oid, const char *name, buffe
  out:
   dout(10) << "getattr " << cid << "/" << oid << " '" << name << "' = " << r << dendl;
   assert(!m_filestore_fail_eio || r != -EIO);
-  return r;
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_mdata_eio(oid)) {
+    return -EIO;
+  } else {
+    return r;
+  }
 }
 
 int FileStore::getattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>& aset, bool user_only) 
@@ -3871,7 +3927,13 @@ int FileStore::getattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>&
  out:
   dout(10) << "getattrs " << cid << "/" << oid << " = " << r << dendl;
   assert(!m_filestore_fail_eio || r != -EIO);
-  return r;
+
+  if (g_conf->filestore_debug_inject_read_err &&
+      debug_mdata_eio(oid)) {
+    return -EIO;
+  } else {
+    return r;
+  }
 }
 
 int FileStore::_setattrs(coll_t cid, const hobject_t& oid, map<string,bufferptr>& aset,

diff --git a/src/os/FileStore.h b/src/os/FileStore.h
index f1b9c1b04c9f5dc38327b0064b862fabf0cbf0d4..536c98b168ba89857385e160001186521f4a4c14 100644 (file)
--- a/src/os/FileStore.h
+++ b/src/os/FileStore.h
@@ -402,6 +402,16 @@ public:
   }
   uuid_d get_fsid() { return fsid; }
 
+  // DEBUG read error injection, an object is removed from both on delete()
+  Mutex read_error_lock;
+  set<hobject_t> data_error_set; // read() will return -EIO
+  set<hobject_t> mdata_error_set; // getattr(),stat() will return -EIO
+  void inject_data_error(const hobject_t &oid);
+  void inject_mdata_error(const hobject_t &oid);
+  void debug_delete_obj(const hobject_t &oid);
+  bool debug_data_eio(const hobject_t &oid);
+  bool debug_mdata_eio(const hobject_t &oid);
+
   int snapshot(const string& name);
 
   // attrs

diff --git a/src/os/ObjectStore.h b/src/os/ObjectStore.h
index ebcf058dce223ed8cacfc5892cadc3860c15b1c2..47279f7644600ade0f4048dd9f9718636b19d17a 100644 (file)
--- a/src/os/ObjectStore.h
+++ b/src/os/ObjectStore.h
@@ -939,6 +939,10 @@ public:
     
   virtual void set_fsid(uuid_d u) = 0;
   virtual uuid_d get_fsid() = 0;
+
+  // DEBUG
+  virtual void inject_data_error(const hobject_t &oid) {}
+  virtual void inject_mdata_error(const hobject_t &oid) {}
 };
 
 

diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index e72ccad3866b55479c11dad43c8b6dc01e36c8e5..60add150d5b821fc2b4673528ddb6b172bedc165 100644 (file)
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -1056,6 +1056,17 @@ int OSD::init()
   r = admin_socket->register_command("getomap", test_ops_hook,
                                "getomap <pool-id> <obj-name>");
   assert(r == 0);
+  r = admin_socket->register_command("truncobj", test_ops_hook,
+                               "truncobj <pool-id> <obj-name> <len>");
+  assert(r == 0);
+
+  r = admin_socket->register_command("injectdataerr", test_ops_hook,
+                                    "injectdataerr <pool-id> <obj-name>");
+  assert(r == 0);
+
+  r = admin_socket->register_command("injectmdataerr", test_ops_hook,
+                                    "injectmdataerr <pool-id> <obj-name>");
+  assert(r == 0);
 
   service.init();
   service.publish_map(osdmap);
@@ -1238,6 +1249,9 @@ int OSD::shutdown()
   cct->get_admin_socket()->unregister_command("rmomapkey");
   cct->get_admin_socket()->unregister_command("setomapheader");
   cct->get_admin_socket()->unregister_command("getomap");
+  cct->get_admin_socket()->unregister_command("truncobj");
+  cct->get_admin_socket()->unregister_command("injectdataerr");
+  cct->get_admin_socket()->unregister_command("injectmdataerr");
   delete test_ops_hook;
   test_ops_hook = NULL;
 
@@ -2504,6 +2518,7 @@ void OSD::check_ops_in_flight()
 //   setomapval <pool-id> <obj-name> <key> <val>
 //   rmomapkey <pool-id> <obj-name> <key>
 //   setomapheader <pool-id> <obj-name> <header>
+//   truncobj <pool-id> <obj-name> <newlen>
 void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
      std::string command, std::string args, ostream &ss)
 {
@@ -2511,7 +2526,10 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
   //Support changing the omap on a single osd by using the Admin Socket to
   //directly request the osd make a change.
   if (command == "setomapval" || command == "rmomapkey" ||
-        command == "setomapheader" || command == "getomap") {
+      command == "setomapheader" || command == "getomap" ||
+      command == "truncobj" || command == "injectmdataerr" ||
+      command == "injectdataerr"
+    ) {
     std::vector<std::string> argv;
     pg_t rawpg, pgid;
     int64_t pool;
@@ -2607,6 +2625,23 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store,
       } else {
           ss << "error=" << r;
       }
+    } else if (command == "truncobj") {
+      if (argc != 4) {
+       ss << "usage: truncobj <pool> <obj-name> <val>";
+       return;
+      }
+      t.truncate(coll_t(pgid), obj, atoi(argv[3].c_str()));
+      r = store->apply_transaction(t);
+      if (r < 0)
+       ss << "error=" << r;
+      else
+       ss << "ok";
+    } else if (command == "injectdataerr") {
+      store->inject_data_error(obj);
+      ss << "ok";
+    } else if (command == "injectmdataerr") {
+      store->inject_mdata_error(obj);
+      ss << "ok";
     }
     return;
   }