FDCache: implement a basic sharding of the FDCache

This is just a basic sharding. A more sophisticated implementation would
rely on something other than luck for keeping the distribution equitable.
The minimum FDCache shard size is 1.

Signed-off-by: Greg Farnum <greg@inktank.com>
Signed-off-by: Somnath Roy <somnath.roy@sandisk.com>

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index ee4353f2af93b9ccd1c299f5d6ab325c70c7bb36..aff4f2563593c3ab2981c0d9c586dcece39e1072 100644 (file)
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -713,6 +713,7 @@ OPTION(filestore_split_multiple, OPT_INT, 2)
 OPTION(filestore_update_to, OPT_INT, 1000)
 OPTION(filestore_blackhole, OPT_BOOL, false)     // drop any new transactions on the floor
 OPTION(filestore_fd_cache_size, OPT_INT, 128)    // FD lru size
+OPTION(filestore_fd_cache_shards, OPT_INT, 16)   // FD number of shards
 OPTION(filestore_dump_file, OPT_STR, "")         // file onto which store transaction dumps
 OPTION(filestore_kill_at, OPT_INT, 0)            // inject a failure at the n'th opportunity
 OPTION(filestore_inject_stall, OPT_INT, 0)       // artificially stall for N seconds in op queue thread

diff --git a/src/os/FDCache.h b/src/os/FDCache.h
index 712355b48192bc4d8c208c97932c1b42ccafed18..39872f0af789391b5446eefd3bf7f48b61ab90fb 100644 (file)
--- a/src/os/FDCache.h
+++ b/src/os/FDCache.h
@@ -23,6 +23,7 @@
 #include "common/Cond.h"
 #include "common/shared_cache.hpp"
 #include "include/compat.h"
+#include "include/intarith.h"
 
 /**
  * FD Cache
@@ -49,32 +50,42 @@ public:
   };
 
 private:
-  SharedLRU<ghobject_t, FD> registry;
   CephContext *cct;
+  const int registry_shards;
+  SharedLRU<ghobject_t, FD> *registry;
 
 public:
-  FDCache(CephContext *cct) : cct(cct) {
+  FDCache(CephContext *cct) : cct(cct),
+  registry_shards(cct->_conf->filestore_fd_cache_shards) {
     assert(cct);
     cct->_conf->add_observer(this);
-    registry.set_size(cct->_conf->filestore_fd_cache_size);
+    registry = new SharedLRU<ghobject_t, FD>[registry_shards];
+    for (int i = 0; i < registry_shards; ++i) {
+      registry[i].set_size(
+          MAX((cct->_conf->filestore_fd_cache_size / registry_shards), 1));
+    }
   }
   ~FDCache() {
     cct->_conf->remove_observer(this);
+    delete[] registry;
   }
   typedef ceph::shared_ptr<FD> FDRef;
 
   FDRef lookup(const ghobject_t &hoid) {
-    return registry.lookup(hoid);
+    int registry_id = hoid.hobj.hash % registry_shards;
+    return registry[registry_id].lookup(hoid);
   }
 
   FDRef add(const ghobject_t &hoid, int fd, bool *existed) {
-    return registry.add(hoid, new FD(fd), existed);
+    int registry_id = hoid.hobj.hash % registry_shards;
+    return registry[registry_id].add(hoid, new FD(fd), existed);
   }
 
   /// clear cached fd for hoid, subsequent lookups will get an empty FD
   void clear(const ghobject_t &hoid) {
-    registry.clear(hoid);
-    assert(!registry.lookup(hoid));
+    int registry_id = hoid.hobj.hash % registry_shards;
+    registry[registry_id].clear(hoid);
+    assert(!registry[registry_id].lookup(hoid));
   }
 
   /// md_config_obs_t
@@ -88,7 +99,9 @@ public:
   void handle_conf_change(const md_config_t *conf,
                          const std::set<std::string> &changed) {
     if (changed.count("filestore_fd_cache_size")) {
-      registry.set_size(conf->filestore_fd_cache_size);
+      for (int i = 0; i < registry_shards; ++i)
+        registry[i].set_size(
+              MAX((conf->filestore_fd_cache_size / registry_shards), 1));
     }
   }