tools: remove RBD specific logic in immutable obj cache

Make the daemon more generic(pool name + object id) so it
can service both RBD and RGW.
Also supports caching partial object(even empty), the caller
needs to append_zero to read buffer if read on those objects.

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>

diff --git a/src/test/immutable_object_cache/test_DomainSocket.cc b/src/test/immutable_object_cache/test_DomainSocket.cc
index a7ae75c3a9d28b7d0f34e657496827ec5788d4ab..d167d96dcaeb2f674d206ea69c12d9ca2e8a276f 100644 (file)
--- a/src/test/immutable_object_cache/test_DomainSocket.cc
+++ b/src/test/immutable_object_cache/test_DomainSocket.cc
@@ -94,7 +94,7 @@ public:
     auto ctx = new FunctionContext([this](bool reg) {
       ASSERT_TRUE(reg);
     });
-    m_cache_client->register_volume("pool_name", "rbd_name", 4096, ctx);
+    m_cache_client->register_client(ctx);
     ASSERT_TRUE(m_cache_client->is_session_work());
   }
 
@@ -153,19 +153,19 @@ public:
          usleep(1);
        }
 
-       m_cache_client->lookup_object("test_pool", "test_rbd", "123456", ctx);
+       m_cache_client->lookup_object("test_pool", "123456", ctx);
        m_send_request_index++;
      }
      m_wait_event.wait();
    }
 
-  bool startup_lookupobject_testing(std::string pool_name, std::string volume_name, std::string object_id) {
+  bool startup_lookupobject_testing(std::string pool_name, std::string object_id) {
     bool hit;
     auto ctx = new FunctionContext([this, &hit](bool req){
        hit = req;
        m_wait_event.signal();
     });
-    m_cache_client->lookup_object(pool_name, volume_name, object_id, ctx);
+    m_cache_client->lookup_object(pool_name, object_id, ctx);
     m_wait_event.wait();
     return hit;
   }
@@ -201,9 +201,9 @@ TEST_F(TestCommunication, test_lookup_object) {
   }
   for (uint64_t i = 50; i < 100; i++) {
     if ((random_hit % i) != 0) {
-      ASSERT_FALSE(startup_lookupobject_testing("test_pool", "testing_volume", std::to_string(i)));
+      ASSERT_FALSE(startup_lookupobject_testing("test_pool", std::to_string(i)));
     } else {
-      ASSERT_TRUE(startup_lookupobject_testing("test_pool", "testing_volume", std::to_string(i)));
+      ASSERT_TRUE(startup_lookupobject_testing("test_pool", std::to_string(i)));
     }
   }
 }

diff --git a/src/tools/immutable_object_cache/CacheClient.cc b/src/tools/immutable_object_cache/CacheClient.cc
index 414c9cbc4cd22c3607bccde92fc57ae45fd2673d..2e7609007f30c20fcd915f6917027a138046a8bc 100644 (file)
--- a/src/tools/immutable_object_cache/CacheClient.cc
+++ b/src/tools/immutable_object_cache/CacheClient.cc
@@ -85,16 +85,10 @@ namespace immutable_obj_cache {
     return 0;
   }
 
-  int CacheClient::register_volume(std::string pool_name, std::string vol_name,
-                                   uint64_t vol_size, Context* on_finish) {
+  int CacheClient::register_client(Context* on_finish) {
     // cache controller will init layout
     rbdsc_req_type_t *message = new rbdsc_req_type_t();
     message->type = RBDSC_REGISTER;
-    memcpy(message->pool_name, pool_name.c_str(), pool_name.size());
-    memcpy(message->vol_name, vol_name.c_str(), vol_name.size());
-    message->vol_size = vol_size;
-    message->offset = 0;
-    message->length = 0;
 
     uint64_t ret;
     boost::system::error_code ec;
@@ -151,12 +145,11 @@ namespace immutable_obj_cache {
   }
 
   // if occur any error, we just return false. Then read from rados.
-  int CacheClient::lookup_object(std::string pool_name, std::string vol_name,
-                                 std::string object_id, Context* on_finish) {
+  int CacheClient::lookup_object(std::string pool_name, std::string object_id,
+                                 Context* on_finish) {
     rbdsc_req_type_t *message = new rbdsc_req_type_t();
     message->type = RBDSC_READ;
     memcpy(message->pool_name, pool_name.c_str(), pool_name.size());
-    memcpy(message->vol_name, vol_name.c_str(), vol_name.size());
     memcpy(message->oid, object_id.c_str(), object_id.size());
     message->vol_size = 0;
     message->offset = 0;

diff --git a/src/tools/immutable_object_cache/CacheClient.h b/src/tools/immutable_object_cache/CacheClient.h
index 26162c45f823fa160e8d20ddbab7c2006facc303..8bf76dd4909e36bbf0c81475c90aa5405bf926ce 100644 (file)
--- a/src/tools/immutable_object_cache/CacheClient.h
+++ b/src/tools/immutable_object_cache/CacheClient.h
@@ -30,8 +30,8 @@ public:
   int stop();
   int connect();
 
-  int register_volume(std::string pool_name, std::string vol_name, uint64_t vol_size, Context* on_finish);
-  int lookup_object(std::string pool_name, std::string vol_name, std::string object_id, Context* on_finish);
+  int register_client(Context* on_finish);
+  int lookup_object(std::string pool_name, std::string object_id, Context* on_finish);
   void get_result(Context* on_finish);
 
 private:

diff --git a/src/tools/immutable_object_cache/CacheController.cc b/src/tools/immutable_object_cache/CacheController.cc
index e6a1e92220f2f78418fa3830d94ebebd555daae2..717852eef9166c872713c490dd75c5683f48b42b 100644 (file)
--- a/src/tools/immutable_object_cache/CacheController.cc
+++ b/src/tools/immutable_object_cache/CacheController.cc
@@ -68,7 +68,7 @@ void CacheController::handle_request(uint64_t session_id, std::string msg){
   switch (io_ctx->type) {
     case RBDSC_REGISTER: {
       // init cache layout for volume
-      m_object_cache_store->init_cache(io_ctx->pool_name, io_ctx->vol_name, io_ctx->vol_size);
+      m_object_cache_store->init_cache();
       io_ctx->type = RBDSC_REGISTER_REPLY;
       m_cache_server->send(session_id, std::string((char*)io_ctx, msg.size()));
 
@@ -76,7 +76,7 @@ void CacheController::handle_request(uint64_t session_id, std::string msg){
     }
     case RBDSC_READ: {
       // lookup object in local cache store
-      int ret = m_object_cache_store->lookup_object(io_ctx->pool_name, io_ctx->vol_name, io_ctx->oid);
+      int ret = m_object_cache_store->lookup_object(io_ctx->pool_name, io_ctx->oid);
       if (ret < 0) {
         io_ctx->type = RBDSC_READ_RADOS;
       } else {

diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.cc b/src/tools/immutable_object_cache/ObjectCacheStore.cc
index 63b742e14810120706c633bf7746b8be9323ad7f..e465f603a9a6f5d0a291b529a16af6dc10acd863 100644 (file)
--- a/src/tools/immutable_object_cache/ObjectCacheStore.cc
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.cc
@@ -67,29 +67,25 @@ int ObjectCacheStore::shutdown() {
   return 0;
 }
 
-int ObjectCacheStore::init_cache(std::string pool_name, std::string vol_name, uint64_t vol_size) {
-  ldout(m_cct, 20) << "pool name = " << pool_name
-                   << " volume name = " << vol_name
-                   << " volume size = " << vol_size << dendl;
-
-  std::string vol_cache_dir = m_cache_root_dir + pool_name + "_" + vol_name;
+int ObjectCacheStore::init_cache() {
+  ldout(m_cct, 20) << dendl;
+  std::string cache_dir = m_cache_root_dir;
 
   int dir = m_dir_num - 1;
   while (dir >= 0) {
-    efs::create_directories(vol_cache_dir + "/" + std::to_string(dir));
+    efs::create_directories(cache_dir + "/" + std::to_string(dir));
     dir --;
   }
   return 0;
 }
 
-int ObjectCacheStore::do_promote(std::string pool_name, std::string vol_name, std::string object_name) {
+int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name) {
   ldout(m_cct, 20) << "to promote object = "
                    << object_name << " from pool: "
                    << pool_name << dendl;
 
   int ret = 0;
   std::string cache_file_name =  pool_name + object_name;
-  std::string vol_cache_dir = pool_name + "_" + vol_name;
   {
     Mutex::Locker _locker(m_ioctxs_lock);
     if (m_ioctxs.find(pool_name) == m_ioctxs.end()) {
@@ -108,20 +104,17 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string vol_name, st
   librados::IoCtx* ioctx = m_ioctxs[pool_name];
 
   librados::bufferlist* read_buf = new librados::bufferlist();
-  uint32_t object_size = 4096*1024; //TODO(): read config from image metadata
 
-  auto ctx = new FunctionContext([this, read_buf, vol_cache_dir, cache_file_name,
-    object_size](int ret) {
-      handle_promote_callback(ret, read_buf, vol_cache_dir, cache_file_name, object_size);
+  auto ctx = new FunctionContext([this, read_buf, cache_file_name](int ret) {
+      handle_promote_callback(ret, read_buf, cache_file_name);
    });
 
-   return promote_object(ioctx, object_name, read_buf, object_size, ctx);
+   return promote_object(ioctx, object_name, read_buf, ctx);
 }
 
 int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf,
-  std::string cache_dir, std::string cache_file_name, uint32_t object_size) {
-  ldout(m_cct, 20) << "cache dir: " << cache_dir
-                   << " cache_file_name: " << cache_file_name << dendl;
+  std::string cache_file_name) {
+  ldout(m_cct, 20) << " cache_file_name: " << cache_file_name << dendl;
 
   // rados read error
   if(ret != -ENOENT && ret < 0) {
@@ -137,20 +130,18 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf,
     ret = 0;
   }
 
-  if (ret < object_size) {
-    // object is partial, fill with '0'
-    read_buf->append(std::string(object_size - ret, '0'));
-  }
+  std::string cache_dir = "";
+  uint32_t file_size = ret;
 
   if (m_dir_num > 0) {
     auto const pos = cache_file_name.find_last_of('.');
-    cache_dir = cache_dir + "/" + std::to_string(stoul(cache_file_name.substr(pos+1)) % m_dir_num);
+    cache_dir = "/" + std::to_string(stoul(cache_file_name.substr(pos+1)) % m_dir_num);
   }
   // write to cache
   ObjectCacheFile cache_file(m_cct, cache_dir + "/" + cache_file_name);
   cache_file.create();
 
-  ret = cache_file.write_object_to_file(*read_buf, object_size);
+  ret = cache_file.write_object_to_file(*read_buf, file_size);
   if (ret < 0) {
     lderr(m_cct) << "fail to write cache file" << dendl;
 
@@ -171,7 +162,7 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf,
 }
 
 int ObjectCacheStore::lookup_object(std::string pool_name,
-    std::string vol_name, std::string object_name) {
+                                    std::string object_name) {
   ldout(m_cct, 20) << "object name = " << object_name
                    << " in pool: " << pool_name << dendl;
 
@@ -180,7 +171,7 @@ int ObjectCacheStore::lookup_object(std::string pool_name,
 
   switch(ret) {
     case OBJ_CACHE_NONE: {
-      pret = do_promote(pool_name, vol_name, object_name);
+      pret = do_promote(pool_name, object_name);
       if (pret < 0) {
         lderr(m_cct) << "fail to start promote" << dendl;
       }
@@ -196,18 +187,19 @@ int ObjectCacheStore::lookup_object(std::string pool_name,
   }
 }
 
-int ObjectCacheStore::promote_object(librados::IoCtx* ioctx, std::string object_name,
-                                     librados::bufferlist* read_buf, uint64_t read_len,
+int ObjectCacheStore::promote_object(librados::IoCtx* ioctx,
+                                     std::string object_name,
+                                     librados::bufferlist* read_buf,
                                      Context* on_finish) {
-  ldout(m_cct, 20) << "object name = " << object_name
-                   << " read len = " << read_len << dendl;
+  ldout(m_cct, 20) << "object name = " << object_name << dendl;
 
   auto ctx = new FunctionContext([on_finish](int ret) {
     on_finish->complete(ret);
   });
 
   librados::AioCompletion* read_completion = create_rados_callback(ctx);
-  int ret = ioctx->aio_read(object_name, read_completion, read_buf, read_len, 0);
+  // issue a zero-sized read req to get full obj
+  int ret = ioctx->aio_read(object_name, read_completion, read_buf, 0, 0);
   if(ret < 0) {
     lderr(m_cct) << "failed to read from rados" << dendl;
   }
@@ -230,16 +222,8 @@ int ObjectCacheStore::do_evict(std::string cache_file) {
   ldout(m_cct, 20) << "file = " << cache_file << dendl;
 
   //TODO(): need a better way to get file path
-  std::string pool_name = "rbd";
-
-  size_t pos1 = cache_file.rfind("rbd_data");
-  pool_name = cache_file.substr(0, pos1);
-
-  pos1 = cache_file.find_first_of('.');
-  size_t pos2 = cache_file.find_last_of('.');
-  std::string vol_name = cache_file.substr(pos1+1, pos2-pos1-1);
 
-  std::string cache_dir = m_cache_root_dir + pool_name + "_" + vol_name;
+  std::string cache_dir = m_cache_root_dir;
 
    if (m_dir_num > 0) {
     auto const pos = cache_file.find_last_of('.');

diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.h b/src/tools/immutable_object_cache/ObjectCacheStore.h
index 4b71e24a9677c4dded8b8b7aba5d30e60fb955c6..e5f2475ca86c79ac10f33afb36cfb6ea7c1f8b5e 100644 (file)
--- a/src/tools/immutable_object_cache/ObjectCacheStore.h
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.h
@@ -32,21 +32,19 @@ class ObjectCacheStore
 
     int shutdown();
 
-    int lookup_object(std::string pool_name, std::string vol_name, std::string object_name);
-
-    int init_cache(std::string pool_name, std::string vol_name, uint64_t vol_size);
+    int init_cache();
+    int lookup_object(std::string pool_name, std::string object_name);
 
   private:
-    void evict_thread_body();
     int evict_objects();
 
-    int do_promote(std::string pool_name, std::string vol_name, std::string object_name);
+    int do_promote(std::string pool_name, std::string object_name);
 
     int promote_object(librados::IoCtx*, std::string object_name,
                        librados::bufferlist* read_buf,
-                       uint64_t length, Context* on_finish);
+                       Context* on_finish);
 
-   int handle_promote_callback(int, bufferlist*, std::string, std::string, uint32_t);
+   int handle_promote_callback(int, bufferlist*, std::string);
    int do_evict(std::string cache_file);
 
     CephContext *m_cct;