From c82f469824bc82eee7c7aae92149e20c767c439a Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Fri, 21 Dec 2018 10:41:32 +0800 Subject: [PATCH] tools: remove RBD specific logic in immutable obj cache Make the daemon more generic(pool name + object id) so it can service both RBD and RGW. Also supports caching partial object(even empty), the caller needs to append_zero to read buffer if read on those objects. Signed-off-by: Yuan Zhou --- .../test_DomainSocket.cc | 12 ++-- .../immutable_object_cache/CacheClient.cc | 13 +--- .../immutable_object_cache/CacheClient.h | 4 +- .../immutable_object_cache/CacheController.cc | 4 +- .../ObjectCacheStore.cc | 62 +++++++------------ .../immutable_object_cache/ObjectCacheStore.h | 12 ++-- 6 files changed, 41 insertions(+), 66 deletions(-) diff --git a/src/test/immutable_object_cache/test_DomainSocket.cc b/src/test/immutable_object_cache/test_DomainSocket.cc index a7ae75c3a9d28..d167d96dcaeb2 100644 --- a/src/test/immutable_object_cache/test_DomainSocket.cc +++ b/src/test/immutable_object_cache/test_DomainSocket.cc @@ -94,7 +94,7 @@ public: auto ctx = new FunctionContext([this](bool reg) { ASSERT_TRUE(reg); }); - m_cache_client->register_volume("pool_name", "rbd_name", 4096, ctx); + m_cache_client->register_client(ctx); ASSERT_TRUE(m_cache_client->is_session_work()); } @@ -153,19 +153,19 @@ public: usleep(1); } - m_cache_client->lookup_object("test_pool", "test_rbd", "123456", ctx); + m_cache_client->lookup_object("test_pool", "123456", ctx); m_send_request_index++; } m_wait_event.wait(); } - bool startup_lookupobject_testing(std::string pool_name, std::string volume_name, std::string object_id) { + bool startup_lookupobject_testing(std::string pool_name, std::string object_id) { bool hit; auto ctx = new FunctionContext([this, &hit](bool req){ hit = req; m_wait_event.signal(); }); - m_cache_client->lookup_object(pool_name, volume_name, object_id, ctx); + m_cache_client->lookup_object(pool_name, object_id, ctx); m_wait_event.wait(); return hit; } @@ -201,9 +201,9 @@ TEST_F(TestCommunication, test_lookup_object) { } for (uint64_t i = 50; i < 100; i++) { if ((random_hit % i) != 0) { - ASSERT_FALSE(startup_lookupobject_testing("test_pool", "testing_volume", std::to_string(i))); + ASSERT_FALSE(startup_lookupobject_testing("test_pool", std::to_string(i))); } else { - ASSERT_TRUE(startup_lookupobject_testing("test_pool", "testing_volume", std::to_string(i))); + ASSERT_TRUE(startup_lookupobject_testing("test_pool", std::to_string(i))); } } } diff --git a/src/tools/immutable_object_cache/CacheClient.cc b/src/tools/immutable_object_cache/CacheClient.cc index 414c9cbc4cd22..2e7609007f30c 100644 --- a/src/tools/immutable_object_cache/CacheClient.cc +++ b/src/tools/immutable_object_cache/CacheClient.cc @@ -85,16 +85,10 @@ namespace immutable_obj_cache { return 0; } - int CacheClient::register_volume(std::string pool_name, std::string vol_name, - uint64_t vol_size, Context* on_finish) { + int CacheClient::register_client(Context* on_finish) { // cache controller will init layout rbdsc_req_type_t *message = new rbdsc_req_type_t(); message->type = RBDSC_REGISTER; - memcpy(message->pool_name, pool_name.c_str(), pool_name.size()); - memcpy(message->vol_name, vol_name.c_str(), vol_name.size()); - message->vol_size = vol_size; - message->offset = 0; - message->length = 0; uint64_t ret; boost::system::error_code ec; @@ -151,12 +145,11 @@ namespace immutable_obj_cache { } // if occur any error, we just return false. Then read from rados. - int CacheClient::lookup_object(std::string pool_name, std::string vol_name, - std::string object_id, Context* on_finish) { + int CacheClient::lookup_object(std::string pool_name, std::string object_id, + Context* on_finish) { rbdsc_req_type_t *message = new rbdsc_req_type_t(); message->type = RBDSC_READ; memcpy(message->pool_name, pool_name.c_str(), pool_name.size()); - memcpy(message->vol_name, vol_name.c_str(), vol_name.size()); memcpy(message->oid, object_id.c_str(), object_id.size()); message->vol_size = 0; message->offset = 0; diff --git a/src/tools/immutable_object_cache/CacheClient.h b/src/tools/immutable_object_cache/CacheClient.h index 26162c45f823f..8bf76dd4909e3 100644 --- a/src/tools/immutable_object_cache/CacheClient.h +++ b/src/tools/immutable_object_cache/CacheClient.h @@ -30,8 +30,8 @@ public: int stop(); int connect(); - int register_volume(std::string pool_name, std::string vol_name, uint64_t vol_size, Context* on_finish); - int lookup_object(std::string pool_name, std::string vol_name, std::string object_id, Context* on_finish); + int register_client(Context* on_finish); + int lookup_object(std::string pool_name, std::string object_id, Context* on_finish); void get_result(Context* on_finish); private: diff --git a/src/tools/immutable_object_cache/CacheController.cc b/src/tools/immutable_object_cache/CacheController.cc index e6a1e92220f2f..717852eef9166 100644 --- a/src/tools/immutable_object_cache/CacheController.cc +++ b/src/tools/immutable_object_cache/CacheController.cc @@ -68,7 +68,7 @@ void CacheController::handle_request(uint64_t session_id, std::string msg){ switch (io_ctx->type) { case RBDSC_REGISTER: { // init cache layout for volume - m_object_cache_store->init_cache(io_ctx->pool_name, io_ctx->vol_name, io_ctx->vol_size); + m_object_cache_store->init_cache(); io_ctx->type = RBDSC_REGISTER_REPLY; m_cache_server->send(session_id, std::string((char*)io_ctx, msg.size())); @@ -76,7 +76,7 @@ void CacheController::handle_request(uint64_t session_id, std::string msg){ } case RBDSC_READ: { // lookup object in local cache store - int ret = m_object_cache_store->lookup_object(io_ctx->pool_name, io_ctx->vol_name, io_ctx->oid); + int ret = m_object_cache_store->lookup_object(io_ctx->pool_name, io_ctx->oid); if (ret < 0) { io_ctx->type = RBDSC_READ_RADOS; } else { diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.cc b/src/tools/immutable_object_cache/ObjectCacheStore.cc index 63b742e148101..e465f603a9a6f 100644 --- a/src/tools/immutable_object_cache/ObjectCacheStore.cc +++ b/src/tools/immutable_object_cache/ObjectCacheStore.cc @@ -67,29 +67,25 @@ int ObjectCacheStore::shutdown() { return 0; } -int ObjectCacheStore::init_cache(std::string pool_name, std::string vol_name, uint64_t vol_size) { - ldout(m_cct, 20) << "pool name = " << pool_name - << " volume name = " << vol_name - << " volume size = " << vol_size << dendl; - - std::string vol_cache_dir = m_cache_root_dir + pool_name + "_" + vol_name; +int ObjectCacheStore::init_cache() { + ldout(m_cct, 20) << dendl; + std::string cache_dir = m_cache_root_dir; int dir = m_dir_num - 1; while (dir >= 0) { - efs::create_directories(vol_cache_dir + "/" + std::to_string(dir)); + efs::create_directories(cache_dir + "/" + std::to_string(dir)); dir --; } return 0; } -int ObjectCacheStore::do_promote(std::string pool_name, std::string vol_name, std::string object_name) { +int ObjectCacheStore::do_promote(std::string pool_name, std::string object_name) { ldout(m_cct, 20) << "to promote object = " << object_name << " from pool: " << pool_name << dendl; int ret = 0; std::string cache_file_name = pool_name + object_name; - std::string vol_cache_dir = pool_name + "_" + vol_name; { Mutex::Locker _locker(m_ioctxs_lock); if (m_ioctxs.find(pool_name) == m_ioctxs.end()) { @@ -108,20 +104,17 @@ int ObjectCacheStore::do_promote(std::string pool_name, std::string vol_name, st librados::IoCtx* ioctx = m_ioctxs[pool_name]; librados::bufferlist* read_buf = new librados::bufferlist(); - uint32_t object_size = 4096*1024; //TODO(): read config from image metadata - auto ctx = new FunctionContext([this, read_buf, vol_cache_dir, cache_file_name, - object_size](int ret) { - handle_promote_callback(ret, read_buf, vol_cache_dir, cache_file_name, object_size); + auto ctx = new FunctionContext([this, read_buf, cache_file_name](int ret) { + handle_promote_callback(ret, read_buf, cache_file_name); }); - return promote_object(ioctx, object_name, read_buf, object_size, ctx); + return promote_object(ioctx, object_name, read_buf, ctx); } int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf, - std::string cache_dir, std::string cache_file_name, uint32_t object_size) { - ldout(m_cct, 20) << "cache dir: " << cache_dir - << " cache_file_name: " << cache_file_name << dendl; + std::string cache_file_name) { + ldout(m_cct, 20) << " cache_file_name: " << cache_file_name << dendl; // rados read error if(ret != -ENOENT && ret < 0) { @@ -137,20 +130,18 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf, ret = 0; } - if (ret < object_size) { - // object is partial, fill with '0' - read_buf->append(std::string(object_size - ret, '0')); - } + std::string cache_dir = ""; + uint32_t file_size = ret; if (m_dir_num > 0) { auto const pos = cache_file_name.find_last_of('.'); - cache_dir = cache_dir + "/" + std::to_string(stoul(cache_file_name.substr(pos+1)) % m_dir_num); + cache_dir = "/" + std::to_string(stoul(cache_file_name.substr(pos+1)) % m_dir_num); } // write to cache ObjectCacheFile cache_file(m_cct, cache_dir + "/" + cache_file_name); cache_file.create(); - ret = cache_file.write_object_to_file(*read_buf, object_size); + ret = cache_file.write_object_to_file(*read_buf, file_size); if (ret < 0) { lderr(m_cct) << "fail to write cache file" << dendl; @@ -171,7 +162,7 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf, } int ObjectCacheStore::lookup_object(std::string pool_name, - std::string vol_name, std::string object_name) { + std::string object_name) { ldout(m_cct, 20) << "object name = " << object_name << " in pool: " << pool_name << dendl; @@ -180,7 +171,7 @@ int ObjectCacheStore::lookup_object(std::string pool_name, switch(ret) { case OBJ_CACHE_NONE: { - pret = do_promote(pool_name, vol_name, object_name); + pret = do_promote(pool_name, object_name); if (pret < 0) { lderr(m_cct) << "fail to start promote" << dendl; } @@ -196,18 +187,19 @@ int ObjectCacheStore::lookup_object(std::string pool_name, } } -int ObjectCacheStore::promote_object(librados::IoCtx* ioctx, std::string object_name, - librados::bufferlist* read_buf, uint64_t read_len, +int ObjectCacheStore::promote_object(librados::IoCtx* ioctx, + std::string object_name, + librados::bufferlist* read_buf, Context* on_finish) { - ldout(m_cct, 20) << "object name = " << object_name - << " read len = " << read_len << dendl; + ldout(m_cct, 20) << "object name = " << object_name << dendl; auto ctx = new FunctionContext([on_finish](int ret) { on_finish->complete(ret); }); librados::AioCompletion* read_completion = create_rados_callback(ctx); - int ret = ioctx->aio_read(object_name, read_completion, read_buf, read_len, 0); + // issue a zero-sized read req to get full obj + int ret = ioctx->aio_read(object_name, read_completion, read_buf, 0, 0); if(ret < 0) { lderr(m_cct) << "failed to read from rados" << dendl; } @@ -230,16 +222,8 @@ int ObjectCacheStore::do_evict(std::string cache_file) { ldout(m_cct, 20) << "file = " << cache_file << dendl; //TODO(): need a better way to get file path - std::string pool_name = "rbd"; - - size_t pos1 = cache_file.rfind("rbd_data"); - pool_name = cache_file.substr(0, pos1); - - pos1 = cache_file.find_first_of('.'); - size_t pos2 = cache_file.find_last_of('.'); - std::string vol_name = cache_file.substr(pos1+1, pos2-pos1-1); - std::string cache_dir = m_cache_root_dir + pool_name + "_" + vol_name; + std::string cache_dir = m_cache_root_dir; if (m_dir_num > 0) { auto const pos = cache_file.find_last_of('.'); diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.h b/src/tools/immutable_object_cache/ObjectCacheStore.h index 4b71e24a9677c..e5f2475ca86c7 100644 --- a/src/tools/immutable_object_cache/ObjectCacheStore.h +++ b/src/tools/immutable_object_cache/ObjectCacheStore.h @@ -32,21 +32,19 @@ class ObjectCacheStore int shutdown(); - int lookup_object(std::string pool_name, std::string vol_name, std::string object_name); - - int init_cache(std::string pool_name, std::string vol_name, uint64_t vol_size); + int init_cache(); + int lookup_object(std::string pool_name, std::string object_name); private: - void evict_thread_body(); int evict_objects(); - int do_promote(std::string pool_name, std::string vol_name, std::string object_name); + int do_promote(std::string pool_name, std::string object_name); int promote_object(librados::IoCtx*, std::string object_name, librados::bufferlist* read_buf, - uint64_t length, Context* on_finish); + Context* on_finish); - int handle_promote_callback(int, bufferlist*, std::string, std::string, uint32_t); + int handle_promote_callback(int, bufferlist*, std::string); int do_evict(std::string cache_file); CephContext *m_cct; -- 2.39.5