From 8179677fe3a27b67b6a13d7647c65979db1eaa04 Mon Sep 17 00:00:00 2001
From: Yuan Zhou <yuan.zhou@intel.com>
Date: Wed, 13 Mar 2019 11:31:42 +0800
Subject: [PATCH] tools: new cache sub-dir rule for immutable obj cache daemon

do not use static sub-dirs for cache, instead we create the sub-dir
dynamically with the first two charactor of crc32(file_name)

Signed-off-by: Yuan Zhou <yuan.zhou@intel.com>
---
 src/common/options.cc                         |  4 --
 .../test_object_store.cc                      |  5 +-
 .../ObjectCacheStore.cc                       | 64 +++++++++++--------
 .../immutable_object_cache/ObjectCacheStore.h |  3 +-
 4 files changed, 40 insertions(+), 36 deletions(-)

diff --git a/src/common/options.cc b/src/common/options.cc
index e6a4d507cce..3a3c3be5a0b 100644
--- a/src/common/options.cc
+++ b/src/common/options.cc
@@ -7413,10 +7413,6 @@ static std::vector<Option> get_immutable_object_cache_options() {
     Option("immutable_object_cache_watermark", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(0.1)
     .set_description("immutable object cache water mark"),
-
-    Option("immutable_object_cache_dir_num", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
-    .set_default(10)
-    .set_description("immutable object cache dir number"),
   });
 }
 
diff --git a/src/test/immutable_object_cache/test_object_store.cc b/src/test/immutable_object_cache/test_object_store.cc
index 94f8f8b0002..5e044322ee0 100644
--- a/src/test/immutable_object_cache/test_object_store.cc
+++ b/src/test/immutable_object_cache/test_object_store.cc
@@ -87,10 +87,7 @@ TEST_F(TestObjectStore, test_1) {
   efs::remove_all(test_cache_path);
 
   init_object_cache_store(m_temp_pool_name, m_temp_volume_name, 1000, true);
-  int temp_index = 0;
-  while(temp_index < 10) {
-    ASSERT_TRUE(efs::exists(cache_path + "/" + std::to_string(temp_index++)));
-  }
+
 
   // TODO add lookup interface testing
 
diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.cc b/src/tools/immutable_object_cache/ObjectCacheStore.cc
index 2ca431dbb39..cdd0e91cc52 100644
--- a/src/tools/immutable_object_cache/ObjectCacheStore.cc
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.cc
@@ -28,9 +28,6 @@ ObjectCacheStore::ObjectCacheStore(CephContext *cct)
   m_cache_watermark =
     m_cct->_conf.get_val<double>("immutable_object_cache_watermark");
 
-  m_dir_num =
-    m_cct->_conf.get_val<uint64_t>("immutable_object_cache_dir_num");
-
   if (m_cache_root_dir.back() != '/') {
     m_cache_root_dir += "/";
   }
@@ -62,15 +59,10 @@ int ObjectCacheStore::init(bool reset) {
   if (reset) {
     std::error_code ec;
     if (efs::exists(m_cache_root_dir)) {
-       int dir = m_dir_num - 1;
-       while (dir >= 0) {
-         if (!efs::remove_all(
-           m_cache_root_dir + "/" + std::to_string(dir), ec)) {
-           lderr(m_cct) << "fail to remove old cache store: " << ec << dendl;
-           return ec.value();
-         }
-         dir--;
-       }
+      // remove all sub folders
+      for(auto& p: efs::directory_iterator(m_cache_root_dir)) {
+        efs::remove_all(p.path());
+      }
     } else {
       if (!efs::create_directories(m_cache_root_dir, ec)) {
         lderr(m_cct) << "fail to create cache store dir: " << ec << dendl;
@@ -92,11 +84,6 @@ int ObjectCacheStore::init_cache() {
   ldout(m_cct, 20) << dendl;
   std::string cache_dir = m_cache_root_dir;
 
-  int dir = m_dir_num - 1;
-  while (dir >= 0) {
-    efs::create_directories(cache_dir + "/" + std::to_string(dir));
-    dir--;
-  }
   return 0;
 }
 
@@ -156,7 +143,15 @@ int ObjectCacheStore::handle_promote_callback(int ret, bufferlist* read_buf,
     ret = 0;
   }
 
-  std::string cache_file_path = std::move(get_cache_file_path(cache_file_name));
+  std::string cache_file_path = std::move(
+    get_cache_file_path(cache_file_name, true));
+
+  if (cache_file_path == "") {
+    lderr(m_cct) << "fail to write cache file" << dendl;
+    m_policy->update_status(cache_file_name, OBJ_CACHE_NONE);
+    delete read_buf;
+    return -ENOSPC;
+  }
 
   ret = read_buf->write_file(cache_file_path.c_str());
   if (ret < 0) {
@@ -269,16 +264,33 @@ std::string ObjectCacheStore::get_cache_file_name(std::string pool_nspace,
          std::to_string(snap_id) + ":" + oid;
 }
 
-std::string ObjectCacheStore::get_cache_file_path(std::string cache_file_name) {
-  std::string cache_file_dir = "";
-  if (m_dir_num > 0) {
-    uint32_t crc = 0;
-    crc = ceph_crc32c(0, (unsigned char *)cache_file_name.c_str(),
-                     cache_file_name.length());
-    cache_file_dir = std::to_string(crc % m_dir_num);
+std::string ObjectCacheStore::get_cache_file_path(std::string cache_file_name,
+                                                  bool mkdir) {
+  ldout(m_cct, 20) << cache_file_name <<dendl;
+
+  uint32_t crc = 0;
+  crc = ceph_crc32c(0, (unsigned char *)cache_file_name.c_str(),
+                    cache_file_name.length());
+
+  std::string cache_file_dir = std::to_string(crc % 100) + "/";
+
+  if (mkdir) {
+    ldout(m_cct, 20) << "creating cache dir: " << cache_file_dir <<dendl;
+    std::error_code ec;
+    std::string new_dir = m_cache_root_dir + cache_file_dir;
+    if (efs::exists(new_dir, ec)) {
+      ldout(m_cct, 20) << "cache dir exists: " << cache_file_dir <<dendl;
+      return new_dir + cache_file_name;
+    }
+
+    if (!efs::create_directories(new_dir, ec)) {
+      ldout(m_cct, 5) << "fail to create cache dir: " << new_dir
+                      << "error: " << ec.message() << dendl;
+      return "";
+    }
   }
 
-  return m_cache_root_dir + cache_file_dir + "/" + cache_file_name;
+  return m_cache_root_dir + cache_file_dir + cache_file_name;
 }
 
 }  // namespace immutable_obj_cache
diff --git a/src/tools/immutable_object_cache/ObjectCacheStore.h b/src/tools/immutable_object_cache/ObjectCacheStore.h
index 7176fdfa413..aaa8e0bc31e 100644
--- a/src/tools/immutable_object_cache/ObjectCacheStore.h
+++ b/src/tools/immutable_object_cache/ObjectCacheStore.h
@@ -36,7 +36,7 @@ class ObjectCacheStore {
  private:
   std::string get_cache_file_name(std::string pool_nspace, uint64_t pool_id,
                                   uint64_t snap_id, std::string oid);
-  std::string get_cache_file_path(std::string cache_file_name);
+  std::string get_cache_file_path(std::string cache_file_name, bool mkdir = false);
   int evict_objects();
   int do_promote(std::string pool_nspace, uint64_t pool_id,
                  uint64_t snap_id, std::string object_name);
@@ -51,7 +51,6 @@ class ObjectCacheStore {
   std::map<uint64_t, librados::IoCtx> m_ioctx_map;
   Mutex m_ioctx_map_lock;
   Policy* m_policy;
-  uint64_t m_dir_num;
   uint64_t m_object_cache_max_size;
   float m_cache_watermark;
   std::string m_cache_root_dir;
-- 
2.39.5