rgw: d3n: fix d3n cached objects on disk lru eviction entering infinite loop

fixes: https://tracker.ceph.com/issues/52457

Signed-off-by: Mark Kogan <mkogan@redhat.com>

diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in
index c7498481d198167dc0f887a54302d7ba0bfd5608..f8d935854b6a20a68436bffffee6719fb9f53b3f 100644 (file)
--- a/src/common/options/rgw.yaml.in
+++ b/src/common/options/rgw.yaml.in
@@ -3134,7 +3134,7 @@ options:
   type: size
   level: advanced
   desc: datacache maximum size on disk in bytes
-  default: 1048576
+  default: 1_G
   services:
   - rgw
   with_legacy: true

diff --git a/src/rgw/rgw_d3n_cacherequest.h b/src/rgw/rgw_d3n_cacherequest.h
index 41e026bcd34538f8b2c94c6b375d9ec7613199ae..d4acc4c0b1f8fc28173448473f588b37cef89370 100644 (file)
--- a/src/rgw/rgw_d3n_cacherequest.h
+++ b/src/rgw/rgw_d3n_cacherequest.h
@@ -26,7 +26,7 @@ struct D3nGetObjData {
 
 struct D3nL1CacheRequest {
   ~D3nL1CacheRequest() {
-    lsubdout(g_ceph_context, rgw_datacache, 30) << "D3nDataCache: " << __func__ << "(): Read From Cache, comlete" << dendl;
+    lsubdout(g_ceph_context, rgw_datacache, 30) << "D3nDataCache: " << __func__ << "(): Read From Cache, complete" << dendl;
   }
 
   // unique_ptr with custom deleter for struct aiocb

diff --git a/src/rgw/rgw_d3n_datacache.cc b/src/rgw/rgw_d3n_datacache.cc
index e2ad497dae977029908bedcdfafd57754c650d65..3ca7e11ea2ad7a93bb6311023efdc8971116a312 100644 (file)
--- a/src/rgw/rgw_d3n_datacache.cc
+++ b/src/rgw/rgw_d3n_datacache.cc
@@ -161,9 +161,8 @@ void D3nDataCache::d3n_libaio_write_completion_cb(D3nCacheAioWriteRequest* c)
   { // update cache_map entries for new chunk in cache
     const std::lock_guard l(d3n_cache_lock);
     auto it = d3n_outstanding_write_list.find(c->oid);
-    if (it != d3n_outstanding_write_list.end()) {
+    if (it != d3n_outstanding_write_list.end())
       d3n_outstanding_write_list.erase(it);
-    }
     chunk_info = new D3nChunkDataInfo;
     chunk_info->oid = c->oid;
     chunk_info->set_ctx(cct);
@@ -211,10 +210,15 @@ done:
 
 void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid)
 {
-  int r = 0;
+  size_t sr = 0;
   uint64_t freed_size = 0, _free_data_cache_size = 0, _outstanding_write_size = 0;
 
-  ldout(cct, 10) << "D3nDataCache::" << __func__ << "(): oid=" << oid << dendl;
+  ldout(cct, 10) << "D3nDataCache::" << __func__ << "(): oid=" << oid << ", len=" << len << dendl;
+  if (len > cct->_conf->rgw_d3n_l1_datacache_size) {
+    ldout(cct, 2) << "D3nDataCache: Warning: object oid=" << oid << ", length=" << len <<
+                     " is larger than the datacache size " << cct->_conf->rgw_d3n_l1_datacache_size << ", not writing to cache" << dendl;
+    return;
+  }
   {
     const std::lock_guard l(d3n_cache_lock);
     std::unordered_map<string, D3nChunkDataInfo*>::iterator iter = d3n_cache_map.find(oid);
@@ -235,27 +239,33 @@ void D3nDataCache::put(bufferlist& bl, unsigned int len, std::string& oid)
     _outstanding_write_size = outstanding_write_size;
   }
   ldout(cct, 20) << "D3nDataCache: Before eviction _free_data_cache_size:" << _free_data_cache_size << ", _outstanding_write_size:" << _outstanding_write_size << ", freed_size:" << freed_size << dendl;
-  while (len >= (_free_data_cache_size - _outstanding_write_size + freed_size)) {
-    ldout(cct, 20) << "D3nDataCache: enter eviction, r=" << r << dendl;
+  while (len > (_free_data_cache_size - _outstanding_write_size + freed_size)) {
+    ldout(cct, 20) << "D3nDataCache: enter eviction" << dendl;
     if (eviction_policy == _eviction_policy::LRU) {
-      r = lru_eviction();
+      sr = lru_eviction();
     } else if (eviction_policy == _eviction_policy::RANDOM) {
-      r = random_eviction();
+      sr = random_eviction();
     } else {
       ldout(cct, 0) << "D3nDataCache: Warning: unknown cache eviction policy, defaulting to lru eviction" << dendl;
-      r = lru_eviction();
+      sr = lru_eviction();
     }
-    if (r < 0)
+    if (sr == 0) {
+      ldout(cct, 2) << "D3nDataCache: Warning: eviction was not able to free disk space, not writing to cache" << dendl;
+      auto it = d3n_outstanding_write_list.find(oid);
+      if (it != d3n_outstanding_write_list.end())
+        d3n_outstanding_write_list.erase(it);
       return;
-    freed_size += r;
+    }
+    ldout(cct, 20) << "D3nDataCache: completed eviction of " << sr << " bytes" << dendl;
+    freed_size += sr;
   }
+  int r = 0;
   r = d3n_libaio_create_write_request(bl, len, oid);
   if (r < 0) {
     const std::lock_guard l(d3n_cache_lock);
     auto it = d3n_outstanding_write_list.find(oid);
-    if (it != d3n_outstanding_write_list.end()) {
+    if (it != d3n_outstanding_write_list.end())
       d3n_outstanding_write_list.erase(it);
-    }
     ldout(cct, 1) << "D3nDataCache: create_aio_write_request fail, r=" << r << dendl;
     return;
   }
@@ -324,7 +334,7 @@ size_t D3nDataCache::random_eviction()
   }
 
   location = cache_location + del_oid;
-  remove(location.c_str());
+  ::remove(location.c_str());
   return freed_size;
 }
 
@@ -356,13 +366,12 @@ size_t D3nDataCache::lru_eviction()
     del_oid = del_entry->oid;
     ldout(cct, 20) << "D3nDataCache: lru_eviction: oid to remove: " << del_oid << dendl;
     std::unordered_map<string, D3nChunkDataInfo*>::iterator iter = d3n_cache_map.find(del_oid);
-    if (iter != d3n_cache_map.end()) {
+    if (iter != d3n_cache_map.end())
       d3n_cache_map.erase(iter); // oid
-    }
   }
   freed_size = del_entry->size;
   delete del_entry;
   location = cache_location + del_oid;
-  remove(location.c_str());
+  ::remove(location.c_str());
   return freed_size;
 }