From: Yehuda Sadeh <yehuda@redhat.com>
Date: Fri, 10 Jun 2016 21:35:01 +0000 (-0700)
Subject: rgw: add obj tombstone cache
X-Git-Tag: v11.0.0~89^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=eb10214920c23b24edd94ca53d0f36c85404644d;p=ceph.git

rgw: add obj tombstone cache

The obj tombstone cache is used in multi-zone environmet to keep
track of removed objects' mtime. This is then used to fetch remote
object only if its newer than the object that was removed, otherwise
we're just fetching ghost of the past.

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>
---

diff --git a/src/common/config_opts.h b/src/common/config_opts.h
index ae0e20772a5e..e455d4194889 100644
--- a/src/common/config_opts.h
+++ b/src/common/config_opts.h
@@ -1381,6 +1381,8 @@ OPTION(rgw_opstate_ratelimit_sec, OPT_INT, 30) // min time between opstate updat
 OPTION(rgw_curl_wait_timeout_ms, OPT_INT, 1000) // timeout for certain curl calls
 OPTION(rgw_copy_obj_progress, OPT_BOOL, true) // should dump progress during long copy operations?
 OPTION(rgw_copy_obj_progress_every_bytes, OPT_INT, 1024 * 1024) // min bytes between copy progress output
+OPTION(rgw_obj_tombstone_cache_size, OPT_INT, 1000) // how many objects in tombstone cache, which is used in multi-zone sync to keep
+                                                    // track of removed objects' mtime
 
 OPTION(rgw_data_log_window, OPT_INT, 30) // data log entries window (in seconds)
 OPTION(rgw_data_log_changes_size, OPT_INT, 1000) // number of in-memory entries to hold for data changes log
diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 372a2479cd3f..db6a3daa5a48 100644
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -3190,6 +3190,7 @@ void RGWRados::finalize()
     cr_registry->put();
   }
   delete binfo_cache;
+  delete obj_tombstone_cache;
 }
 
 /** 
@@ -3776,6 +3777,12 @@ int RGWRados::init_complete()
   binfo_cache = new RGWChainedCacheImpl<bucket_info_entry>;
   binfo_cache->init(this);
 
+  bool need_tombstone_cache = !zone_conn_map.empty();
+
+  if (need_tombstone_cache) {
+    obj_tombstone_cache = new tombstone_cache_t(cct->_conf->rgw_obj_tombstone_cache_size);
+  }
+
   return ret;
 }
 
@@ -6493,7 +6500,7 @@ int RGWRados::fetch_remote_obj(RGWObjectCtx& obj_ctx,
     if (ret < 0)
       return ret;
 
-    if (dest_state->exists) {
+    if (!real_clock::is_zero(dest_state->mtime)) {
       dest_mtime_weight.init(dest_state);
       pmod = &dest_mtime_weight.mtime;
     }
@@ -7588,6 +7595,11 @@ int RGWRados::Object::Delete::delete_obj()
 
   int64_t poolid = ref.ioctx.get_id();
   if (r >= 0) {
+    tombstone_cache_t *obj_tombstone_cache = store->get_tombstone_cache();
+    pair<ceph::real_time, uint32_t> tombstone_entry = make_pair<>(state->mtime, state->zone_short_id);
+    if (obj_tombstone_cache) {
+      obj_tombstone_cache->add(obj, tombstone_entry);
+    }
     r = index_op.complete_del(poolid, ref.ioctx.get_last_version(), state->mtime, params.remove_objs);
   } else {
     int ret = index_op.cancel();
@@ -7808,7 +7820,14 @@ int RGWRados::get_obj_state_impl(RGWObjectCtx *rctx, rgw_obj& obj, RGWObjState *
   if (r == -ENOENT) {
     s->exists = false;
     s->has_attrs = true;
-    s->mtime = real_time();
+    pair<ceph::real_time, uint32_t> tombstone_entry;
+    if (obj_tombstone_cache && obj_tombstone_cache->find(obj, tombstone_entry)) {
+      s->mtime = tombstone_entry.first;
+      s->zone_short_id = tombstone_entry.second;
+      ldout(cct, 20) << __func__ << "(): found obj in tombstone cache: obj=" << obj << " mtime=" << s->mtime << dendl;
+    } else {
+      s->mtime = real_time();
+    }
     return 0;
   }
   if (r < 0)
diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index e2c354033a54..ce498b563f25 100644
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -9,6 +9,7 @@
 #include "common/RefCountedObj.h"
 #include "common/RWLock.h"
 #include "common/ceph_time.h"
+#include "common/lru_map.h"
 #include "rgw_common.h"
 #include "cls/rgw/cls_rgw_types.h"
 #include "cls/version/cls_version_types.h"
@@ -1808,6 +1809,9 @@ protected:
   using RGWChainedCacheImpl_bucket_info_entry = RGWChainedCacheImpl<bucket_info_entry>;
   RGWChainedCacheImpl_bucket_info_entry *binfo_cache;
 
+  using tombstone_cache_t = lru_map<rgw_obj, pair<ceph::real_time, uint32_t> >;
+  tombstone_cache_t *obj_tombstone_cache;
+
   librados::IoCtx gc_pool_ctx;        // .rgw.gc
   librados::IoCtx objexp_pool_ctx;
 
@@ -1842,7 +1846,7 @@ public:
                max_bucket_id(0), cct(NULL),
                next_rados_handle(0),
                handle_lock("rados_handle_lock"),
-               binfo_cache(NULL),
+               binfo_cache(NULL), obj_tombstone_cache(nullptr),
                pools_initialized(false),
                quota_handler(NULL),
                finisher(NULL),
@@ -1957,6 +1961,10 @@ public:
 
   virtual ~RGWRados() = default;
 
+  tombstone_cache_t *get_tombstone_cache() {
+    return obj_tombstone_cache;
+  }
+
   int get_required_alignment(rgw_bucket& bucket, uint64_t *alignment);
   int get_max_chunk_size(rgw_bucket& bucket, uint64_t *max_chunk_size);