rgw: fetch bucket instance info if doesn't exist

when doing data sync. This handles a race in which we try to sync data
of bucket instance before fetched its metadata

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/rgw_data_sync.cc
index 7e52b9ea6f51dd563d28ecb170898b9fe76690c9..774213f6b42a5a2607f839d9ce3a85d3bbaed293 100644 (file)
--- a/src/rgw/rgw_data_sync.cc
+++ b/src/rgw/rgw_data_sync.cc
@@ -691,6 +691,7 @@ class RGWRunBucketSyncCoroutine : public RGWCoroutine {
   RGWBucketInfo bucket_info;
   int shard_id;
   rgw_bucket_shard_sync_info sync_status;
+  RGWMetaSyncEnv sync_env;
 
 public:
   RGWRunBucketSyncCoroutine(RGWHTTPManager *_mgr, RGWAsyncRadosProcessor *_async_rados,
@@ -2195,11 +2196,41 @@ int RGWRunBucketSyncCoroutine::operate()
       int r = call(new RGWGetBucketInstanceInfoCR(async_rados, store, bucket_name, bucket_id, &bucket_info));
       if (r < 0) {
         ldout(store->ctx(), 0) << "ERROR: failed to fetch sync status" << dendl;
-        return r;
+        return set_cr_error(r);
       }
     }
 
-    if (retcode < 0 && retcode != -ENOENT) {
+    if (retcode == -ENOENT) {
+      /* bucket instance info has not been synced in yet, fetch it now */
+      yield {
+        ldout(store->ctx(), 10) << "no local info for bucket " << bucket_name << ":" << bucket_id << ": fetching metadata" << dendl;
+        string raw_key = string("bucket.instance:") + bucket_name + ":" + bucket_id;
+
+        sync_env.init(cct, store, store->rest_master_conn, async_rados, http_manager);
+
+        int r = call(new RGWMetaSyncSingleEntryCR(&sync_env, raw_key,
+                                                  string() /* no marker */,
+                                                  NULL /* no marker tracker */));
+        if (r < 0) {
+          ldout(store->ctx(), 0) << "ERROR: failed to fetch bucket instance info for " << raw_key << dendl;
+          return set_cr_error(r);
+        }
+      }
+
+      if (retcode < 0) {
+        ldout(store->ctx(), 0) << "ERROR: failed to fetch bucket instance info for " << bucket_name << ":" << bucket_id << dendl;
+        return set_cr_error(retcode);
+      }
+      yield {
+        int r = call(new RGWGetBucketInstanceInfoCR(async_rados, store, bucket_name, bucket_id, &bucket_info));
+        if (r < 0) {
+          ldout(store->ctx(), 0) << "ERROR: failed to fetch sync status" << dendl;
+          return set_cr_error(r);
+        }
+      }
+    }
+
+    if (retcode < 0) {
       ldout(store->ctx(), 0) << "ERROR: failed to retrieve bucket info for bucket=" << bucket_name << " bucket_id=" << bucket_id << dendl;
       return set_cr_error(retcode);
     }

diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc
index 38fbcf9abf0ecd801075b9f0682d189166144527..3ececd2af3376af0a61d46b5d7fd60e73ab6606d 100644 (file)
--- a/src/rgw/rgw_sync.cc
+++ b/src/rgw/rgw_sync.cc
@@ -903,94 +903,66 @@ public:
   }
 };
 
-class RGWMetaSyncSingleEntryCR : public RGWCoroutine {
-  RGWMetaSyncEnv *sync_env;
-
-  string raw_key;
-  string entry_marker;
-
-  ssize_t pos;
-  string section;
-  string key;
-
-  int sync_status;
-
-  bufferlist md_bl;
-
-  RGWMetaSyncShardMarkerTrack *marker_tracker;
-
-  int tries;
-
-public:
-  RGWMetaSyncSingleEntryCR(RGWMetaSyncEnv *_sync_env,
-                          const string& _raw_key, const string& _entry_marker, RGWMetaSyncShardMarkerTrack *_marker_tracker) : RGWCoroutine(_sync_env->cct),
-                                                      sync_env(_sync_env),
-                                                     raw_key(_raw_key), entry_marker(_entry_marker),
-                                                      pos(0), sync_status(0),
-                                                      marker_tracker(_marker_tracker), tries(0) {
-  }
-
-  int operate() {
-    reenter(this) {
+int RGWMetaSyncSingleEntryCR::operate() {
+  reenter(this) {
 #define NUM_TRANSIENT_ERROR_RETRIES 10
-      for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) {
-        yield {
-          pos = raw_key.find(':');
-          section = raw_key.substr(0, pos);
-          key = raw_key.substr(pos + 1);
-          ldout(sync_env->cct, 20) << "fetching remote metadata: " << section << ":" << key << (tries == 0 ? "" : " (retry)") << dendl;
-          int ret = call(new RGWReadRemoteMetadataCR(sync_env, section, key, &md_bl));
-          if (ret < 0) {
-            ldout(sync_env->cct, 0) << "ERROR: failed to call RGWReadRemoteMetadataCR()" << dendl;
-            return set_cr_error(sync_status);
-          }
+    for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) {
+      yield {
+        pos = raw_key.find(':');
+        section = raw_key.substr(0, pos);
+        key = raw_key.substr(pos + 1);
+        ldout(sync_env->cct, 20) << "fetching remote metadata: " << section << ":" << key << (tries == 0 ? "" : " (retry)") << dendl;
+        int ret = call(new RGWReadRemoteMetadataCR(sync_env, section, key, &md_bl));
+        if (ret < 0) {
+          ldout(sync_env->cct, 0) << "ERROR: failed to call RGWReadRemoteMetadataCR()" << dendl;
+          return set_cr_error(sync_status);
         }
+      }
 
-        sync_status = retcode;
+      sync_status = retcode;
 
-        if (sync_status == -ENOENT) {
+      if (sync_status == -ENOENT) {
 #warning remove entry from local
-          return set_cr_done();
-        }
+        return set_cr_done();
+      }
 
-        if (sync_status == -EAGAIN && (tries < NUM_TRANSIENT_ERROR_RETRIES - 1)) {
-          ldout(sync_env->cct, 20) << *this << ": failed to fetch remote metadata: " << section << ":" << key << ", will retry" << dendl;
-          continue;
-        }
+      if (sync_status == -EAGAIN && (tries < NUM_TRANSIENT_ERROR_RETRIES - 1)) {
+        ldout(sync_env->cct, 20) << *this << ": failed to fetch remote metadata: " << section << ":" << key << ", will retry" << dendl;
+        continue;
+      }
 
-        if (sync_status < 0) {
+      if (sync_status < 0) {
 #warning need to store entry for non-transient errors
-          ldout(sync_env->cct, 10) << *this << ": failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << dendl;
-          log_error() << "failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << std::endl;
-          return set_cr_error(sync_status);
-        }
-
-        break;
+        ldout(sync_env->cct, 10) << *this << ": failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << dendl;
+        log_error() << "failed to send read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << std::endl;
+        return set_cr_error(sync_status);
       }
 
-      yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl));
+      break;
+    }
 
-      sync_status = retcode;
+    yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl));
 
-      if (sync_status == 0) {
-        yield {
-          /* update marker */
-          int ret = call(marker_tracker->finish(entry_marker));
-          if (ret < 0) {
-            ldout(sync_env->cct, 0) << "ERROR: marker_tracker->finish(" << entry_marker << ") returned ret=" << ret << dendl;
-            return set_cr_error(sync_status);
-          }
+    sync_status = retcode;
+
+    if (sync_status == 0 && marker_tracker) {
+      yield {
+        /* update marker */
+        int ret = call(marker_tracker->finish(entry_marker));
+        if (ret < 0) {
+          ldout(sync_env->cct, 0) << "ERROR: marker_tracker->finish(" << entry_marker << ") returned ret=" << ret << dendl;
+          return set_cr_error(sync_status);
         }
-        sync_status = retcode;
       }
-      if (sync_status < 0) {
-        return set_cr_error(sync_status);
-      }
-      return set_cr_done();
+      sync_status = retcode;
     }
-    return 0;
+    if (sync_status < 0) {
+      return set_cr_error(sync_status);
+    }
+    return set_cr_done();
   }
-};
+  return 0;
+}
 
 class RGWCloneMetaLogCoroutine : public RGWCoroutine {
   RGWMetaSyncEnv *sync_env;

diff --git a/src/rgw/rgw_sync.h b/src/rgw/rgw_sync.h
index f308fbeaad9da60ada6cb7253fe8ddfb2c5ac159..51f3e4ef0419e194c9a42df0410dd91ee6240780 100644 (file)
--- a/src/rgw/rgw_sync.h
+++ b/src/rgw/rgw_sync.h
@@ -87,6 +87,15 @@ struct RGWMetaSyncEnv {
 
   RGWMetaSyncEnv() : cct(NULL), store(NULL), conn(NULL), async_rados(NULL), http_manager(NULL) {}
 
+  void init(CephContext *_cct, RGWRados *_store, RGWRESTConn *_conn,
+            RGWAsyncRadosProcessor *_async_rados, RGWHTTPManager *_http_manager) {
+    cct = _cct;
+    store = _store;
+    conn = _conn;
+    async_rados = _async_rados;
+    http_manager = _http_manager;
+  }
+
   string shard_obj_name(int shard_id);
   string status_oid();
 };
@@ -253,4 +262,37 @@ public:
   }
 };
 
+class RGWMetaSyncShardMarkerTrack;
+
+class RGWMetaSyncSingleEntryCR : public RGWCoroutine {
+  RGWMetaSyncEnv *sync_env;
+
+  string raw_key;
+  string entry_marker;
+
+  ssize_t pos;
+  string section;
+  string key;
+
+  int sync_status;
+
+  bufferlist md_bl;
+
+  RGWMetaSyncShardMarkerTrack *marker_tracker;
+
+  int tries;
+
+public:
+  RGWMetaSyncSingleEntryCR(RGWMetaSyncEnv *_sync_env,
+                          const string& _raw_key, const string& _entry_marker, RGWMetaSyncShardMarkerTrack *_marker_tracker) : RGWCoroutine(_sync_env->cct),
+                                                      sync_env(_sync_env),
+                                                     raw_key(_raw_key), entry_marker(_entry_marker),
+                                                      pos(0), sync_status(0),
+                                                      marker_tracker(_marker_tracker), tries(0) {
+  }
+
+  int operate();
+};
+
+
 #endif