rgw: naively fetch all meta objects

doing it one by one, later will spawn these concurrently

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_coroutine.cc b/src/rgw/rgw_coroutine.cc
index 8140922a5c0387392587563e46701ba7b56aeef0..1c3e98d20fcb64ab7d8e210b0466974e4966b550 100644 (file)
--- a/src/rgw/rgw_coroutine.cc
+++ b/src/rgw/rgw_coroutine.cc
@@ -309,10 +309,11 @@ RGWAioCompletionNotifier *RGWCoroutinesManager::create_completion_notifier(RGWCo
   return new RGWAioCompletionNotifier(&completion_mgr, (void *)stack);
 }
 
-void RGWCoroutine::call(RGWCoroutine *op)
+int RGWCoroutine::call(RGWCoroutine *op)
 {
   int r = stack->call(op, 0);
   assert(r == 0);
+  return 0;
 }
 
 void RGWCoroutine::spawn(RGWCoroutine *op, bool wait)

diff --git a/src/rgw/rgw_coroutine.h b/src/rgw/rgw_coroutine.h
index f5aeed2545d416abda19f3d5e58ba2e96435ff88..5c423236715c142a24146909bbf13c599fdddc2f 100644 (file)
--- a/src/rgw/rgw_coroutine.h
+++ b/src/rgw/rgw_coroutine.h
@@ -112,7 +112,7 @@ public:
     return retcode;
   }
 
-  void call(RGWCoroutine *op); /* call at the same stack we're in */
+  int call(RGWCoroutine *op); /* call at the same stack we're in */
   void spawn(RGWCoroutine *op, bool wait); /* execute on a different stack */
   bool collect(int *ret); /* returns true if needs to be called again */
 };

diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc
index bf4587a7ef859e3d1a64ff2b4c32ebb8d98facc5..739b5faa5c9f32a0129829150f90e4f8f6374686 100644 (file)
--- a/src/rgw/rgw_sync.cc
+++ b/src/rgw/rgw_sync.cc
@@ -1059,7 +1059,6 @@ static string full_sync_index_shard_oid(int shard_id)
   return string(buf);
 }
 
-template <class T>
 class RGWReadRemoteMetadataCR : public RGWCoroutine {
   RGWRados *store;
   RGWHTTPManager *http_manager;
@@ -1087,9 +1086,10 @@ public:
     RGWRESTConn *conn = store->rest_master_conn;
     reenter(this) {
       yield {
-        rgw_http_param_pair pairs[] = { { NULL, NULL } };
+        rgw_http_param_pair pairs[] = { { "key" , key.c_str()},
+                                       { NULL, NULL } };
 
-        string p = string("/metadata") + section + "/" + key;
+        string p = string("/admin/metadata/") + section + "/" + key;
 
         http_op = new RGWRESTReadResource(conn, p, pairs, NULL, http_manager);
 
@@ -1134,6 +1134,8 @@ class RGWMetaSyncShardCR : public RGWCoroutine {
 
   string oid;
 
+  bufferlist md_bl;
+
 public:
   RGWMetaSyncShardCR(RGWRados *_store, RGWHTTPManager *_mgr, RGWAsyncRadosProcessor *_async_rados,
                     rgw_bucket& _pool,
@@ -1147,26 +1149,35 @@ public:
 
   int operate() {
     RGWRESTConn *conn = store->rest_master_conn;
+    string section;
+    string key;
 
 #define OMAP_GET_MAX_ENTRIES 100
     int max_entries = OMAP_GET_MAX_ENTRIES;
+    ssize_t pos;
     reenter(this) {
       if (sync_marker.state == rgw_meta_sync_marker::FullSync) {
         oid = full_sync_index_shard_oid(shard_id);
         do {
-          yield call(new RGWRadosGetOmapKeysCR(store, pool, oid, sync_marker.marker, &entries, max_entries));
+          yield return call(new RGWRadosGetOmapKeysCR(store, pool, oid, sync_marker.marker, &entries, max_entries));
           if (retcode < 0) {
             ldout(store->ctx(), 0) << "ERROR: " << __func__ << "(): RGWRadosGetOmapKeysCR() returned ret=" << retcode << dendl;
             return set_state(RGWCoroutine_Error, retcode);
           }
           iter = entries.begin();
-          yield {
-            for (; iter != entries.end(); ++iter) {
-              ldout(store->ctx(), 20) << __func__ << ": full sync: " << iter->first << dendl;
+          for (; iter != entries.end(); ++iter) {
+            ldout(store->ctx(), 20) << __func__ << ": full sync: " << iter->first << dendl;
               // fetch remote
+           pos = iter->first.find(':');
+           section = iter->first.substr(0, pos);
+           key = iter->first.substr(pos + 1);
+           yield return call(new RGWReadRemoteMetadataCR(store, http_manager, async_rados, 
+                                                            section, key, &md_bl));
               // write local
               // update shard marker
-            }
+           if (retcode < 0) {
+              return set_state(RGWCoroutine_Error, retcode);
+           }
           }
         } while ((int)entries.size() == max_entries);
         // update shard state