rbd-mirror: bootstrap requires the global image id

The global image id can be used to cross-reference images that
have been replicated amongst peers.

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

diff --git a/src/test/rbd_mirror/image_replay.cc b/src/test/rbd_mirror/image_replay.cc
index a51a200d8e3f27b259aa359e4142a3666ceda432..bde46e2cc4f5de0a8ee99c99fdf801a3f50c179d 100644 (file)
--- a/src/test/rbd_mirror/image_replay.cc
+++ b/src/test/rbd_mirror/image_replay.cc
@@ -186,7 +186,8 @@ int main(int argc, const char **argv)
     local->cct()));
   replayer = new rbd::mirror::ImageReplayer<>(threads, local, remote, client_id,
                                              local_pool_id, remote_pool_id,
-                                             remote_image_id);
+                                             remote_image_id,
+                                              "global image id");
 
   replayer->start(&start_cond, &bootstap_params);
   r = start_cond.wait();

diff --git a/src/test/rbd_mirror/test_ImageReplayer.cc b/src/test/rbd_mirror/test_ImageReplayer.cc
index a4d597319aff85be10f3975fd58607d6304c9b17..d5de02843612ce400af61201ab2daea7525f3fb9 100644 (file)
--- a/src/test/rbd_mirror/test_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_ImageReplayer.cc
@@ -116,7 +116,8 @@ public:
     m_replayer = new ImageReplayerT(m_threads,
       rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)),
       rbd::mirror::RadosRef(new librados::Rados(m_remote_ioctx)),
-      m_client_id, m_local_ioctx.get_id(), m_remote_pool_id, m_remote_image_id);
+      m_client_id, m_local_ioctx.get_id(), m_remote_pool_id, m_remote_image_id,
+      "global image id");
   }
 
   void start(rbd::mirror::ImageReplayer<>::BootstrapParams *bootstap_params =
@@ -531,10 +532,11 @@ public:
   ImageReplayer(rbd::mirror::Threads *threads,
                rbd::mirror::RadosRef local, rbd::mirror::RadosRef remote,
                const std::string &client_id, int64_t local_pool_id,
-               int64_t remote_pool_id, const std::string &remote_image_id)
+               int64_t remote_pool_id, const std::string &remote_image_id,
+                const std::string &global_image_id)
     : rbd::mirror::ImageReplayer<>(threads, local, remote, client_id,
                                   local_pool_id, remote_pool_id,
-                                   remote_image_id)
+                                   remote_image_id, global_image_id)
     {}
 
   void set_error(const std::string &state, int r) {

diff --git a/src/test/rbd_mirror/test_PoolWatcher.cc b/src/test/rbd_mirror/test_PoolWatcher.cc
index 5cb68058b2276e4335bb12261c47169ee8c2ed9b..b8ff3117b30db522cfb2df8a571e9cbf78e70f2c 100644 (file)
--- a/src/test/rbd_mirror/test_PoolWatcher.cc
+++ b/src/test/rbd_mirror/test_PoolWatcher.cc
@@ -140,8 +140,14 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
       librbd::RBD rbd;
       rbd.open(ioctx, image, name.c_str());
       image.mirror_image_enable();
+
+      librbd::mirror_image_info_t mirror_image_info;
+      ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
+                                               sizeof(mirror_image_info)));
       image.close();
-      m_mirrored_images[ioctx.get_id()].insert(get_image_id(&ioctx, name));
+
+      m_mirrored_images[ioctx.get_id()].insert(PoolWatcher::ImageIds(
+        get_image_id(&ioctx, name), mirror_image_info.global_id));
     }
     if (image_name != nullptr)
       *image_name = name;
@@ -179,8 +185,14 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
       librbd::RBD rbd;
       rbd.open(cioctx, image, name.c_str());
       image.mirror_image_enable();
+
+      librbd::mirror_image_info_t mirror_image_info;
+      ASSERT_EQ(0, image.mirror_image_get_info(&mirror_image_info,
+                                               sizeof(mirror_image_info)));
       image.close();
-      m_mirrored_images[cioctx.get_id()].insert(get_image_id(&cioctx, name));
+
+      m_mirrored_images[cioctx.get_id()].insert(PoolWatcher::ImageIds(
+        get_image_id(&cioctx, name), mirror_image_info.global_id));
     }
     if (image_name != nullptr)
       *image_name = name;
@@ -198,7 +210,7 @@ TestPoolWatcher() : m_lock("TestPoolWatcherLock"),
   unique_ptr<PoolWatcher> m_pool_watcher;
 
   set<string> m_pools;
-  std::map<int64_t, std::set<std::string> > m_mirrored_images;
+  PoolWatcher::PoolImageIds m_mirrored_images;
 
   uint64_t m_image_number;
   uint64_t m_snap_number;

diff --git a/src/test/rbd_mirror/test_mock_ImageReplayer.cc b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
index 9b78c070c7a1895935dd13b11fd9592922139394..fc1f16b39d49f57f1f85b60a898f5f94aeae499e 100644 (file)
--- a/src/test/rbd_mirror/test_mock_ImageReplayer.cc
+++ b/src/test/rbd_mirror/test_mock_ImageReplayer.cc
@@ -58,6 +58,7 @@ struct BootstrapRequest<librbd::MockImageReplayerImageCtx> {
                                   librbd::MockImageReplayerImageCtx **local_image_ctx,
                                   const std::string &local_image_name,
                                   const std::string &remote_image_id,
+                                  const std::string &global_image_id,
                                   ContextWQ *work_queue, SafeTimer *timer,
                                   Mutex *timer_lock,
                                   const std::string &mirror_uuid,

diff --git a/src/tools/rbd_mirror/ImageReplayer.cc b/src/tools/rbd_mirror/ImageReplayer.cc
index 25c84838c6a99090f7f260be7fb7604d4a18979a..07334f979895c7e82f3f3e2459778a8965092c6a 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.cc
+++ b/src/tools/rbd_mirror/ImageReplayer.cc
@@ -176,7 +176,8 @@ ImageReplayer<I>::ImageReplayer(Threads *threads, RadosRef local, RadosRef remot
                             const std::string &mirror_uuid,
                             int64_t local_pool_id,
                             int64_t remote_pool_id,
-                            const std::string &remote_image_id) :
+                            const std::string &remote_image_id,
+                             const std::string &global_image_id) :
   m_threads(threads),
   m_local(local),
   m_remote(remote),
@@ -184,6 +185,7 @@ ImageReplayer<I>::ImageReplayer(Threads *threads, RadosRef local, RadosRef remot
   m_remote_pool_id(remote_pool_id),
   m_local_pool_id(local_pool_id),
   m_remote_image_id(remote_image_id),
+  m_global_image_id(global_image_id),
   m_name(stringify(remote_pool_id) + "/" + remote_image_id),
   m_lock("rbd::mirror::ImageReplayer " + stringify(remote_pool_id) + " " +
         remote_image_id),
@@ -266,9 +268,9 @@ void ImageReplayer<I>::bootstrap() {
     ImageReplayer, &ImageReplayer<I>::handle_bootstrap>(this);
   BootstrapRequest<I> *request = BootstrapRequest<I>::create(
     m_local_ioctx, m_remote_ioctx, &m_local_image_ctx,
-    m_local_image_name, m_remote_image_id, m_threads->work_queue,
-    m_threads->timer, &m_threads->timer_lock, m_mirror_uuid, m_remote_journaler,
-    &m_client_meta, ctx);
+    m_local_image_name, m_remote_image_id, m_global_image_id,
+    m_threads->work_queue, m_threads->timer, &m_threads->timer_lock,
+    m_mirror_uuid, m_remote_journaler, &m_client_meta, ctx);
   request->send();
 }
 

diff --git a/src/tools/rbd_mirror/ImageReplayer.h b/src/tools/rbd_mirror/ImageReplayer.h
index 8f36fd099f20b676d07a8364f2a3dcaeb10caa7f..ffff469954d455b2762afcae2c74156c5194f8db 100644 (file)
--- a/src/tools/rbd_mirror/ImageReplayer.h
+++ b/src/tools/rbd_mirror/ImageReplayer.h
@@ -68,7 +68,8 @@ public:
 
   ImageReplayer(Threads *threads, RadosRef local, RadosRef remote,
                const std::string &mirror_uuid, int64_t local_pool_id,
-               int64_t remote_pool_id, const std::string &remote_image_id);
+               int64_t remote_pool_id, const std::string &remote_image_id,
+                const std::string &global_image_id);
   virtual ~ImageReplayer();
   ImageReplayer(const ImageReplayer&) = delete;
   ImageReplayer& operator=(const ImageReplayer&) = delete;
@@ -160,7 +161,7 @@ private:
   RadosRef m_local, m_remote;
   std::string m_mirror_uuid;
   int64_t m_remote_pool_id, m_local_pool_id;
-  std::string m_remote_image_id, m_local_image_id;
+  std::string m_remote_image_id, m_local_image_id, m_global_image_id;
   std::string m_local_image_name;
   std::string m_name;
   Mutex m_lock;

diff --git a/src/tools/rbd_mirror/PoolWatcher.cc b/src/tools/rbd_mirror/PoolWatcher.cc
index 2e147f7f34d2a0e4e8b44769ae0dfacce1e2ca1b..0117cabe11f6b51ae6993156b0e761268b6cc25c 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.cc
+++ b/src/tools/rbd_mirror/PoolWatcher.cc
@@ -17,8 +17,6 @@
 #define dout_prefix *_dout << "rbd-mirror: PoolWatcher::" << __func__ << ": "
 
 using std::list;
-using std::map;
-using std::set;
 using std::string;
 using std::unique_ptr;
 using std::vector;
@@ -49,7 +47,7 @@ PoolWatcher::~PoolWatcher()
   m_timer.shutdown();
 }
 
-const map<int64_t, set<string> >& PoolWatcher::get_images() const
+const PoolWatcher::PoolImageIds& PoolWatcher::get_images() const
 {
   assert(m_lock.is_locked());
   return m_images;
@@ -58,7 +56,7 @@ const map<int64_t, set<string> >& PoolWatcher::get_images() const
 void PoolWatcher::refresh_images(bool reschedule)
 {
   dout(20) << "enter" << dendl;
-  map<int64_t, set<string> > images;
+  PoolImageIds images;
   list<pair<int64_t, string> > pools;
   int r = m_cluster->pool_list2(pools);
   if (r < 0) {
@@ -105,7 +103,7 @@ void PoolWatcher::refresh_images(bool reschedule)
       continue;
     }
 
-    std::set<std::string> image_ids;
+    std::set<ImageIds> image_ids;
     std::string last_read = "";
     int max_read = 1024;
     do {
@@ -117,7 +115,7 @@ void PoolWatcher::refresh_images(bool reschedule)
         continue;
       }
       for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) {
-        image_ids.insert(it->first);
+        image_ids.insert(ImageIds(it->first, it->second));
       }
       if (!mirror_images.empty()) {
         last_read = mirror_images.rbegin()->first;

diff --git a/src/tools/rbd_mirror/PoolWatcher.h b/src/tools/rbd_mirror/PoolWatcher.h
index 1358539b726220dde0397bc0c830a7f20bf21d66..0ab45b41e849a51cf9e51bbbb531e89d3a78ae2e 100644 (file)
--- a/src/tools/rbd_mirror/PoolWatcher.h
+++ b/src/tools/rbd_mirror/PoolWatcher.h
@@ -24,12 +24,30 @@ namespace mirror {
  */
 class PoolWatcher {
 public:
+  struct ImageIds {
+    std::string id;
+    std::string global_id;
+
+    ImageIds(const std::string &id, const std::string &global_id = "")
+      : id(id), global_id(global_id) {
+    }
+
+    inline bool operator==(const ImageIds &rhs) const {
+      return (id == rhs.id && global_id == rhs.global_id);
+    }
+    inline bool operator<(const ImageIds &rhs) const {
+      return id < rhs.id;
+    }
+  };
+  typedef std::map<int64_t, std::set<ImageIds> > PoolImageIds;
+
   PoolWatcher(RadosRef cluster, double interval_seconds,
              Mutex &lock, Cond &cond);
   ~PoolWatcher();
   PoolWatcher(const PoolWatcher&) = delete;
   PoolWatcher& operator=(const PoolWatcher&) = delete;
-  const std::map<int64_t, std::set<std::string> >& get_images() const;
+
+  const PoolImageIds& get_images() const;
   void refresh_images(bool reschedule=true);
 
 private:
@@ -40,8 +58,8 @@ private:
   RadosRef m_cluster;
   SafeTimer m_timer;
   double m_interval;
-  // pool id -> image id
-  std::map<int64_t, std::set<std::string> > m_images;
+
+  PoolImageIds m_images;
 };
 
 } // namespace mirror

diff --git a/src/tools/rbd_mirror/Replayer.cc b/src/tools/rbd_mirror/Replayer.cc
index 5aaf18da6ec479341610b493e0d9b6c95542cea9..7935d269162182d37f84662b27edf804be9c7150 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.cc
+++ b/src/tools/rbd_mirror/Replayer.cc
@@ -203,7 +203,7 @@ void Replayer::run()
   }
 
   // Stopping
-  map<int64_t, set<string> > empty_sources;
+  PoolImageIds empty_sources;
   while (true) {
     Mutex::Locker l(m_lock);
     set_sources(empty_sources);
@@ -260,7 +260,7 @@ void Replayer::flush()
   }
 }
 
-void Replayer::set_sources(const map<int64_t, set<string> > &images)
+void Replayer::set_sources(const PoolImageIds &pool_image_ids)
 {
   dout(20) << "enter" << dendl;
 
@@ -268,7 +268,9 @@ void Replayer::set_sources(const map<int64_t, set<string> > &images)
   for (auto it = m_images.begin(); it != m_images.end();) {
     int64_t pool_id = it->first;
     auto &pool_images = it->second;
-    if (images.find(pool_id) == images.end()) {
+
+    // pool has no mirrored images
+    if (pool_image_ids.find(pool_id) == pool_image_ids.end()) {
       for (auto images_it = pool_images.begin();
           images_it != pool_images.end();) {
        if (stop_image_replayer(images_it->second)) {
@@ -280,10 +282,12 @@ void Replayer::set_sources(const map<int64_t, set<string> > &images)
       }
       continue;
     }
+
+    // shut down replayers for non-mirrored images
     for (auto images_it = pool_images.begin();
         images_it != pool_images.end();) {
-      if (images.at(pool_id).find(images_it->first) ==
-         images.at(pool_id).end()) {
+      auto &image_ids = pool_image_ids.at(pool_id);
+      if (image_ids.find(ImageIds(images_it->first)) == image_ids.end()) {
        if (stop_image_replayer(images_it->second)) {
          pool_images.erase(images_it++);
        }
@@ -294,7 +298,8 @@ void Replayer::set_sources(const map<int64_t, set<string> > &images)
     ++it;
   }
 
-  for (const auto &kv : images) {
+  // (re)start new image replayers
+  for (const auto &kv : pool_image_ids) {
     int64_t pool_id = kv.first;
 
     // TODO: clean up once remote peer -> image replayer refactored
@@ -325,13 +330,13 @@ void Replayer::set_sources(const map<int64_t, set<string> > &images)
     // create entry for pool if it doesn't exist
     auto &pool_replayers = m_images[pool_id];
     for (const auto &image_id : kv.second) {
-      auto it = pool_replayers.find(image_id);
+      auto it = pool_replayers.find(image_id.id);
       if (it == pool_replayers.end()) {
        unique_ptr<ImageReplayer<> > image_replayer(new ImageReplayer<>(
           m_threads, m_local, m_remote, mirror_uuid, local_ioctx.get_id(),
-          pool_id, image_id));
+          pool_id, image_id.id, image_id.global_id));
        it = pool_replayers.insert(
-         std::make_pair(image_id, std::move(image_replayer))).first;
+         std::make_pair(image_id.id, std::move(image_replayer))).first;
       }
       start_image_replayer(it->second);
     }

diff --git a/src/tools/rbd_mirror/Replayer.h b/src/tools/rbd_mirror/Replayer.h
index d818f72983640f5a324e6a1cf0cac37c4a9beb16..f7c623b592f29c69df3ee665f6b45b9ae7a7ec25 100644 (file)
--- a/src/tools/rbd_mirror/Replayer.h
+++ b/src/tools/rbd_mirror/Replayer.h
@@ -44,7 +44,10 @@ public:
   void flush();
 
 private:
-  void set_sources(const std::map<int64_t, std::set<std::string> > &images);
+  typedef PoolWatcher::ImageIds ImageIds;
+  typedef PoolWatcher::PoolImageIds PoolImageIds;
+
+  void set_sources(const PoolImageIds &pool_image_ids);
 
   void start_image_replayer(unique_ptr<ImageReplayer<> > &image_replayer);
   bool stop_image_replayer(unique_ptr<ImageReplayer<> > &image_replayer);

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
index 5b112222c1f9fdeba91924e47d11a351454738b2..acb39f00cd5cb01542a65d59aff60102a116f895 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc
@@ -32,15 +32,18 @@ namespace {
 template <typename I>
 struct C_CreateImage : public Context {
   librados::IoCtx &local_io_ctx;
+  std::string global_image_id;
   std::string local_image_name;
   I *remote_image_ctx;
   Context *on_finish;
 
   C_CreateImage(librados::IoCtx &local_io_ctx,
+                const std::string &global_image_id,
                 const std::string &local_image_name, I *remote_image_ctx,
                 Context *on_finish)
-    : local_io_ctx(local_io_ctx), local_image_name(local_image_name),
-      remote_image_ctx(remote_image_ctx), on_finish(on_finish) {
+    : local_io_ctx(local_io_ctx), global_image_id(global_image_id),
+      local_image_name(local_image_name), remote_image_ctx(remote_image_ctx),
+      on_finish(on_finish) {
   }
 
   virtual void finish(int r) override {
@@ -62,7 +65,7 @@ struct C_CreateImage : public Context {
                           remote_image_ctx->stripe_unit,
                           remote_image_ctx->stripe_count,
                           journal_order, journal_splay_width, journal_pool,
-                          "global-image-id");
+                          global_image_id);
     on_finish->complete(r);
   }
 };
@@ -75,6 +78,7 @@ BootstrapRequest<I>::BootstrapRequest(librados::IoCtx &local_io_ctx,
                                       I **local_image_ctx,
                                       const std::string &local_image_name,
                                       const std::string &remote_image_id,
+                                      const std::string &global_image_id,
                                       ContextWQ *work_queue, SafeTimer *timer,
                                       Mutex *timer_lock,
                                       const std::string &mirror_uuid,
@@ -83,9 +87,10 @@ BootstrapRequest<I>::BootstrapRequest(librados::IoCtx &local_io_ctx,
                                       Context *on_finish)
   : m_local_io_ctx(local_io_ctx), m_remote_io_ctx(remote_io_ctx),
     m_local_image_ctx(local_image_ctx), m_local_image_name(local_image_name),
-    m_remote_image_id(remote_image_id), m_work_queue(work_queue),
-    m_timer(timer), m_timer_lock(timer_lock), m_mirror_uuid(mirror_uuid),
-    m_journaler(journaler), m_client_meta(client_meta), m_on_finish(on_finish) {
+    m_remote_image_id(remote_image_id), m_global_image_id(global_image_id),
+    m_work_queue(work_queue), m_timer(timer), m_timer_lock(timer_lock),
+    m_mirror_uuid(mirror_uuid), m_journaler(journaler),
+    m_client_meta(client_meta), m_on_finish(on_finish) {
 }
 
 template <typename I>
@@ -257,7 +262,8 @@ void BootstrapRequest<I>::create_local_image() {
   Context *ctx = create_context_callback<
     BootstrapRequest<I>, &BootstrapRequest<I>::handle_create_local_image>(
       this);
-  m_work_queue->queue(new C_CreateImage<I>(m_local_io_ctx, m_local_image_name,
+  m_work_queue->queue(new C_CreateImage<I>(m_local_io_ctx, m_global_image_id,
+                                           m_local_image_name,
                                            m_remote_image_ctx, ctx), 0);
 }
 

diff --git a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
index 955195161305a556ed5b2f17aebec7c26e39da65..b0822be1e75831388b9d54d77faf6c6736799651 100644 (file)
--- a/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
+++ b/src/tools/rbd_mirror/image_replayer/BootstrapRequest.h
@@ -34,6 +34,7 @@ public:
                                   ImageCtxT **local_image_ctx,
                                   const std::string &local_image_name,
                                   const std::string &remote_image_id,
+                                  const std::string &global_image_id,
                                   ContextWQ *work_queue, SafeTimer *timer,
                                   Mutex *timer_lock,
                                   const std::string &mirror_uuid,
@@ -41,16 +42,17 @@ public:
                                   MirrorPeerClientMeta *client_meta,
                                   Context *on_finish) {
     return new BootstrapRequest(local_io_ctx, remote_io_ctx, local_image_ctx,
-                                local_image_name, remote_image_id, work_queue,
-                                timer, timer_lock, mirror_uuid, journaler,
-                                client_meta, on_finish);
+                                local_image_name, remote_image_id,
+                                global_image_id, work_queue, timer, timer_lock,
+                                mirror_uuid, journaler, client_meta, on_finish);
   }
 
   BootstrapRequest(librados::IoCtx &local_io_ctx,
                    librados::IoCtx &remote_io_ctx,
                    ImageCtxT **local_image_ctx,
                    const std::string &local_image_name,
-                   const std::string &remote_image_id, ContextWQ *work_queue,
+                   const std::string &remote_image_id,
+                   const std::string &global_image_id, ContextWQ *work_queue,
                    SafeTimer *timer, Mutex *timer_lock,
                    const std::string &mirror_uuid, Journaler *journaler,
                    MirrorPeerClientMeta *client_meta, Context *on_finish);
@@ -110,6 +112,7 @@ private:
   std::string m_local_image_name;
   std::string m_local_image_id;
   std::string m_remote_image_id;
+  std::string m_global_image_id;
   ContextWQ *m_work_queue;
   SafeTimer *m_timer;
   Mutex *m_timer_lock;