From: Jason Dillaman <dillaman@redhat.com>
Date: Thu, 16 Feb 2017 13:04:40 +0000 (-0500)
Subject: rbd: new rbd mirror pool promote/demote actions
X-Git-Tag: v12.0.1~112^2~1
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=4692fe246c24ceb3f06d09717c15a4ad7897db80;p=ceph.git

rbd: new rbd mirror pool promote/demote actions

Fixes: http://tracker.ceph.com/issues/18748
Signed-off-by: Jason Dillaman <dillaman@redhat.com>
---

diff --git a/src/test/cli/rbd/help.t b/src/test/cli/rbd/help.t
index 12a68367c6f4..b2a44296f82f 100644
--- a/src/test/cli/rbd/help.t
+++ b/src/test/cli/rbd/help.t
@@ -56,6 +56,7 @@
       mirror image promote        Promote an image to primary for RBD mirroring.
       mirror image resync         Force resync to primary image for RBD mirroring.
       mirror image status         Show RDB mirroring status for an image.
+      mirror pool demote          Demote all primary images in the pool.
       mirror pool disable         Disable RBD mirroring by default within a pool.
       mirror pool enable          Enable RBD mirroring by default within a pool.
       mirror pool info            Show information about the pool mirroring
@@ -63,6 +64,7 @@
       mirror pool peer add        Add a mirroring peer to a pool.
       mirror pool peer remove     Remove a mirroring peer from a pool.
       mirror pool peer set        Update mirroring peer settings.
+      mirror pool promote         Promote all non-primary images in the pool.
       mirror pool status          Show status for all mirrored images in the pool.
       nbd list (nbd ls)           List the nbd devices already used.
       nbd map                     Map image to a nbd device.
@@ -981,6 +983,18 @@
     --format arg         output format [plain, json, or xml]
     --pretty-format      pretty formatting (json and xml)
   
+  rbd help mirror pool demote
+  usage: rbd mirror pool demote [--pool <pool>] 
+                                <pool-name> 
+  
+  Demote all primary images in the pool.
+  
+  Positional arguments
+    <pool-name>          pool name
+  
+  Optional arguments
+    -p [ --pool ] arg    pool name
+  
   rbd help mirror pool disable
   usage: rbd mirror pool disable [--pool <pool>] 
                                  <pool-name> 
@@ -1067,6 +1081,19 @@
   Optional arguments
     -p [ --pool ] arg    pool name
   
+  rbd help mirror pool promote
+  usage: rbd mirror pool promote [--force] [--pool <pool>] 
+                                 <pool-name> 
+  
+  Promote all non-primary images in the pool.
+  
+  Positional arguments
+    <pool-name>          pool name
+  
+  Optional arguments
+    --force              promote even if not cleanly demoted by remote cluster
+    -p [ --pool ] arg    pool name
+  
   rbd help mirror pool status
   usage: rbd mirror pool status [--pool <pool>] [--format <format>] 
                                 [--pretty-format] [--verbose] 
diff --git a/src/tools/rbd/Utils.h b/src/tools/rbd/Utils.h
index bc23a2f0c58c..a718ded0bded 100644
--- a/src/tools/rbd/Utils.h
+++ b/src/tools/rbd/Utils.h
@@ -14,6 +14,24 @@
 namespace rbd {
 namespace utils {
 
+namespace detail {
+
+template <typename T, void(T::*MF)(int)>
+void aio_completion_callback(librbd::completion_t completion,
+                                    void *arg) {
+  librbd::RBD::AioCompletion *aio_completion =
+    reinterpret_cast<librbd::RBD::AioCompletion*>(completion);
+
+  // complete the AIO callback in separate thread context
+  T *t = reinterpret_cast<T *>(arg);
+  int r = aio_completion->get_return_value();
+  aio_completion->release();
+
+  (t->*MF)(r);
+}
+
+} // namespace detail
+
 static const std::string RBD_DIFF_BANNER ("rbd diff v1\n");
 
 static const std::string RBD_IMAGE_BANNER_V2 ("rbd image v2\n");
@@ -59,6 +77,12 @@ struct ProgressContext : public librbd::ProgressContext {
   void fail();
 };
 
+template <typename T, void(T::*MF)(int)>
+librbd::RBD::AioCompletion *create_aio_completion(T *t) {
+  return new librbd::RBD::AioCompletion(
+    t, &detail::aio_completion_callback<T, MF>);
+}
+
 void aio_context_callback(librbd::completion_t completion, void *arg);
 
 int read_string(int fd, unsigned max, std::string *out);
diff --git a/src/tools/rbd/action/MirrorImage.cc b/src/tools/rbd/action/MirrorImage.cc
index e703e2df288c..e1b6d40132c0 100644
--- a/src/tools/rbd/action/MirrorImage.cc
+++ b/src/tools/rbd/action/MirrorImage.cc
@@ -229,7 +229,8 @@ int execute_status(const po::variables_map &vm) {
   }
 
   std::string state = utils::mirror_image_status_state(status);
-  std::string last_update = utils::timestr(status.last_update);
+  std::string last_update = (
+    status.last_update == 0 ? "" : utils::timestr(status.last_update));
 
   if (formatter != nullptr) {
     formatter->open_object_section("image");
diff --git a/src/tools/rbd/action/MirrorPool.cc b/src/tools/rbd/action/MirrorPool.cc
index 94251717eb64..19981cd58784 100644
--- a/src/tools/rbd/action/MirrorPool.cc
+++ b/src/tools/rbd/action/MirrorPool.cc
@@ -4,15 +4,27 @@
 #include "tools/rbd/ArgumentTypes.h"
 #include "tools/rbd/Shell.h"
 #include "tools/rbd/Utils.h"
+#include "include/atomic.h"
+#include "include/Context.h"
 #include "include/stringify.h"
+#include "include/rbd/librbd.hpp"
 #include "common/config.h"
+#include "common/debug.h"
 #include "common/errno.h"
 #include "common/Formatter.h"
 #include "common/TextTable.h"
+#include "common/Throttle.h"
 #include "global/global_context.h"
+#include <functional>
 #include <iostream>
 #include <boost/program_options.hpp>
 #include <boost/regex.hpp>
+#include "include/assert.h"
+
+#define dout_context g_ceph_context
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "rbd::action::MirrorPool: "
 
 namespace rbd {
 namespace action {
@@ -115,6 +127,369 @@ void format_mirror_peers(const std::string &config_path,
   }
 }
 
+class ImageRequestBase {
+public:
+  void send() {
+    dout(20) << this << " " << __func__ << ": image_name=" << m_image_name
+             << dendl;
+
+    auto ctx = new FunctionContext([this](int r) {
+        handle_finalize(r);
+      });
+
+    // will pause here until slots are available
+    m_finalize_ctx = m_throttle.start_op(ctx);
+
+    open_image();
+  }
+
+protected:
+  ImageRequestBase(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                   const std::string &image_name)
+    : m_io_ctx(io_ctx), m_throttle(throttle), m_image_name(image_name) {
+  }
+  virtual ~ImageRequestBase() {
+  }
+
+  virtual bool skip_get_info() const {
+    return false;
+  }
+  virtual void get_info(librbd::Image &image, librbd::mirror_image_info_t *info,
+                        librbd::RBD::AioCompletion *aio_comp) {
+    image.aio_mirror_image_get_info(info, sizeof(librbd::mirror_image_info_t),
+                                    aio_comp);
+  }
+
+  virtual bool skip_action(const librbd::mirror_image_info_t &info) const {
+    return false;
+  }
+  virtual void execute_action(librbd::Image &image,
+                              librbd::RBD::AioCompletion *aio_comp) = 0;
+  virtual void handle_execute_action(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r < 0 && r != -ENOENT) {
+      std::cerr << "rbd: failed to " << get_action_type() << " image "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+      m_ret_val = r;
+    }
+
+    close_image();
+  }
+
+  virtual void finalize_action() {
+  }
+  virtual std::string get_action_type() const = 0;
+
+private:
+  /**
+   * @verbatim
+   *
+   * <start>
+   *    |
+   *    v
+   * OPEN_IMAGE
+   *    |
+   *    v
+   * GET_INFO
+   *    |
+   *    v
+   * EXECUTE_ACTION
+   *    |
+   *    v
+   * CLOSE_IMAGE
+   *    |
+   *    v
+   * FINALIZE_ACTION
+   *    |
+   *    v
+   * <finish>
+   *
+   * @endverbatim
+   */
+
+  librados::IoCtx &m_io_ctx;
+  OrderedThrottle &m_throttle;
+  const std::string m_image_name;
+
+  librbd::Image m_image;
+  Context *m_finalize_ctx;
+
+  librbd::mirror_image_info_t m_mirror_image_info;
+
+  int m_ret_val = 0;
+
+  void open_image() {
+    dout(20) << this << " " << __func__ << dendl;
+
+    librbd::RBD rbd;
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_open_image>(this);
+    rbd.aio_open(m_io_ctx, m_image, m_image_name.c_str(), nullptr,
+                 aio_completion);
+  }
+
+  void handle_open_image(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r < 0) {
+      std::cerr << "rbd: failed to open image "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+      m_finalize_ctx->complete(r);
+      return;
+    }
+
+    get_info();
+  }
+
+  void get_info() {
+    if (skip_get_info()) {
+      execute_action();
+      return;
+    }
+    dout(20) << this << " " << __func__ << dendl;
+
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_get_info>(this);
+    get_info(m_image, &m_mirror_image_info, aio_completion);
+  }
+
+  void handle_get_info(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r == -ENOENT) {
+      close_image();
+      return;
+    } else if (r < 0) {
+      std::cerr << "rbd: failed to retrieve mirror image info for "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+      m_ret_val = r;
+      close_image();
+      return;
+    }
+
+    execute_action();
+  }
+
+  void execute_action() {
+    if (skip_action(m_mirror_image_info)) {
+      close_image();
+      return;
+    }
+    dout(20) << this << " " << __func__ << dendl;
+
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_execute_action>(this);
+    execute_action(m_image, aio_completion);
+  }
+
+  void close_image() {
+    dout(20) << this << " " << __func__ << dendl;
+
+    auto aio_completion = utils::create_aio_completion<
+      ImageRequestBase, &ImageRequestBase::handle_close_image>(this);
+    m_image.aio_close(aio_completion);
+  }
+
+  void handle_close_image(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r < 0) {
+      std::cerr << "rbd: failed to close image "
+                << m_image_name << ": " << cpp_strerror(r) << std::endl;
+    }
+
+    m_finalize_ctx->complete(r);
+  }
+
+  void handle_finalize(int r) {
+    dout(20) << this << " " << __func__ << ": r=" << r << dendl;
+
+    if (r == 0 && m_ret_val < 0) {
+      r = m_ret_val;
+    }
+    if (r >= 0) {
+      finalize_action();
+    }
+    m_throttle.end_op(r);
+  }
+
+};
+
+class PromoteImageRequest : public ImageRequestBase {
+public:
+  PromoteImageRequest(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                      const std::string &image_name, atomic_t *counter,
+                      bool force)
+    : ImageRequestBase(io_ctx, throttle, image_name), m_counter(counter),
+      m_force(force) {
+  }
+
+protected:
+  bool skip_action(const librbd::mirror_image_info_t &info) const override {
+    return info.primary;
+  }
+
+  void execute_action(librbd::Image &image,
+                      librbd::RBD::AioCompletion *aio_comp) override {
+    image.aio_mirror_image_promote(m_force, aio_comp);
+  }
+  void handle_execute_action(int r) override {
+    if (r >= 0) {
+      m_counter->inc();
+    }
+    ImageRequestBase::handle_execute_action(r);
+  }
+
+  std::string get_action_type() const override {
+    return "promote";
+  }
+
+private:
+  atomic_t *m_counter;
+  bool m_force;
+};
+
+class DemoteImageRequest : public ImageRequestBase {
+public:
+  DemoteImageRequest(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                     const std::string &image_name, atomic_t *counter)
+    : ImageRequestBase(io_ctx, throttle, image_name), m_counter(counter) {
+  }
+
+protected:
+  bool skip_action(const librbd::mirror_image_info_t &info) const override {
+    return !info.primary;
+  }
+
+  void execute_action(librbd::Image &image,
+                      librbd::RBD::AioCompletion *aio_comp) override {
+    image.aio_mirror_image_demote(aio_comp);
+  }
+  void handle_execute_action(int r) override {
+    if (r >= 0) {
+      m_counter->inc();
+    }
+    ImageRequestBase::handle_execute_action(r);
+  }
+
+  std::string get_action_type() const override {
+    return "demote";
+  }
+
+private:
+  atomic_t *m_counter;
+};
+
+class StatusImageRequest : public ImageRequestBase {
+public:
+  StatusImageRequest(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                     const std::string &image_name,
+                     at::Format::Formatter formatter)
+    : ImageRequestBase(io_ctx, throttle, image_name),
+      m_formatter(formatter) {
+  }
+
+protected:
+  bool skip_get_info() const override {
+    return true;
+  }
+
+  void execute_action(librbd::Image &image,
+                      librbd::RBD::AioCompletion *aio_comp) override {
+    image.aio_mirror_image_get_status(&m_mirror_image_status,
+                                      sizeof(m_mirror_image_status), aio_comp);
+  }
+
+  void finalize_action() override {
+    std::string state = utils::mirror_image_status_state(m_mirror_image_status);
+    std::string last_update = (
+      m_mirror_image_status.last_update == 0 ?
+        "" : utils::timestr(m_mirror_image_status.last_update));
+
+    if (m_formatter != nullptr) {
+      m_formatter->open_object_section("image");
+      m_formatter->dump_string("name", m_mirror_image_status.name);
+      m_formatter->dump_string("global_id",
+                               m_mirror_image_status.info.global_id);
+      m_formatter->dump_string("state", state);
+      m_formatter->dump_string("description",
+                               m_mirror_image_status.description);
+      m_formatter->dump_string("last_update", last_update);
+      m_formatter->close_section(); // image
+    } else {
+      std::cout << "\n" << m_mirror_image_status.name << ":\n"
+	        << "  global_id:   "
+                << m_mirror_image_status.info.global_id << "\n"
+	        << "  state:       " << state << "\n"
+	        << "  description: "
+                << m_mirror_image_status.description << "\n"
+	        << "  last_update: " << last_update << std::endl;
+    }
+  }
+
+  std::string get_action_type() const override {
+    return "status";
+  }
+
+private:
+  at::Format::Formatter m_formatter;
+  librbd::mirror_image_status_t m_mirror_image_status;
+
+};
+
+template <typename RequestT>
+class ImageRequestAllocator {
+public:
+  template <class... Args>
+  RequestT *operator()(librados::IoCtx &io_ctx, OrderedThrottle &throttle,
+                       const std::string &image_name, Args&&... args) {
+    return new RequestT(io_ctx, throttle, image_name,
+                        std::forward<Args>(args)...);
+  }
+};
+
+template <typename RequestT>
+class ImageRequestGenerator {
+public:
+  template <class... Args>
+  ImageRequestGenerator(librados::IoCtx &io_ctx, Args&&... args)
+    : m_io_ctx(io_ctx),
+      m_factory(std::bind(ImageRequestAllocator<RequestT>(),
+                          std::ref(m_io_ctx), std::ref(m_throttle),
+                          std::placeholders::_1, std::forward<Args>(args)...)),
+      m_throttle(g_conf->rbd_concurrent_management_ops, true) {
+  }
+
+  int execute() {
+    // use the alphabetical list of image names for pool-level
+    // mirror image operations
+    librbd::RBD rbd;
+    int r = rbd.list(m_io_ctx, m_image_names);
+    if (r < 0) {
+      std::cerr << "rbd: failed to list images within pool" << std::endl;
+      return r;
+    }
+
+    for (auto &image_name : m_image_names) {
+      auto request = m_factory(image_name);
+      request->send();
+    }
+
+    return m_throttle.wait_for_ret();
+  }
+private:
+  typedef std::function<RequestT*(const std::string&)>  Factory;
+
+  librados::IoCtx &m_io_ctx;
+  Factory m_factory;
+
+  OrderedThrottle m_throttle;
+
+  std::vector<std::string> m_image_names;
+
+};
+
 } // anonymous namespace
 
 void get_peer_add_arguments(po::options_description *positional,
@@ -492,45 +867,8 @@ int execute_status(const po::variables_map &vm) {
       formatter->open_array_section("images");
     }
 
-    std::string last_read = "";
-    int max_read = 1024;
-    do {
-      map<std::string, librbd::mirror_image_status_t> mirror_images;
-      r = rbd.mirror_image_status_list(io_ctx, last_read, max_read,
-				       &mirror_images);
-      if (r < 0) {
-	std::cerr << "rbd: failed to list mirrored image directory: "
-		  << cpp_strerror(r) << std::endl;
-	return r;
-      }
-      for (auto it = mirror_images.begin(); it != mirror_images.end(); ++it) {
-	librbd::mirror_image_status_t &status = it->second;
-	const std::string &image_name = status.name;
-	std::string &global_image_id = status.info.global_id;
-	std::string state = utils::mirror_image_status_state(status);
-	std::string last_update = utils::timestr(status.last_update);
-
-	if (formatter != nullptr) {
-	  formatter->open_object_section("image");
-	  formatter->dump_string("name", image_name);
-	  formatter->dump_string("global_id", global_image_id);
-	  formatter->dump_string("state", state);
-	  formatter->dump_string("description", status.description);
-	  formatter->dump_string("last_update", last_update);
-	  formatter->close_section(); // image
-	} else {
-	  std::cout << "\n" << image_name << ":\n"
-		    << "  global_id:   " << global_image_id << "\n"
-		    << "  state:       " << state << "\n"
-		    << "  description: " << status.description << "\n"
-		    << "  last_update: " << last_update << std::endl;
-	}
-      }
-      if (!mirror_images.empty()) {
-	last_read = mirror_images.rbegin()->first;
-      }
-      r = mirror_images.size();
-    } while (r == max_read);
+    ImageRequestGenerator<StatusImageRequest> generator(io_ctx, formatter);
+    ret = generator.execute();
 
     if (formatter != nullptr) {
       formatter->close_section(); // images
@@ -545,6 +883,58 @@ int execute_status(const po::variables_map &vm) {
   return ret;
 }
 
+void get_promote_arguments(po::options_description *positional,
+			   po::options_description *options) {
+  options->add_options()
+    ("force", po::bool_switch(),
+     "promote even if not cleanly demoted by remote cluster");
+  at::add_pool_options(positional, options);
+}
+
+int execute_promote(const po::variables_map &vm) {
+  size_t arg_index = 0;
+  std::string pool_name = utils::get_pool_name(vm, &arg_index);
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  int r = utils::init(pool_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  atomic_t counter;
+  ImageRequestGenerator<PromoteImageRequest> generator(io_ctx, &counter,
+                                                       vm["force"].as<bool>());
+  r = generator.execute();
+
+  std::cout << "Promoted " << counter.read() << " mirrored images" << std::endl;
+  return r;
+}
+
+void get_demote_arguments(po::options_description *positional,
+			   po::options_description *options) {
+  at::add_pool_options(positional, options);
+}
+
+int execute_demote(const po::variables_map &vm) {
+  size_t arg_index = 0;
+  std::string pool_name = utils::get_pool_name(vm, &arg_index);
+
+  librados::Rados rados;
+  librados::IoCtx io_ctx;
+  int r = utils::init(pool_name, &rados, &io_ctx);
+  if (r < 0) {
+    return r;
+  }
+
+  atomic_t counter;
+  ImageRequestGenerator<DemoteImageRequest> generator(io_ctx, &counter);
+  r = generator.execute();
+
+  std::cout << "Demoted " << counter.read() << " mirrored images" << std::endl;
+  return r;
+}
+
 Shell::Action action_add(
   {"mirror", "pool", "peer", "add"}, {},
   "Add a mirroring peer to a pool.", "",
@@ -574,6 +964,14 @@ Shell::Action action_status(
   {"mirror", "pool", "status"}, {},
   "Show status for all mirrored images in the pool.", {},
   &get_status_arguments, &execute_status);
+Shell::Action action_promote(
+  {"mirror", "pool", "promote"}, {},
+  "Promote all non-primary images in the pool.", {},
+  &get_promote_arguments, &execute_promote);
+Shell::Action action_demote(
+  {"mirror", "pool", "demote"}, {},
+  "Demote all primary images in the pool.", {},
+  &get_demote_arguments, &execute_demote);
 
 } // namespace mirror_pool
 } // namespace action