journal: allow alternate pool for journal objects

Signed-off-by: Mykola Golub <mgolub@mirantis.com>

diff --git a/src/cls/journal/cls_journal.cc b/src/cls/journal/cls_journal.cc
index 90f0eaa729e46b491d869ec1d773e99e48a5531a..0f2f3e48846cbfdaea979d412f5c58fd8eb061f3 100644 (file)
--- a/src/cls/journal/cls_journal.cc
+++ b/src/cls/journal/cls_journal.cc
@@ -19,6 +19,7 @@ cls_handle_t h_class;
 cls_method_handle_t h_journal_create;
 cls_method_handle_t h_journal_get_order;
 cls_method_handle_t h_journal_get_splay_width;
+cls_method_handle_t h_journal_get_pool_id;
 cls_method_handle_t h_journal_get_minimum_set;
 cls_method_handle_t h_journal_set_minimum_set;
 cls_method_handle_t h_journal_get_active_set;
@@ -35,6 +36,7 @@ static const uint64_t MAX_KEYS_READ = 64;
 
 static const std::string HEADER_KEY_ORDER         = "order";
 static const std::string HEADER_KEY_SPLAY_WIDTH   = "splay_width";
+static const std::string HEADER_KEY_POOL_ID       = "pool_id";
 static const std::string HEADER_KEY_MINIMUM_SET   = "minimum_set";
 static const std::string HEADER_KEY_ACTIVE_SET    = "active_set";
 static const std::string HEADER_KEY_CLIENT_PREFIX = "client_";
@@ -88,10 +90,12 @@ int write_key(cls_method_context_t hctx, const string &key, const T &t) {
 int journal_create(cls_method_context_t hctx, bufferlist *in, bufferlist *out) {
   uint8_t order;
   uint8_t splay_width;
+  int64_t pool_id;
   try {
     bufferlist::iterator iter = in->begin();
     ::decode(order, iter);
     ::decode(splay_width, iter);
+    ::decode(pool_id, iter);
   } catch (const buffer::error &err) {
     CLS_ERR("failed to decode input parameters: %s", err.what());
     return -EINVAL;
@@ -114,6 +118,11 @@ int journal_create(cls_method_context_t hctx, bufferlist *in, bufferlist *out) {
     return r;
   }
 
+  r = write_key(hctx, HEADER_KEY_POOL_ID, pool_id);
+  if (r < 0) {
+    return r;
+  }
+
   uint64_t object_set = 0;
   r = write_key(hctx, HEADER_KEY_ACTIVE_SET, object_set);
   if (r < 0) {
@@ -167,6 +176,26 @@ int journal_get_splay_width(cls_method_context_t hctx, bufferlist *in,
   return 0;
 }
 
+/**
+ * Input:
+ * none
+ *
+ * Output:
+ * pool_id (int64_t)
+ * @returns 0 on success, negative error code on failure
+ */
+int journal_get_pool_id(cls_method_context_t hctx, bufferlist *in,
+                            bufferlist *out) {
+  int64_t pool_id;
+  int r = read_key(hctx, HEADER_KEY_POOL_ID, &pool_id);
+  if (r < 0) {
+    return r;
+  }
+
+  ::encode(pool_id, *out);
+  return 0;
+}
+
 /**
  * Input:
  * none
@@ -549,6 +578,9 @@ void CEPH_CLS_API __cls_init()
   cls_register_cxx_method(h_class, "get_splay_width",
                           CLS_METHOD_RD,
                           journal_get_splay_width, &h_journal_get_splay_width);
+  cls_register_cxx_method(h_class, "get_pool_id",
+                          CLS_METHOD_RD,
+                          journal_get_pool_id, &h_journal_get_pool_id);
   cls_register_cxx_method(h_class, "get_minimum_set",
                           CLS_METHOD_RD,
                           journal_get_minimum_set,

diff --git a/src/cls/journal/cls_journal_client.cc b/src/cls/journal/cls_journal_client.cc
index 8382b336b2a55c59da56bf0c5b45a5b9f170a0a0..a4a268d8d46b3ad1de71fd093af94f36a54594a7 100644 (file)
--- a/src/cls/journal/cls_journal_client.cc
+++ b/src/cls/journal/cls_journal_client.cc
@@ -92,14 +92,15 @@ struct C_ClientList : public C_AioExec {
 struct C_ImmutableMetadata : public C_AioExec {
   uint8_t *order;
   uint8_t *splay_width;
+  int64_t *pool_id;
   Context *on_finish;
   bufferlist outbl;
 
   C_ImmutableMetadata(librados::IoCtx &_ioctx, const std::string &_oid,
                       uint8_t *_order, uint8_t *_splay_width,
-                      Context *_on_finish)
+                     int64_t *_pool_id, Context *_on_finish)
     : C_AioExec(_ioctx, _oid), order(_order), splay_width(_splay_width),
-      on_finish(_on_finish) {
+      pool_id(_pool_id), on_finish(_on_finish) {
   }
 
   void send() {
@@ -107,6 +108,7 @@ struct C_ImmutableMetadata : public C_AioExec {
     bufferlist inbl;
     op.exec("journal", "get_order", inbl);
     op.exec("journal", "get_splay_width", inbl);
+    op.exec("journal", "get_pool_id", inbl);
 
     librados::AioCompletion *rados_completion =
       librados::Rados::aio_create_completion(this, rados_callback, NULL);
@@ -121,6 +123,7 @@ struct C_ImmutableMetadata : public C_AioExec {
         bufferlist::iterator iter = outbl.begin();
         ::decode(*order, iter);
         ::decode(*splay_width, iter);
+        ::decode(*pool_id, iter);
       } catch (const buffer::error &err) {
         r = -EBADMSG;
       }
@@ -175,10 +178,11 @@ struct C_MutableMetadata : public C_AioExec {
 } // anonymous namespace
 
 int create(librados::IoCtx &ioctx, const std::string &oid, uint8_t order,
-           uint8_t splay) {
+           uint8_t splay, int64_t pool_id) {
   bufferlist inbl;
   ::encode(order, inbl);
   ::encode(splay, inbl);
+  ::encode(pool_id, inbl);
 
   bufferlist outbl;
   int r = ioctx.exec(oid, "journal", "create", inbl, outbl);
@@ -190,9 +194,9 @@ int create(librados::IoCtx &ioctx, const std::string &oid, uint8_t order,
 
 void get_immutable_metadata(librados::IoCtx &ioctx, const std::string &oid,
                             uint8_t *order, uint8_t *splay_width,
-                            Context *on_finish) {
+                            int64_t *pool_id, Context *on_finish) {
   C_ImmutableMetadata *metadata = new C_ImmutableMetadata(ioctx, oid, order,
-                                                          splay_width,
+                                                          splay_width, pool_id,
                                                           on_finish);
   metadata->send();
 }

diff --git a/src/cls/journal/cls_journal_client.h b/src/cls/journal/cls_journal_client.h
index 6cdb87a4e09b4bc563832b55542e9a84e6c9ab5d..18ccf7bc2286df708a2b602fecb987926bab1b7a 100644 (file)
--- a/src/cls/journal/cls_journal_client.h
+++ b/src/cls/journal/cls_journal_client.h
@@ -18,11 +18,11 @@ namespace journal {
 namespace client {
 
 int create(librados::IoCtx &ioctx, const std::string &oid, uint8_t order,
-           uint8_t splay);
+           uint8_t splay, int64_t pool_id);
 
 void get_immutable_metadata(librados::IoCtx &ioctx, const std::string &oid,
                             uint8_t *order, uint8_t *splay_width,
-                            Context *on_finish);
+                           int64_t *pool_id, Context *on_finish);
 void get_mutable_metadata(librados::IoCtx &ioctx, const std::string &oid,
                           uint64_t *minimum_set, uint64_t *active_set,
                           std::set<cls::journal::Client> *clients,

diff --git a/src/journal/JournalMetadata.cc b/src/journal/JournalMetadata.cc
index 0da06ed6be59323e60273464b2328a3cd61fa852..409b4b98ed1085ea27777c285ddec8fa67d63e56 100644 (file)
--- a/src/journal/JournalMetadata.cc
+++ b/src/journal/JournalMetadata.cc
@@ -23,8 +23,8 @@ JournalMetadata::JournalMetadata(librados::IoCtx &ioctx,
                                  double commit_interval)
     : RefCountedObject(NULL, 0), m_cct(NULL), m_oid(oid),
       m_client_id(client_id), m_commit_interval(commit_interval), m_order(0),
-      m_splay_width(0), m_initialized(false), m_finisher(NULL), m_timer(NULL),
-      m_timer_lock("JournalMetadata::m_timer_lock"),
+      m_splay_width(0), m_pool_id(-1), m_initialized(false), m_finisher(NULL),
+      m_timer(NULL), m_timer_lock("JournalMetadata::m_timer_lock"),
       m_lock("JournalMetadata::m_lock"), m_commit_tid(0), m_watch_ctx(this),
       m_watch_handle(0), m_minimum_set(0), m_active_set(0),
       m_update_notifications(0), m_commit_position_ctx(NULL),
@@ -59,7 +59,7 @@ void JournalMetadata::init(Context *on_init) {
 
   C_ImmutableMetadata *ctx = new C_ImmutableMetadata(this, on_init);
   client::get_immutable_metadata(m_ioctx, m_oid, &m_order, &m_splay_width,
-                                 ctx);
+                                 &m_pool_id, ctx);
 }
 
 void JournalMetadata::shutdown() {
@@ -479,6 +479,7 @@ std::ostream &operator<<(std::ostream &os,
      << "initialized=" << jm.m_initialized << ", "
      << "order=" << (int)jm.m_order << ", "
      << "splay_width=" << (int)jm.m_splay_width << ", "
+     << "pool_id=" << jm.m_pool_id << ", "
      << "minimum_set=" << jm.m_minimum_set << ", "
      << "active_set=" << jm.m_active_set << ", "
      << "client_id=" << jm.m_client_id << ", "

diff --git a/src/journal/JournalMetadata.h b/src/journal/JournalMetadata.h
index 76ab695ab1d783f7e7cd24b44dfd22985800d98b..d15bbca7ba1d67663bb9b86ed659dd4fa223dd0b 100644 (file)
--- a/src/journal/JournalMetadata.h
+++ b/src/journal/JournalMetadata.h
@@ -63,6 +63,9 @@ public:
   inline uint8_t get_splay_width() const {
     return m_splay_width;
   }
+  inline int64_t get_pool_id() const {
+    return m_pool_id;
+  }
 
   inline Finisher &get_finisher() {
     return *m_finisher;
@@ -259,6 +262,7 @@ private:
 
   uint8_t m_order;
   uint8_t m_splay_width;
+  int64_t m_pool_id;
   bool m_initialized;
 
   Finisher *m_finisher;

diff --git a/src/journal/Journaler.cc b/src/journal/Journaler.cc
index d758690881aa16fd0e830a8ceab4f4936ec3e113..5bf732f974f8386eaeac59701eec86e2c69e40a4 100644 (file)
--- a/src/journal/Journaler.cc
+++ b/src/journal/Journaler.cc
@@ -42,14 +42,13 @@ struct C_DeleteRecorder : public Context {
 
 using namespace cls::journal;
 
-Journaler::Journaler(librados::IoCtx &header_ioctx, librados::IoCtx &data_ioctx,
-                     const std::string &journal_id,
-                     const std::string &client_id)
+Journaler::Journaler(librados::IoCtx &header_ioctx,
+                    const std::string &journal_id,
+                    const std::string &client_id)
   : m_client_id(client_id), m_metadata(NULL), m_player(NULL), m_recorder(NULL),
     m_trimmer(NULL)
 {
   m_header_ioctx.dup(header_ioctx);
-  m_data_ioctx.dup(data_ioctx);
   m_cct = reinterpret_cast<CephContext *>(m_header_ioctx.cct());
 
   m_header_oid = JOURNAL_HEADER_PREFIX + journal_id;
@@ -59,8 +58,6 @@ Journaler::Journaler(librados::IoCtx &header_ioctx, librados::IoCtx &data_ioctx,
   m_metadata = new JournalMetadata(m_header_ioctx, m_header_oid, m_client_id,
                                    5);
   m_metadata->get();
-
-  m_trimmer = new JournalTrimmer(m_data_ioctx, m_object_oid_prefix, m_metadata);
 }
 
 Journaler::~Journaler() {
@@ -74,10 +71,34 @@ Journaler::~Journaler() {
 }
 
 void Journaler::init(Context *on_init) {
-  m_metadata->init(on_init);
+  m_metadata->init(new C_InitJournaler(this, on_init));
+}
+
+int Journaler::init_complete() {
+  int64_t pool_id = m_metadata->get_pool_id();
+
+  if (pool_id < 0 || pool_id == m_header_ioctx.get_id()) {
+    ldout(m_cct, 20) << "using image pool for journal data" << dendl;
+    m_data_ioctx.dup(m_header_ioctx);
+  } else {
+    ldout(m_cct, 20) << "using pool id=" << pool_id << " for journal data"
+                    << dendl;
+    librados::Rados rados(m_header_ioctx);
+    int r = rados.ioctx_create2(pool_id, m_data_ioctx);
+    if (r < 0) {
+      if (r == -ENOENT) {
+       ldout(m_cct, 1) << "pool id=" << pool_id << " no longer exists"
+                       << dendl;
+      }
+      return r;
+    }
+  }
+  m_trimmer = new JournalTrimmer(m_data_ioctx, m_object_oid_prefix,
+                                 m_metadata);
+  return 0;
 }
 
-int Journaler::create(uint8_t order, uint8_t splay_width) {
+int Journaler::create(uint8_t order, uint8_t splay_width, int64_t pool_id) {
   if (order > 64 || order < 12) {
     lderr(m_cct) << "order must be in the range [12, 64]" << dendl;
     return -EDOM;
@@ -87,7 +108,8 @@ int Journaler::create(uint8_t order, uint8_t splay_width) {
   }
 
   ldout(m_cct, 5) << "creating new journal: " << m_header_oid << dendl;
-  int r = client::create(m_header_ioctx, m_header_oid, order, splay_width);
+  int r = client::create(m_header_ioctx, m_header_oid, order, splay_width,
+                        pool_id);
   if (r < 0) {
     lderr(m_cct) << "failed to create journal: " << cpp_strerror(r) << dendl;
     return r;

diff --git a/src/journal/Journaler.h b/src/journal/Journaler.h
index 676e994a5df7893ec97946c893e61c28506c7d62..fa90b40e8e799e91f25e47f23ac63b112407b8bd 100644 (file)
--- a/src/journal/Journaler.h
+++ b/src/journal/Journaler.h
@@ -6,13 +6,13 @@
 
 #include "include/int_types.h"
 #include "include/buffer.h"
+#include "include/Context.h"
 #include "include/rados/librados.hpp"
 #include "journal/Future.h"
 #include <string>
 #include <map>
 #include "include/assert.h"
 
-class Context;
 class SafeTimer;
 
 namespace journal {
@@ -26,11 +26,11 @@ class ReplayHandler;
 
 class Journaler {
 public:
-  Journaler(librados::IoCtx &header_ioctx, librados::IoCtx &data_ioctx,
-            const std::string &journal_id, const std::string &client_id);
+  Journaler(librados::IoCtx &header_ioctx, const std::string &journal_id,
+           const std::string &client_id);
   ~Journaler();
 
-  int create(uint8_t order, uint8_t splay_width);
+  int create(uint8_t order, uint8_t splay_width, int64_t pool_id);
   int remove();
 
   void init(Context *on_init);
@@ -52,6 +52,20 @@ public:
   void committed(const Future &future);
 
 private:
+  struct C_InitJournaler : public Context {
+    Journaler *journaler;
+    Context *on_safe;
+    C_InitJournaler(Journaler *_journaler, Context *_on_safe)
+      : journaler(_journaler), on_safe(_on_safe) {
+    }
+    virtual void finish(int r) {
+      if (r == 0) {
+       r = journaler->init_complete();
+      }
+      on_safe->complete(r);
+    }
+  };
+
   librados::IoCtx m_header_ioctx;
   librados::IoCtx m_data_ioctx;
   CephContext *m_cct;
@@ -65,6 +79,7 @@ private:
   JournalRecorder *m_recorder;
   JournalTrimmer *m_trimmer;
 
+  int init_complete();
   void create_player(ReplayHandler *replay_handler);
 
   friend std::ostream &operator<<(std::ostream &os,

diff --git a/src/test/cls_journal/test_cls_journal.cc b/src/test/cls_journal/test_cls_journal.cc
index 18375645b3792e494d56bbd4b861f2d061cbf27b..b6405ff0fa99143a5e2aa69fd3c4a35fffbd318f 100644 (file)
--- a/src/test/cls_journal/test_cls_journal.cc
+++ b/src/test/cls_journal/test_cls_journal.cc
@@ -47,16 +47,19 @@ TEST_F(TestClsJournal, Create) {
 
   uint8_t order = 1;
   uint8_t splay_width = 2;
-  ASSERT_EQ(0, client::create(ioctx, oid, order, splay_width));
+  int64_t pool_id = ioctx.get_id();
+  ASSERT_EQ(0, client::create(ioctx, oid, order, splay_width, pool_id));
 
   uint8_t read_order;
   uint8_t read_splay_width;
+  int64_t read_pool_id;
   C_SaferCond cond;
   client::get_immutable_metadata(ioctx, oid, &read_order, &read_splay_width,
-                                 &cond);
+                                 &read_pool_id, &cond);
   ASSERT_EQ(0, cond.wait());
   ASSERT_EQ(order, read_order);
   ASSERT_EQ(splay_width, read_splay_width);
+  ASSERT_EQ(pool_id, read_pool_id);
 }
 
 TEST_F(TestClsJournal, MinimumSet) {
@@ -65,7 +68,7 @@ TEST_F(TestClsJournal, MinimumSet) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4, ioctx.get_id()));
 
   librados::ObjectWriteOperation op1;
   client::set_active_set(&op1, 300);
@@ -92,7 +95,7 @@ TEST_F(TestClsJournal, MinimumSetStale) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4, ioctx.get_id()));
 
   librados::ObjectWriteOperation op1;
   client::set_active_set(&op1, 300);
@@ -123,7 +126,7 @@ TEST_F(TestClsJournal, MinimumSetOrderConstraint) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4, ioctx.get_id()));
 
   librados::ObjectWriteOperation op1;
   client::set_minimum_set(&op1, 123);
@@ -145,7 +148,7 @@ TEST_F(TestClsJournal, ActiveSet) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4, ioctx.get_id()));
 
   uint64_t active_set = 234;
   librados::ObjectWriteOperation op1;
@@ -168,7 +171,7 @@ TEST_F(TestClsJournal, ActiveSetStale) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4, ioctx.get_id()));
 
   librados::ObjectWriteOperation op1;
   client::set_active_set(&op1, 345);
@@ -185,8 +188,8 @@ TEST_F(TestClsJournal, CreateDuplicate) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4));
-  ASSERT_EQ(-EEXIST, client::create(ioctx, oid, 3, 5));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 4, ioctx.get_id()));
+  ASSERT_EQ(-EEXIST, client::create(ioctx, oid, 3, 5, ioctx.get_id()));
 }
 
 TEST_F(TestClsJournal, ClientRegister) {
@@ -241,7 +244,7 @@ TEST_F(TestClsJournal, ClientCommit) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 2));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 2, ioctx.get_id()));
   ASSERT_EQ(0, client::client_register(ioctx, oid, "id1", "desc1"));
 
   cls::journal::EntryPositions entry_positions;
@@ -269,7 +272,7 @@ TEST_F(TestClsJournal, ClientCommitInvalid) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 2, 2));
+  ASSERT_EQ(0, client::create(ioctx, oid, 2, 2, ioctx.get_id()));
   ASSERT_EQ(0, client::client_register(ioctx, oid, "id1", "desc1"));
 
   cls::journal::EntryPositions entry_positions;
@@ -304,7 +307,7 @@ TEST_F(TestClsJournal, ClientList) {
 
   std::string oid = get_temp_image_name();
 
-  ASSERT_EQ(0, client::create(ioctx, oid, 12, 5));
+  ASSERT_EQ(0, client::create(ioctx, oid, 12, 5, ioctx.get_id()));
 
   std::set<Client> expected_clients;
   librados::ObjectWriteOperation op1;

diff --git a/src/test/journal/RadosTestFixture.cc b/src/test/journal/RadosTestFixture.cc
index 36282fd3f1b3a57ae9a149c6b7f3a307e627a4a5..d5f1b3262b6adbf6e7f96216a56fbeba9a49078c 100644 (file)
--- a/src/test/journal/RadosTestFixture.cc
+++ b/src/test/journal/RadosTestFixture.cc
@@ -40,7 +40,7 @@ void RadosTestFixture::TearDown() {
 
 int RadosTestFixture::create(const std::string &oid, uint8_t order,
                              uint8_t splay_width) {
-  return cls::journal::client::create(m_ioctx, oid, order, splay_width);
+  return cls::journal::client::create(m_ioctx, oid, order, splay_width, -1);
 }
 
 int RadosTestFixture::append(const std::string &oid, const bufferlist &bl) {

diff --git a/src/test/journal/test_Journaler.cc b/src/test/journal/test_Journaler.cc
index 41b3071761fd6a5576c2fbe030e33b2a94983355..4b81bbf9933275edd73967c84d238e3beb7d5cb5 100644 (file)
--- a/src/test/journal/test_Journaler.cc
+++ b/src/test/journal/test_Journaler.cc
@@ -20,8 +20,7 @@ public:
   virtual void SetUp() {
     RadosTestFixture::SetUp();
     m_journal_id = get_temp_journal_id();
-    m_journaler = new journal::Journaler(m_ioctx, m_ioctx, m_journal_id,
-                                         CLIENT_ID);
+    m_journaler = new journal::Journaler(m_ioctx, m_journal_id, CLIENT_ID);
   }
 
   virtual void TearDown() {
@@ -30,7 +29,7 @@ public:
   }
 
   int create_journal(uint8_t order, uint8_t splay_width) {
-    return m_journaler->create(order, splay_width);
+    return m_journaler->create(order, splay_width, -1);
   }
 
   int init_journaler() {
@@ -40,7 +39,7 @@ public:
   }
 
   int register_client(const std::string &client_id, const std::string &desc) {
-    journal::Journaler journaler(m_ioctx, m_ioctx, m_journal_id, client_id);
+    journal::Journaler journaler(m_ioctx, m_journal_id, client_id);
     return journaler.register_client(desc);
   }