rgw: can list mdlog entries

Signed-off-by: Yehuda Sadeh <yehuda@redhat.com>

diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc
index 621e55aad07504038a50609b29587c88aeeb05b6..9e4116fe976908bbd1859f08d00dde8b130a2709 100644 (file)
--- a/src/rgw/rgw_metadata.cc
+++ b/src/rgw/rgw_metadata.cc
@@ -10,69 +10,78 @@
 
 #define dout_subsys ceph_subsys_rgw
 
-struct LogStatusDump {
-  RGWMDLogStatus status;
-
-  explicit LogStatusDump(RGWMDLogStatus _status) : status(_status) {}
-  void dump(Formatter *f) const {
-    string s;
-    switch (status) {
-      case MDLOG_STATUS_WRITE:
-        s = "write";
-        break;
-      case MDLOG_STATUS_SETATTRS:
-        s = "set_attrs";
-        break;
-      case MDLOG_STATUS_REMOVE:
-        s = "remove";
-        break;
-      case MDLOG_STATUS_COMPLETE:
-        s = "complete";
-        break;
-      case MDLOG_STATUS_ABORT:
-        s = "abort";
-        break;
-      default:
-        s = "unknown";
-        break;
-    }
-    encode_json("status", s, f);
-  }
-};
-
-struct RGWMetadataLogData {
-  obj_version read_version;
-  obj_version write_version;
-  RGWMDLogStatus status;
-  
-  RGWMetadataLogData() : status(MDLOG_STATUS_UNKNOWN) {}
-
-  void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
-    ::encode(read_version, bl);
-    ::encode(write_version, bl);
-    uint32_t s = (uint32_t)status;
-    ::encode(s, bl);
-    ENCODE_FINISH(bl);
+void LogStatusDump::dump(Formatter *f) const {
+  string s;
+  switch (status) {
+    case MDLOG_STATUS_WRITE:
+      s = "write";
+      break;
+    case MDLOG_STATUS_SETATTRS:
+      s = "set_attrs";
+      break;
+    case MDLOG_STATUS_REMOVE:
+      s = "remove";
+      break;
+    case MDLOG_STATUS_COMPLETE:
+      s = "complete";
+      break;
+    case MDLOG_STATUS_ABORT:
+      s = "abort";
+      break;
+    default:
+      s = "unknown";
+      break;
   }
-
-  void decode(bufferlist::iterator& bl) {
-     DECODE_START(1, bl);
-     ::decode(read_version, bl);
-     ::decode(write_version, bl);
-     uint32_t s;
-     ::decode(s, bl);
-     status = (RGWMDLogStatus)s;
-     DECODE_FINISH(bl);
+  encode_json("status", s, f);
+}
+
+void RGWMetadataLogData::encode(bufferlist& bl) const {
+  ENCODE_START(1, 1, bl);
+  ::encode(read_version, bl);
+  ::encode(write_version, bl);
+  uint32_t s = (uint32_t)status;
+  ::encode(s, bl);
+  ENCODE_FINISH(bl);
+}
+
+void RGWMetadataLogData::decode(bufferlist::iterator& bl) {
+   DECODE_START(1, bl);
+   ::decode(read_version, bl);
+   ::decode(write_version, bl);
+   uint32_t s;
+   ::decode(s, bl);
+   status = (RGWMDLogStatus)s;
+   DECODE_FINISH(bl);
+}
+
+void RGWMetadataLogData::dump(Formatter *f) const {
+  encode_json("read_version", read_version, f);
+  encode_json("write_version", write_version, f);
+  encode_json("status", LogStatusDump(status), f);
+}
+
+void decode_json_obj(RGWMDLogStatus& status, JSONObj *obj) {
+  string s = obj->get_data();
+  if (s == "complete") {
+    status = MDLOG_STATUS_COMPLETE;
+  } else if (s == "write") {
+    status = MDLOG_STATUS_WRITE;
+  } else if (s == "remove") {
+    status = MDLOG_STATUS_REMOVE;
+  } else if (s == "set_attrs") {
+    status = MDLOG_STATUS_SETATTRS;
+  } else if (s == "abort") {
+    status = MDLOG_STATUS_ABORT;
+  } else {
+    status = MDLOG_STATUS_UNKNOWN;
   }
+}
 
-  void dump(Formatter *f) const {
-    encode_json("read_version", read_version, f);
-    encode_json("write_version", write_version, f);
-    encode_json("status", LogStatusDump(status), f);
-  }
-};
-WRITE_CLASS_ENCODER(RGWMetadataLogData)
+void RGWMetadataLogData::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("read_version", read_version, obj);
+  JSONDecoder::decode_json("write_version", write_version, obj);
+  JSONDecoder::decode_json("status", status, obj);
+}
 
 
 int RGWMetadataLog::add_entry(RGWRados *store, RGWMetadataHandler *handler, const string& section, const string& key, bufferlist& bl) {

diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h
index 8c950f2de3fbfbb700663b4c01b6f4b82d22b7a3..de40bd8862037aa46deebec0613f992661faa703 100644 (file)
--- a/src/rgw/rgw_metadata.h
+++ b/src/rgw/rgw_metadata.h
@@ -178,7 +178,26 @@ public:
   int unlock(int shard_id, string& zone_id, string& owner_id);
 };
 
-struct RGWMetadataLogData;
+struct LogStatusDump {
+  RGWMDLogStatus status;
+
+  explicit LogStatusDump(RGWMDLogStatus _status) : status(_status) {}
+  void dump(Formatter *f) const;
+};
+
+struct RGWMetadataLogData {
+  obj_version read_version;
+  obj_version write_version;
+  RGWMDLogStatus status;
+  
+  RGWMetadataLogData() : status(MDLOG_STATUS_UNKNOWN) {}
+
+  void encode(bufferlist& bl) const;
+  void decode(bufferlist::iterator& bl);
+  void dump(Formatter *f) const;
+  void decode_json(JSONObj *obj);
+};
+WRITE_CLASS_ENCODER(RGWMetadataLogData)
 
 class RGWMetadataManager {
   map<string, RGWMetadataHandler *> handlers;

diff --git a/src/rgw/rgw_rest_conn.h b/src/rgw/rgw_rest_conn.h
index 7598a2a40932ffb14eaa13f2bab37b1b05cf4f09..6c7a2bb9beeb00f06683402d02ef13cf04c277e9 100644 (file)
--- a/src/rgw/rgw_rest_conn.h
+++ b/src/rgw/rgw_rest_conn.h
@@ -29,6 +29,11 @@ static int parse_decode_json(CephContext *cct, T& t, bufferlist& bl)
   return 0;
 }
 
+struct rgw_http_param_pair {
+  const char *key;
+  const char *val;
+};
+
 class RGWRESTConn
 {
   CephContext *cct;
@@ -60,6 +65,8 @@ public:
 
   template <class T>
   int get_json_resource(const string& resource, list<pair<string, string> > *params, T& t);
+  template <class T>
+  int get_json_resource(const string& resource, const rgw_http_param_pair *pp, T& t);
 };
 
 
@@ -80,4 +87,19 @@ int RGWRESTConn::get_json_resource(const string& resource, list<pair<string, str
   return 0;
 }
 
+template<class T>
+int RGWRESTConn::get_json_resource(const string& resource,  const rgw_http_param_pair *pp, T& t)
+{
+  list<pair<string, string> > params;
+
+  while (pp && pp->key) {
+    string k = pp->key;
+    string v = (pp->val ? pp->val : "");
+    params.push_back(make_pair(k, v));
+    ++pp;
+  }
+
+  return get_json_resource(resource, &params, t);
+}
+
 #endif

diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc
index 13d7ae2bc137940f30a7a62c4402111bf154ba49..6c5594dc299e278bfceb0cef8c1dc4f6a1ccf1e0 100644 (file)
--- a/src/rgw/rgw_sync.cc
+++ b/src/rgw/rgw_sync.cc
@@ -3,6 +3,7 @@
 #include "rgw_common.h"
 #include "rgw_rados.h"
 #include "rgw_sync.h"
+#include "rgw_metadata.h"
 
 
 #define dout_subsys ceph_subsys_rgw
@@ -11,45 +12,89 @@ void rgw_mdlog_info::decode_json(JSONObj *obj) {
   JSONDecoder::decode_json("num_objects", num_shards, obj);
 }
 
-template <class T>
-static int parse_decode_json(T& t, bufferlist& bl)
+struct rgw_mdlog_entry {
+  string id;
+  string section;
+  string name;
+  utime_t timestamp;
+  RGWMetadataLogData log_data;
+
+  void decode_json(JSONObj *obj);
+};
+
+struct rgw_mdlog_shard_data {
+  string marker;
+  bool truncated;
+  vector<rgw_mdlog_entry> entries;
+
+  void decode_json(JSONObj *obj);
+};
+
+
+void rgw_mdlog_entry::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("id", id, obj);
+  JSONDecoder::decode_json("section", section, obj);
+  JSONDecoder::decode_json("name", name, obj);
+  JSONDecoder::decode_json("timestamp", timestamp, obj);
+  JSONDecoder::decode_json("log_data", log_data, obj);
+}
+
+void rgw_mdlog_shard_data::decode_json(JSONObj *obj) {
+  JSONDecoder::decode_json("marker", marker, obj);
+  JSONDecoder::decode_json("truncated", truncated, obj);
+  JSONDecoder::decode_json("entries", entries, obj);
+};
+
+int RGWRemoteMetaLog::init()
 {
-  JSONParser p;
-  int ret = p.parse(bl.c_str(), bl.length());
+  conn = store->rest_master_conn;
+
+  rgw_http_param_pair pairs[] = { { "type", "metadata" },
+                                  { NULL, NULL } };
+
+  int ret = conn->get_json_resource("/admin/log", pairs, log_info);
   if (ret < 0) {
-    cout << "failed to parse JSON" << std::endl;
+    ldout(store->ctx(), 0) << "ERROR: failed to fetch mdlog info" << dendl;
     return ret;
   }
 
-  try {
-    decode_json_obj(t, &p);
-  } catch (JSONDecoder::err& e) {
-    cout << "failed to decode JSON input: " << e.message << std::endl;
-    return -EINVAL;
+  ldout(store->ctx(), 20) << "remote mdlog, num_shards=" << log_info.num_shards << dendl;
+
+  for (int i = 0; i < log_info.num_shards; i++) {
+    int ret = list_shard(i);
+    if (ret < 0) {
+      ldout(store->ctx(), 10) << "failed to list shard: ret=" << ret << dendl;
+    }
   }
+
   return 0;
 }
 
-
-int RGWRemoteMetaLog::init()
+int RGWRemoteMetaLog::list_shard(int shard_id)
 {
   conn = store->rest_master_conn;
 
-  list<pair<string, string> > params;
-  params.push_back(make_pair("type", "metadata"));
+  char buf[32];
+  snprintf(buf, sizeof(buf), "%d", shard_id);
+
+  rgw_http_param_pair pairs[] = { { "type", "metadata" },
+                                  { "id", buf },
+                                  { NULL, NULL } };
 
-  int ret = conn->get_json_resource("/admin/log", &params, log_info);
+  rgw_mdlog_shard_data data;
+  int ret = conn->get_json_resource("/admin/log", pairs, data);
   if (ret < 0) {
-    ldout(store->ctx(), 0) << "ERROR: failed to fetch mdlog info" << dendl;
+    ldout(store->ctx(), 0) << "ERROR: failed to fetch mdlog data" << dendl;
     return ret;
   }
 
-  ldout(store->ctx(), 20) << "remote mdlog, num_shards=" << log_info.num_shards << dendl;
+  ldout(store->ctx(), 20) << "remote mdlog, shard_id=" << shard_id << " num of shard entries: " << data.entries.size() << dendl;
 
   return 0;
 }
 
 
+
 int RGWMetadataSync::init()
 {
   if (store->is_meta_master()) {