rgw: keep max_marker on bucket index

and expose it via the REST interface

Signed-off-by: Yehuda Sadeh <yehuda@inktank.com>

diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc
index 5b52bbe2269cad4d04b3e1cab0d8e83bb5907deb..c438c42f28205c46df4f6389bb8248c0ba5ed74d 100644 (file)
--- a/src/cls/rgw/cls_rgw.cc
+++ b/src/cls/rgw/cls_rgw.cc
@@ -108,7 +108,8 @@ static void bi_log_index_key(cls_method_context_t hctx, string& key, string& id,
 
 static int log_index_operation(cls_method_context_t hctx, string& obj, RGWModifyOp op,
                                string& tag, utime_t& timestamp,
-                               rgw_bucket_entry_ver& ver, RGWPendingState state, uint64_t index_ver)
+                               rgw_bucket_entry_ver& ver, RGWPendingState state, uint64_t index_ver,
+                               string& max_marker)
 {
   bufferlist bl;
 
@@ -127,6 +128,9 @@ static int log_index_operation(cls_method_context_t hctx, string& obj, RGWModify
 
   ::encode(entry, bl);
 
+  if (entry.id > max_marker)
+    max_marker = entry.id;
+
   return cls_cxx_map_set_val(hctx, key, &bl);
 }
 
@@ -470,7 +474,8 @@ int rgw_bucket_prepare_op(cls_method_context_t hctx, bufferlist *in, bufferlist
   }
 
   if (op.log_op) {
-    rc = log_index_operation(hctx, op.name, op.op, op.tag, entry.meta.mtime, entry.ver, info.state, header.ver);
+    rc = log_index_operation(hctx, op.name, op.op, op.tag, entry.meta.mtime,
+                             entry.ver, info.state, header.ver, header.max_marker);
     if (rc < 0)
       return rc;
   }
@@ -584,7 +589,8 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist
   bufferlist op_bl;
   if (cancel) {
     if (op.log_op) {
-      rc = log_index_operation(hctx, op.name, op.op, op.tag, entry.meta.mtime, entry.ver, CLS_RGW_STATE_COMPLETE, header.ver);
+      rc = log_index_operation(hctx, op.name, op.op, op.tag, entry.meta.mtime, entry.ver,
+                               CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker);
       if (rc < 0)
         return rc;
     }
@@ -643,7 +649,8 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist
   }
 
   if (op.log_op) {
-    rc = log_index_operation(hctx, op.name, op.op, op.tag, entry.meta.mtime, entry.ver, CLS_RGW_STATE_COMPLETE, header.ver);
+    rc = log_index_operation(hctx, op.name, op.op, op.tag, entry.meta.mtime, entry.ver,
+                             CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker);
     if (rc < 0)
       return rc;
   }
@@ -664,7 +671,7 @@ int rgw_bucket_complete_op(cls_method_context_t hctx, bufferlist *in, bufferlist
 
     if (op.log_op) {
       rc = log_index_operation(hctx, op.name, CLS_RGW_OP_DEL, op.tag, remove_entry.meta.mtime,
-                               remove_entry.ver, CLS_RGW_STATE_COMPLETE, header.ver);
+                               remove_entry.ver, CLS_RGW_STATE_COMPLETE, header.ver, header.max_marker);
       if (rc < 0)
         continue;
     }

diff --git a/src/cls/rgw/cls_rgw_types.h b/src/cls/rgw/cls_rgw_types.h
index 3f7abaff857bf6138cae128a4f6257e206ff5a81..b5270da33081ec5d897dca296c162a34912fdabc 100644 (file)
--- a/src/cls/rgw/cls_rgw_types.h
+++ b/src/cls/rgw/cls_rgw_types.h
@@ -321,15 +321,17 @@ struct rgw_bucket_dir_header {
   uint64_t tag_timeout;
   uint64_t ver;
   uint64_t master_ver;
+  string max_marker;
 
   rgw_bucket_dir_header() : tag_timeout(0), ver(0), master_ver(0) {}
 
   void encode(bufferlist &bl) const {
-    ENCODE_START(4, 2, bl);
+    ENCODE_START(5, 2, bl);
     ::encode(stats, bl);
     ::encode(tag_timeout, bl);
     ::encode(ver, bl);
     ::encode(master_ver, bl);
+    ::encode(max_marker, bl);
     ENCODE_FINISH(bl);
   }
   void decode(bufferlist::iterator &bl) {
@@ -346,6 +348,9 @@ struct rgw_bucket_dir_header {
     } else {
       ver = 0;
     }
+    if (struct_v >= 5) {
+      ::decode(max_marker, bl);
+    }
     DECODE_FINISH(bl);
   }
   void dump(Formatter *f) const;

diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc
index 7a266b9c00af909031fd81bee5bc5110d27ca505..24e27c17a5bfc6ad855df37af54934b95b9efa43 100644 (file)
--- a/src/rgw/rgw_admin.cc
+++ b/src/rgw/rgw_admin.cc
@@ -500,7 +500,8 @@ int bucket_stats(rgw_bucket& bucket, Formatter *formatter)
 
   map<RGWObjCategory, RGWBucketStats> stats;
   uint64_t bucket_ver, master_ver;
-  int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats);
+  string max_marker;
+  int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, &max_marker);
   if (ret < 0) {
     cerr << "error getting bucket stats ret=" << ret << std::endl;
     return ret;
@@ -516,6 +517,7 @@ int bucket_stats(rgw_bucket& bucket, Formatter *formatter)
   formatter->dump_int("mtime", mtime);
   formatter->dump_int("ver", bucket_ver);
   formatter->dump_int("master_ver", master_ver);
+  formatter->dump_string("max_marker", max_marker);
   dump_bucket_usage(stats, formatter);
   formatter->close_section();
 

diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc
index f7efd266047fb271a4b30d9216671c891fe373b6..fc364cfec8dd0aeef746467a0a5ff0f77c66f015 100644 (file)
--- a/src/rgw/rgw_bucket.cc
+++ b/src/rgw/rgw_bucket.cc
@@ -304,7 +304,7 @@ int rgw_remove_bucket(RGWRados *store, rgw_bucket& bucket, bool delete_children)
 
   uint64_t bucket_ver, master_ver;
 
-  ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats);
+  ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, NULL);
   if (ret < 0)
     return ret;
 
@@ -874,7 +874,8 @@ static int bucket_stats(RGWRados *store, std::string&  bucket_name, Formatter *f
   bucket = bucket_info.bucket;
 
   uint64_t bucket_ver, master_ver;
-  int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats);
+  string max_marker;
+  int ret = store->get_bucket_stats(bucket, &bucket_ver, &master_ver, stats, &max_marker);
   if (ret < 0) {
     cerr << "error getting bucket stats ret=" << ret << std::endl;
     return ret;
@@ -890,6 +891,7 @@ static int bucket_stats(RGWRados *store, std::string&  bucket_name, Formatter *f
   formatter->dump_int("ver", bucket_ver);
   formatter->dump_int("master_ver", master_ver);
   formatter->dump_int("mtime", mtime);
+  formatter->dump_string("max_marker", max_marker);
   dump_bucket_usage(stats, formatter);
   formatter->close_section();
 

diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc
index 40867432fe43bbb2f75879b1066fdcdc43f385bc..d1fdce17d58513c121a27b87816795eab37d4050 100644 (file)
--- a/src/rgw/rgw_rados.cc
+++ b/src/rgw/rgw_rados.cc
@@ -4479,7 +4479,8 @@ int RGWRados::obj_stat(void *ctx, rgw_obj& obj, uint64_t *psize, time_t *pmtime,
   return 0;
 }
 
-int RGWRados::get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWBucketStats>& stats)
+int RGWRados::get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWBucketStats>& stats,
+                               string *max_marker)
 {
   rgw_bucket_dir_header header;
   int r = cls_bucket_head(bucket, header);
@@ -4493,6 +4494,9 @@ int RGWRados::get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_
   *bucket_ver = header.ver;
   *master_ver = header.master_ver;
 
+  if (max_marker)
+    *max_marker = header.max_marker;
+
   return 0;
 }
 

diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h
index 3c91153db773f3eea2d7cce28824d6809d5ade47..3e747d2b8fb17eee908ca01fa2b5df45bcd1e456 100644 (file)
--- a/src/rgw/rgw_rados.h
+++ b/src/rgw/rgw_rados.h
@@ -1279,7 +1279,8 @@ public:
   }
 
   int decode_policy(bufferlist& bl, ACLOwner *owner);
-  int get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWBucketStats>& stats);
+  int get_bucket_stats(rgw_bucket& bucket, uint64_t *bucket_ver, uint64_t *master_ver, map<RGWObjCategory, RGWBucketStats>& stats,
+                       string *max_marker);
   void get_bucket_instance_entry(rgw_bucket& bucket, string& entry);
   void get_bucket_meta_oid(rgw_bucket& bucket, string& oid);
 

diff --git a/src/rgw/rgw_rest_log.cc b/src/rgw/rgw_rest_log.cc
index 614175f57b5a8eff5ef3e4df3073456bab7a31a2..544adbe796556b3751b5dcfcd6c109f409d5040a 100644 (file)
--- a/src/rgw/rgw_rest_log.cc
+++ b/src/rgw/rgw_rest_log.cc
@@ -345,6 +345,55 @@ void RGWOp_BILog_List::send_response_end() {
   flusher.flush();
 }
       
+void RGWOp_BILog_Info::execute() {
+  string bucket_name = s->info.args.get("bucket"),
+         bucket_instance = s->info.args.get("bucket-instance");
+  RGWBucketInfo bucket_info;
+
+  if (bucket_name.empty() && bucket_instance.empty()) {
+    dout(5) << "ERROR: neither bucket nor bucket instance specified" << dendl;
+    http_ret = -EINVAL;
+    return;
+  }
+
+  if (!bucket_instance.empty()) {
+    http_ret = store->get_bucket_instance_info(NULL, bucket_instance, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket instance info for bucket instance id=" << bucket_instance << dendl;
+      return;
+    }
+  } else { /* !bucket_name.empty() */
+    http_ret = store->get_bucket_info(NULL, bucket_name, bucket_info, NULL, NULL);
+    if (http_ret < 0) {
+      dout(5) << "could not get bucket info for bucket=" << bucket_name << dendl;
+      return;
+    }
+  }
+  map<RGWObjCategory, RGWBucketStats> stats;
+  int ret =  store->get_bucket_stats(bucket_info.bucket, &bucket_ver, &master_ver, stats, &max_marker);
+  if (ret < 0 && ret != -ENOENT) {
+    http_ret = ret;
+    return;
+  }
+}
+
+void RGWOp_BILog_Info::send_response() {
+  set_req_state_err(s, http_ret);
+  dump_errno(s);
+  end_header(s);
+
+  if (http_ret < 0)
+    return;
+
+  s->formatter->open_object_section("info");
+  encode_json("bucket_ver", bucket_ver, s->formatter);
+  encode_json("master_ver", master_ver, s->formatter);
+  encode_json("max_marker", max_marker, s->formatter);
+  s->formatter->close_section();
+
+  flusher.flush();
+}
+
 void RGWOp_BILog_Delete::execute() {
   string bucket_name = s->info.args.get("bucket"),
          start_marker = s->info.args.get("start-marker"),
@@ -613,7 +662,11 @@ RGWOp *RGWHandler_Log::op_get() {
       return new RGWOp_MDLog_Info;
     }
   } else if (type.compare("bucket-index") == 0) {
-    return new RGWOp_BILog_List;
+    if (s->info.args.exists("info")) {
+      return new RGWOp_BILog_Info;
+    } else {
+      return new RGWOp_BILog_List;
+    }
   } else if (type.compare("data") == 0) {
     if (s->info.args.exists("id")) {
       if (s->info.args.exists("info")) {

diff --git a/src/rgw/rgw_rest_log.h b/src/rgw/rgw_rest_log.h
index ec340d66482af96aad66c92bc50a52d9e928c7d5..38c6b5fb4abfffd95ad44e428b68e4c267603778 100644 (file)
--- a/src/rgw/rgw_rest_log.h
+++ b/src/rgw/rgw_rest_log.h
@@ -17,10 +17,9 @@
 #include "rgw_metadata.h"
 
 class RGWOp_BILog_List : public RGWRESTOp {
-  int http_ret;
   bool sent_header;
 public:
-  RGWOp_BILog_List() : http_ret(0), sent_header(false) {}
+  RGWOp_BILog_List() : sent_header(false) {}
   ~RGWOp_BILog_List() {}
 
   int check_caps(RGWUserCaps& caps) {
@@ -34,7 +33,28 @@ public:
   virtual void send_response_end();
   void execute();
   virtual const char *name() {
-    return "list bucket index log";
+    return "list_bucket_index_log";
+  }
+};
+
+class RGWOp_BILog_Info : public RGWRESTOp {
+  uint64_t bucket_ver;
+  uint64_t master_ver;
+  string max_marker;
+public:
+  RGWOp_BILog_Info() : bucket_ver(0), master_ver(0) {}
+  ~RGWOp_BILog_Info() {}
+
+  int check_caps(RGWUserCaps& caps) {
+    return caps.check_cap("bilog", RGW_CAP_READ);
+  }
+  int verify_permission() {
+    return check_caps(s->user.caps);
+  }
+  virtual void send_response();
+  void execute();
+  virtual const char *name() {
+    return "bucket_index_log_info";
   }
 };