rgw/cloud-restore: admin CLI for restore list and status

Also added stats as part of radosgw-admin bucket stats command

Signed-off-by: Jiffin Tony Thottan <thottanjiffin@gmail.com>

diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst
index 1c171dffdb52c1ccb79c08113dbfbf8d5133e050..ae317aef7dfb5bbe49210de83d2e79a2298a8ca8 100644 (file)
--- a/doc/man/8/radosgw-admin.rst
+++ b/doc/man/8/radosgw-admin.rst
@@ -491,6 +491,12 @@ as follows:
 :command:`topic dump`
   Dump (in JSON format) all pending bucket notifications of a persistent topic
 
+:command:`restore list`
+  List restore status of each object in a bucket
+
+:command:`restore status`
+  Show restoration status of an object in the bucket
+
 Options
 =======
 
@@ -880,6 +886,14 @@ Options
    operations such as finding orphans or checking the bucket index.
    The default is 32.
 
+.. option:: --restore-status
+
+   Filter objects return by the 'restore list' command by status.
+
+.. option:: --show-restore-stats
+
+   Shows restore stats in a bucket stat command. Here the bucket name need be provided.
+
 Quota Options
 =============
 

diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc
index e858f0c4b499836bb3110fdd69e829c4cfc17979..f4248da41ce38746893d09106bf4f0e832883a00 100644 (file)
--- a/src/rgw/driver/rados/rgw_bucket.cc
+++ b/src/rgw/driver/rados/rgw_bucket.cc
@@ -1597,6 +1597,69 @@ static int bucket_stats(rgw::sal::Driver* driver,
   return 0;
 }
 
+static int bucket_restore_stats(rgw::sal::Driver* driver,
+                        const std::string& tenant_name,
+                        const std::string& bucket_name, Formatter* formatter,
+                        const DoutPrefixProvider* dpp, optional_yield y) {
+  std::unique_ptr<rgw::sal::Bucket> bucket;
+  int restore_completed_count = 0;
+  int restore_in_progress_count = 0;
+  int restore_failed_count = 0;
+  int ret = driver->load_bucket(dpp, rgw_bucket(tenant_name, bucket_name),
+                                &bucket, y);
+  if (ret < 0) {
+    return ret;
+  }
+  rgw::sal::Bucket::ListParams params;
+  rgw::sal::Bucket::ListResults results;
+  params.list_versions = bucket->versioned();
+  params.allow_unordered = true;
+  do {
+    ret = bucket->list(dpp, params, listing_max_entries, results, null_yield);
+    if (ret < 0) {
+      cerr << "ERROR: driver->list_objects(): " << cpp_strerror(-ret) << std::endl;
+      return ret;
+    }
+    for (vector<rgw_bucket_dir_entry>::iterator iter = results.objs.begin(); iter != results.objs.end(); ++iter) {
+      std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(iter->key.name);
+      if (obj) {
+        ret = obj->get_obj_attrs(null_yield, dpp);
+        if (ret < 0) {
+          cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl;
+          return ret;
+        }
+        for (map<string, bufferlist>::iterator getattriter = obj->get_attrs().begin(); getattriter != obj->get_attrs().end(); ++getattriter) {
+          bufferlist& bl = getattriter->second;
+          if (getattriter->first == RGW_ATTR_RESTORE_STATUS) {
+            rgw::sal::RGWRestoreStatus rs;
+            try {
+              decode(rs, bl);
+            } catch (const JSONDecoder::err& e) {
+              cerr << "failed to decode JSON input: " << e.what() << std::endl;
+              return -EINVAL;
+            }
+            if (rs == rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) {
+              restore_in_progress_count ++;
+            } else  if (rs == rgw::sal::RGWRestoreStatus::CloudRestored) {
+              restore_completed_count ++;
+            } else if (rs == rgw::sal::RGWRestoreStatus::RestoreFailed) {
+              restore_failed_count ++;
+            }
+          }
+        }
+      }
+    }
+  } while (results.is_truncated);
+  formatter->open_object_section("");
+  formatter->open_object_section("restore_stats");
+  formatter->dump_int("restore_completed_count", restore_completed_count);
+  formatter->dump_int("restore_in_progress_count", restore_in_progress_count);
+  formatter->dump_int("restore_failed_count", restore_failed_count);
+  formatter->close_section();
+  formatter->close_section();
+  return 0;
+}
+
 int RGWBucketAdminOp::limit_check(rgw::sal::Driver* driver,
                                  RGWBucketAdminOpState& op_state,
                                  const std::list<std::string>& user_ids,
@@ -1816,6 +1879,12 @@ int RGWBucketAdminOp::info(rgw::sal::Driver* driver,
     if (ret < 0) {
       return ret;
     }
+    if (op_state.restore_stats) {
+      ret = bucket_restore_stats(driver, user_id.tenant, bucket_name, formatter, dpp, y);
+      if (ret < 0) {
+        return ret;
+      }
+    }
   } else if (op_state.is_user_op()) {
     const rgw_user& uid = op_state.get_user_id();
     auto user = driver->get_user(uid);
@@ -1874,7 +1943,6 @@ int RGWBucketAdminOp::info(rgw::sal::Driver* driver,
       }
     }
     driver->meta_list_keys_complete(handle);
-
     formatter->close_section();
   }
 

diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h
index fb559961b2816e7f94e1f9d60a949c9a76accf3d..2bf40f6f45b86780e9180a88ac1f82b30074c9ce 100644 (file)
--- a/src/rgw/driver/rados/rgw_bucket.h
+++ b/src/rgw/driver/rados/rgw_bucket.h
@@ -237,6 +237,7 @@ struct RGWBucketAdminOpState {
   bool sync_bucket;
   bool dump_keys;
   bool hide_progress;
+  bool restore_stats;
   int max_aio = 0;
   ceph::timespan min_age = std::chrono::hours::zero();
 
@@ -246,6 +247,7 @@ struct RGWBucketAdminOpState {
   RGWRateLimitInfo ratelimit_info;
 
   void set_fetch_stats(bool value) { stat_buckets = value; }
+  void set_restore_stats(bool value) { restore_stats = value; }
   void set_check_objects(bool value) { check_objects = value; }
   void set_fix_index(bool value) { fix_index = value; }
   void set_delete_children(bool value) { delete_child_objects = value; }

diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc
index 61525fa9be7a96dfab374bdce842e520eb7887c8..26c71026ab0b54f2974882a62cc76f4cd7a99915 100644 (file)
--- a/src/rgw/radosgw-admin/radosgw-admin.cc
+++ b/src/rgw/radosgw-admin/radosgw-admin.cc
@@ -358,6 +358,9 @@ void usage()
   cout << "  notification list                list bucket notifications configuration\n";
   cout << "  notification get                 get a bucket notifications configuration\n";
   cout << "  notification rm                  remove a bucket notifications configuration\n";
+  cout << "  restore status                   shows restoration status of object in a bucket\n";
+  cout << "  restore list                     list restore status of each object in the bucket\n";
+  cout <<"                                    can be filtered with help of --restore-status which shows objects with specified status\n";
   cout << "options:\n";
   cout << "   --tenant=<tenant>                 tenant name\n";
   cout << "   --user_ns=<namespace>             namespace of user (oidc in case of users authenticated with oidc provider)\n";
@@ -540,6 +543,7 @@ void usage()
   cout << "\nBucket list objects options:\n";
   cout << "   --max-entries                 max number of entries listed (default 1000)\n";
   cout << "   --marker                      the marker used to specify on which entry the listing begins, default none (i.e., very first entry)\n";
+  cout << "   --show-restore-stats          if the flag is in present it will show restores stats in the bucket stats command\n";
   cout << "\n";
   generic_client_usage();
 }
@@ -920,6 +924,8 @@ enum class OPT {
   ACCOUNT_STATS,
   ACCOUNT_RM,
   ACCOUNT_LIST,
+  RESTORE_STATUS,
+  RESTORE_LIST,
 };
 
 }
@@ -1183,6 +1189,8 @@ static SimpleCmd::Commands all_cmds = {
   { "account stats", OPT::ACCOUNT_STATS },
   { "account rm", OPT::ACCOUNT_RM },
   { "account list", OPT::ACCOUNT_LIST },
+  { "restore status", OPT::RESTORE_STATUS },
+  { "restore list", OPT::RESTORE_LIST },
 };
 
 static SimpleCmd::Aliases cmd_aliases = {
@@ -3818,6 +3826,8 @@ int main(int argc, const char **argv)
   bool raw_storage_op = false;
 
   std::optional<std::string> rgw_obj_fs; // radoslist field separator
+  std::optional<std::string> restore_status_filter;
+  int show_restore_stats = false;
 
   init_realm_param(cct.get(), realm_id, opt_realm_id, "rgw_realm_id");
   init_realm_param(cct.get(), zonegroup_id, opt_zonegroup_id, "rgw_zonegroup_id");
@@ -4385,6 +4395,10 @@ int main(int argc, const char **argv)
       enable_features.insert(val);
     } else if (ceph_argparse_witharg(args, i, &val, "--disable-feature", (char*)NULL)) {
       disable_features.insert(val);
+    } else if (ceph_argparse_witharg(args, i, &val, "--restore-status", (char*)NULL)) {
+      restore_status_filter = val;
+    } else if (ceph_argparse_binary_flag(args, i, &show_restore_stats, NULL, "--show-restore-stats", (char*)NULL)){
+      // do nothing
     } else if (strncmp(*i, "-", 1) == 0) {
       cerr << "ERROR: invalid flag " << *i << std::endl;
       return EINVAL;
@@ -4567,6 +4581,8 @@ int main(int argc, const char **argv)
        OPT::PUBSUB_TOPIC_STATS  ,
        OPT::PUBSUB_TOPIC_DUMP  ,
                         OPT::SCRIPT_GET,
+       OPT::RESTORE_STATUS,
+       OPT::RESTORE_LIST,
     };
 
     std::set<OPT> gc_ops_list = {
@@ -7631,6 +7647,7 @@ int main(int argc, const char **argv)
       bucket_op.max_entries = max_entries;
     else
       bucket_op.max_entries = 0; /* for backward compatibility */
+    bucket_op.set_restore_stats(bool(show_restore_stats));
 
     int r = RGWBucketAdminOp::info(driver, bucket_op, stream_flusher, null_yield, dpp());
     if (r < 0) {
@@ -12209,7 +12226,124 @@ next:
       }
     }
   }
-
+  if (opt_cmd == OPT::RESTORE_STATUS ||
+      opt_cmd == OPT::RESTORE_LIST) {
+        int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket);
+        if (ret < 0) {
+          cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
+          return -ret;
+        }
+        if (opt_cmd == OPT::RESTORE_STATUS) {
+          if (!object.empty()) {
+            std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(object);
+            obj->set_instance(object_version);
+            ret = obj->get_obj_attrs(null_yield, dpp());
+            if (ret < 0) {
+              cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl;
+              return -ret;
+            }
+            formatter->open_object_section("object restore status");
+            formatter->dump_string("name", object);
+            map<string, bufferlist>::iterator iter;
+            for (iter = obj->get_attrs().begin(); iter != obj->get_attrs().end(); ++iter) {
+              bufferlist& bl = iter->second;
+              if (iter->first == RGW_ATTR_RESTORE_STATUS) {
+                rgw::sal::RGWRestoreStatus rs;
+                try {
+                  decode(rs, bl);
+                } catch (const JSONDecoder::err& e) {
+                  cerr << "failed to decode JSON input: " << e.what() << std::endl;
+                  return EINVAL;
+                }
+                formatter->dump_string("RestoreStatus", rgw::sal::rgw_restore_status_dump(rs));
+              } else if (iter->first == RGW_ATTR_RESTORE_TYPE) {
+                rgw::sal::RGWRestoreType rt;
+                try {
+                  decode(rt, bl);
+                } catch (const JSONDecoder::err& e) {
+                  cerr << "failed to decode JSON input: " << e.what() << std::endl;
+                  return EINVAL;
+                }
+                formatter->dump_string("RestoreType", rgw::sal::rgw_restore_type_dump(rt));
+              } else if (iter->first == RGW_ATTR_RESTORE_EXPIRY_DATE) {
+                decode_dump<ceph::real_time>("RestoreExpiryDate", bl, formatter.get());
+              } else if (iter->first == RGW_ATTR_RESTORE_TIME) {
+                decode_dump<ceph::real_time>("RestoreTime", bl, formatter.get());
+              } else if (iter->first == RGW_ATTR_RESTORE_VERSIONED_EPOCH) {
+                uint64_t versioned_epoch;
+                try {
+                  decode(versioned_epoch, bl);
+                } catch (const JSONDecoder::err& e) {
+                  cerr << "failed to decode JSON input: " << e.what() << std::endl;
+                  return EINVAL;
+                }
+                formatter->dump_unsigned("RestoreVersionedEpoch", versioned_epoch);
+              }
+            }
+            formatter->close_section();
+            formatter->flush(cout);
+          }
+        } else if (opt_cmd == OPT::RESTORE_LIST) {
+          int count = 0;
+          int restore_entries;
+          string prefix;
+          string delim;
+          string marker;
+          string ns;
+          rgw::sal::Bucket::ListParams params;
+          rgw::sal::Bucket::ListResults results;
+          params.prefix = prefix;
+          params.delim = delim;
+          params.marker = rgw_obj_key(marker);
+          params.ns = ns;
+          params.enforce_ns = true;
+          if (max_entries_specified) {
+            restore_entries = max_entries;
+          } else {
+            restore_entries = 1000;
+          }
+          formatter->open_object_section("restore_list");
+          do {
+            ret = bucket->list(dpp(), params, restore_entries - count, results, null_yield);
+            if (ret < 0) {
+              cerr << "ERROR: driver->list_objects(): " << cpp_strerror(-ret) << std::endl;
+              return -ret;
+            }
+            count += results.objs.size();
+            for (vector<rgw_bucket_dir_entry>::iterator iter = results.objs.begin(); iter != results.objs.end(); ++iter) {
+              std::unique_ptr<rgw::sal::Object> obj = bucket->get_object(iter->key.name);
+              if (obj) {
+                obj->set_instance(object_version);
+                ret = obj->get_obj_attrs(null_yield, dpp());
+                if (ret < 0) {
+                  cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl;
+                  return -ret;
+                }
+                for (map<string, bufferlist>::iterator getattriter = obj->get_attrs().begin(); getattriter != obj->get_attrs().end(); ++getattriter) {
+                  bufferlist& bl = getattriter->second;
+                  if (getattriter->first == RGW_ATTR_RESTORE_STATUS) {
+                    rgw::sal::RGWRestoreStatus rs;
+                    try {
+                      decode(rs, bl);
+                    } catch (const JSONDecoder::err& e) {
+                      cerr << "failed to decode JSON input: " << e.what() << std::endl;
+                      return EINVAL;
+                    }
+                    if (restore_status_filter) {
+                      if (restore_status_filter == rgw::sal::rgw_restore_status_dump(rs)) {
+                        formatter->dump_string(iter->key.name, rgw::sal::rgw_restore_status_dump(rs));
+                      }
+                    } else {
+                        formatter->dump_string(iter->key.name, rgw::sal::rgw_restore_status_dump(rs));
+                    }
+                  }
+                }
+              }
+            }
+          } while (results.is_truncated && count < restore_entries);
+          formatter->close_section();
+          formatter->flush(cout);
+        }
+      }
   return 0;
 }
-