From: Jiffin Tony Thottan Date: Tue, 15 Apr 2025 06:22:26 +0000 (+0530) Subject: rgw/cloud-restore: admin CLI for restore list and status X-Git-Tag: testing/wip-vshankar-testing-20260219.125903~6^2~7^2~1 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=02c925c4c1af2858b1f0f5d4c8f5cc032ea97093;p=ceph-ci.git rgw/cloud-restore: admin CLI for restore list and status Also added stats as part of radosgw-admin bucket stats command Signed-off-by: Jiffin Tony Thottan --- diff --git a/doc/man/8/radosgw-admin.rst b/doc/man/8/radosgw-admin.rst index 1c171dffdb5..ae317aef7df 100644 --- a/doc/man/8/radosgw-admin.rst +++ b/doc/man/8/radosgw-admin.rst @@ -491,6 +491,12 @@ as follows: :command:`topic dump` Dump (in JSON format) all pending bucket notifications of a persistent topic +:command:`restore list` + List restore status of each object in a bucket + +:command:`restore status` + Show restoration status of an object in the bucket + Options ======= @@ -880,6 +886,14 @@ Options operations such as finding orphans or checking the bucket index. The default is 32. +.. option:: --restore-status + + Filter objects return by the 'restore list' command by status. + +.. option:: --show-restore-stats + + Shows restore stats in a bucket stat command. Here the bucket name need be provided. + Quota Options ============= diff --git a/src/rgw/driver/rados/rgw_bucket.cc b/src/rgw/driver/rados/rgw_bucket.cc index e858f0c4b49..f4248da41ce 100644 --- a/src/rgw/driver/rados/rgw_bucket.cc +++ b/src/rgw/driver/rados/rgw_bucket.cc @@ -1597,6 +1597,69 @@ static int bucket_stats(rgw::sal::Driver* driver, return 0; } +static int bucket_restore_stats(rgw::sal::Driver* driver, + const std::string& tenant_name, + const std::string& bucket_name, Formatter* formatter, + const DoutPrefixProvider* dpp, optional_yield y) { + std::unique_ptr bucket; + int restore_completed_count = 0; + int restore_in_progress_count = 0; + int restore_failed_count = 0; + int ret = driver->load_bucket(dpp, rgw_bucket(tenant_name, bucket_name), + &bucket, y); + if (ret < 0) { + return ret; + } + rgw::sal::Bucket::ListParams params; + rgw::sal::Bucket::ListResults results; + params.list_versions = bucket->versioned(); + params.allow_unordered = true; + do { + ret = bucket->list(dpp, params, listing_max_entries, results, null_yield); + if (ret < 0) { + cerr << "ERROR: driver->list_objects(): " << cpp_strerror(-ret) << std::endl; + return ret; + } + for (vector::iterator iter = results.objs.begin(); iter != results.objs.end(); ++iter) { + std::unique_ptr obj = bucket->get_object(iter->key.name); + if (obj) { + ret = obj->get_obj_attrs(null_yield, dpp); + if (ret < 0) { + cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl; + return ret; + } + for (map::iterator getattriter = obj->get_attrs().begin(); getattriter != obj->get_attrs().end(); ++getattriter) { + bufferlist& bl = getattriter->second; + if (getattriter->first == RGW_ATTR_RESTORE_STATUS) { + rgw::sal::RGWRestoreStatus rs; + try { + decode(rs, bl); + } catch (const JSONDecoder::err& e) { + cerr << "failed to decode JSON input: " << e.what() << std::endl; + return -EINVAL; + } + if (rs == rgw::sal::RGWRestoreStatus::RestoreAlreadyInProgress) { + restore_in_progress_count ++; + } else if (rs == rgw::sal::RGWRestoreStatus::CloudRestored) { + restore_completed_count ++; + } else if (rs == rgw::sal::RGWRestoreStatus::RestoreFailed) { + restore_failed_count ++; + } + } + } + } + } + } while (results.is_truncated); + formatter->open_object_section(""); + formatter->open_object_section("restore_stats"); + formatter->dump_int("restore_completed_count", restore_completed_count); + formatter->dump_int("restore_in_progress_count", restore_in_progress_count); + formatter->dump_int("restore_failed_count", restore_failed_count); + formatter->close_section(); + formatter->close_section(); + return 0; +} + int RGWBucketAdminOp::limit_check(rgw::sal::Driver* driver, RGWBucketAdminOpState& op_state, const std::list& user_ids, @@ -1816,6 +1879,12 @@ int RGWBucketAdminOp::info(rgw::sal::Driver* driver, if (ret < 0) { return ret; } + if (op_state.restore_stats) { + ret = bucket_restore_stats(driver, user_id.tenant, bucket_name, formatter, dpp, y); + if (ret < 0) { + return ret; + } + } } else if (op_state.is_user_op()) { const rgw_user& uid = op_state.get_user_id(); auto user = driver->get_user(uid); @@ -1874,7 +1943,6 @@ int RGWBucketAdminOp::info(rgw::sal::Driver* driver, } } driver->meta_list_keys_complete(handle); - formatter->close_section(); } diff --git a/src/rgw/driver/rados/rgw_bucket.h b/src/rgw/driver/rados/rgw_bucket.h index fb559961b28..2bf40f6f45b 100644 --- a/src/rgw/driver/rados/rgw_bucket.h +++ b/src/rgw/driver/rados/rgw_bucket.h @@ -237,6 +237,7 @@ struct RGWBucketAdminOpState { bool sync_bucket; bool dump_keys; bool hide_progress; + bool restore_stats; int max_aio = 0; ceph::timespan min_age = std::chrono::hours::zero(); @@ -246,6 +247,7 @@ struct RGWBucketAdminOpState { RGWRateLimitInfo ratelimit_info; void set_fetch_stats(bool value) { stat_buckets = value; } + void set_restore_stats(bool value) { restore_stats = value; } void set_check_objects(bool value) { check_objects = value; } void set_fix_index(bool value) { fix_index = value; } void set_delete_children(bool value) { delete_child_objects = value; } diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index 61525fa9be7..26c71026ab0 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -358,6 +358,9 @@ void usage() cout << " notification list list bucket notifications configuration\n"; cout << " notification get get a bucket notifications configuration\n"; cout << " notification rm remove a bucket notifications configuration\n"; + cout << " restore status shows restoration status of object in a bucket\n"; + cout << " restore list list restore status of each object in the bucket\n"; + cout <<" can be filtered with help of --restore-status which shows objects with specified status\n"; cout << "options:\n"; cout << " --tenant= tenant name\n"; cout << " --user_ns= namespace of user (oidc in case of users authenticated with oidc provider)\n"; @@ -540,6 +543,7 @@ void usage() cout << "\nBucket list objects options:\n"; cout << " --max-entries max number of entries listed (default 1000)\n"; cout << " --marker the marker used to specify on which entry the listing begins, default none (i.e., very first entry)\n"; + cout << " --show-restore-stats if the flag is in present it will show restores stats in the bucket stats command\n"; cout << "\n"; generic_client_usage(); } @@ -920,6 +924,8 @@ enum class OPT { ACCOUNT_STATS, ACCOUNT_RM, ACCOUNT_LIST, + RESTORE_STATUS, + RESTORE_LIST, }; } @@ -1183,6 +1189,8 @@ static SimpleCmd::Commands all_cmds = { { "account stats", OPT::ACCOUNT_STATS }, { "account rm", OPT::ACCOUNT_RM }, { "account list", OPT::ACCOUNT_LIST }, + { "restore status", OPT::RESTORE_STATUS }, + { "restore list", OPT::RESTORE_LIST }, }; static SimpleCmd::Aliases cmd_aliases = { @@ -3818,6 +3826,8 @@ int main(int argc, const char **argv) bool raw_storage_op = false; std::optional rgw_obj_fs; // radoslist field separator + std::optional restore_status_filter; + int show_restore_stats = false; init_realm_param(cct.get(), realm_id, opt_realm_id, "rgw_realm_id"); init_realm_param(cct.get(), zonegroup_id, opt_zonegroup_id, "rgw_zonegroup_id"); @@ -4385,6 +4395,10 @@ int main(int argc, const char **argv) enable_features.insert(val); } else if (ceph_argparse_witharg(args, i, &val, "--disable-feature", (char*)NULL)) { disable_features.insert(val); + } else if (ceph_argparse_witharg(args, i, &val, "--restore-status", (char*)NULL)) { + restore_status_filter = val; + } else if (ceph_argparse_binary_flag(args, i, &show_restore_stats, NULL, "--show-restore-stats", (char*)NULL)){ + // do nothing } else if (strncmp(*i, "-", 1) == 0) { cerr << "ERROR: invalid flag " << *i << std::endl; return EINVAL; @@ -4567,6 +4581,8 @@ int main(int argc, const char **argv) OPT::PUBSUB_TOPIC_STATS , OPT::PUBSUB_TOPIC_DUMP , OPT::SCRIPT_GET, + OPT::RESTORE_STATUS, + OPT::RESTORE_LIST, }; std::set gc_ops_list = { @@ -7631,6 +7647,7 @@ int main(int argc, const char **argv) bucket_op.max_entries = max_entries; else bucket_op.max_entries = 0; /* for backward compatibility */ + bucket_op.set_restore_stats(bool(show_restore_stats)); int r = RGWBucketAdminOp::info(driver, bucket_op, stream_flusher, null_yield, dpp()); if (r < 0) { @@ -12209,7 +12226,124 @@ next: } } } - + if (opt_cmd == OPT::RESTORE_STATUS || + opt_cmd == OPT::RESTORE_LIST) { + int ret = init_bucket(tenant, bucket_name, bucket_id, &bucket); + if (ret < 0) { + cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + if (opt_cmd == OPT::RESTORE_STATUS) { + if (!object.empty()) { + std::unique_ptr obj = bucket->get_object(object); + obj->set_instance(object_version); + ret = obj->get_obj_attrs(null_yield, dpp()); + if (ret < 0) { + cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + formatter->open_object_section("object restore status"); + formatter->dump_string("name", object); + map::iterator iter; + for (iter = obj->get_attrs().begin(); iter != obj->get_attrs().end(); ++iter) { + bufferlist& bl = iter->second; + if (iter->first == RGW_ATTR_RESTORE_STATUS) { + rgw::sal::RGWRestoreStatus rs; + try { + decode(rs, bl); + } catch (const JSONDecoder::err& e) { + cerr << "failed to decode JSON input: " << e.what() << std::endl; + return EINVAL; + } + formatter->dump_string("RestoreStatus", rgw::sal::rgw_restore_status_dump(rs)); + } else if (iter->first == RGW_ATTR_RESTORE_TYPE) { + rgw::sal::RGWRestoreType rt; + try { + decode(rt, bl); + } catch (const JSONDecoder::err& e) { + cerr << "failed to decode JSON input: " << e.what() << std::endl; + return EINVAL; + } + formatter->dump_string("RestoreType", rgw::sal::rgw_restore_type_dump(rt)); + } else if (iter->first == RGW_ATTR_RESTORE_EXPIRY_DATE) { + decode_dump("RestoreExpiryDate", bl, formatter.get()); + } else if (iter->first == RGW_ATTR_RESTORE_TIME) { + decode_dump("RestoreTime", bl, formatter.get()); + } else if (iter->first == RGW_ATTR_RESTORE_VERSIONED_EPOCH) { + uint64_t versioned_epoch; + try { + decode(versioned_epoch, bl); + } catch (const JSONDecoder::err& e) { + cerr << "failed to decode JSON input: " << e.what() << std::endl; + return EINVAL; + } + formatter->dump_unsigned("RestoreVersionedEpoch", versioned_epoch); + } + } + formatter->close_section(); + formatter->flush(cout); + } + } else if (opt_cmd == OPT::RESTORE_LIST) { + int count = 0; + int restore_entries; + string prefix; + string delim; + string marker; + string ns; + rgw::sal::Bucket::ListParams params; + rgw::sal::Bucket::ListResults results; + params.prefix = prefix; + params.delim = delim; + params.marker = rgw_obj_key(marker); + params.ns = ns; + params.enforce_ns = true; + if (max_entries_specified) { + restore_entries = max_entries; + } else { + restore_entries = 1000; + } + formatter->open_object_section("restore_list"); + do { + ret = bucket->list(dpp(), params, restore_entries - count, results, null_yield); + if (ret < 0) { + cerr << "ERROR: driver->list_objects(): " << cpp_strerror(-ret) << std::endl; + return -ret; + } + count += results.objs.size(); + for (vector::iterator iter = results.objs.begin(); iter != results.objs.end(); ++iter) { + std::unique_ptr obj = bucket->get_object(iter->key.name); + if (obj) { + obj->set_instance(object_version); + ret = obj->get_obj_attrs(null_yield, dpp()); + if (ret < 0) { + cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + for (map::iterator getattriter = obj->get_attrs().begin(); getattriter != obj->get_attrs().end(); ++getattriter) { + bufferlist& bl = getattriter->second; + if (getattriter->first == RGW_ATTR_RESTORE_STATUS) { + rgw::sal::RGWRestoreStatus rs; + try { + decode(rs, bl); + } catch (const JSONDecoder::err& e) { + cerr << "failed to decode JSON input: " << e.what() << std::endl; + return EINVAL; + } + if (restore_status_filter) { + if (restore_status_filter == rgw::sal::rgw_restore_status_dump(rs)) { + formatter->dump_string(iter->key.name, rgw::sal::rgw_restore_status_dump(rs)); + } + } else { + formatter->dump_string(iter->key.name, rgw::sal::rgw_restore_status_dump(rs)); + } + } + } + } + } + } while (results.is_truncated && count < restore_entries); + formatter->close_section(); + formatter->flush(cout); + } + } return 0; } -