From: Yehuda Sadeh Date: Tue, 18 Jun 2019 23:34:04 +0000 (-0700) Subject: rgw: more api fixes X-Git-Tag: v15.1.0~1898^2^2~42 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=4019f8921fb05ed27d7c0351c819901904c9a81f;p=ceph-ci.git rgw: more api fixes Signed-off-by: Yehuda Sadeh --- diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index cad85003254..3dbe33f4760 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -59,6 +59,9 @@ extern "C" { #include "services/svc_sync_modules.h" #include "services/svc_cls.h" +#include "services/svc_bilog_rados.h" +#include "services/svc_datalog_rados.h" +#include "services/svc_mdlog.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw @@ -1663,13 +1666,13 @@ int set_bucket_sync_enabled(RGWRados *store, int opt_cmd, const string& tenant_n int shard_id = bucket_info.num_shards? 0 : -1; if (opt_cmd == OPT_BUCKET_SYNC_DISABLE) { - r = store->stop_bi_log_entries(bucket_info, -1); + r = store->svc.bilog_rados->log_stop(bucket_info, -1); if (r < 0) { lderr(store->ctx()) << "ERROR: failed writing stop bilog" << dendl; return r; } } else { - r = store->resync_bi_log_entries(bucket_info, -1); + r = store->svc.bilog_rados->log_start(bucket_info, -1); if (r < 0) { lderr(store->ctx()) << "ERROR: failed writing resync bilog" << dendl; return r; @@ -2628,7 +2631,7 @@ int create_new_bucket_instance(RGWRados *store, new_bucket_info.num_shards = new_num_shards; new_bucket_info.objv_tracker.clear(); - int ret = store->init_bucket_index(new_bucket_info, new_bucket_info.num_shards); + int ret = store->svc.bi->init_index(new_bucket_info); if (ret < 0) { cerr << "ERROR: failed to init new bucket indexes: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -3440,10 +3443,6 @@ int main(int argc, const char **argv) } } - rgw_user_init(store); - rgw_bucket_init(store->meta_mgr); - rgw_otp_init(store); - rgw_http_client_init(g_ceph_context); struct rgw_curl_setup { @@ -5206,7 +5205,7 @@ int main(int argc, const char **argv) cerr << "failed to parse policy: " << e.what() << std::endl; return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, path, assume_role_doc, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, path, assume_role_doc, tenant); ret = role.create(true); if (ret < 0) { return -ret; @@ -5220,7 +5219,7 @@ int main(int argc, const char **argv) cerr << "ERROR: empty role name" << std::endl; return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); ret = role.delete_obj(); if (ret < 0) { return -ret; @@ -5234,7 +5233,7 @@ int main(int argc, const char **argv) cerr << "ERROR: empty role name" << std::endl; return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); ret = role.get(); if (ret < 0) { return -ret; @@ -5262,7 +5261,7 @@ int main(int argc, const char **argv) return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); ret = role.get(); if (ret < 0) { return -ret; @@ -5310,7 +5309,7 @@ int main(int argc, const char **argv) return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); ret = role.get(); if (ret < 0) { return -ret; @@ -5329,7 +5328,7 @@ int main(int argc, const char **argv) cerr << "ERROR: Role name is empty" << std::endl; return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); ret = role.get(); if (ret < 0) { return -ret; @@ -5349,7 +5348,7 @@ int main(int argc, const char **argv) cerr << "ERROR: policy name is empty" << std::endl; return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); int ret = role.get(); if (ret < 0) { return -ret; @@ -5373,7 +5372,7 @@ int main(int argc, const char **argv) cerr << "ERROR: policy name is empty" << std::endl; return -EINVAL; } - RGWRole role(g_ceph_context, store, role_name, tenant); + RGWRole role(g_ceph_context, store->pctl, role_name, tenant); ret = role.get(); if (ret < 0) { return -ret; @@ -5436,7 +5435,7 @@ int main(int argc, const char **argv) } else { /* list users in groups of max-keys, then perform user-bucket * limit-check on each group */ - ret = store->meta_mgr->list_keys_init(metadata_key, &handle); + ret = store->ctl.meta.mgr->list_keys_init(metadata_key, &handle); if (ret < 0) { cerr << "ERROR: buckets limit check can't get user metadata_key: " << cpp_strerror(-ret) << std::endl; @@ -5444,7 +5443,7 @@ int main(int argc, const char **argv) } do { - ret = store->meta_mgr->list_keys_next(handle, max, user_ids, + ret = store->ctl.meta.mgr->list_keys_next(handle, max, user_ids, &truncated); if (ret < 0 && ret != -ENOENT) { cerr << "ERROR: buckets limit check lists_keys_next(): " @@ -5460,7 +5459,7 @@ int main(int argc, const char **argv) } user_ids.clear(); } while (truncated); - store->meta_mgr->list_keys_complete(handle); + store->ctl.meta.mgr->list_keys_complete(handle); } return -ret; } /* OPT_BUCKET_LIMIT_CHECK */ @@ -6751,13 +6750,12 @@ next: return EINVAL; } - string user_str = user_id.to_str(); if (reset_stats) { if (!bucket_name.empty()){ cerr << "ERROR: recalculate doesn't work on buckets" << std::endl; return EINVAL; } - ret = store->ctl.user->reset_bucket_stats(user_id); + ret = store->ctl.user->reset_stats(user_id); if (ret < 0) { cerr << "ERROR: could not clear user stats: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6772,7 +6770,7 @@ next: cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; return -ret; } - int ret = store->ctl.bucket->sync_user_stats(user_id, bucket_info); + ret = store->ctl.bucket->sync_user_stats(user_id, bucket_info); if (ret < 0) { cerr << "ERROR: could not sync bucket stats: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6786,8 +6784,10 @@ next: } } - cls_user_header header; - int ret = store->cls_user_get_header(user_str, &header); + RGWStorageStats stats; + ceph::real_time last_stats_sync; + ceph::real_time last_stats_update; + int ret = store->ctl.user->read_stats(user_id, &stats, last_stats_sync, last_stats_update); if (ret < 0) { if (ret == -ENOENT) { /* in case of ENOENT */ cerr << "User has not been initialized or user does not exist" << std::endl; @@ -6797,12 +6797,17 @@ next: return -ret; } - encode_json("header", header, formatter); - formatter->flush(cout); + + { + Formatter::ObjectSection os(*formatter, "result"); + encode_json("stats", stats, formatter); + encode_json("last_stats_sync", last_stats_sync, formatter); + encode_json("last_stats_update", last_stats_update, formatter); + } } if (opt_cmd == OPT_METADATA_GET) { - int ret = store->meta_mgr->get(metadata_key, formatter); + int ret = store->ctl.meta.mgr->get(metadata_key, formatter); if (ret < 0) { cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6818,7 +6823,7 @@ next: cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl; return -ret; } - ret = store->meta_mgr->put(metadata_key, bl, RGWMDLogSyncType::APPLY_ALWAYS); + ret = store->ctl.meta.mgr->put(metadata_key, bl, RGWMDLogSyncType::APPLY_ALWAYS); if (ret < 0) { cerr << "ERROR: can't put key: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6826,7 +6831,7 @@ next: } if (opt_cmd == OPT_METADATA_RM) { - int ret = store->meta_mgr->remove(metadata_key); + int ret = store->ctl.meta.mgr->remove(metadata_key); if (ret < 0) { cerr << "ERROR: can't remove key: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6839,7 +6844,7 @@ next: } void *handle; int max = 1000; - int ret = store->meta_mgr->list_keys_init(metadata_key, marker, &handle); + int ret = store->ctl.meta.mgr->list_keys_init(metadata_key, marker, &handle); if (ret < 0) { cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6857,7 +6862,7 @@ next: do { list keys; left = (max_entries_specified ? max_entries - count : max); - ret = store->meta_mgr->list_keys_next(handle, left, keys, &truncated); + ret = store->ctl.meta.mgr->list_keys_next(handle, left, keys, &truncated); if (ret < 0 && ret != -ENOENT) { cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; return -ret; @@ -6876,13 +6881,13 @@ next: encode_json("truncated", truncated, formatter); encode_json("count", count, formatter); if (truncated) { - encode_json("marker", store->meta_mgr->get_marker(handle), formatter); + encode_json("marker", store->ctl.meta.mgr->get_marker(handle), formatter); } formatter->close_section(); } formatter->flush(cout); - store->meta_mgr->list_keys_complete(handle); + store->ctl.meta.mgr->list_keys_complete(handle); } if (opt_cmd == OPT_MDLOG_LIST) { @@ -6906,7 +6911,7 @@ next: std::cerr << "No --period given, using current period=" << period_id << std::endl; } - RGWMetadataLog *meta_log = store->meta_mgr->get_log(period_id); + RGWMetadataLog *meta_log = store->svc.mdlog->get_log(period_id); formatter->open_array_section("entries"); for (; i < g_ceph_context->_conf->rgw_md_log_max_shards; i++) { @@ -6925,7 +6930,7 @@ next: for (list::iterator iter = entries.begin(); iter != entries.end(); ++iter) { cls_log_entry& entry = *iter; - store->meta_mgr->dump_log_entry(entry, formatter); + store->ctl.meta.mgr->dump_log_entry(entry, formatter); } formatter->flush(cout); } while (truncated); @@ -6952,7 +6957,7 @@ next: std::cerr << "No --period given, using current period=" << period_id << std::endl; } - RGWMetadataLog *meta_log = store->meta_mgr->get_log(period_id); + RGWMetadataLog *meta_log = store->svc.mdlog->get_log(period_id); formatter->open_array_section("entries"); @@ -6973,7 +6978,7 @@ next: if (opt_cmd == OPT_MDLOG_AUTOTRIM) { // need a full history for purging old mdlog periods - store->meta_mgr->init_oldest_log_period(); + store->svc.mdlog->init_oldest_log_period(); RGWCoroutinesManager crs(store->ctx(), store->get_cr_registry()); RGWHTTPManager http(store->ctx(), crs.get_completion_mgr()); @@ -7011,7 +7016,7 @@ next: std::cerr << "missing --period argument" << std::endl; return EINVAL; } - RGWMetadataLog *meta_log = store->meta_mgr->get_log(period_id); + RGWMetadataLog *meta_log = store->svc.mdlog->get_log(period_id); ret = meta_log->trim(shard_id, start_time.to_real_time(), end_time.to_real_time(), start_marker, end_marker); if (ret < 0) { @@ -7362,7 +7367,7 @@ next: do { list entries; - ret = store->list_bi_log_entries(bucket_info, shard_id, marker, max_entries - count, entries, &truncated); + ret = store->svc.bilog_rados->log_list(bucket_info, shard_id, marker, max_entries - count, entries, &truncated); if (ret < 0) { cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl; return -ret; @@ -7500,7 +7505,7 @@ next: cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl; return -ret; } - ret = store->trim_bi_log_entries(bucket_info, shard_id, start_marker, end_marker); + ret = store->svc.bilog_rados->trim_entries(bucket_info, shard_id, start_marker, end_marker); if (ret < 0) { cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl; return -ret; @@ -7519,7 +7524,7 @@ next: return -ret; } map markers; - ret = store->get_bi_log_status(bucket_info, shard_id, markers); + ret = store->svc.bilog_rados->get_log_status(bucket_info, shard_id, &markers); if (ret < 0) { cerr << "ERROR: get_bi_log_status(): " << cpp_strerror(-ret) << std::endl; return -ret; @@ -7731,7 +7736,7 @@ next: real_time mtime = real_clock::now(); string oid = store->svc.cls->mfa.get_mfa_oid(user_id); - int ret = store->meta_mgr->mutate(rgw_otp_get_handler(), oid, mtime, &objv_tracker, + int ret = store->ctl.meta.mgr->mutate(rgw_otp_get_handler(), oid, mtime, &objv_tracker, MDLOG_STATUS_WRITE, RGWMDLogSyncType::APPLY_ALWAYS, [&] { return store->svc.cls->mfa.create_mfa(user_id, config, &objv_tracker, mtime, null_yield); @@ -7766,7 +7771,7 @@ next: real_time mtime = real_clock::now(); string oid = store->svc.cls->mfa.get_mfa_oid(user_id); - int ret = store->meta_mgr->mutate(rgw_otp_get_handler(), oid, mtime, &objv_tracker, + int ret = store->ctl.meta.mgr->mutate(rgw_otp_get_handler(), oid, mtime, &objv_tracker, MDLOG_STATUS_WRITE, RGWMDLogSyncType::APPLY_ALWAYS, [&] { return store->svc.cls->mfa.remove_mfa(user_id, totp_serial, &objv_tracker, mtime, null_yield); @@ -7909,7 +7914,7 @@ next: real_time mtime = real_clock::now(); string oid = store->svc.cls->mfa.get_mfa_oid(user_id); - ret = store->meta_mgr->mutate(rgw_otp_get_handler(), oid, mtime, &objv_tracker, + ret = store->ctl.meta.mgr->mutate(rgw_otp_get_handler(), oid, mtime, &objv_tracker, MDLOG_STATUS_WRITE, RGWMDLogSyncType::APPLY_ALWAYS, [&] { return store->svc.cls->mfa.create_mfa(user_id, config, &objv_tracker, mtime, null_yield); diff --git a/src/rgw/rgw_bucket.h b/src/rgw/rgw_bucket.h index 494c42eecd8..27c4184d7dc 100644 --- a/src/rgw/rgw_bucket.h +++ b/src/rgw/rgw_bucket.h @@ -464,6 +464,13 @@ namespace rgw { struct BucketChangeObserver; } +struct RGWDataChangesLogMarker { + int shard; + string marker; + + RGWDataChangesLogMarker() : shard(0) {} +}; + class RGWDataChangesLog { CephContext *cct; rgw::BucketChangeObserver *observer = nullptr; @@ -546,12 +553,9 @@ public: int lock_exclusive(int shard_id, timespan duration, string& zone_id, string& owner_id); int unlock(int shard_id, string& zone_id, string& owner_id); #endif - struct LogMarker { - int shard; - string marker; - LogMarker() : shard(0) {} - }; + using LogMarker = RGWDataChangesLogMarker; + int list_entries(const real_time& start_time, const real_time& end_time, int max_entries, list& entries, LogMarker& marker, bool *ptruncated); diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index e3ab2c883fc..4a9f4b3b374 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -372,7 +372,7 @@ int RGWOrphanSearch::build_buckets_instance_index() void *handle; int max = 1000; string section = "bucket.instance"; - int ret = store->meta_mgr->list_keys_init(section, &handle); + int ret = store->ctl.meta.mgr->list_keys_init(section, &handle); if (ret < 0) { lderr(store->ctx()) << "ERROR: can't get key: " << cpp_strerror(-ret) << dendl; return -ret; @@ -389,7 +389,7 @@ int RGWOrphanSearch::build_buckets_instance_index() do { list keys; - ret = store->meta_mgr->list_keys_next(handle, max, keys, &truncated); + ret = store->ctl.meta.mgr->list_keys_next(handle, max, keys, &truncated); if (ret < 0) { lderr(store->ctx()) << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << dendl; return -ret; @@ -419,7 +419,7 @@ int RGWOrphanSearch::build_buckets_instance_index() lderr(store->ctx()) << __func__ << ": ERROR: log_oids() returned ret=" << ret << dendl; return ret; } - store->meta_mgr->list_keys_complete(handle); + store->ctl.meta.mgr->list_keys_complete(handle); return 0; } diff --git a/src/rgw/services/svc_datalog_rados.cc b/src/rgw/services/svc_datalog_rados.cc index 4a94ea789f4..a87a1cfcb58 100644 --- a/src/rgw/services/svc_datalog_rados.cc +++ b/src/rgw/services/svc_datalog_rados.cc @@ -68,6 +68,13 @@ int RGWSI_DataLog_RADOS::list_entries(int shard, const real_time& start_time, co entries, marker, out_marker, truncated); } +int RGWSI_DataLog_RADOS::list_entries(const real_time& start_time, const real_time& end_time, int max_entries, + list& entries, RGWDataChangesLogMarker& marker, bool *ptruncated) +{ + return log->list_entries(start_time, end_time, max_entries, + entries, marker, ptruncated); +} + int RGWSI_DataLog_RADOS::trim_entries(int shard_id, const real_time& start_time, const real_time& end_time, const string& start_marker, const string& end_marker) { diff --git a/src/rgw/services/svc_datalog_rados.h b/src/rgw/services/svc_datalog_rados.h index 777dff1e73e..b3d14adfd26 100644 --- a/src/rgw/services/svc_datalog_rados.h +++ b/src/rgw/services/svc_datalog_rados.h @@ -21,6 +21,7 @@ class RGWDataChangesLog; class RGWDataChangesLogInfo; +struct RGWDataChangesLogMarker; struct rgw_data_change_log_entry; namespace rgw { @@ -63,6 +64,8 @@ public: const string& marker, string *out_marker, bool *truncated); + int list_entries(const real_time& start_time, const real_time& end_time, int max_entries, + list& entries, RGWDataChangesLogMarker& marker, bool *ptruncated); int trim_entries(int shard_id, const real_time& start_time, const real_time& end_time, const string& start_marker, const string& end_marker); int trim_entries(const real_time& start_time, const real_time& end_time,