From 022d2eb3bb63304e8f020edfe9a308d909bd6526 Mon Sep 17 00:00:00 2001 From: Daniel Gryniewicz Date: Fri, 7 Aug 2020 09:20:08 -0400 Subject: [PATCH] Zipper - Split RadosStore from base SAL Signed-off-by: Daniel Gryniewicz --- src/rgw/CMakeLists.txt | 1 + src/rgw/rgw_bucket.cc | 1 + src/rgw/rgw_cr_rados.h | 1 + src/rgw/rgw_dencoder.cc | 1 + src/rgw/rgw_frontend.h | 1 + src/rgw/rgw_gc.h | 1 + src/rgw/rgw_lc.cc | 1 + src/rgw/rgw_lc.h | 3 +- src/rgw/rgw_multi.cc | 1 + src/rgw/rgw_notify.cc | 1 + src/rgw/rgw_object_expirer_core.cc | 1 + src/rgw/rgw_object_expirer_core.h | 1 + src/rgw/rgw_op.cc | 1 + src/rgw/rgw_orphan.cc | 1 + src/rgw/rgw_period_pusher.cc | 1 + src/rgw/rgw_pubsub.cc | 1 + src/rgw/rgw_putobj_processor.cc | 1 + src/rgw/rgw_quota.cc | 1 + src/rgw/rgw_realm_reloader.cc | 1 + src/rgw/rgw_reshard.cc | 1 + src/rgw/rgw_rest.cc | 1 + src/rgw/rgw_rest_bucket.cc | 1 + src/rgw/rgw_rest_config.cc | 1 + src/rgw/rgw_rest_conn.cc | 1 + src/rgw/rgw_rest_metadata.cc | 1 + src/rgw/rgw_rest_oidc_provider.cc | 1 + src/rgw/rgw_rest_pubsub.cc | 1 + src/rgw/rgw_rest_pubsub_common.cc | 1 + src/rgw/rgw_rest_realm.cc | 1 + src/rgw/rgw_rest_role.cc | 1 + src/rgw/rgw_rest_s3.cc | 1 + src/rgw/rgw_rest_swift.cc | 1 + src/rgw/rgw_rest_usage.cc | 1 + src/rgw/rgw_rest_user.cc | 1 + src/rgw/rgw_rest_user_policy.cc | 1 + src/rgw/rgw_sal.cc | 953 ------------------------ src/rgw/rgw_sal.h | 281 +------ src/rgw/rgw_sal_rados.cc | 978 +++++++++++++++++++++++++ src/rgw/rgw_sal_rados.h | 297 ++++++++ src/rgw/rgw_sts.cc | 1 + src/rgw/rgw_swift_auth.cc | 1 + src/rgw/rgw_sync.h | 5 +- src/rgw/rgw_sync_module_es_rest.cc | 1 + src/rgw/rgw_sync_module_pubsub_rest.cc | 1 + src/rgw/rgw_tools.cc | 1 + src/rgw/rgw_torrent.cc | 1 + src/rgw/rgw_user.cc | 1 + src/test/rgw/test_rgw_iam_policy.cc | 1 + 48 files changed, 1325 insertions(+), 1234 deletions(-) create mode 100644 src/rgw/rgw_sal_rados.cc create mode 100644 src/rgw/rgw_sal_rados.h diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 13ef176724e..a9e6130d576 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -131,6 +131,7 @@ set(librgw_common_srcs rgw_rest_s3.cc rgw_role.cc rgw_sal.cc + rgw_sal_rados.cc rgw_string.cc rgw_tag.cc rgw_tag_s3.cc diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index aac26a0db03..68ed28efb4b 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -51,6 +51,7 @@ #include "cls/user/cls_user_types.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_cr_rados.h b/src/rgw/rgw_cr_rados.h index 0e8a94154f0..87636fd1d08 100644 --- a/src/rgw/rgw_cr_rados.h +++ b/src/rgw/rgw_cr_rados.h @@ -8,6 +8,7 @@ #include "include/ceph_assert.h" #include "rgw_coroutine.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "common/WorkQueue.h" #include "common/Throttle.h" diff --git a/src/rgw/rgw_dencoder.cc b/src/rgw/rgw_dencoder.cc index c6156aac39f..612d6d1e8b7 100644 --- a/src/rgw/rgw_dencoder.cc +++ b/src/rgw/rgw_dencoder.cc @@ -2,6 +2,7 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "rgw_common.h" +#include "rgw_rados.h" #include "rgw_zone.h" #include "rgw_log.h" #include "rgw_acl.h" diff --git a/src/rgw/rgw_frontend.h b/src/rgw/rgw_frontend.h index 88ee0a3f7ff..e728a68292f 100644 --- a/src/rgw/rgw_frontend.h +++ b/src/rgw/rgw_frontend.h @@ -15,6 +15,7 @@ #include "rgw_civetweb_log.h" #include "civetweb/civetweb.h" #include "rgw_auth_registry.h" +#include "rgw_sal_rados.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_gc.h b/src/rgw/rgw_gc.h index d45b54d5404..2345db85146 100644 --- a/src/rgw/rgw_gc.h +++ b/src/rgw/rgw_gc.h @@ -12,6 +12,7 @@ #include "common/Thread.h" #include "rgw_common.h" #include "rgw_sal.h" +#include "rgw_rados.h" #include "cls/rgw/cls_rgw_types.h" #include diff --git a/src/rgw/rgw_lc.cc b/src/rgw/rgw_lc.cc index f7aface2fd7..61394ccd299 100644 --- a/src/rgw/rgw_lc.cc +++ b/src/rgw/rgw_lc.cc @@ -28,6 +28,7 @@ #include "rgw_string.h" #include "rgw_multi.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" // this seems safe to use, at least for now--arguably, we should // prefer header-only fmt, in general diff --git a/src/rgw/rgw_lc.h b/src/rgw/rgw_lc.h index a1718003640..7ab3296e995 100644 --- a/src/rgw/rgw_lc.h +++ b/src/rgw/rgw_lc.h @@ -20,6 +20,7 @@ #include "cls/rgw/cls_rgw_types.h" #include "rgw_tag.h" #include "rgw_sal.h" +#include "rgw_rados.h" #include #include @@ -533,7 +534,7 @@ public: int remove_bucket_config(RGWBucketInfo& bucket_info, const map& bucket_attrs); - CephContext *get_cct() const override { return store->ctx(); } + CephContext *get_cct() const override { return cct; } unsigned get_subsys() const; std::ostream& gen_prefix(std::ostream& out) const; diff --git a/src/rgw/rgw_multi.cc b/src/rgw/rgw_multi.cc index 3bb7b260b0c..873bb90f1d2 100644 --- a/src/rgw/rgw_multi.cc +++ b/src/rgw/rgw_multi.cc @@ -12,6 +12,7 @@ #include "rgw_multi.h" #include "rgw_op.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "services/svc_sys_obj.h" #include "services/svc_tier_rados.h" diff --git a/src/rgw/rgw_notify.cc b/src/rgw/rgw_notify.cc index 6270a4e2794..3f56da71ac6 100644 --- a/src/rgw/rgw_notify.cc +++ b/src/rgw/rgw_notify.cc @@ -10,6 +10,7 @@ #include "rgw_pubsub.h" #include "rgw_pubsub_push.h" #include "rgw_perf_counters.h" +#include "rgw_sal_rados.h" #include "common/dout.h" #include diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/rgw_object_expirer_core.cc index be5b402b690..d390d491013 100644 --- a/src/rgw/rgw_object_expirer_core.cc +++ b/src/rgw/rgw_object_expirer_core.cc @@ -30,6 +30,7 @@ #include "rgw_usage.h" #include "rgw_object_expirer_core.h" #include "rgw_zone.h" +#include "rgw_sal_rados.h" #include "services/svc_rados.h" #include "services/svc_zone.h" diff --git a/src/rgw/rgw_object_expirer_core.h b/src/rgw/rgw_object_expirer_core.h index 9037c523e94..f76fe56cb65 100644 --- a/src/rgw/rgw_object_expirer_core.h +++ b/src/rgw/rgw_object_expirer_core.h @@ -30,6 +30,7 @@ #include "include/str_list.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" class RGWSI_RADOS; class RGWSI_Zone; diff --git a/src/rgw/rgw_op.cc b/src/rgw/rgw_op.cc index a608aa157fc..8116cdc41c8 100644 --- a/src/rgw/rgw_op.cc +++ b/src/rgw/rgw_op.cc @@ -49,6 +49,7 @@ #include "rgw_perf_counters.h" #include "rgw_notify.h" #include "rgw_notify_event_type.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" #include "services/svc_quota.h" diff --git a/src/rgw/rgw_orphan.cc b/src/rgw/rgw_orphan.cc index 01bfebff5e7..20a2d85862d 100644 --- a/src/rgw/rgw_orphan.cc +++ b/src/rgw/rgw_orphan.cc @@ -13,6 +13,7 @@ #include "rgw_orphan.h" #include "rgw_zone.h" #include "rgw_bucket.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" #include "services/svc_sys_obj.h" diff --git a/src/rgw/rgw_period_pusher.cc b/src/rgw/rgw_period_pusher.cc index 29c13643c35..d8742c32e21 100644 --- a/src/rgw/rgw_period_pusher.cc +++ b/src/rgw/rgw_period_pusher.cc @@ -8,6 +8,7 @@ #include "rgw_cr_rest.h" #include "rgw_zone.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" diff --git a/src/rgw/rgw_pubsub.cc b/src/rgw/rgw_pubsub.cc index f73f1d121d6..0df56cbf773 100644 --- a/src/rgw/rgw_pubsub.cc +++ b/src/rgw/rgw_pubsub.cc @@ -4,6 +4,7 @@ #include "services/svc_zone.h" #include "rgw_b64.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "rgw_pubsub.h" #include "rgw_tools.h" #include "rgw_xml.h" diff --git a/src/rgw/rgw_putobj_processor.cc b/src/rgw/rgw_putobj_processor.cc index 498581298ed..19077e14649 100644 --- a/src/rgw/rgw_putobj_processor.cc +++ b/src/rgw/rgw_putobj_processor.cc @@ -18,6 +18,7 @@ #include "rgw_multi.h" #include "rgw_compression.h" #include "services/svc_sys_obj.h" +#include "rgw_sal_rados.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_quota.cc b/src/rgw/rgw_quota.cc index 5fb894ca0c5..cc0754f61ed 100644 --- a/src/rgw/rgw_quota.cc +++ b/src/rgw/rgw_quota.cc @@ -22,6 +22,7 @@ #include "rgw_common.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "rgw_quota.h" #include "rgw_bucket.h" #include "rgw_user.h" diff --git a/src/rgw/rgw_realm_reloader.cc b/src/rgw/rgw_realm_reloader.cc index 4dbd378dd05..d815ad14f3a 100644 --- a/src/rgw/rgw_realm_reloader.cc +++ b/src/rgw/rgw_realm_reloader.cc @@ -8,6 +8,7 @@ #include "rgw_rest.h" #include "rgw_user.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" diff --git a/src/rgw/rgw_reshard.cc b/src/rgw/rgw_reshard.cc index 2f79f366ac2..27ed1435096 100644 --- a/src/rgw/rgw_reshard.cc +++ b/src/rgw/rgw_reshard.cc @@ -8,6 +8,7 @@ #include "rgw_bucket.h" #include "rgw_reshard.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "cls/rgw/cls_rgw_client.h" #include "cls/lock/cls_lock_client.h" #include "common/errno.h" diff --git a/src/rgw/rgw_rest.cc b/src/rgw/rgw_rest.cc index 55bc28ba05d..bd1fcf69fde 100644 --- a/src/rgw/rgw_rest.cc +++ b/src/rgw/rgw_rest.cc @@ -26,6 +26,7 @@ #include "rgw_client_io.h" #include "rgw_resolve.h" +#include "rgw_sal_rados.h" #include diff --git a/src/rgw/rgw_rest_bucket.cc b/src/rgw/rgw_rest_bucket.cc index 06b793c814b..6b0433f3b54 100644 --- a/src/rgw/rgw_rest_bucket.cc +++ b/src/rgw/rgw_rest_bucket.cc @@ -4,6 +4,7 @@ #include "rgw_op.h" #include "rgw_bucket.h" #include "rgw_rest_bucket.h" +#include "rgw_sal_rados.h" #include "include/str_list.h" diff --git a/src/rgw/rgw_rest_config.cc b/src/rgw/rgw_rest_config.cc index cf1efef4e61..bdc64823384 100644 --- a/src/rgw/rgw_rest_config.cc +++ b/src/rgw/rgw_rest_config.cc @@ -21,6 +21,7 @@ #include "rgw_rest_s3.h" #include "rgw_rest_config.h" #include "rgw_client_io.h" +#include "rgw_sal_rados.h" #include "common/errno.h" #include "include/ceph_assert.h" diff --git a/src/rgw/rgw_rest_conn.cc b/src/rgw/rgw_rest_conn.cc index 32e16884a78..bb7ecda23f4 100644 --- a/src/rgw/rgw_rest_conn.cc +++ b/src/rgw/rgw_rest_conn.cc @@ -4,6 +4,7 @@ #include "rgw_zone.h" #include "rgw_rest_conn.h" #include "rgw_sal.h" +#include "rgw_rados.h" #include "services/svc_zone.h" diff --git a/src/rgw/rgw_rest_metadata.cc b/src/rgw/rgw_rest_metadata.cc index fd18f74698c..64daa046e52 100644 --- a/src/rgw/rgw_rest_metadata.cc +++ b/src/rgw/rgw_rest_metadata.cc @@ -21,6 +21,7 @@ #include "rgw_rest_metadata.h" #include "rgw_client_io.h" #include "rgw_mdlog_types.h" +#include "rgw_sal_rados.h" #include "common/errno.h" #include "common/strtol.h" #include "rgw/rgw_b64.h" diff --git a/src/rgw/rgw_rest_oidc_provider.cc b/src/rgw/rgw_rest_oidc_provider.cc index a72e3c64b78..3dbaeae786f 100644 --- a/src/rgw/rgw_rest_oidc_provider.cc +++ b/src/rgw/rgw_rest_oidc_provider.cc @@ -16,6 +16,7 @@ #include "rgw_role.h" #include "rgw_rest_oidc_provider.h" #include "rgw_oidc_provider.h" +#include "rgw_sal_rados.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/rgw_rest_pubsub.cc index e3c9f330f3e..9853934c697 100644 --- a/src/rgw/rgw_rest_pubsub.cc +++ b/src/rgw/rgw_rest_pubsub.cc @@ -15,6 +15,7 @@ #include "rgw_arn.h" #include "rgw_auth_s3.h" #include "rgw_notify.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" #define dout_context g_ceph_context diff --git a/src/rgw/rgw_rest_pubsub_common.cc b/src/rgw/rgw_rest_pubsub_common.cc index ba09a3073dc..9145eafa8d8 100644 --- a/src/rgw/rgw_rest_pubsub_common.cc +++ b/src/rgw/rgw_rest_pubsub_common.cc @@ -5,6 +5,7 @@ #include "rgw_rest_pubsub_common.h" #include "common/dout.h" #include "rgw_url.h" +#include "rgw_sal_rados.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_rest_realm.cc b/src/rgw/rgw_rest_realm.cc index 9a14d8d5ff0..558e23db473 100644 --- a/src/rgw/rgw_rest_realm.cc +++ b/src/rgw/rgw_rest_realm.cc @@ -6,6 +6,7 @@ #include "rgw_rest_s3.h" #include "rgw_rest_config.h" #include "rgw_zone.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" #include "services/svc_mdlog.h" diff --git a/src/rgw/rgw_rest_role.cc b/src/rgw/rgw_rest_role.cc index cb3b2c8566b..a5812f174a8 100644 --- a/src/rgw/rgw_rest_role.cc +++ b/src/rgw/rgw_rest_role.cc @@ -15,6 +15,7 @@ #include "rgw_rest.h" #include "rgw_role.h" #include "rgw_rest_role.h" +#include "rgw_sal_rados.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_rest_s3.cc b/src/rgw/rgw_rest_s3.cc index 0f93a88d75e..281f259a69b 100644 --- a/src/rgw/rgw_rest_s3.cc +++ b/src/rgw/rgw_rest_s3.cc @@ -62,6 +62,7 @@ #include "rgw_rest_sts.h" #include "rgw_rest_iam.h" #include "rgw_sts.h" +#include "rgw_sal_rados.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_rest_swift.cc b/src/rgw/rgw_rest_swift.cc index ff1c24d6775..3468584b169 100644 --- a/src/rgw/rgw_rest_swift.cc +++ b/src/rgw/rgw_rest_swift.cc @@ -27,6 +27,7 @@ #include "rgw_process.h" #include "rgw_zone.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" diff --git a/src/rgw/rgw_rest_usage.cc b/src/rgw/rgw_rest_usage.cc index d3c722623fb..51e028813bc 100644 --- a/src/rgw/rgw_rest_usage.cc +++ b/src/rgw/rgw_rest_usage.cc @@ -4,6 +4,7 @@ #include "rgw_op.h" #include "rgw_usage.h" #include "rgw_rest_usage.h" +#include "rgw_sal_rados.h" #include "include/str_list.h" diff --git a/src/rgw/rgw_rest_user.cc b/src/rgw/rgw_rest_user.cc index 9a67577bdd1..64865b34084 100644 --- a/src/rgw/rgw_rest_user.cc +++ b/src/rgw/rgw_rest_user.cc @@ -6,6 +6,7 @@ #include "rgw_op.h" #include "rgw_user.h" #include "rgw_rest_user.h" +#include "rgw_sal_rados.h" #include "include/str_list.h" #include "include/ceph_assert.h" diff --git a/src/rgw/rgw_rest_user_policy.cc b/src/rgw/rgw_rest_user_policy.cc index 3802a873a02..e3c261439a5 100644 --- a/src/rgw/rgw_rest_user_policy.cc +++ b/src/rgw/rgw_rest_user_policy.cc @@ -15,6 +15,7 @@ #include "rgw_op.h" #include "rgw_rest.h" #include "rgw_rest_user_policy.h" +#include "rgw_sal_rados.h" #include "services/svc_zone.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index bee7f670c92..e9a4b385bd9 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -19,960 +19,7 @@ #include #include -#include "common/Clock.h" -#include "common/errno.h" - #include "rgw_sal.h" -#include "rgw_bucket.h" -#include "rgw_multi.h" -#include "rgw_acl_s3.h" - -/* Stuff for RGWRadosStore. Move to separate file when store split out */ -#include "rgw_zone.h" -#include "rgw_rest_conn.h" -#include "services/svc_sys_obj.h" -#include "services/svc_zone.h" #define dout_subsys ceph_subsys_rgw -namespace rgw::sal { - -int RGWRadosUser::list_buckets(const string& marker, const string& end_marker, - uint64_t max, bool need_stats, RGWBucketList &buckets) -{ - RGWUserBuckets ulist; - bool is_truncated = false; - int ret; - - buckets.clear(); - ret = store->ctl()->user->list_buckets(info.user_id, marker, end_marker, max, - need_stats, &ulist, &is_truncated); - if (ret < 0) - return ret; - - buckets.set_truncated(is_truncated); - for (const auto& ent : ulist.get_buckets()) { - buckets.add(std::unique_ptr(new RGWRadosBucket(this->store, ent.second, this))); - } - - return 0; -} - -RGWBucket* RGWRadosUser::create_bucket(rgw_bucket& bucket, - ceph::real_time creation_time) -{ - return NULL; -} - -int RGWRadosUser::load_by_id(optional_yield y) - -{ - return store->ctl()->user->get_info_by_uid(info.user_id, &info, y); -} - -std::unique_ptr RGWRadosStore::get_object(const rgw_obj_key& k) -{ - return std::unique_ptr(new RGWRadosObject(this, k)); -} - -/* Placeholder */ -RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) -{ - return nullptr; -} - -int RGWRadosBucket::remove_bucket(bool delete_children, std::string prefix, std::string delimiter, bool forward_to_master, req_info* req_info, optional_yield y) -{ - int ret; - - // Refresh info - ret = get_bucket_info(y); - if (ret < 0) - return ret; - - ListParams params; - params.list_versions = true; - params.allow_unordered = true; - - ListResults results; - - bool is_truncated = false; - do { - results.objs.clear(); - - ret = list(params, 1000, results, y); - if (ret < 0) - return ret; - - if (!results.objs.empty() && !delete_children) { - lderr(store->ctx()) << "ERROR: could not remove non-empty bucket " << info.bucket.name << - dendl; - return -ENOTEMPTY; - } - - for (const auto& obj : results.objs) { - rgw_obj_key key(obj.key); - /* xxx dang */ - ret = rgw_remove_object(store, info, info.bucket, key); - if (ret < 0 && ret != -ENOENT) { - return ret; - } - } - } while(is_truncated); - - /* If there's a prefix, then we are aborting multiparts as well */ - if (!prefix.empty()) { - ret = abort_bucket_multiparts(store, store->ctx(), info, prefix, delimiter); - if (ret < 0) { - return ret; - } - } - - ret = store->ctl()->bucket->sync_user_stats(info.owner, info); - if ( ret < 0) { - ldout(store->ctx(), 1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl; - } - - RGWObjVersionTracker ot; - - // if we deleted children above we will force delete, as any that - // remain is detrius from a prior bug - ret = store->getRados()->delete_bucket(info, ot, null_yield, !delete_children); - if (ret < 0) { - lderr(store->ctx()) << "ERROR: could not remove bucket " << - info.bucket.name << dendl; - return ret; - } - - ret = store->ctl()->bucket->unlink_bucket(info.owner, info.bucket, null_yield, false); - if (ret < 0) { - lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl; - } - - if (forward_to_master) { - bufferlist in_data; - ret = store->forward_request_to_master(owner, &ot.read_version, in_data, nullptr, *req_info); - if (ret < 0) { - if (ret == -ENOENT) { - /* adjust error, we want to return with NoSuchBucket and not - * NoSuchKey */ - ret = -ERR_NO_SUCH_BUCKET; - } - return ret; - } - } - - return ret; -} - -int RGWRadosBucket::get_bucket_info(optional_yield y) -{ - auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); - RGWSI_MetaBackend_CtxParams bectx_params = RGWSI_MetaBackend_CtxParams_SObj(&obj_ctx); - RGWObjVersionTracker ep_ot; - int ret = store->ctl()->bucket->read_bucket_info(info.bucket, &info, y, - RGWBucketCtl::BucketInstance::GetParams() - .set_mtime(&mtime) - .set_attrs(&attrs.attrs) - .set_bectx_params(bectx_params), - &ep_ot); - if (ret == 0) { - bucket_version = ep_ot.read_version; - ent.placement_rule = info.placement_rule; - } - return ret; -} - -int RGWRadosBucket::load_by_name(const std::string& tenant, const std::string& bucket_name, const std::string bucket_instance_id, RGWSysObjectCtx *rctx, optional_yield y) -{ - info.bucket.tenant = tenant; - info.bucket.name = bucket_name; - info.bucket.bucket_id = bucket_instance_id; - ent.bucket = info.bucket; - - if (bucket_instance_id.empty()) { - return get_bucket_info(y); - } - - return store->getRados()->get_bucket_instance_info(*rctx, info.bucket, info, NULL, &attrs.attrs, y); -} - -int RGWRadosBucket::get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, - std::string *bucket_ver, std::string *master_ver, - std::map& stats, - std::string *max_marker, bool *syncstopped) -{ - return store->getRados()->get_bucket_stats(bucket_info, shard_id, bucket_ver, master_ver, stats, max_marker, syncstopped); -} - -int RGWRadosBucket::read_bucket_stats(optional_yield y) -{ - int ret = store->ctl()->bucket->read_bucket_stats(info.bucket, &ent, y); - info.placement_rule = ent.placement_rule; - return ret; -} - -int RGWRadosBucket::sync_user_stats() -{ - return store->ctl()->bucket->sync_user_stats(owner->get_id(), info); -} - -int RGWRadosBucket::update_container_stats(void) -{ - int ret; - map m; - - m[info.bucket.name] = ent; - ret = store->getRados()->update_containers_stats(m); - if (!ret) - return -EEXIST; - if (ret < 0) - return ret; - - map::iterator iter = m.find(info.bucket.name); - if (iter == m.end()) - return -EINVAL; - - ent.count = iter->second.count; - ent.size = iter->second.size; - ent.size_rounded = iter->second.size_rounded; - ent.creation_time = iter->second.creation_time; - ent.placement_rule = std::move(iter->second.placement_rule); - - info.creation_time = ent.creation_time; - info.placement_rule = ent.placement_rule; - - return 0; -} - -int RGWRadosBucket::check_bucket_shards(void) -{ - return store->getRados()->check_bucket_shards(info, info.bucket, get_count()); -} - -int RGWRadosBucket::link(RGWUser* new_user, optional_yield y) -{ - RGWBucketEntryPoint ep; - ep.bucket = info.bucket; - ep.owner = new_user->get_user(); - ep.creation_time = get_creation_time(); - ep.linked = true; - map ep_attrs; - rgw_ep_info ep_data{ep, ep_attrs}; - - return store->ctl()->bucket->link_bucket(new_user->get_user(), info.bucket, - ceph::real_time(), y, true, &ep_data); -} - -int RGWRadosBucket::unlink(RGWUser* new_user, optional_yield y) -{ - return -1; -} - -int RGWRadosBucket::chown(RGWUser* new_user, RGWUser* old_user, optional_yield y) -{ - string obj_marker; - - return store->ctl()->bucket->chown(store, info, new_user->get_user(), - old_user->get_display_name(), obj_marker, y); -} - -int RGWRadosBucket::put_instance_info(bool exclusive, ceph::real_time _mtime) -{ - mtime = _mtime; - return store->getRados()->put_bucket_instance_info(info, exclusive, mtime, &attrs.attrs); -} - -/* Make sure to call get_bucket_info() if you need it first */ -bool RGWRadosBucket::is_owner(RGWUser* user) -{ - return (info.owner.compare(user->get_user()) == 0); -} - -int RGWRadosBucket::check_empty(optional_yield y) -{ - return store->getRados()->check_bucket_empty(info, y); -} - -int RGWRadosBucket::check_quota(RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, bool check_size_only) -{ - return store->getRados()->check_quota(owner->get_user(), get_key(), - user_quota, bucket_quota, obj_size, check_size_only); -} - -int RGWRadosBucket::set_instance_attrs(RGWAttrs& attrs, optional_yield y) -{ - return store->ctl()->bucket->set_bucket_instance_attrs(get_info(), - attrs.attrs, &get_info().objv_tracker, y); -} - -int RGWRadosBucket::try_refresh_info(ceph::real_time *pmtime) -{ - return store->getRados()->try_refresh_bucket_info(info, pmtime, &attrs.attrs); -} - -int RGWRadosBucket::set_acl(RGWAccessControlPolicy &acl, optional_yield y) -{ - bufferlist aclbl; - - acls = acl; - acl.encode(aclbl); - - return store->ctl()->bucket->set_acl(acl.get_owner(), info.bucket, info, aclbl, null_yield); -} - -std::unique_ptr RGWRadosBucket::get_object(const rgw_obj_key& k) -{ - return std::unique_ptr(new RGWRadosObject(this->store, k, this)); -} - -int RGWRadosBucket::list(ListParams& params, int max, ListResults& results, optional_yield y) -{ - RGWRados::Bucket target(store->getRados(), get_info()); - if (params.shard_id >= 0) { - target.set_shard_id(params.shard_id); - } - RGWRados::Bucket::List list_op(&target); - - list_op.params.prefix = params.prefix; - list_op.params.delim = params.delim; - list_op.params.marker = params.marker; - list_op.params.end_marker = params.end_marker; - list_op.params.list_versions = params.list_versions; - list_op.params.allow_unordered = params.allow_unordered; - - int ret = list_op.list_objects(max, &results.objs, &results.common_prefixes, &results.is_truncated, y); - if (ret >= 0) { - results.next_marker = list_op.get_next_marker(); - } - - return ret; -} - -std::unique_ptr RGWRadosStore::get_user(const rgw_user &u) -{ - return std::unique_ptr(new RGWRadosUser(this, u)); -} - -//RGWBucket *RGWRadosStore::create_bucket(RGWUser &u, const rgw_bucket &b) -//{ - //if (!bucket) { - //bucket = new RGWRadosBucket(this, u, b); - //} -// - //return bucket; -//} -// -void RGWRadosStore::finalize(void) -{ - if (rados) - rados->finalize(); -} - -int RGWObject::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) -{ - if (ofs < 0) { - ofs += obj_size; - if (ofs < 0) - ofs = 0; - end = obj_size - 1; - } else if (end < 0) { - end = obj_size - 1; - } - - if (obj_size > 0) { - if (ofs >= (off_t)obj_size) { - return -ERANGE; - } - if (end >= (off_t)obj_size) { - end = obj_size - 1; - } - } - return 0; -} - -int RGWRadosObject::get_obj_state(RGWObjectCtx *rctx, RGWBucket& bucket, RGWObjState **state, optional_yield y, bool follow_olh) -{ - rgw_obj obj(bucket.get_key(), key.name); - - return store->getRados()->get_obj_state(rctx, bucket.get_info(), obj, state, follow_olh, y); -} - -int RGWRadosObject::read_attrs(RGWRados::Object::Read &read_op, optional_yield y, rgw_obj *target_obj) -{ - read_op.params.attrs = &attrs.attrs; - read_op.params.target_obj = target_obj; - read_op.params.obj_size = &obj_size; - read_op.params.lastmod = &mtime; - - return read_op.prepare(y); -} - -int RGWRadosObject::set_obj_attrs(RGWObjectCtx* rctx, RGWAttrs* setattrs, RGWAttrs* delattrs, optional_yield y, rgw_obj* target_obj) -{ - map empty; - rgw_obj target = get_obj(); - - if (!target_obj) - target_obj = ⌖ - - return store->getRados()->set_attrs(rctx, - bucket->get_info(), - *target_obj, - setattrs ? setattrs->attrs : empty, - delattrs ? &delattrs->attrs : nullptr, - y); -} - -int RGWRadosObject::get_obj_attrs(RGWObjectCtx *rctx, optional_yield y, rgw_obj* target_obj) -{ - RGWRados::Object op_target(store->getRados(), bucket->get_info(), *rctx, get_obj()); - RGWRados::Object::Read read_op(&op_target); - - return read_attrs(read_op, y, target_obj); -} - -int RGWRadosObject::modify_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, bufferlist& attr_val, optional_yield y) -{ - rgw_obj target = get_obj(); - int r = get_obj_attrs(rctx, y, &target); - if (r < 0) { - return r; - } - set_atomic(rctx); - attrs.attrs[attr_name] = attr_val; - return set_obj_attrs(rctx, &attrs, nullptr, y, &target); -} - -int RGWRadosObject::delete_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, optional_yield y) -{ - RGWAttrs rmattr; - bufferlist bl; - - set_atomic(rctx); - rmattr.attrs[attr_name] = bl; - return set_obj_attrs(rctx, nullptr, &rmattr, y); -} - -int RGWRadosObject::copy_obj_data(RGWObjectCtx& rctx, RGWBucket* dest_bucket, - RGWObject* dest_obj, - uint16_t olh_epoch, - std::string* petag, - const DoutPrefixProvider *dpp, - optional_yield y) -{ - map attrset; - RGWRados::Object op_target(store->getRados(), dest_bucket->get_info(), rctx, get_obj()); - RGWRados::Object::Read read_op(&op_target); - - int ret = read_attrs(read_op, y); - if (ret < 0) - return ret; - - attrset = attrs.attrs; - - attrset.erase(RGW_ATTR_ID_TAG); - attrset.erase(RGW_ATTR_TAIL_TAG); - - return store->getRados()->copy_obj_data(rctx, dest_bucket, - dest_bucket->get_info().placement_rule, read_op, - obj_size - 1, dest_obj, NULL, mtime, attrset, 0, - real_time(), NULL, dpp, y); -} - -void RGWRadosObject::set_atomic(RGWObjectCtx *rctx) const -{ - rgw_obj obj = get_obj(); - store->getRados()->set_atomic(rctx, obj); -} - -void RGWRadosObject::set_prefetch_data(RGWObjectCtx *rctx) -{ - rgw_obj obj = get_obj(); - store->getRados()->set_prefetch_data(rctx, obj); -} - -bool RGWRadosObject::is_expired() { - map::iterator iter = attrs.find(RGW_ATTR_DELETE_AT); - if (iter != attrs.end()) { - utime_t delete_at; - try { - auto bufit = iter->second.cbegin(); - decode(delete_at, bufit); - } catch (buffer::error& err) { - ldout(store->ctx(), 0) << "ERROR: " << __func__ << ": failed to decode " RGW_ATTR_DELETE_AT " attr" << dendl; - return false; - } - - if (delete_at <= ceph_clock_now() && !delete_at.is_zero()) { - return true; - } - } - - return false; -} - -void RGWRadosObject::gen_rand_obj_instance_name() -{ - store->getRados()->gen_rand_obj_instance_name(&key); -} - -int RGWRadosObject::omap_get_vals_by_keys(const std::string& oid, - const std::set& keys, - std::map *vals) -{ - int ret; - rgw_raw_obj head_obj; - librados::IoCtx cur_ioctx; - rgw_obj obj = get_obj(); - - store->getRados()->obj_to_raw(bucket->get_placement_rule(), obj, &head_obj); - ret = store->get_obj_head_ioctx(bucket->get_info(), obj, &cur_ioctx); - if (ret < 0) { - return ret; - } - - return cur_ioctx.omap_get_vals_by_keys(oid, keys, vals); -} - -std::unique_ptr RGWRadosObject::get_read_op(RGWObjectCtx *ctx) -{ - return std::unique_ptr(new RGWRadosObject::RadosReadOp(this, ctx)); -} - -RGWRadosObject::RadosReadOp::RadosReadOp(RGWRadosObject *_source, RGWObjectCtx *_rctx) : - source(_source), - rctx(_rctx), - op_target(_source->store->getRados(), - _source->get_bucket()->get_info(), - *static_cast(rctx), - _source->get_obj()), - parent_op(&op_target) -{ } - -int RGWRadosObject::RadosReadOp::prepare(optional_yield y) -{ - uint64_t obj_size; - - parent_op.conds.mod_ptr = params.mod_ptr; - parent_op.conds.unmod_ptr = params.unmod_ptr; - parent_op.conds.high_precision_time = params.high_precision_time; - parent_op.conds.mod_zone_id = params.mod_zone_id; - parent_op.conds.mod_pg_ver = params.mod_pg_ver; - parent_op.conds.if_match = params.if_match; - parent_op.conds.if_nomatch = params.if_nomatch; - parent_op.params.lastmod = params.lastmod; - parent_op.params.target_obj = params.target_obj; - parent_op.params.obj_size = &obj_size; - parent_op.params.attrs = &source->get_attrs().attrs; - - int ret = parent_op.prepare(y); - if (ret < 0) - return ret; - - source->set_key(parent_op.state.obj.key); - source->set_obj_size(obj_size); - result.head_obj = parent_op.state.head_obj; - - return ret; -} - -int RGWRadosObject::RadosReadOp::read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y) -{ - return parent_op.read(ofs, end, bl, y); -} - -int RGWRadosObject::RadosReadOp::get_manifest(RGWObjManifest **pmanifest, - optional_yield y) -{ - return op_target.get_manifest(pmanifest, y); -} - -int RGWRadosObject::delete_object(RGWObjectCtx* obj_ctx, ACLOwner obj_owner, ACLOwner bucket_owner, ceph::real_time unmod_since, bool high_precision_time, uint64_t epoch, string& version_id, optional_yield y) -{ - int ret = 0; - RGWRados::Object del_target(store->getRados(), bucket->get_info(), *obj_ctx, get_obj()); - RGWRados::Object::Delete del_op(&del_target); - - del_op.params.olh_epoch = epoch; - del_op.params.marker_version_id = version_id; - del_op.params.bucket_owner = bucket_owner.get_id(); - del_op.params.versioning_status = bucket->get_info().versioning_status(); - del_op.params.obj_owner = obj_owner; - del_op.params.unmod_since = unmod_since; - del_op.params.high_precision_time = high_precision_time; - - ret = del_op.delete_obj(y); - if (ret >= 0) { - delete_marker = del_op.result.delete_marker; - version_id = del_op.result.version_id; - } - - return ret; -} - - -int RGWRadosObject::copy_object(RGWObjectCtx& obj_ctx, - RGWUser* user, - req_info *info, - const rgw_zone_id& source_zone, - rgw::sal::RGWObject* dest_object, - rgw::sal::RGWBucket* dest_bucket, - rgw::sal::RGWBucket* src_bucket, - const rgw_placement_rule& dest_placement, - ceph::real_time *src_mtime, - ceph::real_time *mtime, - const ceph::real_time *mod_ptr, - const ceph::real_time *unmod_ptr, - bool high_precision_time, - const char *if_match, - const char *if_nomatch, - AttrsMod attrs_mod, - bool copy_if_newer, - RGWAttrs& attrs, - RGWObjCategory category, - uint64_t olh_epoch, - boost::optional delete_at, - string *version_id, - string *tag, - string *etag, - void (*progress_cb)(off_t, void *), - void *progress_data, - const DoutPrefixProvider *dpp, - optional_yield y) -{ - return store->getRados()->copy_obj(obj_ctx, - user->get_id(), - info, - source_zone, - dest_object, - this, - dest_bucket, - src_bucket, - dest_placement, - src_mtime, - mtime, - mod_ptr, - unmod_ptr, - high_precision_time, - if_match, - if_nomatch, - static_cast(attrs_mod), - copy_if_newer, - attrs.attrs, - category, - olh_epoch, - (delete_at ? *delete_at : real_time()), - version_id, - tag, - etag, - progress_cb, - progress_data, - dpp, - y); -} - -int RGWRadosObject::RadosReadOp::iterate(int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y) -{ - return parent_op.iterate(ofs, end, cb, y); -} - -int RGWRadosStore::get_bucket(RGWUser* u, const rgw_bucket& b, std::unique_ptr* bucket) -{ - int ret; - RGWBucket* bp; - - bp = new RGWRadosBucket(this, b, u); - ret = bp->get_bucket_info(null_yield); - if (ret < 0) { - delete bp; - return ret; - } - - bucket->reset(bp); - return 0; -} - -int RGWRadosStore::get_bucket(RGWUser* u, const RGWBucketInfo& i, std::unique_ptr* bucket) -{ - RGWBucket* bp; - - bp = new RGWRadosBucket(this, i, u); - /* Don't need to fetch the bucket info, use the provided one */ - - bucket->reset(bp); - return 0; -} - -int RGWRadosStore::get_bucket(RGWUser* u, const std::string& tenant, const std::string&name, std::unique_ptr* bucket) -{ - rgw_bucket b; - - b.tenant = tenant; - b.name = name; - - return get_bucket(u, b, bucket); -} - -static int decode_policy(CephContext *cct, - bufferlist& bl, - RGWAccessControlPolicy *policy) -{ - auto iter = bl.cbegin(); - try { - policy->decode(iter); - } catch (buffer::error& err) { - ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl; - return -EIO; - } - if (cct->_conf->subsys.should_gather()) { - ldout(cct, 15) << __func__ << " Read AccessControlPolicy"; - RGWAccessControlPolicy_S3 *s3policy = static_cast(policy); - s3policy->to_xml(*_dout); - *_dout << dendl; - } - return 0; -} - -static int rgw_op_get_bucket_policy_from_attr(RGWRadosStore *store, - RGWUser& user, - map& bucket_attrs, - RGWAccessControlPolicy *policy) -{ - map::iterator aiter = bucket_attrs.find(RGW_ATTR_ACL); - - if (aiter != bucket_attrs.end()) { - int ret = decode_policy(store->ctx(), aiter->second, policy); - if (ret < 0) - return ret; - } else { - ldout(store->ctx(), 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl; - /* object exists, but policy is broken */ - int r = user.load_by_id(null_yield); - if (r < 0) - return r; - - policy->create_default(user.get_user(), user.get_display_name()); - } - return 0; -} - -bool RGWRadosStore::is_meta_master() -{ - return svc()->zone->is_meta_master(); -} - -int RGWRadosStore::forward_request_to_master(RGWUser* user, obj_version *objv, - bufferlist& in_data, - JSONParser *jp, req_info& info) -{ - if (is_meta_master()) { - /* We're master, don't forward */ - return 0; - } - - if (!svc()->zone->get_master_conn()) { - ldout(ctx(), 0) << "rest connection is invalid" << dendl; - return -EINVAL; - } - ldout(ctx(), 0) << "sending request to master zonegroup" << dendl; - bufferlist response; - string uid_str = user->get_id().to_str(); -#define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response - int ret = svc()->zone->get_master_conn()->forward(rgw_user(uid_str), info, - objv, MAX_REST_RESPONSE, - &in_data, &response); - if (ret < 0) - return ret; - - ldout(ctx(), 20) << "response: " << response.c_str() << dendl; - if (jp && !jp->parse(response.c_str(), response.length())) { - ldout(ctx(), 0) << "failed parsing response from master zonegroup" << dendl; - return -EINVAL; - } - - return 0; -} - -int RGWRadosStore::defer_gc(RGWObjectCtx *rctx, RGWBucket* bucket, RGWObject* obj, optional_yield y) -{ - return rados->defer_gc(rctx, bucket->get_info(), obj->get_obj(), y); -} - -int RGWRadosStore::create_bucket(RGWUser& u, const rgw_bucket& b, - const string& zonegroup_id, - rgw_placement_rule& placement_rule, - string& swift_ver_location, - const RGWQuotaInfo * pquota_info, - map& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool *existed, - req_info& req_info, - std::unique_ptr* bucket_out) -{ - int ret; - bufferlist in_data; - RGWBucketInfo master_info; - rgw_bucket *pmaster_bucket; - uint32_t *pmaster_num_shards; - real_time creation_time; - RGWAccessControlPolicy old_policy(ctx()); - std::unique_ptr bucket; - obj_version objv, *pobjv = NULL; - - /* If it exists, look it up; otherwise create it */ - ret = get_bucket(&u, b, &bucket); - if (ret < 0 && ret != -ENOENT) - return ret; - - if (ret != -ENOENT) { - *existed = true; - if (swift_ver_location.empty()) { - swift_ver_location = bucket->get_info().swift_ver_location; - } - placement_rule.inherit_from(bucket->get_info().placement_rule); - int r = rgw_op_get_bucket_policy_from_attr(this, u, bucket->get_attrs().attrs, - &old_policy); - if (r >= 0) { - if (old_policy.get_owner().get_id().compare(u.get_id()) != 0) { - bucket_out->swap(bucket); - ret = -EEXIST; - return ret; - } - } - } else { - bucket = std::unique_ptr(new RGWRadosBucket(this, b, &u)); - *existed = false; - bucket->set_attrs(attrs); - } - - if (!svc()->zone->is_meta_master()) { - JSONParser jp; - ret = forward_request_to_master(&u, NULL, in_data, &jp, req_info); - if (ret < 0) { - return ret; - } - - JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp); - JSONDecoder::decode_json("object_ver", objv, &jp); - JSONDecoder::decode_json("bucket_info", master_info, &jp); - ldpp_dout(this, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl; - std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time); - ldpp_dout(this, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl; - pmaster_bucket= &master_info.bucket; - creation_time = master_info.creation_time; - pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards; - pobjv = &objv; - if (master_info.obj_lock_enabled()) { - info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; - } - } else { - pmaster_bucket = NULL; - pmaster_num_shards = NULL; - if (obj_lock_enabled) - info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; - } - - std::string zid = zonegroup_id; - if (zid.empty()) { - zid = svc()->zone->get_zonegroup().get_id(); - } - - if (*existed) { - rgw_placement_rule selected_placement_rule; - ret = svc()->zone->select_bucket_placement(u.get_info(), - zid, placement_rule, - &selected_placement_rule, nullptr); - if (selected_placement_rule != info.placement_rule) { - ret = -EEXIST; - bucket_out->swap(bucket); - return ret; - } - } else { - - ret = getRados()->create_bucket(u.get_info(), bucket->get_key(), - zid, placement_rule, swift_ver_location, - pquota_info, attrs, - info, pobjv, &ep_objv, creation_time, - pmaster_bucket, pmaster_num_shards, exclusive); - if (ret == -EEXIST) { - *existed = true; - } else if (ret != 0) { - return ret; - } - } - - bucket->set_version(ep_objv); - bucket->get_info() = info; - - bucket_out->swap(bucket); - - return ret; -} - -} // namespace rgw::sal - -rgw::sal::RGWRadosStore *RGWStoreManager::init_storage_provider(CephContext *cct, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_cache) -{ - RGWRados *rados = new RGWRados; - rgw::sal::RGWRadosStore *store = new rgw::sal::RGWRadosStore(); - - store->setRados(rados); - rados->set_store(store); - - if ((*rados).set_use_cache(use_cache) - .set_run_gc_thread(use_gc_thread) - .set_run_lc_thread(use_lc_thread) - .set_run_quota_threads(quota_threads) - .set_run_sync_thread(run_sync_thread) - .set_run_reshard_thread(run_reshard_thread) - .initialize(cct) < 0) { - delete store; - return NULL; - } - - return store; -} - -rgw::sal::RGWRadosStore *RGWStoreManager::init_raw_storage_provider(CephContext *cct) -{ - RGWRados *rados = new RGWRados; - rgw::sal::RGWRadosStore *store = new rgw::sal::RGWRadosStore(); - - store->setRados(rados); - rados->set_store(store); - - rados->set_context(cct); - - int ret = rados->init_svc(true); - if (ret < 0) { - ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl; - delete store; - return nullptr; - } - - if (rados->init_rados() < 0) { - delete store; - return nullptr; - } - - return store; -} - -int rgw::sal::RGWRadosStore::get_obj_head_ioctx(const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx *ioctx) -{ - return rados->get_obj_head_ioctx(bucket_info, obj, ioctx); -} - -void RGWStoreManager::close_storage(rgw::sal::RGWRadosStore *store) -{ - if (!store) - return; - - store->finalize(); - - delete store; -} diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 5d00eaf8a14..6f7ebe01052 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -15,8 +15,12 @@ #pragma once -#include "rgw_rados.h" #include "rgw_user.h" +#include "rgw_obj_manifest.h" + +class RGWGetDataCB; +struct RGWObjState; +class RGWAccessListFilter; namespace rgw { namespace sal { @@ -443,280 +447,5 @@ class RGWObject { } }; - -class RGWRadosStore; - -class RGWRadosUser : public RGWUser { - private: - RGWRadosStore *store; - - public: - RGWRadosUser(RGWRadosStore *_st, const rgw_user& _u) : RGWUser(_u), store(_st) { } - RGWRadosUser(RGWRadosStore *_st, const RGWUserInfo& _i) : RGWUser(_i), store(_st) { } - RGWRadosUser(RGWRadosStore *_st) : store(_st) { } - RGWRadosUser() {} - - int list_buckets(const std::string& marker, const std::string& end_marker, - uint64_t max, bool need_stats, RGWBucketList& buckets); - RGWBucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time); - - /* Placeholders */ - virtual int load_by_id(optional_yield y); - - friend class RGWRadosBucket; -}; - -class RGWRadosObject : public RGWObject { - private: - RGWRadosStore *store; - RGWAccessControlPolicy acls; - - public: - - struct RadosReadOp : public ReadOp { - private: - RGWRadosObject* source; - RGWObjectCtx* rctx; - RGWRados::Object op_target; - RGWRados::Object::Read parent_op; - - public: - RadosReadOp(RGWRadosObject *_source, RGWObjectCtx *_rctx); - - virtual int prepare(optional_yield y) override; - virtual int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y) override; - virtual int iterate(int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y) override; - virtual int get_manifest(RGWObjManifest **pmanifest, optional_yield y) override; - }; - - RGWRadosObject() = default; - - RGWRadosObject(RGWRadosStore *_st, const rgw_obj_key& _k) - : RGWObject(_k), - store(_st), - acls() { - } - RGWRadosObject(RGWRadosStore *_st, const rgw_obj_key& _k, RGWBucket* _b) - : RGWObject(_k, _b), - store(_st), - acls() { - } - RGWRadosObject(RGWRadosObject& _o) = default; - - int read(off_t offset, off_t length, std::iostream& stream) { return length; } - int write(off_t offset, off_t length, std::iostream& stream) { return length; } - virtual int delete_object(RGWObjectCtx* obj_ctx, ACLOwner obj_owner, - ACLOwner bucket_owner, ceph::real_time unmod_since, - bool high_precision_time, uint64_t epoch, - std::string& version_id,optional_yield y) override; - virtual int copy_object(RGWObjectCtx& obj_ctx, RGWUser* user, - req_info *info, const rgw_zone_id& source_zone, - rgw::sal::RGWObject* dest_object, rgw::sal::RGWBucket* dest_bucket, - rgw::sal::RGWBucket* src_bucket, - const rgw_placement_rule& dest_placement, - ceph::real_time *src_mtime, ceph::real_time *mtime, - const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr, - bool high_precision_time, - const char *if_match, const char *if_nomatch, - AttrsMod attrs_mod, bool copy_if_newer, RGWAttrs& attrs, - RGWObjCategory category, uint64_t olh_epoch, - boost::optional delete_at, - string *version_id, string *tag, string *etag, - void (*progress_cb)(off_t, void *), void *progress_data, - const DoutPrefixProvider *dpp, optional_yield y) override; - RGWAccessControlPolicy& get_acl(void) { return acls; } - int set_acl(const RGWAccessControlPolicy& acl) { acls = acl; return 0; } - virtual void set_atomic(RGWObjectCtx *rctx) const; - virtual void set_prefetch_data(RGWObjectCtx *rctx); - - virtual int get_obj_state(RGWObjectCtx *rctx, RGWBucket& bucket, RGWObjState **state, optional_yield y, bool follow_olh = true) override; - virtual int set_obj_attrs(RGWObjectCtx* rctx, RGWAttrs* setattrs, RGWAttrs* delattrs, optional_yield y, rgw_obj* target_obj = NULL) override; - virtual int get_obj_attrs(RGWObjectCtx *rctx, optional_yield y, rgw_obj* target_obj = NULL) override; - virtual int modify_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, bufferlist& attr_val, optional_yield y) override; - virtual int delete_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, optional_yield y) override; - virtual int copy_obj_data(RGWObjectCtx& rctx, RGWBucket* dest_bucket, RGWObject* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider *dpp, optional_yield y) override; - virtual bool is_expired() override; - virtual void gen_rand_obj_instance_name() override; - virtual std::unique_ptr clone() { - return std::unique_ptr(new RGWRadosObject(*this)); - } - - /* OPs */ - virtual std::unique_ptr get_read_op(RGWObjectCtx *) override; - - /* OMAP */ - virtual int omap_get_vals_by_keys(const std::string& oid, - const std::set& keys, - std::map *vals) override; - - private: - int read_attrs(RGWRados::Object::Read &read_op, optional_yield y, rgw_obj *target_obj = nullptr); -}; - -class RGWRadosBucket : public RGWBucket { - private: - RGWRadosStore *store; - RGWAccessControlPolicy acls; - - public: - RGWRadosBucket(RGWRadosStore *_st) - : store(_st), - acls() { - } - - RGWRadosBucket(RGWRadosStore *_st, const rgw_bucket& _b) - : RGWBucket(_b), - store(_st), - acls() { - } - - RGWRadosBucket(RGWRadosStore *_st, const RGWBucketEnt& _e) - : RGWBucket(_e), - store(_st), - acls() { - } - - RGWRadosBucket(RGWRadosStore *_st, const RGWBucketInfo& _i) - : RGWBucket(_i), - store(_st), - acls() { - } - - RGWRadosBucket(RGWRadosStore *_st, const rgw_bucket& _b, RGWUser* _u) - : RGWBucket(_b, _u), - store(_st), - acls() { - } - - RGWRadosBucket(RGWRadosStore *_st, const RGWBucketEnt& _e, RGWUser* _u) - : RGWBucket(_e, _u), - store(_st), - acls() { - } - - RGWRadosBucket(RGWRadosStore *_st, const RGWBucketInfo& _i, RGWUser* _u) - : RGWBucket(_i, _u), - store(_st), - acls() { - } - - ~RGWRadosBucket() { } - - virtual int load_by_name(const std::string& tenant, const std::string& bucket_name, const std::string bucket_instance_id, RGWSysObjectCtx *rctx, optional_yield y) override; - virtual std::unique_ptr get_object(const rgw_obj_key& k) override; - RGWBucketList* list(void) { return new RGWBucketList(); } - virtual int list(ListParams&, int, ListResults&, optional_yield y) override; - RGWObject* create_object(const rgw_obj_key& key /* Attributes */) override; - virtual int remove_bucket(bool delete_children, std::string prefix, std::string delimiter, bool forward_to_master, req_info* req_info, optional_yield y) override; - RGWAccessControlPolicy& get_acl(void) { return acls; } - virtual int set_acl(RGWAccessControlPolicy& acl, optional_yield y) override; - virtual int get_bucket_info(optional_yield y) override; - virtual int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, - std::string *bucket_ver, std::string *master_ver, - std::map& stats, - std::string *max_marker = nullptr, - bool *syncstopped = nullptr) override; - virtual int read_bucket_stats(optional_yield y) override; - virtual int sync_user_stats() override; - virtual int update_container_stats(void) override; - virtual int check_bucket_shards(void) override; - virtual int link(RGWUser* new_user, optional_yield y) override; - virtual int unlink(RGWUser* new_user, optional_yield y) override; - virtual int chown(RGWUser* new_user, RGWUser* old_user, optional_yield y) override; - virtual int put_instance_info(bool exclusive, ceph::real_time mtime) override; - virtual bool is_owner(RGWUser* user) override; - virtual int check_empty(optional_yield y) override; - virtual int check_quota(RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, bool check_size_only = false) override; - virtual int set_instance_attrs(RGWAttrs& attrs, optional_yield y) override; - virtual int try_refresh_info(ceph::real_time *pmtime) override; - virtual std::unique_ptr clone() { - return std::unique_ptr(new RGWRadosBucket(*this)); - } - - friend class RGWRadosStore; -}; - -class RGWRadosStore : public RGWStore { - private: - RGWRados *rados; - RGWUserCtl *user_ctl; - - public: - RGWRadosStore() - : rados(nullptr) { - } - ~RGWRadosStore() { - delete rados; - } - - virtual std::unique_ptr get_user(const rgw_user& u); - virtual std::unique_ptr get_object(const rgw_obj_key& k) override; - virtual int get_bucket(RGWUser* u, const rgw_bucket& b, std::unique_ptr* bucket) override; - virtual int get_bucket(RGWUser* u, const RGWBucketInfo& i, std::unique_ptr* bucket) override; - virtual int get_bucket(RGWUser* u, const std::string& tenant, const std::string&name, std::unique_ptr* bucket) override; - virtual int create_bucket(RGWUser& u, const rgw_bucket& b, - const std::string& zonegroup_id, - rgw_placement_rule& placement_rule, - std::string& swift_ver_location, - const RGWQuotaInfo * pquota_info, - map& attrs, - RGWBucketInfo& info, - obj_version& ep_objv, - bool exclusive, - bool obj_lock_enabled, - bool *existed, - req_info& req_info, - std::unique_ptr* bucket); - virtual RGWBucketList* list_buckets(void) { return new RGWBucketList(); } - virtual bool is_meta_master() override; - virtual int forward_request_to_master(RGWUser* user, obj_version *objv, - bufferlist& in_data, JSONParser *jp, req_info& info) override; - virtual int defer_gc(RGWObjectCtx *rctx, RGWBucket* bucket, RGWObject* obj, - optional_yield y) override; - - void setRados(RGWRados * st) { rados = st; } - RGWRados *getRados(void) { return rados; } - - RGWServices *svc() { return &rados->svc; } - const RGWServices *svc() const { return &rados->svc; } - RGWCtl *ctl() { return &rados->ctl; } - const RGWCtl *ctl() const { return &rados->ctl; } - - void setUserCtl(RGWUserCtl *_ctl) { user_ctl = _ctl; } - - void finalize(void) override; - - virtual CephContext *ctx(void) { return rados->ctx(); } - - - int get_obj_head_ioctx(const RGWBucketInfo& bucket_info, const rgw_obj& obj, - librados::IoCtx *ioctx); - - // implements DoutPrefixProvider - std::ostream& gen_prefix(std::ostream& out) const { return out << "RGWRadosStore "; } - CephContext* get_cct() const override { return rados->ctx(); } - unsigned get_subsys() const override { return ceph_subsys_rgw; } - -}; - } } // namespace rgw::sal - -class RGWStoreManager { -public: - RGWStoreManager() {} - static rgw::sal::RGWRadosStore *get_storage(CephContext *cct, bool use_gc_thread, bool use_lc_thread, bool quota_threads, - bool run_sync_thread, bool run_reshard_thread, bool use_cache = true) { - rgw::sal::RGWRadosStore *store = init_storage_provider(cct, use_gc_thread, use_lc_thread, - quota_threads, run_sync_thread, run_reshard_thread, use_cache); - return store; - } - static rgw::sal::RGWRadosStore *get_raw_storage(CephContext *cct) { - rgw::sal::RGWRadosStore *rados = init_raw_storage_provider(cct); - return rados; - } - static rgw::sal::RGWRadosStore *init_storage_provider(CephContext *cct, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_metadata_cache); - static rgw::sal::RGWRadosStore *init_raw_storage_provider(CephContext *cct); - static void close_storage(rgw::sal::RGWRadosStore *store); - -}; diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/rgw_sal_rados.cc new file mode 100644 index 00000000000..06028c303b0 --- /dev/null +++ b/src/rgw/rgw_sal_rados.cc @@ -0,0 +1,978 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#include +#include +#include +#include +#include + +#include "common/Clock.h" +#include "common/errno.h" + +#include "rgw_sal.h" +#include "rgw_sal_rados.h" +#include "rgw_bucket.h" +#include "rgw_multi.h" +#include "rgw_acl_s3.h" + +/* Stuff for RGWRadosStore. Move to separate file when store split out */ +#include "rgw_zone.h" +#include "rgw_rest_conn.h" +#include "services/svc_sys_obj.h" +#include "services/svc_zone.h" + +#define dout_subsys ceph_subsys_rgw + +namespace rgw::sal { + +int RGWRadosUser::list_buckets(const string& marker, const string& end_marker, + uint64_t max, bool need_stats, RGWBucketList &buckets) +{ + RGWUserBuckets ulist; + bool is_truncated = false; + int ret; + + buckets.clear(); + ret = store->ctl()->user->list_buckets(info.user_id, marker, end_marker, max, + need_stats, &ulist, &is_truncated); + if (ret < 0) + return ret; + + buckets.set_truncated(is_truncated); + for (const auto& ent : ulist.get_buckets()) { + buckets.add(std::unique_ptr(new RGWRadosBucket(this->store, ent.second, this))); + } + + return 0; +} + +RGWBucket* RGWRadosUser::create_bucket(rgw_bucket& bucket, + ceph::real_time creation_time) +{ + return NULL; +} + +int RGWRadosUser::load_by_id(optional_yield y) + +{ + return store->ctl()->user->get_info_by_uid(info.user_id, &info, y); +} + +std::unique_ptr RGWRadosStore::get_object(const rgw_obj_key& k) +{ + return std::unique_ptr(new RGWRadosObject(this, k)); +} + +/* Placeholder */ +RGWObject *RGWRadosBucket::create_object(const rgw_obj_key &key) +{ + return nullptr; +} + +int RGWRadosBucket::remove_bucket(bool delete_children, std::string prefix, std::string delimiter, bool forward_to_master, req_info* req_info, optional_yield y) +{ + int ret; + + // Refresh info + ret = get_bucket_info(y); + if (ret < 0) + return ret; + + ListParams params; + params.list_versions = true; + params.allow_unordered = true; + + ListResults results; + + bool is_truncated = false; + do { + results.objs.clear(); + + ret = list(params, 1000, results, y); + if (ret < 0) + return ret; + + if (!results.objs.empty() && !delete_children) { + lderr(store->ctx()) << "ERROR: could not remove non-empty bucket " << info.bucket.name << + dendl; + return -ENOTEMPTY; + } + + for (const auto& obj : results.objs) { + rgw_obj_key key(obj.key); + /* xxx dang */ + ret = rgw_remove_object(store, info, info.bucket, key); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + } + } while(is_truncated); + + /* If there's a prefix, then we are aborting multiparts as well */ + if (!prefix.empty()) { + ret = abort_bucket_multiparts(store, store->ctx(), info, prefix, delimiter); + if (ret < 0) { + return ret; + } + } + + ret = store->ctl()->bucket->sync_user_stats(info.owner, info); + if ( ret < 0) { + ldout(store->ctx(), 1) << "WARNING: failed sync user stats before bucket delete. ret=" << ret << dendl; + } + + RGWObjVersionTracker ot; + + // if we deleted children above we will force delete, as any that + // remain is detrius from a prior bug + ret = store->getRados()->delete_bucket(info, ot, null_yield, !delete_children); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: could not remove bucket " << + info.bucket.name << dendl; + return ret; + } + + ret = store->ctl()->bucket->unlink_bucket(info.owner, info.bucket, null_yield, false); + if (ret < 0) { + lderr(store->ctx()) << "ERROR: unable to remove user bucket information" << dendl; + } + + if (forward_to_master) { + bufferlist in_data; + ret = store->forward_request_to_master(owner, &ot.read_version, in_data, nullptr, *req_info); + if (ret < 0) { + if (ret == -ENOENT) { + /* adjust error, we want to return with NoSuchBucket and not + * NoSuchKey */ + ret = -ERR_NO_SUCH_BUCKET; + } + return ret; + } + } + + return ret; +} + +int RGWRadosBucket::get_bucket_info(optional_yield y) +{ + auto obj_ctx = store->svc()->sysobj->init_obj_ctx(); + RGWSI_MetaBackend_CtxParams bectx_params = RGWSI_MetaBackend_CtxParams_SObj(&obj_ctx); + RGWObjVersionTracker ep_ot; + int ret = store->ctl()->bucket->read_bucket_info(info.bucket, &info, y, + RGWBucketCtl::BucketInstance::GetParams() + .set_mtime(&mtime) + .set_attrs(&attrs.attrs) + .set_bectx_params(bectx_params), + &ep_ot); + if (ret == 0) { + bucket_version = ep_ot.read_version; + ent.placement_rule = info.placement_rule; + } + return ret; +} + +int RGWRadosBucket::load_by_name(const std::string& tenant, const std::string& bucket_name, const std::string bucket_instance_id, RGWSysObjectCtx *rctx, optional_yield y) +{ + info.bucket.tenant = tenant; + info.bucket.name = bucket_name; + info.bucket.bucket_id = bucket_instance_id; + ent.bucket = info.bucket; + + if (bucket_instance_id.empty()) { + return get_bucket_info(y); + } + + return store->getRados()->get_bucket_instance_info(*rctx, info.bucket, info, NULL, &attrs.attrs, y); +} + +int RGWRadosBucket::get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map& stats, + std::string *max_marker, bool *syncstopped) +{ + return store->getRados()->get_bucket_stats(bucket_info, shard_id, bucket_ver, master_ver, stats, max_marker, syncstopped); +} + +int RGWRadosBucket::read_bucket_stats(optional_yield y) +{ + int ret = store->ctl()->bucket->read_bucket_stats(info.bucket, &ent, y); + info.placement_rule = ent.placement_rule; + return ret; +} + +int RGWRadosBucket::sync_user_stats() +{ + return store->ctl()->bucket->sync_user_stats(owner->get_id(), info); +} + +int RGWRadosBucket::update_container_stats(void) +{ + int ret; + map m; + + m[info.bucket.name] = ent; + ret = store->getRados()->update_containers_stats(m); + if (!ret) + return -EEXIST; + if (ret < 0) + return ret; + + map::iterator iter = m.find(info.bucket.name); + if (iter == m.end()) + return -EINVAL; + + ent.count = iter->second.count; + ent.size = iter->second.size; + ent.size_rounded = iter->second.size_rounded; + ent.creation_time = iter->second.creation_time; + ent.placement_rule = std::move(iter->second.placement_rule); + + info.creation_time = ent.creation_time; + info.placement_rule = ent.placement_rule; + + return 0; +} + +int RGWRadosBucket::check_bucket_shards(void) +{ + return store->getRados()->check_bucket_shards(info, info.bucket, get_count()); +} + +int RGWRadosBucket::link(RGWUser* new_user, optional_yield y) +{ + RGWBucketEntryPoint ep; + ep.bucket = info.bucket; + ep.owner = new_user->get_user(); + ep.creation_time = get_creation_time(); + ep.linked = true; + map ep_attrs; + rgw_ep_info ep_data{ep, ep_attrs}; + + return store->ctl()->bucket->link_bucket(new_user->get_user(), info.bucket, + ceph::real_time(), y, true, &ep_data); +} + +int RGWRadosBucket::unlink(RGWUser* new_user, optional_yield y) +{ + return -1; +} + +int RGWRadosBucket::chown(RGWUser* new_user, RGWUser* old_user, optional_yield y) +{ + string obj_marker; + + return store->ctl()->bucket->chown(store, info, new_user->get_user(), + old_user->get_display_name(), obj_marker, y); +} + +int RGWRadosBucket::put_instance_info(bool exclusive, ceph::real_time _mtime) +{ + mtime = _mtime; + return store->getRados()->put_bucket_instance_info(info, exclusive, mtime, &attrs.attrs); +} + +/* Make sure to call get_bucket_info() if you need it first */ +bool RGWRadosBucket::is_owner(RGWUser* user) +{ + return (info.owner.compare(user->get_user()) == 0); +} + +int RGWRadosBucket::check_empty(optional_yield y) +{ + return store->getRados()->check_bucket_empty(info, y); +} + +int RGWRadosBucket::check_quota(RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, bool check_size_only) +{ + return store->getRados()->check_quota(owner->get_user(), get_key(), + user_quota, bucket_quota, obj_size, check_size_only); +} + +int RGWRadosBucket::set_instance_attrs(RGWAttrs& attrs, optional_yield y) +{ + return store->ctl()->bucket->set_bucket_instance_attrs(get_info(), + attrs.attrs, &get_info().objv_tracker, y); +} + +int RGWRadosBucket::try_refresh_info(ceph::real_time *pmtime) +{ + return store->getRados()->try_refresh_bucket_info(info, pmtime, &attrs.attrs); +} + +int RGWRadosBucket::set_acl(RGWAccessControlPolicy &acl, optional_yield y) +{ + bufferlist aclbl; + + acls = acl; + acl.encode(aclbl); + + return store->ctl()->bucket->set_acl(acl.get_owner(), info.bucket, info, aclbl, null_yield); +} + +std::unique_ptr RGWRadosBucket::get_object(const rgw_obj_key& k) +{ + return std::unique_ptr(new RGWRadosObject(this->store, k, this)); +} + +int RGWRadosBucket::list(ListParams& params, int max, ListResults& results, optional_yield y) +{ + RGWRados::Bucket target(store->getRados(), get_info()); + if (params.shard_id >= 0) { + target.set_shard_id(params.shard_id); + } + RGWRados::Bucket::List list_op(&target); + + list_op.params.prefix = params.prefix; + list_op.params.delim = params.delim; + list_op.params.marker = params.marker; + list_op.params.end_marker = params.end_marker; + list_op.params.list_versions = params.list_versions; + list_op.params.allow_unordered = params.allow_unordered; + + int ret = list_op.list_objects(max, &results.objs, &results.common_prefixes, &results.is_truncated, y); + if (ret >= 0) { + results.next_marker = list_op.get_next_marker(); + } + + return ret; +} + +std::unique_ptr RGWRadosStore::get_user(const rgw_user &u) +{ + return std::unique_ptr(new RGWRadosUser(this, u)); +} + +//RGWBucket *RGWRadosStore::create_bucket(RGWUser &u, const rgw_bucket &b) +//{ + //if (!bucket) { + //bucket = new RGWRadosBucket(this, u, b); + //} +// + //return bucket; +//} +// +void RGWRadosStore::finalize(void) +{ + if (rados) + rados->finalize(); +} + +int RGWObject::range_to_ofs(uint64_t obj_size, int64_t &ofs, int64_t &end) +{ + if (ofs < 0) { + ofs += obj_size; + if (ofs < 0) + ofs = 0; + end = obj_size - 1; + } else if (end < 0) { + end = obj_size - 1; + } + + if (obj_size > 0) { + if (ofs >= (off_t)obj_size) { + return -ERANGE; + } + if (end >= (off_t)obj_size) { + end = obj_size - 1; + } + } + return 0; +} + +int RGWRadosObject::get_obj_state(RGWObjectCtx *rctx, RGWBucket& bucket, RGWObjState **state, optional_yield y, bool follow_olh) +{ + rgw_obj obj(bucket.get_key(), key.name); + + return store->getRados()->get_obj_state(rctx, bucket.get_info(), obj, state, follow_olh, y); +} + +int RGWRadosObject::read_attrs(RGWRados::Object::Read &read_op, optional_yield y, rgw_obj *target_obj) +{ + read_op.params.attrs = &attrs.attrs; + read_op.params.target_obj = target_obj; + read_op.params.obj_size = &obj_size; + read_op.params.lastmod = &mtime; + + return read_op.prepare(y); +} + +int RGWRadosObject::set_obj_attrs(RGWObjectCtx* rctx, RGWAttrs* setattrs, RGWAttrs* delattrs, optional_yield y, rgw_obj* target_obj) +{ + map empty; + rgw_obj target = get_obj(); + + if (!target_obj) + target_obj = ⌖ + + return store->getRados()->set_attrs(rctx, + bucket->get_info(), + *target_obj, + setattrs ? setattrs->attrs : empty, + delattrs ? &delattrs->attrs : nullptr, + y); +} + +int RGWRadosObject::get_obj_attrs(RGWObjectCtx *rctx, optional_yield y, rgw_obj* target_obj) +{ + RGWRados::Object op_target(store->getRados(), bucket->get_info(), *rctx, get_obj()); + RGWRados::Object::Read read_op(&op_target); + + return read_attrs(read_op, y, target_obj); +} + +int RGWRadosObject::modify_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, bufferlist& attr_val, optional_yield y) +{ + rgw_obj target = get_obj(); + int r = get_obj_attrs(rctx, y, &target); + if (r < 0) { + return r; + } + set_atomic(rctx); + attrs.attrs[attr_name] = attr_val; + return set_obj_attrs(rctx, &attrs, nullptr, y, &target); +} + +int RGWRadosObject::delete_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, optional_yield y) +{ + RGWAttrs rmattr; + bufferlist bl; + + set_atomic(rctx); + rmattr.attrs[attr_name] = bl; + return set_obj_attrs(rctx, nullptr, &rmattr, y); +} + +int RGWRadosObject::copy_obj_data(RGWObjectCtx& rctx, RGWBucket* dest_bucket, + RGWObject* dest_obj, + uint16_t olh_epoch, + std::string* petag, + const DoutPrefixProvider *dpp, + optional_yield y) +{ + map attrset; + RGWRados::Object op_target(store->getRados(), dest_bucket->get_info(), rctx, get_obj()); + RGWRados::Object::Read read_op(&op_target); + + int ret = read_attrs(read_op, y); + if (ret < 0) + return ret; + + attrset = attrs.attrs; + + attrset.erase(RGW_ATTR_ID_TAG); + attrset.erase(RGW_ATTR_TAIL_TAG); + + return store->getRados()->copy_obj_data(rctx, dest_bucket, + dest_bucket->get_info().placement_rule, read_op, + obj_size - 1, dest_obj, NULL, mtime, attrset, 0, + real_time(), NULL, dpp, y); +} + +void RGWRadosObject::set_atomic(RGWObjectCtx *rctx) const +{ + rgw_obj obj = get_obj(); + store->getRados()->set_atomic(rctx, obj); +} + +void RGWRadosObject::set_prefetch_data(RGWObjectCtx *rctx) +{ + rgw_obj obj = get_obj(); + store->getRados()->set_prefetch_data(rctx, obj); +} + +bool RGWRadosObject::is_expired() { + map::iterator iter = attrs.find(RGW_ATTR_DELETE_AT); + if (iter != attrs.end()) { + utime_t delete_at; + try { + auto bufit = iter->second.cbegin(); + decode(delete_at, bufit); + } catch (buffer::error& err) { + ldout(store->ctx(), 0) << "ERROR: " << __func__ << ": failed to decode " RGW_ATTR_DELETE_AT " attr" << dendl; + return false; + } + + if (delete_at <= ceph_clock_now() && !delete_at.is_zero()) { + return true; + } + } + + return false; +} + +void RGWRadosObject::gen_rand_obj_instance_name() +{ + store->getRados()->gen_rand_obj_instance_name(&key); +} + +int RGWRadosObject::omap_get_vals_by_keys(const std::string& oid, + const std::set& keys, + std::map *vals) +{ + int ret; + rgw_raw_obj head_obj; + librados::IoCtx cur_ioctx; + rgw_obj obj = get_obj(); + + store->getRados()->obj_to_raw(bucket->get_placement_rule(), obj, &head_obj); + ret = store->get_obj_head_ioctx(bucket->get_info(), obj, &cur_ioctx); + if (ret < 0) { + return ret; + } + + return cur_ioctx.omap_get_vals_by_keys(oid, keys, vals); +} + +std::unique_ptr RGWRadosObject::get_read_op(RGWObjectCtx *ctx) +{ + return std::unique_ptr(new RGWRadosObject::RadosReadOp(this, ctx)); +} + +RGWRadosObject::RadosReadOp::RadosReadOp(RGWRadosObject *_source, RGWObjectCtx *_rctx) : + source(_source), + rctx(_rctx), + op_target(_source->store->getRados(), + _source->get_bucket()->get_info(), + *static_cast(rctx), + _source->get_obj()), + parent_op(&op_target) +{ } + +int RGWRadosObject::RadosReadOp::prepare(optional_yield y) +{ + uint64_t obj_size; + + parent_op.conds.mod_ptr = params.mod_ptr; + parent_op.conds.unmod_ptr = params.unmod_ptr; + parent_op.conds.high_precision_time = params.high_precision_time; + parent_op.conds.mod_zone_id = params.mod_zone_id; + parent_op.conds.mod_pg_ver = params.mod_pg_ver; + parent_op.conds.if_match = params.if_match; + parent_op.conds.if_nomatch = params.if_nomatch; + parent_op.params.lastmod = params.lastmod; + parent_op.params.target_obj = params.target_obj; + parent_op.params.obj_size = &obj_size; + parent_op.params.attrs = &source->get_attrs().attrs; + + int ret = parent_op.prepare(y); + if (ret < 0) + return ret; + + source->set_key(parent_op.state.obj.key); + source->set_obj_size(obj_size); + result.head_obj = parent_op.state.head_obj; + + return ret; +} + +int RGWRadosObject::RadosReadOp::read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y) +{ + return parent_op.read(ofs, end, bl, y); +} + +int RGWRadosObject::RadosReadOp::get_manifest(RGWObjManifest **pmanifest, + optional_yield y) +{ + return op_target.get_manifest(pmanifest, y); +} + +int RGWRadosObject::delete_object(RGWObjectCtx* obj_ctx, ACLOwner obj_owner, ACLOwner bucket_owner, ceph::real_time unmod_since, bool high_precision_time, uint64_t epoch, string& version_id, optional_yield y) +{ + int ret = 0; + RGWRados::Object del_target(store->getRados(), bucket->get_info(), *obj_ctx, get_obj()); + RGWRados::Object::Delete del_op(&del_target); + + del_op.params.olh_epoch = epoch; + del_op.params.marker_version_id = version_id; + del_op.params.bucket_owner = bucket_owner.get_id(); + del_op.params.versioning_status = bucket->get_info().versioning_status(); + del_op.params.obj_owner = obj_owner; + del_op.params.unmod_since = unmod_since; + del_op.params.high_precision_time = high_precision_time; + + ret = del_op.delete_obj(y); + if (ret >= 0) { + delete_marker = del_op.result.delete_marker; + version_id = del_op.result.version_id; + } + + return ret; +} + +int RGWRadosObject::copy_object(RGWObjectCtx& obj_ctx, + RGWUser* user, + req_info *info, + const rgw_zone_id& source_zone, + rgw::sal::RGWObject* dest_object, + rgw::sal::RGWBucket* dest_bucket, + rgw::sal::RGWBucket* src_bucket, + const rgw_placement_rule& dest_placement, + ceph::real_time *src_mtime, + ceph::real_time *mtime, + const ceph::real_time *mod_ptr, + const ceph::real_time *unmod_ptr, + bool high_precision_time, + const char *if_match, + const char *if_nomatch, + AttrsMod attrs_mod, + bool copy_if_newer, + RGWAttrs& attrs, + RGWObjCategory category, + uint64_t olh_epoch, + boost::optional delete_at, + string *version_id, + string *tag, + string *etag, + void (*progress_cb)(off_t, void *), + void *progress_data, + const DoutPrefixProvider *dpp, + optional_yield y) +{ + return store->getRados()->copy_obj(obj_ctx, + user->get_id(), + info, + source_zone, + dest_object, + this, + dest_bucket, + src_bucket, + dest_placement, + src_mtime, + mtime, + mod_ptr, + unmod_ptr, + high_precision_time, + if_match, + if_nomatch, + static_cast(attrs_mod), + copy_if_newer, + attrs.attrs, + category, + olh_epoch, + (delete_at ? *delete_at : real_time()), + version_id, + tag, + etag, + progress_cb, + progress_data, + dpp, + y); +} + +int RGWRadosObject::RadosReadOp::iterate(int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y) +{ + return parent_op.iterate(ofs, end, cb, y); +} + +int RGWRadosStore::get_bucket(RGWUser* u, const rgw_bucket& b, std::unique_ptr* bucket) +{ + int ret; + RGWBucket* bp; + + bp = new RGWRadosBucket(this, b, u); + ret = bp->get_bucket_info(null_yield); + if (ret < 0) { + delete bp; + return ret; + } + + bucket->reset(bp); + return 0; +} + +int RGWRadosStore::get_bucket(RGWUser* u, const RGWBucketInfo& i, std::unique_ptr* bucket) +{ + RGWBucket* bp; + + bp = new RGWRadosBucket(this, i, u); + /* Don't need to fetch the bucket info, use the provided one */ + + bucket->reset(bp); + return 0; +} + +int RGWRadosStore::get_bucket(RGWUser* u, const std::string& tenant, const std::string&name, std::unique_ptr* bucket) +{ + rgw_bucket b; + + b.tenant = tenant; + b.name = name; + + return get_bucket(u, b, bucket); +} + +static int decode_policy(CephContext *cct, + bufferlist& bl, + RGWAccessControlPolicy *policy) +{ + auto iter = bl.cbegin(); + try { + policy->decode(iter); + } catch (buffer::error& err) { + ldout(cct, 0) << "ERROR: could not decode policy, caught buffer::error" << dendl; + return -EIO; + } + if (cct->_conf->subsys.should_gather()) { + ldout(cct, 15) << __func__ << " Read AccessControlPolicy"; + RGWAccessControlPolicy_S3 *s3policy = static_cast(policy); + s3policy->to_xml(*_dout); + *_dout << dendl; + } + return 0; +} + +static int rgw_op_get_bucket_policy_from_attr(RGWRadosStore *store, + RGWUser& user, + map& bucket_attrs, + RGWAccessControlPolicy *policy) +{ + map::iterator aiter = bucket_attrs.find(RGW_ATTR_ACL); + + if (aiter != bucket_attrs.end()) { + int ret = decode_policy(store->ctx(), aiter->second, policy); + if (ret < 0) + return ret; + } else { + ldout(store->ctx(), 0) << "WARNING: couldn't find acl header for bucket, generating default" << dendl; + /* object exists, but policy is broken */ + int r = user.load_by_id(null_yield); + if (r < 0) + return r; + + policy->create_default(user.get_user(), user.get_display_name()); + } + return 0; +} + +bool RGWRadosStore::is_meta_master() +{ + return svc()->zone->is_meta_master(); +} + +int RGWRadosStore::forward_request_to_master(RGWUser* user, obj_version *objv, + bufferlist& in_data, + JSONParser *jp, req_info& info) +{ + if (is_meta_master()) { + /* We're master, don't forward */ + return 0; + } + + if (!svc()->zone->get_master_conn()) { + ldout(ctx(), 0) << "rest connection is invalid" << dendl; + return -EINVAL; + } + ldout(ctx(), 0) << "sending request to master zonegroup" << dendl; + bufferlist response; + string uid_str = user->get_id().to_str(); +#define MAX_REST_RESPONSE (128 * 1024) // we expect a very small response + int ret = svc()->zone->get_master_conn()->forward(rgw_user(uid_str), info, + objv, MAX_REST_RESPONSE, + &in_data, &response); + if (ret < 0) + return ret; + + ldout(ctx(), 20) << "response: " << response.c_str() << dendl; + if (jp && !jp->parse(response.c_str(), response.length())) { + ldout(ctx(), 0) << "failed parsing response from master zonegroup" << dendl; + return -EINVAL; + } + + return 0; +} + +int RGWRadosStore::defer_gc(RGWObjectCtx *rctx, RGWBucket* bucket, RGWObject* obj, optional_yield y) +{ + return rados->defer_gc(rctx, bucket->get_info(), obj->get_obj(), y); +} + +int RGWRadosStore::create_bucket(RGWUser& u, const rgw_bucket& b, + const string& zonegroup_id, + rgw_placement_rule& placement_rule, + string& swift_ver_location, + const RGWQuotaInfo * pquota_info, + map& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool *existed, + req_info& req_info, + std::unique_ptr* bucket_out) +{ + int ret; + bufferlist in_data; + RGWBucketInfo master_info; + rgw_bucket *pmaster_bucket; + uint32_t *pmaster_num_shards; + real_time creation_time; + RGWAccessControlPolicy old_policy(ctx()); + std::unique_ptr bucket; + obj_version objv, *pobjv = NULL; + + /* If it exists, look it up; otherwise create it */ + ret = get_bucket(&u, b, &bucket); + if (ret < 0 && ret != -ENOENT) + return ret; + + if (ret != -ENOENT) { + *existed = true; + if (swift_ver_location.empty()) { + swift_ver_location = bucket->get_info().swift_ver_location; + } + placement_rule.inherit_from(bucket->get_info().placement_rule); + int r = rgw_op_get_bucket_policy_from_attr(this, u, bucket->get_attrs().attrs, + &old_policy); + if (r >= 0) { + if (old_policy.get_owner().get_id().compare(u.get_id()) != 0) { + bucket_out->swap(bucket); + ret = -EEXIST; + return ret; + } + } + } else { + bucket = std::unique_ptr(new RGWRadosBucket(this, b, &u)); + *existed = false; + bucket->set_attrs(attrs); + } + + if (!svc()->zone->is_meta_master()) { + JSONParser jp; + ret = forward_request_to_master(&u, NULL, in_data, &jp, req_info); + if (ret < 0) { + return ret; + } + + JSONDecoder::decode_json("entry_point_object_ver", ep_objv, &jp); + JSONDecoder::decode_json("object_ver", objv, &jp); + JSONDecoder::decode_json("bucket_info", master_info, &jp); + ldpp_dout(this, 20) << "parsed: objv.tag=" << objv.tag << " objv.ver=" << objv.ver << dendl; + std::time_t ctime = ceph::real_clock::to_time_t(master_info.creation_time); + ldpp_dout(this, 20) << "got creation time: << " << std::put_time(std::localtime(&ctime), "%F %T") << dendl; + pmaster_bucket= &master_info.bucket; + creation_time = master_info.creation_time; + pmaster_num_shards = &master_info.layout.current_index.layout.normal.num_shards; + pobjv = &objv; + if (master_info.obj_lock_enabled()) { + info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; + } + } else { + pmaster_bucket = NULL; + pmaster_num_shards = NULL; + if (obj_lock_enabled) + info.flags = BUCKET_VERSIONED | BUCKET_OBJ_LOCK_ENABLED; + } + + std::string zid = zonegroup_id; + if (zid.empty()) { + zid = svc()->zone->get_zonegroup().get_id(); + } + + if (*existed) { + rgw_placement_rule selected_placement_rule; + ret = svc()->zone->select_bucket_placement(u.get_info(), + zid, placement_rule, + &selected_placement_rule, nullptr); + if (selected_placement_rule != info.placement_rule) { + ret = -EEXIST; + bucket_out->swap(bucket); + return ret; + } + } else { + + ret = getRados()->create_bucket(u.get_info(), bucket->get_key(), + zid, placement_rule, swift_ver_location, + pquota_info, attrs, + info, pobjv, &ep_objv, creation_time, + pmaster_bucket, pmaster_num_shards, exclusive); + if (ret == -EEXIST) { + *existed = true; + } else if (ret != 0) { + return ret; + } + } + + bucket->set_version(ep_objv); + bucket->get_info() = info; + + bucket_out->swap(bucket); + + return ret; +} + +} // namespace rgw::sal + +rgw::sal::RGWRadosStore *RGWStoreManager::init_storage_provider(CephContext *cct, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_cache) +{ + RGWRados *rados = new RGWRados; + rgw::sal::RGWRadosStore *store = new rgw::sal::RGWRadosStore(); + + store->setRados(rados); + rados->set_store(store); + + if ((*rados).set_use_cache(use_cache) + .set_run_gc_thread(use_gc_thread) + .set_run_lc_thread(use_lc_thread) + .set_run_quota_threads(quota_threads) + .set_run_sync_thread(run_sync_thread) + .set_run_reshard_thread(run_reshard_thread) + .initialize(cct) < 0) { + delete store; + return NULL; + } + + return store; +} + +rgw::sal::RGWRadosStore *RGWStoreManager::init_raw_storage_provider(CephContext *cct) +{ + RGWRados *rados = new RGWRados; + rgw::sal::RGWRadosStore *store = new rgw::sal::RGWRadosStore(); + + store->setRados(rados); + rados->set_store(store); + + rados->set_context(cct); + + int ret = rados->init_svc(true); + if (ret < 0) { + ldout(cct, 0) << "ERROR: failed to init services (ret=" << cpp_strerror(-ret) << ")" << dendl; + delete store; + return nullptr; + } + + if (rados->init_rados() < 0) { + delete store; + return nullptr; + } + + return store; +} + +int rgw::sal::RGWRadosStore::get_obj_head_ioctx(const RGWBucketInfo& bucket_info, const rgw_obj& obj, librados::IoCtx *ioctx) +{ + return rados->get_obj_head_ioctx(bucket_info, obj, ioctx); +} + +void RGWStoreManager::close_storage(rgw::sal::RGWRadosStore *store) +{ + if (!store) + return; + + store->finalize(); + + delete store; +} diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/rgw_sal_rados.h new file mode 100644 index 00000000000..82eb1c1def0 --- /dev/null +++ b/src/rgw/rgw_sal_rados.h @@ -0,0 +1,297 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2020 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + +#pragma once + +#include "rgw_sal.h" +#include "rgw_rados.h" + +namespace rgw { namespace sal { + +class RGWRadosStore; + +class RGWRadosUser : public RGWUser { + private: + RGWRadosStore *store; + + public: + RGWRadosUser(RGWRadosStore *_st, const rgw_user& _u) : RGWUser(_u), store(_st) { } + RGWRadosUser(RGWRadosStore *_st, const RGWUserInfo& _i) : RGWUser(_i), store(_st) { } + RGWRadosUser(RGWRadosStore *_st) : store(_st) { } + RGWRadosUser() {} + + int list_buckets(const std::string& marker, const std::string& end_marker, + uint64_t max, bool need_stats, RGWBucketList& buckets); + RGWBucket* create_bucket(rgw_bucket& bucket, ceph::real_time creation_time); + + /* Placeholders */ + virtual int load_by_id(optional_yield y); + + friend class RGWRadosBucket; +}; + +class RGWRadosObject : public RGWObject { + private: + RGWRadosStore *store; + RGWAccessControlPolicy acls; + + public: + + struct RadosReadOp : public ReadOp { + private: + RGWRadosObject* source; + RGWObjectCtx* rctx; + RGWRados::Object op_target; + RGWRados::Object::Read parent_op; + + public: + RadosReadOp(RGWRadosObject *_source, RGWObjectCtx *_rctx); + + virtual int prepare(optional_yield y) override; + virtual int read(int64_t ofs, int64_t end, bufferlist& bl, optional_yield y) override; + virtual int iterate(int64_t ofs, int64_t end, RGWGetDataCB *cb, optional_yield y) override; + virtual int get_manifest(RGWObjManifest **pmanifest, optional_yield y) override; + }; + + RGWRadosObject() = default; + + RGWRadosObject(RGWRadosStore *_st, const rgw_obj_key& _k) + : RGWObject(_k), + store(_st), + acls() { + } + RGWRadosObject(RGWRadosStore *_st, const rgw_obj_key& _k, RGWBucket* _b) + : RGWObject(_k, _b), + store(_st), + acls() { + } + RGWRadosObject(RGWRadosObject& _o) = default; + + int read(off_t offset, off_t length, std::iostream& stream) { return length; } + int write(off_t offset, off_t length, std::iostream& stream) { return length; } + virtual int delete_object(RGWObjectCtx* obj_ctx, ACLOwner obj_owner, + ACLOwner bucket_owner, ceph::real_time unmod_since, + bool high_precision_time, uint64_t epoch, + std::string& version_id,optional_yield y) override; + virtual int copy_object(RGWObjectCtx& obj_ctx, RGWUser* user, + req_info *info, const rgw_zone_id& source_zone, + rgw::sal::RGWObject* dest_object, rgw::sal::RGWBucket* dest_bucket, + rgw::sal::RGWBucket* src_bucket, + const rgw_placement_rule& dest_placement, + ceph::real_time *src_mtime, ceph::real_time *mtime, + const ceph::real_time *mod_ptr, const ceph::real_time *unmod_ptr, + bool high_precision_time, + const char *if_match, const char *if_nomatch, + AttrsMod attrs_mod, bool copy_if_newer, RGWAttrs& attrs, + RGWObjCategory category, uint64_t olh_epoch, + boost::optional delete_at, + string *version_id, string *tag, string *etag, + void (*progress_cb)(off_t, void *), void *progress_data, + const DoutPrefixProvider *dpp, optional_yield y) override; + RGWAccessControlPolicy& get_acl(void) { return acls; } + int set_acl(const RGWAccessControlPolicy& acl) { acls = acl; return 0; } + virtual void set_atomic(RGWObjectCtx *rctx) const; + virtual void set_prefetch_data(RGWObjectCtx *rctx); + + virtual int get_obj_state(RGWObjectCtx *rctx, RGWBucket& bucket, RGWObjState **state, optional_yield y, bool follow_olh = true) override; + virtual int set_obj_attrs(RGWObjectCtx* rctx, RGWAttrs* setattrs, RGWAttrs* delattrs, optional_yield y, rgw_obj* target_obj = NULL) override; + virtual int get_obj_attrs(RGWObjectCtx *rctx, optional_yield y, rgw_obj* target_obj = NULL) override; + virtual int modify_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, bufferlist& attr_val, optional_yield y) override; + virtual int delete_obj_attrs(RGWObjectCtx *rctx, const char *attr_name, optional_yield y) override; + virtual int copy_obj_data(RGWObjectCtx& rctx, RGWBucket* dest_bucket, RGWObject* dest_obj, uint16_t olh_epoch, std::string* petag, const DoutPrefixProvider *dpp, optional_yield y) override; + virtual bool is_expired() override; + virtual void gen_rand_obj_instance_name() override; + virtual std::unique_ptr clone() { + return std::unique_ptr(new RGWRadosObject(*this)); + } + + /* OPs */ + virtual std::unique_ptr get_read_op(RGWObjectCtx *) override; + + /* OMAP */ + virtual int omap_get_vals_by_keys(const std::string& oid, + const std::set& keys, + std::map *vals) override; + + private: + int read_attrs(RGWRados::Object::Read &read_op, optional_yield y, rgw_obj *target_obj = nullptr); +}; + +class RGWRadosBucket : public RGWBucket { + private: + RGWRadosStore *store; + RGWAccessControlPolicy acls; + + public: + RGWRadosBucket(RGWRadosStore *_st) + : store(_st), + acls() { + } + + RGWRadosBucket(RGWRadosStore *_st, const rgw_bucket& _b) + : RGWBucket(_b), + store(_st), + acls() { + } + + RGWRadosBucket(RGWRadosStore *_st, const RGWBucketEnt& _e) + : RGWBucket(_e), + store(_st), + acls() { + } + + RGWRadosBucket(RGWRadosStore *_st, const RGWBucketInfo& _i) + : RGWBucket(_i), + store(_st), + acls() { + } + + RGWRadosBucket(RGWRadosStore *_st, const rgw_bucket& _b, RGWUser* _u) + : RGWBucket(_b, _u), + store(_st), + acls() { + } + + RGWRadosBucket(RGWRadosStore *_st, const RGWBucketEnt& _e, RGWUser* _u) + : RGWBucket(_e, _u), + store(_st), + acls() { + } + + RGWRadosBucket(RGWRadosStore *_st, const RGWBucketInfo& _i, RGWUser* _u) + : RGWBucket(_i, _u), + store(_st), + acls() { + } + + ~RGWRadosBucket() { } + + virtual int load_by_name(const std::string& tenant, const std::string& bucket_name, const std::string bucket_instance_id, RGWSysObjectCtx *rctx, optional_yield y) override; + virtual std::unique_ptr get_object(const rgw_obj_key& k) override; + RGWBucketList* list(void) { return new RGWBucketList(); } + virtual int list(ListParams&, int, ListResults&, optional_yield y) override; + RGWObject* create_object(const rgw_obj_key& key /* Attributes */) override; + virtual int remove_bucket(bool delete_children, std::string prefix, std::string delimiter, bool forward_to_master, req_info* req_info, optional_yield y) override; + RGWAccessControlPolicy& get_acl(void) { return acls; } + virtual int set_acl(RGWAccessControlPolicy& acl, optional_yield y) override; + virtual int get_bucket_info(optional_yield y) override; + virtual int get_bucket_stats(RGWBucketInfo& bucket_info, int shard_id, + std::string *bucket_ver, std::string *master_ver, + std::map& stats, + std::string *max_marker = nullptr, + bool *syncstopped = nullptr) override; + virtual int read_bucket_stats(optional_yield y) override; + virtual int sync_user_stats() override; + virtual int update_container_stats(void) override; + virtual int check_bucket_shards(void) override; + virtual int link(RGWUser* new_user, optional_yield y) override; + virtual int unlink(RGWUser* new_user, optional_yield y) override; + virtual int chown(RGWUser* new_user, RGWUser* old_user, optional_yield y) override; + virtual int put_instance_info(bool exclusive, ceph::real_time mtime) override; + virtual bool is_owner(RGWUser* user) override; + virtual int check_empty(optional_yield y) override; + virtual int check_quota(RGWQuotaInfo& user_quota, RGWQuotaInfo& bucket_quota, uint64_t obj_size, bool check_size_only = false) override; + virtual int set_instance_attrs(RGWAttrs& attrs, optional_yield y) override; + virtual int try_refresh_info(ceph::real_time *pmtime) override; + virtual std::unique_ptr clone() { + return std::unique_ptr(new RGWRadosBucket(*this)); + } + + friend class RGWRadosStore; +}; + +class RGWRadosStore : public RGWStore { + private: + RGWRados *rados; + RGWUserCtl *user_ctl; + + public: + RGWRadosStore() + : rados(nullptr) { + } + ~RGWRadosStore() { + delete rados; + } + + virtual std::unique_ptr get_user(const rgw_user& u); + virtual std::unique_ptr get_object(const rgw_obj_key& k) override; + virtual int get_bucket(RGWUser* u, const rgw_bucket& b, std::unique_ptr* bucket) override; + virtual int get_bucket(RGWUser* u, const RGWBucketInfo& i, std::unique_ptr* bucket) override; + virtual int get_bucket(RGWUser* u, const std::string& tenant, const std::string&name, std::unique_ptr* bucket) override; + virtual int create_bucket(RGWUser& u, const rgw_bucket& b, + const std::string& zonegroup_id, + rgw_placement_rule& placement_rule, + std::string& swift_ver_location, + const RGWQuotaInfo * pquota_info, + map& attrs, + RGWBucketInfo& info, + obj_version& ep_objv, + bool exclusive, + bool obj_lock_enabled, + bool *existed, + req_info& req_info, + std::unique_ptr* bucket); + virtual RGWBucketList* list_buckets(void) { return new RGWBucketList(); } + virtual bool is_meta_master() override; + virtual int forward_request_to_master(RGWUser* user, obj_version *objv, + bufferlist& in_data, JSONParser *jp, req_info& info) override; + virtual int defer_gc(RGWObjectCtx *rctx, RGWBucket* bucket, RGWObject* obj, + optional_yield y) override; + + void setRados(RGWRados * st) { rados = st; } + RGWRados *getRados(void) { return rados; } + + RGWServices *svc() { return &rados->svc; } + const RGWServices *svc() const { return &rados->svc; } + RGWCtl *ctl() { return &rados->ctl; } + const RGWCtl *ctl() const { return &rados->ctl; } + + void setUserCtl(RGWUserCtl *_ctl) { user_ctl = _ctl; } + + void finalize(void) override; + + virtual CephContext *ctx(void) { return rados->ctx(); } + + + int get_obj_head_ioctx(const RGWBucketInfo& bucket_info, const rgw_obj& obj, + librados::IoCtx *ioctx); + + // implements DoutPrefixProvider + std::ostream& gen_prefix(std::ostream& out) const { return out << "RGWRadosStore "; } + CephContext* get_cct() const override { return rados->ctx(); } + unsigned get_subsys() const override { return ceph_subsys_rgw; } + +}; + +} } // namespace rgw::sal + +class RGWStoreManager { +public: + RGWStoreManager() {} + static rgw::sal::RGWRadosStore *get_storage(CephContext *cct, bool use_gc_thread, bool use_lc_thread, bool quota_threads, + bool run_sync_thread, bool run_reshard_thread, bool use_cache = true) { + rgw::sal::RGWRadosStore *store = init_storage_provider(cct, use_gc_thread, use_lc_thread, + quota_threads, run_sync_thread, run_reshard_thread, use_cache); + return store; + } + static rgw::sal::RGWRadosStore *get_raw_storage(CephContext *cct) { + rgw::sal::RGWRadosStore *rados = init_raw_storage_provider(cct); + return rados; + } + static rgw::sal::RGWRadosStore *init_storage_provider(CephContext *cct, bool use_gc_thread, bool use_lc_thread, bool quota_threads, bool run_sync_thread, bool run_reshard_thread, bool use_metadata_cache); + static rgw::sal::RGWRadosStore *init_raw_storage_provider(CephContext *cct); + static void close_storage(rgw::sal::RGWRadosStore *store); + +}; diff --git a/src/rgw/rgw_sts.cc b/src/rgw/rgw_sts.cc index b4084c08640..2415f59fac3 100644 --- a/src/rgw/rgw_sts.cc +++ b/src/rgw/rgw_sts.cc @@ -26,6 +26,7 @@ #include "rgw_iam_policy.h" #include "rgw_sts.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_swift_auth.cc b/src/rgw/rgw_swift_auth.cc index 242d24294ef..bf0e03f5306 100644 --- a/src/rgw/rgw_swift_auth.cc +++ b/src/rgw/rgw_swift_auth.cc @@ -19,6 +19,7 @@ #include "rgw_client_io.h" #include "rgw_http_client.h" +#include "rgw_sal_rados.h" #include "include/str_list.h" #define dout_context g_ceph_context diff --git a/src/rgw/rgw_sync.h b/src/rgw/rgw_sync.h index fe8f4cff93a..e7e4f5b4e78 100644 --- a/src/rgw/rgw_sync.h +++ b/src/rgw/rgw_sync.h @@ -14,13 +14,10 @@ #include "rgw_metadata.h" #include "rgw_meta_sync_status.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "rgw_sync_trace.h" #include "rgw_mdlog.h" -namespace rgw { namespace sal { - class RGWRadosStore; -} } - #define ERROR_LOGGER_SHARDS 32 #define RGW_SYNC_ERROR_LOG_SHARD_PREFIX "sync.error-log" diff --git a/src/rgw/rgw_sync_module_es_rest.cc b/src/rgw/rgw_sync_module_es_rest.cc index 675e89196ab..bf9617bbc4b 100644 --- a/src/rgw/rgw_sync_module_es_rest.cc +++ b/src/rgw/rgw_sync_module_es_rest.cc @@ -7,6 +7,7 @@ #include "rgw_op.h" #include "rgw_rest.h" #include "rgw_rest_s3.h" +#include "rgw_sal_rados.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_sync_module_pubsub_rest.cc b/src/rgw/rgw_sync_module_pubsub_rest.cc index f2fff54cf76..0c24414be51 100644 --- a/src/rgw/rgw_sync_module_pubsub_rest.cc +++ b/src/rgw/rgw_sync_module_pubsub_rest.cc @@ -14,6 +14,7 @@ #include "rgw_arn.h" #include "rgw_zone.h" #include "services/svc_zone.h" +#include "rgw_sal_rados.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index 5ea5c72fc3e..10b2e69893d 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -19,6 +19,7 @@ #include "rgw_aio_throttle.h" #include "rgw_compression.h" #include "rgw_zone.h" +#include "rgw_sal_rados.h" #include "osd/osd_types.h" #include "services/svc_sys_obj.h" diff --git a/src/rgw/rgw_torrent.cc b/src/rgw/rgw_torrent.cc index 939f37c5421..22ee5b46ab6 100644 --- a/src/rgw/rgw_torrent.cc +++ b/src/rgw/rgw_torrent.cc @@ -8,6 +8,7 @@ #include "rgw_torrent.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "include/str_list.h" #include "include/rados/librados.hpp" diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 9622d8fa1e3..ad4f6b06127 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -12,6 +12,7 @@ #include "common/ceph_json.h" #include "common/RWLock.h" #include "rgw_sal.h" +#include "rgw_sal_rados.h" #include "rgw_zone.h" #include "rgw_acl.h" diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc index 7652e3c9f3e..cf512cae93a 100644 --- a/src/test/rgw/test_rgw_iam_policy.cc +++ b/src/test/rgw/test_rgw_iam_policy.cc @@ -26,6 +26,7 @@ #include "rgw/rgw_auth.h" #include "rgw/rgw_iam_policy.h" #include "rgw/rgw_op.h" +#include "rgw_sal_rados.h" using std::string; -- 2.39.5