From 5ba9842b92562e33b3770c02d4e62db63383c3da Mon Sep 17 00:00:00 2001 From: "Kaleb S. KEITHLEY" Date: Wed, 16 Nov 2022 07:36:47 -0500 Subject: [PATCH] rgw: refactor selected files for better above- vs below-the-line Based on https://github.com/ceph/ceph/pull/48272, separate selected methods into new files for above-the-line vs below-the-line linkage. This is more of the work to prepare for eventual merging of the loadable module implementation. "Utility" functions that don't reference the Store are moved, e.g. from rgw_zone.cc, into the new rgw_zone_utils.cc. Methods in the new *_utils.cc files are above-the-line. Signed-off-by: Kaleb S. KEITHLEY --- src/rgw/CMakeLists.txt | 97 +- src/rgw/rgw_bucket.cc | 3059 +------------ src/rgw/rgw_common.cc | 12 + src/rgw/rgw_lua.cc | 5 +- src/rgw/rgw_metadata.cc | 539 +-- src/rgw/rgw_multipart_meta_filter.cc | 32 + src/rgw/rgw_obj_manifest.cc | 377 +- src/rgw/rgw_period.cc | 350 ++ src/rgw/rgw_rados.cc | 56 - src/rgw/rgw_realm.cc | 265 ++ src/rgw/rgw_sync.cc | 2588 +---------- src/rgw/rgw_tools.cc | 419 -- src/rgw/rgw_user.cc | 2783 +----------- src/rgw/rgw_zone.cc | 3969 ++++------------- src/rgw/services/svc_bi.h | 2 +- src/rgw/services/svc_bi_rados.cc | 6 +- src/rgw/services/svc_bi_rados.h | 6 +- src/rgw/services/svc_bilog_rados.h | 2 +- src/rgw/services/svc_bucket.h | 2 +- src/rgw/services/svc_bucket_sobj.cc | 8 +- src/rgw/services/svc_bucket_sobj.h | 2 +- src/rgw/services/svc_bucket_sync.h | 2 +- src/rgw/services/svc_bucket_sync_sobj.cc | 6 +- src/rgw/services/svc_bucket_sync_sobj.h | 2 +- src/rgw/services/svc_cls.cc | 2 +- src/rgw/services/svc_cls.h | 2 +- src/rgw/services/svc_config_key.h | 2 +- src/rgw/services/svc_config_key_rados.h | 2 +- src/rgw/services/svc_finisher.h | 2 +- src/rgw/services/svc_mdlog.cc | 10 +- src/rgw/services/svc_mdlog.h | 6 +- src/rgw/services/svc_meta.cc | 2 +- src/rgw/services/svc_meta.h | 2 +- src/rgw/services/svc_meta_be.cc | 2 +- src/rgw/services/svc_meta_be.h | 4 +- src/rgw/services/svc_meta_be_otp.cc | 6 +- src/rgw/services/svc_meta_be_otp.h | 2 +- src/rgw/services/svc_meta_be_sobj.cc | 6 +- src/rgw/services/svc_meta_be_sobj.h | 2 +- src/rgw/services/svc_notify.cc | 4 +- src/rgw/services/svc_notify.h | 2 +- src/rgw/services/svc_otp.cc | 2 +- src/rgw/services/svc_otp.h | 2 +- src/rgw/services/svc_quota.cc | 2 +- src/rgw/services/svc_quota.h | 2 +- src/rgw/services/svc_rados.cc | 4 +- src/rgw/services/svc_rados.h | 2 +- src/rgw/services/svc_role_rados.h | 6 +- src/rgw/services/svc_sync_modules.cc | 4 +- src/rgw/services/svc_sync_modules.h | 4 +- src/rgw/services/svc_sys_obj.cc | 2 +- src/rgw/services/svc_sys_obj.h | 2 +- src/rgw/services/svc_sys_obj_cache.cc | 4 +- src/rgw/services/svc_sys_obj_cache.h | 4 +- src/rgw/services/svc_sys_obj_core.cc | 2 +- src/rgw/services/svc_sys_obj_core.h | 2 +- src/rgw/services/svc_sys_obj_core_types.h | 2 +- src/rgw/services/svc_sys_obj_types.h | 2 +- src/rgw/services/svc_tier_rados.h | 2 +- src/rgw/services/svc_user.h | 2 +- src/rgw/services/svc_user_rados.cc | 10 +- src/rgw/services/svc_user_rados.h | 2 +- src/rgw/services/svc_zone.cc | 6 +- src/rgw/services/svc_zone.h | 2 +- src/rgw/services/svc_zone_utils.cc | 2 +- src/rgw/services/svc_zone_utils.h | 2 +- src/rgw/store/dbstore/CMakeLists.txt | 8 +- src/rgw/store/dbstore/common/dbstore.h | 10 +- src/rgw/{ => store/rados}/cls_fifo_legacy.cc | 0 src/rgw/{ => store/rados}/cls_fifo_legacy.h | 0 src/rgw/store/rados/rgw_bucket.cc | 2971 ++++++++++++ src/rgw/{ => store/rados}/rgw_bucket.h | 0 src/rgw/{ => store/rados}/rgw_bucket_sync.cc | 0 src/rgw/{ => store/rados}/rgw_bucket_sync.h | 0 src/rgw/{ => store/rados}/rgw_cr_rados.cc | 0 src/rgw/{ => store/rados}/rgw_cr_rados.h | 0 src/rgw/{ => store/rados}/rgw_cr_tools.cc | 0 src/rgw/{ => store/rados}/rgw_cr_tools.h | 0 .../{ => store/rados}/rgw_d3n_datacache.cc | 0 src/rgw/{ => store/rados}/rgw_d3n_datacache.h | 0 src/rgw/{ => store/rados}/rgw_data_sync.cc | 0 src/rgw/{ => store/rados}/rgw_data_sync.h | 0 src/rgw/{ => store/rados}/rgw_datalog.cc | 0 src/rgw/{ => store/rados}/rgw_datalog.h | 0 .../{ => store/rados}/rgw_datalog_notify.cc | 0 .../{ => store/rados}/rgw_datalog_notify.h | 2 +- .../{ => store/rados}/rgw_etag_verifier.cc | 0 src/rgw/{ => store/rados}/rgw_etag_verifier.h | 0 src/rgw/{ => store/rados}/rgw_gc.cc | 0 src/rgw/{ => store/rados}/rgw_gc.h | 0 src/rgw/{ => store/rados}/rgw_gc_log.cc | 0 src/rgw/{ => store/rados}/rgw_lc_tier.cc | 0 src/rgw/{ => store/rados}/rgw_lc_tier.h | 0 src/rgw/{ => store/rados}/rgw_log_backing.cc | 0 src/rgw/{ => store/rados}/rgw_log_backing.h | 0 src/rgw/store/rados/rgw_metadata.cc | 233 + src/rgw/{ => store/rados}/rgw_metadata.h | 2 +- src/rgw/{ => store/rados}/rgw_notify.cc | 0 src/rgw/{ => store/rados}/rgw_notify.h | 0 src/rgw/store/rados/rgw_obj_manifest.cc | 404 ++ src/rgw/{ => store/rados}/rgw_obj_manifest.h | 0 .../rados}/rgw_object_expirer_core.cc | 0 .../rados}/rgw_object_expirer_core.h | 0 src/rgw/{ => store/rados}/rgw_otp.cc | 0 src/rgw/{ => store/rados}/rgw_otp.h | 0 src/rgw/store/rados/rgw_period.cc | 324 ++ src/rgw/{ => store/rados}/rgw_rest_pubsub.cc | 0 src/rgw/{ => store/rados}/rgw_rest_pubsub.h | 0 .../rados}/rgw_rest_pubsub_common.cc | 0 .../rados}/rgw_rest_pubsub_common.h | 0 src/rgw/{ => store/rados}/rgw_rest_realm.cc | 0 src/rgw/{ => store/rados}/rgw_rest_realm.h | 0 src/rgw/{ => store/rados}/rgw_rest_user.cc | 0 src/rgw/{ => store/rados}/rgw_rest_user.h | 0 src/rgw/{ => store/rados}/rgw_sal_rados.cc | 0 src/rgw/{ => store/rados}/rgw_sal_rados.h | 0 src/rgw/{ => store/rados}/rgw_service.cc | 0 src/rgw/{ => store/rados}/rgw_service.h | 2 +- src/rgw/store/rados/rgw_sync.cc | 2567 +++++++++++ src/rgw/{ => store/rados}/rgw_sync.h | 0 .../{ => store/rados}/rgw_sync_counters.cc | 0 src/rgw/{ => store/rados}/rgw_sync_counters.h | 0 .../{ => store/rados}/rgw_sync_error_repo.cc | 0 .../{ => store/rados}/rgw_sync_error_repo.h | 0 src/rgw/{ => store/rados}/rgw_sync_module.cc | 0 src/rgw/{ => store/rados}/rgw_sync_module.h | 0 .../{ => store/rados}/rgw_sync_module_aws.cc | 0 .../{ => store/rados}/rgw_sync_module_aws.h | 0 .../{ => store/rados}/rgw_sync_module_es.cc | 0 .../{ => store/rados}/rgw_sync_module_es.h | 0 .../rados}/rgw_sync_module_es_rest.cc | 0 .../rados}/rgw_sync_module_es_rest.h | 0 .../{ => store/rados}/rgw_sync_module_log.cc | 0 .../{ => store/rados}/rgw_sync_module_log.h | 0 .../rados}/rgw_sync_module_pubsub.cc | 0 .../rados}/rgw_sync_module_pubsub.h | 0 src/rgw/{ => store/rados}/rgw_sync_trace.cc | 0 src/rgw/{ => store/rados}/rgw_sync_trace.h | 0 src/rgw/store/rados/rgw_tools.cc | 414 ++ src/rgw/{ => store/rados}/rgw_tools.h | 0 src/rgw/{ => store/rados}/rgw_trim_bilog.cc | 0 src/rgw/{ => store/rados}/rgw_trim_bilog.h | 2 +- src/rgw/{ => store/rados}/rgw_trim_datalog.cc | 0 src/rgw/{ => store/rados}/rgw_trim_datalog.h | 0 src/rgw/{ => store/rados}/rgw_trim_mdlog.cc | 0 src/rgw/{ => store/rados}/rgw_trim_mdlog.h | 0 src/rgw/store/rados/rgw_user.cc | 2768 ++++++++++++ src/rgw/{ => store/rados}/rgw_user.h | 0 src/rgw/store/rados/rgw_zone.cc | 1287 ++++++ src/rgw/{ => store/rados}/rgw_zone.h | 2 + src/test/CMakeLists.txt | 15 +- src/test/librgw_file_marker.cc | 4 +- src/test/librgw_file_nfsns.cc | 4 +- src/test/librgw_file_xattr.cc | 2 +- src/test/rgw/CMakeLists.txt | 42 +- src/test/rgw/bench_rgw_ratelimit.cc | 4 +- src/test/rgw/bench_rgw_ratelimit_gc.cc | 4 +- src/test/rgw/test_cls_fifo_legacy.cc | 4 +- src/test/rgw/test_http_manager.cc | 4 +- src/test/rgw/test_log_backing.cc | 4 +- src/test/rgw/test_rgw_amqp.cc | 2 +- src/test/rgw/test_rgw_arn.cc | 2 +- src/test/rgw/test_rgw_bencode.cc | 2 +- src/test/rgw/test_rgw_bucket_sync_cache.cc | 2 +- src/test/rgw/test_rgw_common.h | 6 +- src/test/rgw/test_rgw_compression.cc | 2 +- src/test/rgw/test_rgw_crypto.cc | 6 +- src/test/rgw/test_rgw_dmclock_scheduler.cc | 4 +- src/test/rgw/test_rgw_gc_log.cc | 2 +- src/test/rgw/test_rgw_iam_policy.cc | 6 +- src/test/rgw/test_rgw_kms.cc | 4 +- src/test/rgw/test_rgw_lua.cc | 14 +- src/test/rgw/test_rgw_manifest.cc | 4 +- src/test/rgw/test_rgw_obj.cc | 6 +- src/test/rgw/test_rgw_period_history.cc | 6 +- src/test/rgw/test_rgw_putobj.cc | 2 +- src/test/rgw/test_rgw_ratelimit.cc | 2 +- src/test/rgw/test_rgw_reshard.cc | 2 +- src/test/rgw/test_rgw_reshard_wait.cc | 2 +- src/test/rgw/test_rgw_string.cc | 2 +- src/test/rgw/test_rgw_throttle.cc | 2 +- src/test/rgw/test_rgw_url.cc | 2 +- src/test/rgw/test_rgw_xml.cc | 2 +- src/test/test_cors.cc | 4 +- src/test/test_rgw_admin_log.cc | 10 +- src/test/test_rgw_admin_meta.cc | 4 +- src/test/test_rgw_ldap.cc | 4 +- src/test/test_rgw_token.cc | 4 +- src/tools/ceph-dencoder/CMakeLists.txt | 2 + src/tools/ceph-dencoder/rgw_types.h | 22 +- 190 files changed, 13039 insertions(+), 12901 deletions(-) create mode 100644 src/rgw/rgw_multipart_meta_filter.cc create mode 100644 src/rgw/rgw_period.cc create mode 100644 src/rgw/rgw_realm.cc rename src/rgw/{ => store/rados}/cls_fifo_legacy.cc (100%) rename src/rgw/{ => store/rados}/cls_fifo_legacy.h (100%) create mode 100644 src/rgw/store/rados/rgw_bucket.cc rename src/rgw/{ => store/rados}/rgw_bucket.h (100%) rename src/rgw/{ => store/rados}/rgw_bucket_sync.cc (100%) rename src/rgw/{ => store/rados}/rgw_bucket_sync.h (100%) rename src/rgw/{ => store/rados}/rgw_cr_rados.cc (100%) rename src/rgw/{ => store/rados}/rgw_cr_rados.h (100%) rename src/rgw/{ => store/rados}/rgw_cr_tools.cc (100%) rename src/rgw/{ => store/rados}/rgw_cr_tools.h (100%) rename src/rgw/{ => store/rados}/rgw_d3n_datacache.cc (100%) rename src/rgw/{ => store/rados}/rgw_d3n_datacache.h (100%) rename src/rgw/{ => store/rados}/rgw_data_sync.cc (100%) rename src/rgw/{ => store/rados}/rgw_data_sync.h (100%) rename src/rgw/{ => store/rados}/rgw_datalog.cc (100%) rename src/rgw/{ => store/rados}/rgw_datalog.h (100%) rename src/rgw/{ => store/rados}/rgw_datalog_notify.cc (100%) rename src/rgw/{ => store/rados}/rgw_datalog_notify.h (96%) rename src/rgw/{ => store/rados}/rgw_etag_verifier.cc (100%) rename src/rgw/{ => store/rados}/rgw_etag_verifier.h (100%) rename src/rgw/{ => store/rados}/rgw_gc.cc (100%) rename src/rgw/{ => store/rados}/rgw_gc.h (100%) rename src/rgw/{ => store/rados}/rgw_gc_log.cc (100%) rename src/rgw/{ => store/rados}/rgw_lc_tier.cc (100%) rename src/rgw/{ => store/rados}/rgw_lc_tier.h (100%) rename src/rgw/{ => store/rados}/rgw_log_backing.cc (100%) rename src/rgw/{ => store/rados}/rgw_log_backing.h (100%) create mode 100644 src/rgw/store/rados/rgw_metadata.cc rename src/rgw/{ => store/rados}/rgw_metadata.h (99%) rename src/rgw/{ => store/rados}/rgw_notify.cc (100%) rename src/rgw/{ => store/rados}/rgw_notify.h (100%) create mode 100644 src/rgw/store/rados/rgw_obj_manifest.cc rename src/rgw/{ => store/rados}/rgw_obj_manifest.h (100%) rename src/rgw/{ => store/rados}/rgw_object_expirer_core.cc (100%) rename src/rgw/{ => store/rados}/rgw_object_expirer_core.h (100%) rename src/rgw/{ => store/rados}/rgw_otp.cc (100%) rename src/rgw/{ => store/rados}/rgw_otp.h (100%) create mode 100644 src/rgw/store/rados/rgw_period.cc rename src/rgw/{ => store/rados}/rgw_rest_pubsub.cc (100%) rename src/rgw/{ => store/rados}/rgw_rest_pubsub.h (100%) rename src/rgw/{ => store/rados}/rgw_rest_pubsub_common.cc (100%) rename src/rgw/{ => store/rados}/rgw_rest_pubsub_common.h (100%) rename src/rgw/{ => store/rados}/rgw_rest_realm.cc (100%) rename src/rgw/{ => store/rados}/rgw_rest_realm.h (100%) rename src/rgw/{ => store/rados}/rgw_rest_user.cc (100%) rename src/rgw/{ => store/rados}/rgw_rest_user.h (100%) rename src/rgw/{ => store/rados}/rgw_sal_rados.cc (100%) rename src/rgw/{ => store/rados}/rgw_sal_rados.h (100%) rename src/rgw/{ => store/rados}/rgw_service.cc (100%) rename src/rgw/{ => store/rados}/rgw_service.h (99%) create mode 100644 src/rgw/store/rados/rgw_sync.cc rename src/rgw/{ => store/rados}/rgw_sync.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_counters.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_counters.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_error_repo.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_error_repo.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_module.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_module.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_aws.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_aws.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_es.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_es.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_es_rest.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_es_rest.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_log.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_log.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_pubsub.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_module_pubsub.h (100%) rename src/rgw/{ => store/rados}/rgw_sync_trace.cc (100%) rename src/rgw/{ => store/rados}/rgw_sync_trace.h (100%) create mode 100644 src/rgw/store/rados/rgw_tools.cc rename src/rgw/{ => store/rados}/rgw_tools.h (100%) rename src/rgw/{ => store/rados}/rgw_trim_bilog.cc (100%) rename src/rgw/{ => store/rados}/rgw_trim_bilog.h (99%) rename src/rgw/{ => store/rados}/rgw_trim_datalog.cc (100%) rename src/rgw/{ => store/rados}/rgw_trim_datalog.h (100%) rename src/rgw/{ => store/rados}/rgw_trim_mdlog.cc (100%) rename src/rgw/{ => store/rados}/rgw_trim_mdlog.h (100%) create mode 100644 src/rgw/store/rados/rgw_user.cc rename src/rgw/{ => store/rados}/rgw_user.h (100%) create mode 100644 src/rgw/store/rados/rgw_zone.cc rename src/rgw/{ => store/rados}/rgw_zone.h (99%) diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index 02c1fa575583..4b6c34b87a19 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -48,7 +48,6 @@ set(librgw_common_srcs services/svc_user_rados.cc services/svc_zone.cc services/svc_zone_utils.cc - rgw_service.cc rgw_acl.cc rgw_acl_s3.cc rgw_acl_swift.cc @@ -60,59 +59,38 @@ set(librgw_common_srcs rgw_basic_types.cc rgw_bucket.cc rgw_bucket_layout.cc - rgw_bucket_sync.cc rgw_cache.cc - rgw_d3n_datacache.cc rgw_common.cc rgw_compression.cc - rgw_etag_verifier.cc rgw_cors.cc rgw_cors_s3.cc rgw_env.cc rgw_es_query.cc rgw_formats.cc - rgw_gc.cc - rgw_gc_log.cc rgw_http_client.cc rgw_keystone.cc rgw_ldap.cc rgw_lc.cc rgw_lc_s3.cc - rgw_lc_tier.cc rgw_metadata.cc rgw_multi.cc rgw_multi_del.cc + rgw_multipart_meta_filter.cc rgw_obj_manifest.cc + rgw_period.cc rgw_pubsub.cc + rgw_realm.cc rgw_sync.cc - rgw_data_sync.cc - rgw_sync_counters.cc - rgw_sync_error_repo.cc - rgw_sync_module.cc - rgw_sync_module_aws.cc - rgw_sync_module_es.cc - rgw_sync_module_es_rest.cc - rgw_sync_module_log.cc - rgw_sync_module_pubsub.cc rgw_sync_policy.cc rgw_pubsub_push.cc - rgw_notify.cc rgw_notify_event_type.cc rgw_sync_module_pubsub_rest.cc - rgw_sync_trace.cc - rgw_trim_bilog.cc - rgw_trim_datalog.cc - rgw_trim_mdlog.cc rgw_period_history.cc rgw_period_puller.cc rgw_reshard.cc rgw_coroutine.cc - rgw_cr_rados.cc rgw_cr_rest.cc - rgw_cr_tools.cc - rgw_object_expirer_core.cc rgw_op.cc - rgw_otp.cc rgw_policy_s3.cc rgw_public_access.cc rgw_putobj.cc @@ -127,23 +105,17 @@ set(librgw_common_srcs rgw_rest_conn.cc rgw_rest_log.cc rgw_rest_metadata.cc - rgw_rest_pubsub.cc - rgw_rest_pubsub_common.cc rgw_rest_ratelimit.cc - rgw_rest_realm.cc rgw_rest_role.cc rgw_rest_s3.cc - rgw_rest_user.cc rgw_s3select.cc rgw_role.cc rgw_sal.cc rgw_sal_filter.cc - rgw_sal_rados.cc rgw_string.cc rgw_tag.cc rgw_tag_s3.cc rgw_tools.cc - rgw_log_backing.cc rgw_user.cc rgw_website.cc rgw_xml.cc @@ -163,9 +135,6 @@ set(librgw_common_srcs rgw_kmip_client.cc rgw_url.cc rgw_oidc_provider.cc - rgw_datalog.cc - rgw_datalog_notify.cc - cls_fifo_legacy.cc rgw_log.cc rgw_lua_request.cc rgw_lua_utils.cc @@ -173,7 +142,49 @@ set(librgw_common_srcs rgw_lua_data_filter.cc rgw_bucket_encryption.cc rgw_tracer.cc - rgw_lua_background.cc) + rgw_lua_background.cc + store/rados/cls_fifo_legacy.cc + store/rados/rgw_bucket.cc + store/rados/rgw_bucket_sync.cc + store/rados/rgw_cr_rados.cc + store/rados/rgw_cr_tools.cc + store/rados/rgw_d3n_datacache.cc + store/rados/rgw_datalog.cc + store/rados/rgw_datalog_notify.cc + store/rados/rgw_data_sync.cc + store/rados/rgw_etag_verifier.cc + store/rados/rgw_gc.cc + store/rados/rgw_gc_log.cc + store/rados/rgw_lc_tier.cc + store/rados/rgw_log_backing.cc + store/rados/rgw_metadata.cc + store/rados/rgw_notify.cc + store/rados/rgw_obj_manifest.cc + store/rados/rgw_object_expirer_core.cc + store/rados/rgw_otp.cc + store/rados/rgw_period.cc + store/rados/rgw_rest_pubsub.cc + store/rados/rgw_rest_pubsub_common.cc + store/rados/rgw_rest_realm.cc + store/rados/rgw_rest_user.cc + store/rados/rgw_sal_rados.cc + store/rados/rgw_service.cc + store/rados/rgw_sync.cc + store/rados/rgw_sync_counters.cc + store/rados/rgw_sync_error_repo.cc + store/rados/rgw_sync_module.cc + store/rados/rgw_sync_module_aws.cc + store/rados/rgw_sync_module_es.cc + store/rados/rgw_sync_module_es_rest.cc + store/rados/rgw_sync_module_log.cc + store/rados/rgw_sync_module_pubsub.cc + store/rados/rgw_sync_trace.cc + store/rados/rgw_tools.cc + store/rados/rgw_trim_bilog.cc + store/rados/rgw_trim_datalog.cc + store/rados/rgw_trim_mdlog.cc + store/rados/rgw_user.cc + store/rados/rgw_zone.cc) list(APPEND librgw_common_srcs store/immutable_config/store.cc @@ -244,7 +255,8 @@ target_link_libraries(rgw_common ${LUA_LIBRARIES} spawn) target_include_directories(rgw_common - PUBLIC "services" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/services" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/store/rados" PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw" PUBLIC "${LUA_INCLUDE_DIR}") if(WITH_RADOSGW_KAFKA_ENDPOINT) @@ -325,7 +337,7 @@ set(rgw_a_srcs rgw_rest_log.cc rgw_rest_metadata.cc rgw_rest_ratelimit.cc - rgw_rest_realm.cc + store/rados/rgw_rest_realm.cc rgw_rest_sts.cc rgw_rest_swift.cc rgw_rest_usage.cc @@ -345,10 +357,13 @@ add_library(rgw_a STATIC ${rgw_a_srcs}) target_compile_definitions(rgw_a PUBLIC "-DCLS_CLIENT_HIDE_IOCTX") -target_include_directories(rgw_a PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src") -target_include_directories(rgw_a PRIVATE "${CMAKE_SOURCE_DIR}/src/libkmip") -target_include_directories(rgw_a SYSTEM PUBLIC "../rapidjson/include") -target_include_directories(rgw_a PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw") + +target_include_directories(rgw_a + SYSTEM PUBLIC "../rapidjson/include" + PUBLIC "${CMAKE_SOURCE_DIR}/src/dmclock/support/src" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/store/rados" + PRIVATE "${CMAKE_SOURCE_DIR}/src/libkmip") if(WITH_RADOSGW_AMQP_ENDPOINT) find_package(RabbitMQ REQUIRED) diff --git a/src/rgw/rgw_bucket.cc b/src/rgw/rgw_bucket.cc index b9f1eb3de22a..7d8dba72bd8c 100644 --- a/src/rgw/rgw_bucket.cc +++ b/src/rgw/rgw_bucket.cc @@ -1,69 +1,15 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp -#include -#include -#include -#include -#include - -#include - -#include "common/errno.h" -#include "common/ceph_json.h" -#include "include/scope_guard.h" - -#include "rgw_datalog.h" -#include "rgw_rados.h" -#include "rgw_zone.h" -#include "rgw_acl.h" -#include "rgw_acl_s3.h" -#include "rgw_tag_s3.h" - -#include "include/types.h" #include "rgw_bucket.h" -#include "rgw_user.h" -#include "rgw_string.h" -#include "rgw_multi.h" -#include "rgw_op.h" -#include "rgw_bucket_sync.h" -#include "services/svc_zone.h" -#include "services/svc_sys_obj.h" -#include "services/svc_bucket.h" -#include "services/svc_bucket_sync.h" -#include "services/svc_meta.h" -#include "services/svc_meta_be_sobj.h" -#include "services/svc_user.h" -#include "services/svc_cls.h" -#include "services/svc_bilog_rados.h" - -#include "include/rados/librados.hpp" -// until everything is moved from rgw_common -#include "rgw_common.h" -#include "rgw_reshard.h" -#include "rgw_lc.h" -#include "rgw_bucket_layout.h" +#define dout_subsys ceph_subsys_rgw // stolen from src/cls/version/cls_version.cc #define VERSION_ATTR "ceph.objclass.version" -#include "cls/user/cls_user_types.h" - -#include "rgw_sal_rados.h" - -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_rgw - -// seconds for timeout during RGWBucket::check_object_index -constexpr uint64_t BUCKET_TAG_QUICK_TIMEOUT = 30; - using namespace std; -// default number of entries to list with each bucket listing call -// (use marker to bridge between calls) -static constexpr size_t listing_max_entries = 1000; - void init_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, const char *ip, const char *m, const char *id) { b->tenant = t; @@ -74,91 +20,6 @@ void init_bucket(rgw_bucket *b, const char *t, const char *n, const char *dp, co b->explicit_placement.index_pool = rgw_pool(ip); } -/* - * The tenant_name is always returned on purpose. May be empty, of course. - */ -static void parse_bucket(const string& bucket, - string *tenant_name, - string *bucket_name, - string *bucket_instance = nullptr /* optional */) -{ - /* - * expected format: [tenant/]bucket:bucket_instance - */ - int pos = bucket.find('/'); - if (pos >= 0) { - *tenant_name = bucket.substr(0, pos); - } else { - tenant_name->clear(); - } - string bn = bucket.substr(pos + 1); - pos = bn.find (':'); - if (pos < 0) { - *bucket_name = std::move(bn); - return; - } - *bucket_name = bn.substr(0, pos); - if (bucket_instance) { - *bucket_instance = bn.substr(pos + 1); - } - - /* - * deal with the possible tenant:bucket:bucket_instance case - */ - if (tenant_name->empty()) { - pos = bucket_instance->find(':'); - if (pos >= 0) { - *tenant_name = *bucket_name; - *bucket_name = bucket_instance->substr(0, pos); - *bucket_instance = bucket_instance->substr(pos + 1); - } - } -} - -/* - * Note that this is not a reversal of parse_bucket(). That one deals - * with the syntax we need in metadata and such. This one deals with - * the representation in RADOS pools. We chose '/' because it's not - * acceptable in bucket names and thus qualified buckets cannot conflict - * with the legacy or S3 buckets. - */ -std::string rgw_make_bucket_entry_name(const std::string& tenant_name, - const std::string& bucket_name) { - std::string bucket_entry; - - if (bucket_name.empty()) { - bucket_entry.clear(); - } else if (tenant_name.empty()) { - bucket_entry = bucket_name; - } else { - bucket_entry = tenant_name + "/" + bucket_name; - } - - return bucket_entry; -} - -/* - * Tenants are separated from buckets in URLs by a colon in S3. - * This function is not to be used on Swift URLs, not even for COPY arguments. - */ -void rgw_parse_url_bucket(const string &bucket, const string& auth_tenant, - string &tenant_name, string &bucket_name) { - - int pos = bucket.find(':'); - if (pos >= 0) { - /* - * N.B.: We allow ":bucket" syntax with explicit empty tenant in order - * to refer to the legacy tenant, in case users in new named tenants - * want to access old global buckets. - */ - tenant_name = bucket.substr(0, pos); - bucket_name = bucket.substr(pos + 1); - } else { - tenant_name = auth_tenant; - bucket_name = bucket; - } -} - // parse key in format: [tenant/]name:instance[:shard_id] int rgw_bucket_parse_bucket_key(CephContext *cct, const string& key, rgw_bucket *bucket, int *shard_id) @@ -214,2899 +75,47 @@ int rgw_bucket_parse_bucket_key(CephContext *cct, const string& key, return 0; } -static void dump_mulipart_index_results(list& objs_to_unlink, - Formatter *f) -{ - for (const auto& o : objs_to_unlink) { - f->dump_string("object", o.name); - } -} - -void check_bad_user_bucket_mapping(rgw::sal::Store* store, rgw::sal::User* user, - bool fix, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - rgw::sal::BucketList user_buckets; - string marker; - - CephContext *cct = store->ctx(); - - size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; - - do { - int ret = user->list_buckets(dpp, marker, string(), max_entries, false, user_buckets, y); - if (ret < 0) { - ldout(store->ctx(), 0) << "failed to read user buckets: " - << cpp_strerror(-ret) << dendl; - return; - } - - map>& buckets = user_buckets.get_buckets(); - for (auto i = buckets.begin(); - i != buckets.end(); - ++i) { - marker = i->first; - - auto& bucket = i->second; - - std::unique_ptr actual_bucket; - int r = store->get_bucket(dpp, user, user->get_tenant(), bucket->get_name(), &actual_bucket, null_yield); - if (r < 0) { - ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl; - continue; - } - - if (actual_bucket->get_name().compare(bucket->get_name()) != 0 || - actual_bucket->get_tenant().compare(bucket->get_tenant()) != 0 || - actual_bucket->get_marker().compare(bucket->get_marker()) != 0 || - actual_bucket->get_bucket_id().compare(bucket->get_bucket_id()) != 0) { - cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl; - if (fix) { - cout << "fixing" << std::endl; - r = actual_bucket->chown(dpp, user, nullptr, null_yield); - if (r < 0) { - cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl; - } - } - } - } - } while (user_buckets.is_truncated()); -} - -// returns true if entry is in the empty namespace. note: function -// type conforms to type RGWBucketListNameFilter -bool rgw_bucket_object_check_filter(const std::string& oid) -{ - const static std::string empty_ns; - rgw_obj_key key; // thrown away but needed for parsing - return rgw_obj_key::oid_to_key_in_ns(oid, &key, empty_ns); -} - -int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::Bucket* bucket, rgw_obj_key& key) -{ - if (key.instance.empty()) { - key.instance = "null"; - } - - std::unique_ptr object = bucket->get_object(key); - - return object->delete_object(dpp, null_yield); -} - -static void set_err_msg(std::string *sink, std::string msg) -{ - if (sink && !msg.empty()) - *sink = msg; -} - -int RGWBucket::init(rgw::sal::Store* _store, RGWBucketAdminOpState& op_state, - optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg) -{ - if (!_store) { - set_err_msg(err_msg, "no storage!"); - return -EINVAL; - } - - store = _store; - - std::string bucket_name = op_state.get_bucket_name(); - - if (bucket_name.empty() && op_state.get_user_id().empty()) - return -EINVAL; - - user = store->get_user(op_state.get_user_id()); - std::string tenant = user->get_tenant(); - - // split possible tenant/name - auto pos = bucket_name.find('/'); - if (pos != string::npos) { - tenant = bucket_name.substr(0, pos); - bucket_name = bucket_name.substr(pos + 1); - } - - int r = store->get_bucket(dpp, user.get(), tenant, bucket_name, &bucket, y); - if (r < 0) { - set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket_name); - return r; - } - - op_state.set_bucket(bucket->clone()); - - if (!rgw::sal::User::empty(user.get())) { - r = user->load_user(dpp, y); - if (r < 0) { - set_err_msg(err_msg, "failed to fetch user info"); - return r; - } - } - - op_state.display_name = user->get_display_name(); - - clear_failure(); - return 0; -} - -bool rgw_find_bucket_by_id(const DoutPrefixProvider *dpp, CephContext *cct, rgw::sal::Store* store, - const string& marker, const string& bucket_id, rgw_bucket* bucket_out) -{ - void *handle = NULL; - bool truncated = false; - string s; - - int ret = store->meta_list_keys_init(dpp, "bucket.instance", marker, &handle); - if (ret < 0) { - cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; - store->meta_list_keys_complete(handle); - return -ret; - } - do { - list keys; - ret = store->meta_list_keys_next(dpp, handle, 1000, keys, &truncated); - if (ret < 0) { - cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; - store->meta_list_keys_complete(handle); - return -ret; - } - for (list::iterator iter = keys.begin(); iter != keys.end(); ++iter) { - s = *iter; - ret = rgw_bucket_parse_bucket_key(cct, s, bucket_out, nullptr); - if (ret < 0) { - continue; - } - if (bucket_id == bucket_out->bucket_id) { - store->meta_list_keys_complete(handle); - return true; - } - } - } while (truncated); - store->meta_list_keys_complete(handle); - return false; -} - -int RGWBucket::chown(RGWBucketAdminOpState& op_state, const string& marker, - optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg) -{ - int ret = bucket->chown(dpp, user.get(), user.get(), y, &marker); - if (ret < 0) { - set_err_msg(err_msg, "Failed to change object ownership: " + cpp_strerror(-ret)); - } - - return ret; -} - -int RGWBucket::set_quota(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, std::string *err_msg) -{ - bucket = op_state.get_bucket()->clone(); - - bucket->get_info().quota = op_state.quota; - int r = bucket->put_info(dpp, false, real_time()); - if (r < 0) { - set_err_msg(err_msg, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r)); - return r; - } - return r; -} - -int RGWBucket::remove_object(const DoutPrefixProvider *dpp, RGWBucketAdminOpState& op_state, std::string *err_msg) -{ - std::string object_name = op_state.get_object_name(); - - rgw_obj_key key(object_name); - - bucket = op_state.get_bucket()->clone(); - - int ret = rgw_remove_object(dpp, store, bucket.get(), key); - if (ret < 0) { - set_err_msg(err_msg, "unable to remove object" + cpp_strerror(-ret)); - return ret; - } - - return 0; -} - -static void dump_bucket_index(const vector& objs, Formatter *f) -{ - for (auto iter = objs.begin(); iter != objs.end(); ++iter) { - f->dump_string("object", iter->key.name); - } -} - -static void dump_bucket_usage(map& stats, Formatter *formatter) -{ - map::iterator iter; - - formatter->open_object_section("usage"); - for (iter = stats.begin(); iter != stats.end(); ++iter) { - RGWStorageStats& s = iter->second; - formatter->open_object_section(to_string(iter->first)); - s.dump(formatter); - formatter->close_section(); - } - formatter->close_section(); -} - -static void dump_index_check(map existing_stats, - map calculated_stats, - Formatter *formatter) -{ - formatter->open_object_section("check_result"); - formatter->open_object_section("existing_header"); - dump_bucket_usage(existing_stats, formatter); - formatter->close_section(); - formatter->open_object_section("calculated_header"); - dump_bucket_usage(calculated_stats, formatter); - formatter->close_section(); - formatter->close_section(); -} - -int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - const DoutPrefixProvider *dpp, - std::string *err_msg) -{ - const bool fix_index = op_state.will_fix_index(); - - bucket = op_state.get_bucket()->clone(); - - rgw::sal::Bucket::ListParams params; - params.list_versions = true; - params.ns = RGW_OBJ_NS_MULTIPART; - - std::map meta_objs; - std::map all_objs; - bool is_truncated; - do { - rgw::sal::Bucket::ListResults results; - int r = bucket->list(dpp, params, listing_max_entries, results, null_yield); - if (r < 0) { - set_err_msg(err_msg, "failed to list objects in bucket=" + bucket->get_name() + - " err=" + cpp_strerror(-r)); - - return r; - } - is_truncated = results.is_truncated; - - for (const auto& o : results.objs) { - rgw_obj_index_key key = o.key; - rgw_obj obj(bucket->get_key(), key); - std::string oid = obj.get_oid(); - - int pos = oid.find_last_of('.'); - if (pos < 0) { - /* obj has no suffix */ - all_objs[key] = oid; - } else { - /* obj has suffix */ - std::string name = oid.substr(0, pos); - std::string suffix = oid.substr(pos + 1); - - if (suffix.compare("meta") == 0) { - meta_objs[name] = true; - } else { - all_objs[key] = name; - } - } - } - } while (is_truncated); - - std::list objs_to_unlink; - Formatter *f = flusher.get_formatter(); - - f->open_array_section("invalid_multipart_entries"); - - for (const auto& o : all_objs) { - const std::string& name = o.second; - if (meta_objs.find(name) == meta_objs.end()) { - objs_to_unlink.push_back(o.first); - } - - if (objs_to_unlink.size() > listing_max_entries) { - if (fix_index) { - // note: under rados this removes directly from rados index objects - int r = bucket->remove_objs_from_index(dpp, objs_to_unlink); - if (r < 0) { - set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " + - cpp_strerror(-r)); - return r; - } - } - - dump_mulipart_index_results(objs_to_unlink, f); - flusher.flush(); - objs_to_unlink.clear(); - } - } - - if (fix_index) { - // note: under rados this removes directly from rados index objects - int r = bucket->remove_objs_from_index(dpp, objs_to_unlink); - if (r < 0) { - set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " + - cpp_strerror(-r)); - - return r; - } - } - - dump_mulipart_index_results(objs_to_unlink, f); - f->close_section(); - flusher.flush(); - - return 0; -} - -int RGWBucket::check_object_index(const DoutPrefixProvider *dpp, - RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y, - std::string *err_msg) -{ - - bool fix_index = op_state.will_fix_index(); - - if (!fix_index) { - set_err_msg(err_msg, "check-objects flag requires fix index enabled"); - return -EINVAL; - } - - // use a quicker/shorter tag timeout during this process - bucket->set_tag_timeout(dpp, BUCKET_TAG_QUICK_TIMEOUT); - - rgw::sal::Bucket::ListResults results; - results.is_truncated = true; - - Formatter *formatter = flusher.get_formatter(); - formatter->open_object_section("objects"); - - while (results.is_truncated) { - rgw::sal::Bucket::ListParams params; - params.marker = results.next_marker; - params.force_check_filter = rgw_bucket_object_check_filter; - - int r = bucket->list(dpp, params, listing_max_entries, results, y); - - if (r == -ENOENT) { - break; - } else if (r < 0) { - set_err_msg(err_msg, "ERROR: failed operation r=" + cpp_strerror(-r)); - } - - dump_bucket_index(results.objs, formatter); - flusher.flush(); - } - - formatter->close_section(); - - // restore normal tag timeout for bucket - bucket->set_tag_timeout(dpp, 0); - - return 0; -} - - -int RGWBucket::check_index(const DoutPrefixProvider *dpp, - RGWBucketAdminOpState& op_state, - map& existing_stats, - map& calculated_stats, - std::string *err_msg) -{ - bool fix_index = op_state.will_fix_index(); - - int r = bucket->check_index(dpp, existing_stats, calculated_stats); - if (r < 0) { - set_err_msg(err_msg, "failed to check index error=" + cpp_strerror(-r)); - return r; - } - - if (fix_index) { - r = bucket->rebuild_index(dpp); - if (r < 0) { - set_err_msg(err_msg, "failed to rebuild index err=" + cpp_strerror(-r)); - return r; - } - } - - return 0; -} +/* + * Note that this is not a reversal of parse_bucket(). That one deals + * with the syntax we need in metadata and such. This one deals with + * the representation in RADOS pools. We chose '/' because it's not + * acceptable in bucket names and thus qualified buckets cannot conflict + * with the legacy or S3 buckets. + */ +std::string rgw_make_bucket_entry_name(const std::string& tenant_name, + const std::string& bucket_name) { + std::string bucket_entry; -int RGWBucket::sync(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, std::string *err_msg) -{ - if (!store->is_meta_master()) { - set_err_msg(err_msg, "ERROR: failed to update bucket sync: only allowed on meta master zone"); - return -EINVAL; - } - bool sync = op_state.will_sync_bucket(); - if (sync) { - bucket->get_info().flags &= ~BUCKET_DATASYNC_DISABLED; + if (bucket_name.empty()) { + bucket_entry.clear(); + } else if (tenant_name.empty()) { + bucket_entry = bucket_name; } else { - bucket->get_info().flags |= BUCKET_DATASYNC_DISABLED; - } - - // when writing this metadata, RGWSI_BucketIndex_RADOS::handle_overwrite() - // will write the corresponding datalog and bilog entries - int r = bucket->put_info(dpp, false, real_time()); - if (r < 0) { - set_err_msg(err_msg, "ERROR: failed writing bucket instance info:" + cpp_strerror(-r)); - return r; - } - - return 0; -} - - -int RGWBucket::policy_bl_to_stream(bufferlist& bl, ostream& o) -{ - RGWAccessControlPolicy_S3 policy(g_ceph_context); - int ret = decode_bl(bl, policy); - if (ret < 0) { - ldout(store->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl; + bucket_entry = tenant_name + "/" + bucket_name; } - policy.to_xml(o); - return 0; -} -int rgw_object_get_attr(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, rgw::sal::Object* obj, - const char* attr_name, bufferlist& out_bl, optional_yield y) -{ - std::unique_ptr rop = obj->get_read_op(); - - return rop->get_attr(dpp, attr_name, out_bl, y); + return bucket_entry; } -int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, RGWAccessControlPolicy& policy, optional_yield y, const DoutPrefixProvider *dpp) -{ - int ret; - std::string object_name = op_state.get_object_name(); - - bucket = op_state.get_bucket()->clone(); - - if (!object_name.empty()) { - bufferlist bl; - std::unique_ptr obj = bucket->get_object(rgw_obj_key(object_name)); - - ret = rgw_object_get_attr(dpp, store, obj.get(), RGW_ATTR_ACL, bl, y); - if (ret < 0){ - return ret; - } +/* + * Tenants are separated from buckets in URLs by a colon in S3. + * This function is not to be used on Swift URLs, not even for COPY arguments. + */ +void rgw_parse_url_bucket(const string &bucket, const string& auth_tenant, + string &tenant_name, string &bucket_name) { - ret = decode_bl(bl, policy); - if (ret < 0) { - ldout(store->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl; - } - return ret; - } - - map::iterator aiter = bucket->get_attrs().find(RGW_ATTR_ACL); - if (aiter == bucket->get_attrs().end()) { - return -ENOENT; - } - - ret = decode_bl(aiter->second, policy); - if (ret < 0) { - ldout(store->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl; - } - - return ret; -} - - -int RGWBucketAdminOp::get_policy(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, - RGWAccessControlPolicy& policy, const DoutPrefixProvider *dpp) -{ - RGWBucket bucket; - - int ret = bucket.init(store, op_state, null_yield, dpp); - if (ret < 0) - return ret; - - ret = bucket.get_policy(op_state, policy, null_yield, dpp); - if (ret < 0) - return ret; - - return 0; -} - -/* Wrappers to facilitate RESTful interface */ - - -int RGWBucketAdminOp::get_policy(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, const DoutPrefixProvider *dpp) -{ - RGWAccessControlPolicy policy(store->ctx()); - - int ret = get_policy(store, op_state, policy, dpp); - if (ret < 0) - return ret; - - Formatter *formatter = flusher.get_formatter(); - - flusher.start(0); - - formatter->open_object_section("policy"); - policy.dump(formatter); - formatter->close_section(); - - flusher.flush(); - - return 0; -} - -int RGWBucketAdminOp::dump_s3_policy(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, - ostream& os, const DoutPrefixProvider *dpp) -{ - RGWAccessControlPolicy_S3 policy(store->ctx()); - - int ret = get_policy(store, op_state, policy, dpp); - if (ret < 0) - return ret; - - policy.to_xml(os); - - return 0; -} - -int RGWBucketAdminOp::unlink(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp) -{ - RGWBucket bucket; - - int ret = bucket.init(store, op_state, null_yield, dpp); - if (ret < 0) - return ret; - - return static_cast(store)->ctl()->bucket->unlink_bucket(op_state.get_user_id(), op_state.get_bucket()->get_info().bucket, null_yield, dpp, true); -} - -int RGWBucketAdminOp::link(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, string *err) -{ - if (!op_state.is_user_op()) { - set_err_msg(err, "empty user id"); - return -EINVAL; - } - - RGWBucket bucket; - int ret = bucket.init(store, op_state, null_yield, dpp, err); - if (ret < 0) - return ret; - - string bucket_id = op_state.get_bucket_id(); - std::string display_name = op_state.get_user_display_name(); - std::unique_ptr loc_bucket; - std::unique_ptr old_bucket; - - loc_bucket = op_state.get_bucket()->clone(); - - if (!bucket_id.empty() && bucket_id != loc_bucket->get_bucket_id()) { - set_err_msg(err, - "specified bucket id does not match " + loc_bucket->get_bucket_id()); - return -EINVAL; - } - - old_bucket = loc_bucket->clone(); - - loc_bucket->get_key().tenant = op_state.get_user_id().tenant; - - if (!op_state.new_bucket_name.empty()) { - auto pos = op_state.new_bucket_name.find('/'); - if (pos != string::npos) { - loc_bucket->get_key().tenant = op_state.new_bucket_name.substr(0, pos); - loc_bucket->get_key().name = op_state.new_bucket_name.substr(pos + 1); - } else { - loc_bucket->get_key().name = op_state.new_bucket_name; - } - } - - RGWObjVersionTracker objv_tracker; - RGWObjVersionTracker old_version = loc_bucket->get_info().objv_tracker; - - map::iterator aiter = loc_bucket->get_attrs().find(RGW_ATTR_ACL); - if (aiter == loc_bucket->get_attrs().end()) { - // should never happen; only pre-argonaut buckets lacked this. - ldpp_dout(dpp, 0) << "WARNING: can't bucket link because no acl on bucket=" << old_bucket << dendl; - set_err_msg(err, - "While crossing the Anavros you have displeased the goddess Hera." - " You must sacrifice your ancient bucket " + loc_bucket->get_bucket_id()); - return -EINVAL; - } - bufferlist& aclbl = aiter->second; - RGWAccessControlPolicy policy; - ACLOwner owner; - try { - auto iter = aclbl.cbegin(); - decode(policy, iter); - owner = policy.get_owner(); - } catch (buffer::error& e) { - set_err_msg(err, "couldn't decode policy"); - return -EIO; - } - - int r = static_cast(store)->ctl()->bucket->unlink_bucket(owner.get_id(), old_bucket->get_info().bucket, null_yield, dpp, false); - if (r < 0) { - set_err_msg(err, "could not unlink policy from user " + owner.get_id().to_str()); - return r; - } - - // now update the user for the bucket... - if (display_name.empty()) { - ldpp_dout(dpp, 0) << "WARNING: user " << op_state.get_user_id() << " has no display name set" << dendl; - } - - RGWAccessControlPolicy policy_instance; - policy_instance.create_default(op_state.get_user_id(), display_name); - owner = policy_instance.get_owner(); - - aclbl.clear(); - policy_instance.encode(aclbl); - - bool exclusive = false; - loc_bucket->get_info().owner = op_state.get_user_id(); - if (*loc_bucket != *old_bucket) { - loc_bucket->get_info().bucket = loc_bucket->get_key(); - loc_bucket->get_info().objv_tracker.version_for_read()->ver = 0; - exclusive = true; - } - - r = loc_bucket->put_info(dpp, exclusive, ceph::real_time()); - if (r < 0) { - set_err_msg(err, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r)); - return r; - } - - /* link to user */ - RGWBucketEntryPoint ep; - ep.bucket = loc_bucket->get_info().bucket; - ep.owner = op_state.get_user_id(); - ep.creation_time = loc_bucket->get_info().creation_time; - ep.linked = true; - rgw::sal::Attrs ep_attrs; - rgw_ep_info ep_data{ep, ep_attrs}; - - r = static_cast(store)->ctl()->bucket->link_bucket(op_state.get_user_id(), loc_bucket->get_info().bucket, loc_bucket->get_info().creation_time, null_yield, dpp, true, &ep_data); - if (r < 0) { - set_err_msg(err, "failed to relink bucket"); - return r; - } - - if (*loc_bucket != *old_bucket) { - // like RGWRados::delete_bucket -- excepting no bucket_index work. - r = static_cast(store)->ctl()->bucket->remove_bucket_entrypoint_info( - old_bucket->get_key(), null_yield, dpp, - RGWBucketCtl::Bucket::RemoveParams() - .set_objv_tracker(&ep_data.ep_objv)); - if (r < 0) { - set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name()); - return r; - } - r = static_cast(store)->ctl()->bucket->remove_bucket_instance_info( - old_bucket->get_key(), old_bucket->get_info(), - null_yield, dpp, - RGWBucketCtl::BucketInstance::RemoveParams() - .set_objv_tracker(&ep_data.ep_objv)); - if (r < 0) { - set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name()); - return r; - } - } - - return 0; -} - -int RGWBucketAdminOp::chown(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const string& marker, const DoutPrefixProvider *dpp, string *err) -{ - RGWBucket bucket; - - int ret = bucket.init(store, op_state, null_yield, dpp, err); - if (ret < 0) - return ret; - - return bucket.chown(op_state, marker, null_yield, dpp, err); - -} - -int RGWBucketAdminOp::check_index(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y, const DoutPrefixProvider *dpp) -{ - int ret; - map existing_stats; - map calculated_stats; - - - RGWBucket bucket; - - ret = bucket.init(store, op_state, null_yield, dpp); - if (ret < 0) - return ret; - - Formatter *formatter = flusher.get_formatter(); - flusher.start(0); - - ret = bucket.check_bad_index_multipart(op_state, flusher, dpp); - if (ret < 0) - return ret; - - ret = bucket.check_object_index(dpp, op_state, flusher, y); - if (ret < 0) - return ret; - - ret = bucket.check_index(dpp, op_state, existing_stats, calculated_stats); - if (ret < 0) - return ret; - - dump_index_check(existing_stats, calculated_stats, formatter); - flusher.flush(); - - return 0; -} - -int RGWBucketAdminOp::remove_bucket(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, - optional_yield y, const DoutPrefixProvider *dpp, - bool bypass_gc, bool keep_index_consistent) -{ - std::unique_ptr bucket; - std::unique_ptr user = store->get_user(op_state.get_user_id()); - - int ret = store->get_bucket(dpp, user.get(), user->get_tenant(), op_state.get_bucket_name(), - &bucket, y); - if (ret < 0) - return ret; - - if (bypass_gc) - ret = bucket->remove_bucket_bypass_gc(op_state.get_max_aio(), keep_index_consistent, y, dpp); - else - ret = bucket->remove_bucket(dpp, op_state.will_delete_children(), - false, nullptr, y); - - return ret; -} - -int RGWBucketAdminOp::remove_object(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp) -{ - RGWBucket bucket; - - int ret = bucket.init(store, op_state, null_yield, dpp); - if (ret < 0) - return ret; - - return bucket.remove_object(dpp, op_state); -} - -int RGWBucketAdminOp::sync_bucket(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, string *err_msg) -{ - RGWBucket bucket; - int ret = bucket.init(store, op_state, null_yield, dpp, err_msg); - if (ret < 0) - { - return ret; - } - return bucket.sync(op_state, dpp, err_msg); -} - -static int bucket_stats(rgw::sal::Store* store, - const std::string& tenant_name, - const std::string& bucket_name, - Formatter *formatter, - const DoutPrefixProvider *dpp) -{ - std::unique_ptr bucket; - map stats; - - real_time mtime; - int ret = store->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield); - if (ret < 0) { - return ret; - } - - const auto& index = bucket->get_info().get_current_index(); - if (is_layout_indexless(index)) { - cerr << "error, indexless buckets do not maintain stats; bucket=" << - bucket->get_name() << std::endl; - return -EINVAL; - } - - std::string bucket_ver, master_ver; - std::string max_marker; - ret = bucket->read_stats(dpp, index, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker); - if (ret < 0) { - cerr << "error getting bucket stats bucket=" << bucket->get_name() << " ret=" << ret << std::endl; - return ret; - } - - utime_t ut(mtime); - utime_t ctime_ut(bucket->get_creation_time()); - - formatter->open_object_section("stats"); - formatter->dump_string("bucket", bucket->get_name()); - formatter->dump_int("num_shards", - bucket->get_info().layout.current_index.layout.normal.num_shards); - formatter->dump_string("tenant", bucket->get_tenant()); - formatter->dump_string("zonegroup", bucket->get_info().zonegroup); - formatter->dump_string("placement_rule", bucket->get_info().placement_rule.to_str()); - ::encode_json("explicit_placement", bucket->get_key().explicit_placement, formatter); - formatter->dump_string("id", bucket->get_bucket_id()); - formatter->dump_string("marker", bucket->get_marker()); - formatter->dump_stream("index_type") << bucket->get_info().layout.current_index.layout.type; - ::encode_json("owner", bucket->get_info().owner, formatter); - formatter->dump_string("ver", bucket_ver); - formatter->dump_string("master_ver", master_ver); - ut.gmtime(formatter->dump_stream("mtime")); - ctime_ut.gmtime(formatter->dump_stream("creation_time")); - formatter->dump_string("max_marker", max_marker); - dump_bucket_usage(stats, formatter); - encode_json("bucket_quota", bucket->get_info().quota, formatter); - - // bucket tags - auto iter = bucket->get_attrs().find(RGW_ATTR_TAGS); - if (iter != bucket->get_attrs().end()) { - RGWObjTagSet_S3 tagset; - bufferlist::const_iterator piter{&iter->second}; - try { - tagset.decode(piter); - tagset.dump(formatter); - } catch (buffer::error& err) { - cerr << "ERROR: caught buffer:error, couldn't decode TagSet" << std::endl; - } - } - - // TODO: bucket CORS - // TODO: bucket LC - formatter->close_section(); - - return 0; -} - -int RGWBucketAdminOp::limit_check(rgw::sal::Store* store, - RGWBucketAdminOpState& op_state, - const std::list& user_ids, - RGWFormatterFlusher& flusher, optional_yield y, - const DoutPrefixProvider *dpp, - bool warnings_only) -{ - int ret = 0; - const size_t max_entries = - store->ctx()->_conf->rgw_list_buckets_max_chunk; - - const size_t safe_max_objs_per_shard = - store->ctx()->_conf->rgw_safe_max_objects_per_shard; - - uint16_t shard_warn_pct = - store->ctx()->_conf->rgw_shard_warning_threshold; - if (shard_warn_pct > 100) - shard_warn_pct = 90; - - Formatter *formatter = flusher.get_formatter(); - flusher.start(0); - - formatter->open_array_section("users"); - - for (const auto& user_id : user_ids) { - - formatter->open_object_section("user"); - formatter->dump_string("user_id", user_id); - formatter->open_array_section("buckets"); - - string marker; - rgw::sal::BucketList buckets; - do { - std::unique_ptr user = store->get_user(rgw_user(user_id)); - - ret = user->list_buckets(dpp, marker, string(), max_entries, false, buckets, y); - - if (ret < 0) - return ret; - - map>& m_buckets = buckets.get_buckets(); - - for (const auto& iter : m_buckets) { - auto& bucket = iter.second; - uint64_t num_objects = 0; - - marker = bucket->get_name(); /* Casey's location for marker update, - * as we may now not reach the end of - * the loop body */ - - ret = bucket->load_bucket(dpp, null_yield); - if (ret < 0) - continue; - - const auto& index = bucket->get_info().get_current_index(); - if (is_layout_indexless(index)) { - continue; // indexless buckets don't have stats - } - - /* need stats for num_entries */ - string bucket_ver, master_ver; - std::map stats; - ret = bucket->read_stats(dpp, index, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, nullptr); - - if (ret < 0) - continue; - - for (const auto& s : stats) { - num_objects += s.second.num_objects; - } - - const uint32_t num_shards = rgw::num_shards(index.layout.normal); - uint64_t objs_per_shard = - (num_shards) ? num_objects/num_shards : num_objects; - { - bool warn; - stringstream ss; - uint64_t fill_pct = objs_per_shard * 100 / safe_max_objs_per_shard; - if (fill_pct > 100) { - ss << "OVER " << fill_pct << "%"; - warn = true; - } else if (fill_pct >= shard_warn_pct) { - ss << "WARN " << fill_pct << "%"; - warn = true; - } else { - ss << "OK"; - warn = false; - } - - if (warn || !warnings_only) { - formatter->open_object_section("bucket"); - formatter->dump_string("bucket", bucket->get_name()); - formatter->dump_string("tenant", bucket->get_tenant()); - formatter->dump_int("num_objects", num_objects); - formatter->dump_int("num_shards", num_shards); - formatter->dump_int("objects_per_shard", objs_per_shard); - formatter->dump_string("fill_status", ss.str()); - formatter->close_section(); - } - } - } - formatter->flush(cout); - } while (buckets.is_truncated()); /* foreach: bucket */ - - formatter->close_section(); - formatter->close_section(); - formatter->flush(cout); - - } /* foreach: user_id */ - - formatter->close_section(); - formatter->flush(cout); - - return ret; -} /* RGWBucketAdminOp::limit_check */ - -int RGWBucketAdminOp::info(rgw::sal::Store* store, - RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - RGWBucket bucket; - int ret = 0; - const std::string& bucket_name = op_state.get_bucket_name(); - if (!bucket_name.empty()) { - ret = bucket.init(store, op_state, null_yield, dpp); - if (-ENOENT == ret) - return -ERR_NO_SUCH_BUCKET; - else if (ret < 0) - return ret; - } - - Formatter *formatter = flusher.get_formatter(); - flusher.start(0); - - CephContext *cct = store->ctx(); - - const size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; - - const bool show_stats = op_state.will_fetch_stats(); - const rgw_user& user_id = op_state.get_user_id(); - if (op_state.is_user_op()) { - formatter->open_array_section("buckets"); - - rgw::sal::BucketList buckets; - std::unique_ptr user = store->get_user(op_state.get_user_id()); - std::string marker; - const std::string empty_end_marker; - constexpr bool no_need_stats = false; // set need_stats to false - - do { - ret = user->list_buckets(dpp, marker, empty_end_marker, max_entries, - no_need_stats, buckets, y); - if (ret < 0) { - return ret; - } - - const std::string* marker_cursor = nullptr; - map>& m = buckets.get_buckets(); - - for (const auto& i : m) { - const std::string& obj_name = i.first; - if (!bucket_name.empty() && bucket_name != obj_name) { - continue; - } - - if (show_stats) { - bucket_stats(store, user_id.tenant, obj_name, formatter, dpp); - } else { - formatter->dump_string("bucket", obj_name); - } - - marker_cursor = &obj_name; - } // for loop - if (marker_cursor) { - marker = *marker_cursor; - } - - flusher.flush(); - } while (buckets.is_truncated()); - - formatter->close_section(); - } else if (!bucket_name.empty()) { - ret = bucket_stats(store, user_id.tenant, bucket_name, formatter, dpp); - if (ret < 0) { - return ret; - } - } else { - void *handle = nullptr; - bool truncated = true; - - formatter->open_array_section("buckets"); - ret = store->meta_list_keys_init(dpp, "bucket", string(), &handle); - while (ret == 0 && truncated) { - std::list buckets; - constexpr int max_keys = 1000; - ret = store->meta_list_keys_next(dpp, handle, max_keys, buckets, - &truncated); - for (auto& bucket_name : buckets) { - if (show_stats) { - bucket_stats(store, user_id.tenant, bucket_name, formatter, dpp); - } else { - formatter->dump_string("bucket", bucket_name); - } - } - } - store->meta_list_keys_complete(handle); - - formatter->close_section(); - } - - flusher.flush(); - - return 0; -} - -int RGWBucketAdminOp::set_quota(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp) -{ - RGWBucket bucket; - - int ret = bucket.init(store, op_state, null_yield, dpp); - if (ret < 0) - return ret; - return bucket.set_quota(op_state, dpp); -} - -inline auto split_tenant(const std::string& bucket_name){ - auto p = bucket_name.find('/'); - if(p != std::string::npos) { - return std::make_pair(bucket_name.substr(0,p), bucket_name.substr(p+1)); - } - return std::make_pair(std::string(), bucket_name); -} - -using bucket_instance_ls = std::vector; -void get_stale_instances(rgw::sal::Store* store, const std::string& bucket_name, - const vector& lst, - bucket_instance_ls& stale_instances, - const DoutPrefixProvider *dpp) -{ - - bucket_instance_ls other_instances; -// first iterate over the entries, and pick up the done buckets; these -// are guaranteed to be stale - for (const auto& bucket_instance : lst){ - RGWBucketInfo binfo; - std::unique_ptr bucket; - rgw_bucket rbucket; - rgw_bucket_parse_bucket_key(store->ctx(), bucket_instance, &rbucket, nullptr); - int r = store->get_bucket(dpp, nullptr, rbucket, &bucket, null_yield); - if (r < 0){ - // this can only happen if someone deletes us right when we're processing - ldpp_dout(dpp, -1) << "Bucket instance is invalid: " << bucket_instance - << cpp_strerror(-r) << dendl; - continue; - } - binfo = bucket->get_info(); - if (binfo.reshard_status == cls_rgw_reshard_status::DONE) - stale_instances.emplace_back(std::move(binfo)); - else { - other_instances.emplace_back(std::move(binfo)); - } - } - - // Read the cur bucket info, if the bucket doesn't exist we can simply return - // all the instances - auto [tenant, bname] = split_tenant(bucket_name); - RGWBucketInfo cur_bucket_info; - std::unique_ptr cur_bucket; - int r = store->get_bucket(dpp, nullptr, tenant, bname, &cur_bucket, null_yield); - if (r < 0) { - if (r == -ENOENT) { - // bucket doesn't exist, everything is stale then - stale_instances.insert(std::end(stale_instances), - std::make_move_iterator(other_instances.begin()), - std::make_move_iterator(other_instances.end())); - } else { - // all bets are off if we can't read the bucket, just return the sureshot stale instances - ldpp_dout(dpp, -1) << "error: reading bucket info for bucket: " - << bname << cpp_strerror(-r) << dendl; - } - return; - } - - // Don't process further in this round if bucket is resharding - cur_bucket_info = cur_bucket->get_info(); - if (cur_bucket_info.reshard_status == cls_rgw_reshard_status::IN_PROGRESS) - return; - - other_instances.erase(std::remove_if(other_instances.begin(), other_instances.end(), - [&cur_bucket_info](const RGWBucketInfo& b){ - return (b.bucket.bucket_id == cur_bucket_info.bucket.bucket_id || - b.bucket.bucket_id == cur_bucket_info.new_bucket_instance_id); - }), - other_instances.end()); - - // check if there are still instances left - if (other_instances.empty()) { - return; - } - - // Now we have a bucket with instances where the reshard status is none, this - // usually happens when the reshard process couldn't complete, lockdown the - // bucket and walk through these instances to make sure no one else interferes - // with these - { - RGWBucketReshardLock reshard_lock(static_cast(store), cur_bucket->get_info(), true); - r = reshard_lock.lock(dpp); - if (r < 0) { - // most likely bucket is under reshard, return the sureshot stale instances - ldpp_dout(dpp, 5) << __func__ - << "failed to take reshard lock; reshard underway likey" << dendl; - return; - } - auto sg = make_scope_guard([&reshard_lock](){ reshard_lock.unlock();} ); - // this should be fast enough that we may not need to renew locks and check - // exit status?, should we read the values of the instances again? - stale_instances.insert(std::end(stale_instances), - std::make_move_iterator(other_instances.begin()), - std::make_move_iterator(other_instances.end())); - } - - return; -} - -static int process_stale_instances(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - const DoutPrefixProvider *dpp, - std::function process_f) -{ - std::string marker; - void *handle; - Formatter *formatter = flusher.get_formatter(); - static constexpr auto default_max_keys = 1000; - - int ret = store->meta_list_keys_init(dpp, "bucket.instance", marker, &handle); - if (ret < 0) { - cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; - return ret; - } - - bool truncated; - - formatter->open_array_section("keys"); - auto g = make_scope_guard([&store, &handle, &formatter]() { - store->meta_list_keys_complete(handle); - formatter->close_section(); // keys - formatter->flush(cout); - }); - - do { - list keys; - - ret = store->meta_list_keys_next(dpp, handle, default_max_keys, keys, &truncated); - if (ret < 0 && ret != -ENOENT) { - cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; - return ret; - } if (ret != -ENOENT) { - // partition the list of buckets by buckets as the listing is un sorted, - // since it would minimize the reads to bucket_info - std::unordered_map> bucket_instance_map; - for (auto &key: keys) { - auto pos = key.find(':'); - if(pos != std::string::npos) - bucket_instance_map[key.substr(0,pos)].emplace_back(std::move(key)); - } - for (const auto& kv: bucket_instance_map) { - bucket_instance_ls stale_lst; - get_stale_instances(store, kv.first, kv.second, stale_lst, dpp); - process_f(stale_lst, formatter, store); - } - } - } while (truncated); - - return 0; -} - -int RGWBucketAdminOp::list_stale_instances(rgw::sal::Store* store, - RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - const DoutPrefixProvider *dpp) -{ - auto process_f = [](const bucket_instance_ls& lst, - Formatter *formatter, - rgw::sal::Store*){ - for (const auto& binfo: lst) - formatter->dump_string("key", binfo.bucket.get_key()); - }; - return process_stale_instances(store, op_state, flusher, dpp, process_f); -} - - -int RGWBucketAdminOp::clear_stale_instances(rgw::sal::Store* store, - RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - const DoutPrefixProvider *dpp) -{ - auto process_f = [dpp](const bucket_instance_ls& lst, - Formatter *formatter, - rgw::sal::Store* store){ - for (const auto &binfo: lst) { - std::unique_ptr bucket; - store->get_bucket(nullptr, binfo, &bucket); - int ret = bucket->purge_instance(dpp); - if (ret == 0){ - auto md_key = "bucket.instance:" + binfo.bucket.get_key(); - ret = store->meta_remove(dpp, md_key, null_yield); - } - formatter->open_object_section("delete_status"); - formatter->dump_string("bucket_instance", binfo.bucket.get_key()); - formatter->dump_int("status", -ret); - formatter->close_section(); - } - }; - - return process_stale_instances(store, op_state, flusher, dpp, process_f); -} - -static int fix_single_bucket_lc(rgw::sal::Store* store, - const std::string& tenant_name, - const std::string& bucket_name, - const DoutPrefixProvider *dpp) -{ - std::unique_ptr bucket; - int ret = store->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield); - if (ret < 0) { - // TODO: Should we handle the case where the bucket could've been removed between - // listing and fetching? - return ret; - } - - return rgw::lc::fix_lc_shard_entry(dpp, store, store->get_rgwlc()->get_lc(), bucket.get()); -} - -static void format_lc_status(Formatter* formatter, - const std::string& tenant_name, - const std::string& bucket_name, - int status) -{ - formatter->open_object_section("bucket_entry"); - std::string entry = tenant_name.empty() ? bucket_name : tenant_name + "/" + bucket_name; - formatter->dump_string("bucket", entry); - formatter->dump_int("status", status); - formatter->close_section(); // bucket_entry -} - -static void process_single_lc_entry(rgw::sal::Store* store, - Formatter *formatter, - const std::string& tenant_name, - const std::string& bucket_name, - const DoutPrefixProvider *dpp) -{ - int ret = fix_single_bucket_lc(store, tenant_name, bucket_name, dpp); - format_lc_status(formatter, tenant_name, bucket_name, -ret); -} - -int RGWBucketAdminOp::fix_lc_shards(rgw::sal::Store* store, - RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - const DoutPrefixProvider *dpp) -{ - std::string marker; - void *handle; - Formatter *formatter = flusher.get_formatter(); - static constexpr auto default_max_keys = 1000; - - bool truncated; - if (const std::string& bucket_name = op_state.get_bucket_name(); - ! bucket_name.empty()) { - const rgw_user user_id = op_state.get_user_id(); - process_single_lc_entry(store, formatter, user_id.tenant, bucket_name, dpp); - formatter->flush(cout); - } else { - int ret = store->meta_list_keys_init(dpp, "bucket", marker, &handle); - if (ret < 0) { - std::cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; - return ret; - } - - { - formatter->open_array_section("lc_fix_status"); - auto sg = make_scope_guard([&store, &handle, &formatter](){ - store->meta_list_keys_complete(handle); - formatter->close_section(); // lc_fix_status - formatter->flush(cout); - }); - do { - list keys; - ret = store->meta_list_keys_next(dpp, handle, default_max_keys, keys, &truncated); - if (ret < 0 && ret != -ENOENT) { - std::cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; - return ret; - } if (ret != -ENOENT) { - for (const auto &key:keys) { - auto [tenant_name, bucket_name] = split_tenant(key); - process_single_lc_entry(store, formatter, tenant_name, bucket_name, dpp); - } - } - formatter->flush(cout); // regularly flush every 1k entries - } while (truncated); - } - - } - return 0; - -} - -static bool has_object_expired(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - rgw::sal::Bucket* bucket, - const rgw_obj_key& key, utime_t& delete_at) -{ - std::unique_ptr obj = bucket->get_object(key); - bufferlist delete_at_bl; - - int ret = rgw_object_get_attr(dpp, store, obj.get(), RGW_ATTR_DELETE_AT, delete_at_bl, null_yield); - if (ret < 0) { - return false; // no delete at attr, proceed - } - - ret = decode_bl(delete_at_bl, delete_at); - if (ret < 0) { - return false; // failed to parse - } - - if (delete_at <= ceph_clock_now() && !delete_at.is_zero()) { - return true; - } - - return false; -} - -static int fix_bucket_obj_expiry(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - rgw::sal::Bucket* bucket, - RGWFormatterFlusher& flusher, bool dry_run) -{ - if (bucket->get_key().bucket_id == bucket->get_key().marker) { - ldpp_dout(dpp, -1) << "Not a resharded bucket skipping" << dendl; - return 0; // not a resharded bucket, move along - } - - Formatter *formatter = flusher.get_formatter(); - formatter->open_array_section("expired_deletion_status"); - auto sg = make_scope_guard([&formatter] { - formatter->close_section(); - formatter->flush(std::cout); - }); - - rgw::sal::Bucket::ListParams params; - rgw::sal::Bucket::ListResults results; - - params.list_versions = bucket->versioned(); - params.allow_unordered = true; - - do { - int ret = bucket->list(dpp, params, listing_max_entries, results, null_yield); - if (ret < 0) { - ldpp_dout(dpp, -1) << "ERROR failed to list objects in the bucket" << dendl; - return ret; - } - for (const auto& obj : results.objs) { - rgw_obj_key key(obj.key); - utime_t delete_at; - if (has_object_expired(dpp, store, bucket, key, delete_at)) { - formatter->open_object_section("object_status"); - formatter->dump_string("object", key.name); - formatter->dump_stream("delete_at") << delete_at; - - if (!dry_run) { - ret = rgw_remove_object(dpp, store, bucket, key); - formatter->dump_int("status", ret); - } - - formatter->close_section(); // object_status - } - } - formatter->flush(cout); // regularly flush every 1k entries - } while (results.is_truncated); - - return 0; -} - -int RGWBucketAdminOp::fix_obj_expiry(rgw::sal::Store* store, - RGWBucketAdminOpState& op_state, - RGWFormatterFlusher& flusher, - const DoutPrefixProvider *dpp, bool dry_run) -{ - RGWBucket admin_bucket; - int ret = admin_bucket.init(store, op_state, null_yield, dpp); - if (ret < 0) { - ldpp_dout(dpp, -1) << "failed to initialize bucket" << dendl; - return ret; - } - std::unique_ptr bucket; - ret = store->get_bucket(nullptr, admin_bucket.get_bucket_info(), &bucket); - if (ret < 0) { - return ret; - } - - return fix_bucket_obj_expiry(dpp, store, bucket.get(), flusher, dry_run); -} - -void RGWBucketCompleteInfo::dump(Formatter *f) const { - encode_json("bucket_info", info, f); - encode_json("attrs", attrs, f); -} - -void RGWBucketCompleteInfo::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("bucket_info", info, obj); - JSONDecoder::decode_json("attrs", attrs, obj); -} - -class RGWBucketMetadataHandler : public RGWBucketMetadataHandlerBase { -public: - struct Svc { - RGWSI_Bucket *bucket{nullptr}; - } svc; - - struct Ctl { - RGWBucketCtl *bucket{nullptr}; - } ctl; - - RGWBucketMetadataHandler() {} - - void init(RGWSI_Bucket *bucket_svc, - RGWBucketCtl *bucket_ctl) override { - base_init(bucket_svc->ctx(), - bucket_svc->get_ep_be_handler().get()); - svc.bucket = bucket_svc; - ctl.bucket = bucket_ctl; - } - - string get_type() override { return "bucket"; } - - RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) override { - RGWBucketEntryPoint be; - - try { - decode_json_obj(be, jo); - } catch (JSONDecoder::err& e) { - return nullptr; - } - - return new RGWBucketEntryMetadataObject(be, objv, mtime); - } - - int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { - RGWObjVersionTracker ot; - RGWBucketEntryPoint be; - - real_time mtime; - map attrs; - - RGWSI_Bucket_EP_Ctx ctx(op->ctx()); - - int ret = svc.bucket->read_bucket_entrypoint_info(ctx, entry, &be, &ot, &mtime, &attrs, y, dpp); - if (ret < 0) - return ret; - - RGWBucketEntryMetadataObject *mdo = new RGWBucketEntryMetadataObject(be, ot.read_version, mtime, std::move(attrs)); - - *obj = mdo; - - return 0; - } - - int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, - const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) override; - - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp) override { - RGWBucketEntryPoint be; - - real_time orig_mtime; - - RGWSI_Bucket_EP_Ctx ctx(op->ctx()); - - int ret = svc.bucket->read_bucket_entrypoint_info(ctx, entry, &be, &objv_tracker, &orig_mtime, nullptr, y, dpp); - if (ret < 0) - return ret; - - /* - * We're unlinking the bucket but we don't want to update the entrypoint here - we're removing - * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal - * will incorrectly fail. - */ - ret = ctl.bucket->unlink_bucket(be.owner, be.bucket, y, dpp, false); - if (ret < 0) { - ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl; - } - - ret = svc.bucket->remove_bucket_entrypoint_info(ctx, entry, &objv_tracker, y, dpp); - if (ret < 0) { - ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl; - } - /* idempotent */ - return 0; - } - - int call(std::function f) { - return call(nullopt, f); - } - - int call(std::optional bectx_params, - std::function f) { - return be_handler->call(bectx_params, [&](RGWSI_MetaBackend_Handler::Op *op) { - RGWSI_Bucket_EP_Ctx ctx(op->ctx()); - return f(ctx); - }); - } -}; - -class RGWMetadataHandlerPut_Bucket : public RGWMetadataHandlerPut_SObj -{ - RGWBucketMetadataHandler *bhandler; - RGWBucketEntryMetadataObject *obj; -public: - RGWMetadataHandlerPut_Bucket(RGWBucketMetadataHandler *_handler, - RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, - RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type, from_remote_zone), - bhandler(_handler) { - obj = static_cast(_obj); - } - ~RGWMetadataHandlerPut_Bucket() {} - - void encode_obj(bufferlist *bl) override { - obj->get_ep().encode(*bl); - } - - int put_checked(const DoutPrefixProvider *dpp) override; - int put_post(const DoutPrefixProvider *dpp) override; -}; - -int RGWBucketMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, - const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) -{ - RGWMetadataHandlerPut_Bucket put_op(this, op, entry, obj, objv_tracker, y, type, from_remote_zone); - return do_put_operate(&put_op, dpp); -} - -int RGWMetadataHandlerPut_Bucket::put_checked(const DoutPrefixProvider *dpp) -{ - RGWBucketEntryMetadataObject *orig_obj = static_cast(old_obj); - - if (orig_obj) { - obj->set_pattrs(&orig_obj->get_attrs()); - } - - auto& be = obj->get_ep(); - auto mtime = obj->get_mtime(); - auto pattrs = obj->get_pattrs(); - - RGWSI_Bucket_EP_Ctx ctx(op->ctx()); - - return bhandler->svc.bucket->store_bucket_entrypoint_info(ctx, entry, - be, - false, - mtime, - pattrs, - &objv_tracker, - y, - dpp); -} - -int RGWMetadataHandlerPut_Bucket::put_post(const DoutPrefixProvider *dpp) -{ - auto& be = obj->get_ep(); - - int ret; - - /* link bucket */ - if (be.linked) { - ret = bhandler->ctl.bucket->link_bucket(be.owner, be.bucket, be.creation_time, y, dpp, false); - } else { - ret = bhandler->ctl.bucket->unlink_bucket(be.owner, be.bucket, y, dpp, false); - } - - return ret; -} - -static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) { - - char md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; - unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; - bufferlist bl; - - Formatter *f = new JSONFormatter(false); - be->dump(f); - f->flush(bl); - - MD5 hash; - // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes - hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); - hash.Update((const unsigned char *)bl.c_str(), bl.length()); - hash.Final(m); - - buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, md5); - - delete f; - - md5_digest = md5; -} - -#define ARCHIVE_META_ATTR RGW_ATTR_PREFIX "zone.archive.info" - -struct archive_meta_info { - rgw_bucket orig_bucket; - - bool from_attrs(CephContext *cct, map& attrs) { - auto iter = attrs.find(ARCHIVE_META_ATTR); - if (iter == attrs.end()) { - return false; - } - - auto bliter = iter->second.cbegin(); - try { - decode(bliter); - } catch (buffer::error& err) { - ldout(cct, 0) << "ERROR: failed to decode archive meta info" << dendl; - return false; - } - - return true; - } - - void store_in_attrs(map& attrs) const { - encode(attrs[ARCHIVE_META_ATTR]); - } - - void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); - encode(orig_bucket, bl); - ENCODE_FINISH(bl); - } - - void decode(bufferlist::const_iterator& bl) { - DECODE_START(1, bl); - decode(orig_bucket, bl); - DECODE_FINISH(bl); - } -}; -WRITE_CLASS_ENCODER(archive_meta_info) - -class RGWArchiveBucketMetadataHandler : public RGWBucketMetadataHandler { -public: - RGWArchiveBucketMetadataHandler() {} - - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp) override { - auto cct = svc.bucket->ctx(); - - RGWSI_Bucket_EP_Ctx ctx(op->ctx()); - - ldpp_dout(dpp, 5) << "SKIP: bucket removal is not allowed on archive zone: bucket:" << entry << " ... proceeding to rename" << dendl; - - string tenant_name, bucket_name; - parse_bucket(entry, &tenant_name, &bucket_name); - rgw_bucket entry_bucket; - entry_bucket.tenant = tenant_name; - entry_bucket.name = bucket_name; - - real_time mtime; - - /* read original entrypoint */ - - RGWBucketEntryPoint be; - map attrs; - int ret = svc.bucket->read_bucket_entrypoint_info(ctx, entry, &be, &objv_tracker, &mtime, &attrs, y, dpp); - if (ret < 0) { - return ret; - } - - string bi_meta_name = RGWSI_Bucket::get_bi_meta_key(be.bucket); - - /* read original bucket instance info */ - - map attrs_m; - ceph::real_time orig_mtime; - RGWBucketInfo old_bi; - - ret = ctl.bucket->read_bucket_instance_info(be.bucket, &old_bi, y, dpp, RGWBucketCtl::BucketInstance::GetParams() - .set_mtime(&orig_mtime) - .set_attrs(&attrs_m)); - if (ret < 0) { - return ret; - } - - archive_meta_info ami; - - if (!ami.from_attrs(svc.bucket->ctx(), attrs_m)) { - ami.orig_bucket = old_bi.bucket; - ami.store_in_attrs(attrs_m); - } - - /* generate a new bucket instance. We could have avoided this if we could just point a new - * bucket entry point to the old bucket instance, however, due to limitation in the way - * we index buckets under the user, bucket entrypoint and bucket instance of the same - * bucket need to have the same name, so we need to copy the old bucket instance into - * to a new entry with the new name - */ - - string new_bucket_name; - - RGWBucketInfo new_bi = old_bi; - RGWBucketEntryPoint new_be = be; - - string md5_digest; - - get_md5_digest(&new_be, md5_digest); - new_bucket_name = ami.orig_bucket.name + "-deleted-" + md5_digest; - - new_bi.bucket.name = new_bucket_name; - new_bi.objv_tracker.clear(); - - new_be.bucket.name = new_bucket_name; - - ret = ctl.bucket->store_bucket_instance_info(be.bucket, new_bi, y, dpp, RGWBucketCtl::BucketInstance::PutParams() - .set_exclusive(false) - .set_mtime(orig_mtime) - .set_attrs(&attrs_m) - .set_orig_info(&old_bi)); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket instance info for bucket=" << new_bi.bucket << " ret=" << ret << dendl; - return ret; - } - - /* store a new entrypoint */ - - RGWObjVersionTracker ot; - ot.generate_new_write_ver(cct); - - ret = svc.bucket->store_bucket_entrypoint_info(ctx, RGWSI_Bucket::get_entrypoint_meta_key(new_be.bucket), - new_be, true, mtime, &attrs, nullptr, y, dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be.bucket << " ret=" << ret << dendl; - return ret; - } - - /* link new bucket */ - - ret = ctl.bucket->link_bucket(new_be.owner, new_be.bucket, new_be.creation_time, y, dpp, false); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to link new bucket for bucket=" << new_be.bucket << " ret=" << ret << dendl; - return ret; - } - - /* clean up old stuff */ - - ret = ctl.bucket->unlink_bucket(be.owner, entry_bucket, y, dpp, false); - if (ret < 0) { - ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl; - } - - // if (ret == -ECANCELED) it means that there was a race here, and someone - // wrote to the bucket entrypoint just before we removed it. The question is - // whether it was a newly created bucket entrypoint ... in which case we - // should ignore the error and move forward, or whether it is a higher version - // of the same bucket instance ... in which we should retry - ret = svc.bucket->remove_bucket_entrypoint_info(ctx, - RGWSI_Bucket::get_entrypoint_meta_key(be.bucket), - &objv_tracker, - y, - dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be.bucket << " ret=" << ret << dendl; - return ret; - } - - ret = ctl.bucket->remove_bucket_instance_info(be.bucket, old_bi, y, dpp); - if (ret < 0) { - ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl; - } - - - /* idempotent */ - - return 0; - } - - int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) override { - if (entry.find("-deleted-") != string::npos) { - RGWObjVersionTracker ot; - RGWMetadataObject *robj; - int ret = do_get(op, entry, &robj, y, dpp); - if (ret != -ENOENT) { - if (ret < 0) { - return ret; - } - ot.read_version = robj->get_version(); - delete robj; - - ret = do_remove(op, entry, ot, y, dpp); - if (ret < 0) { - return ret; - } - } - } - - return RGWBucketMetadataHandler::do_put(op, entry, obj, - objv_tracker, y, dpp, type, from_remote_zone); - } - -}; - -class RGWBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandlerBase { - int read_bucket_instance_entry(RGWSI_Bucket_BI_Ctx& ctx, - const string& entry, - RGWBucketCompleteInfo *bi, - ceph::real_time *pmtime, - optional_yield y, - const DoutPrefixProvider *dpp) { - return svc.bucket->read_bucket_instance_info(ctx, - entry, - &bi->info, - pmtime, &bi->attrs, - y, - dpp); - } - -public: - struct Svc { - RGWSI_Zone *zone{nullptr}; - RGWSI_Bucket *bucket{nullptr}; - RGWSI_BucketIndex *bi{nullptr}; - } svc; - - rgw::sal::Store* store; - - RGWBucketInstanceMetadataHandler(rgw::sal::Store* store) - : store(store) {} - - void init(RGWSI_Zone *zone_svc, - RGWSI_Bucket *bucket_svc, - RGWSI_BucketIndex *bi_svc) override { - base_init(bucket_svc->ctx(), - bucket_svc->get_bi_be_handler().get()); - svc.zone = zone_svc; - svc.bucket = bucket_svc; - svc.bi = bi_svc; - } - - string get_type() override { return "bucket.instance"; } - - RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) override { - RGWBucketCompleteInfo bci; - - try { - decode_json_obj(bci, jo); - } catch (JSONDecoder::err& e) { - return nullptr; - } - - return new RGWBucketInstanceMetadataObject(bci, objv, mtime); - } - - int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { - RGWBucketCompleteInfo bci; - real_time mtime; - - RGWSI_Bucket_BI_Ctx ctx(op->ctx()); - - int ret = svc.bucket->read_bucket_instance_info(ctx, entry, &bci.info, &mtime, &bci.attrs, y, dpp); - if (ret < 0) - return ret; - - RGWBucketInstanceMetadataObject *mdo = new RGWBucketInstanceMetadataObject(bci, bci.info.objv_tracker.read_version, mtime); - - *obj = mdo; - - return 0; - } - - int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp, - RGWMDLogSyncType sync_type, bool from_remote_zone) override; - - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp) override { - RGWBucketCompleteInfo bci; - - RGWSI_Bucket_BI_Ctx ctx(op->ctx()); - - int ret = read_bucket_instance_entry(ctx, entry, &bci, nullptr, y, dpp); - if (ret < 0 && ret != -ENOENT) - return ret; - - return svc.bucket->remove_bucket_instance_info(ctx, entry, bci.info, &bci.info.objv_tracker, y, dpp); - } - - int call(std::function f) { - return call(nullopt, f); - } - - int call(std::optional bectx_params, - std::function f) { - return be_handler->call(bectx_params, [&](RGWSI_MetaBackend_Handler::Op *op) { - RGWSI_Bucket_BI_Ctx ctx(op->ctx()); - return f(ctx); - }); - } -}; - -class RGWMetadataHandlerPut_BucketInstance : public RGWMetadataHandlerPut_SObj -{ - CephContext *cct; - RGWBucketInstanceMetadataHandler *bihandler; - RGWBucketInstanceMetadataObject *obj; -public: - RGWMetadataHandlerPut_BucketInstance(CephContext *_cct, - RGWBucketInstanceMetadataHandler *_handler, - RGWSI_MetaBackend_Handler::Op *_op, string& entry, - RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, - RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, _op, entry, obj, objv_tracker, y, type, from_remote_zone), - cct(_cct), bihandler(_handler) { - obj = static_cast(_obj); - - auto& bci = obj->get_bci(); - obj->set_pattrs(&bci.attrs); - } - - void encode_obj(bufferlist *bl) override { - obj->get_bucket_info().encode(*bl); - } - - int put_check(const DoutPrefixProvider *dpp) override; - int put_checked(const DoutPrefixProvider *dpp) override; - int put_post(const DoutPrefixProvider *dpp) override; -}; - -int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, - string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, - const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) -{ - RGWMetadataHandlerPut_BucketInstance put_op(svc.bucket->ctx(), this, op, entry, obj, - objv_tracker, y, type, from_remote_zone); - return do_put_operate(&put_op, dpp); -} - -void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout, - const RGWZone& zone, - std::optional shards, - std::optional type) { - layout.current_index.gen = 0; - layout.current_index.layout.normal.hash_type = rgw::BucketHashType::Mod; - - layout.current_index.layout.type = - type.value_or(rgw::BucketIndexType::Normal); - - if (shards) { - layout.current_index.layout.normal.num_shards = *shards; - } else if (cct->_conf->rgw_override_bucket_index_max_shards > 0) { - layout.current_index.layout.normal.num_shards = - cct->_conf->rgw_override_bucket_index_max_shards; - } else { - layout.current_index.layout.normal.num_shards = - zone.bucket_index_max_shards; - } - - if (layout.current_index.layout.type == rgw::BucketIndexType::Normal) { - layout.logs.push_back(log_layout_from_index(0, layout.current_index)); - } -} - -int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dpp) -{ - int ret; - - RGWBucketCompleteInfo& bci = obj->get_bci(); - - RGWBucketInstanceMetadataObject *orig_obj = static_cast(old_obj); - - RGWBucketCompleteInfo *old_bci = (orig_obj ? &orig_obj->get_bci() : nullptr); - - const bool exists = (!!orig_obj); - - if (from_remote_zone) { - // don't sync bucket layout changes - if (!exists) { - // replace peer's layout with default-constructed, then apply our defaults - bci.info.layout = rgw::BucketLayout{}; - init_default_bucket_layout(cct, bci.info.layout, - bihandler->svc.zone->get_zone(), - std::nullopt, std::nullopt); - } else { - bci.info.layout = old_bci->info.layout; - } - } - - if (!exists || old_bci->info.bucket.bucket_id != bci.info.bucket.bucket_id) { - /* a new bucket, we need to select a new bucket placement for it */ - string tenant_name; - string bucket_name; - string bucket_instance; - parse_bucket(entry, &tenant_name, &bucket_name, &bucket_instance); - - RGWZonePlacementInfo rule_info; - bci.info.bucket.name = bucket_name; - bci.info.bucket.bucket_id = bucket_instance; - bci.info.bucket.tenant = tenant_name; - // if the sync module never writes data, don't require the zone to specify all placement targets - if (bihandler->svc.zone->sync_module_supports_writes()) { - ret = bihandler->svc.zone->select_bucket_location_by_rule(dpp, bci.info.placement_rule, &rule_info, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: select_bucket_placement() returned " << ret << dendl; - return ret; - } - } - bci.info.layout.current_index.layout.type = rule_info.index_type; - } else { - /* existing bucket, keep its placement */ - bci.info.bucket.explicit_placement = old_bci->info.bucket.explicit_placement; - bci.info.placement_rule = old_bci->info.placement_rule; - } - - /* record the read version (if any), store the new version */ - bci.info.objv_tracker.read_version = objv_tracker.read_version; - bci.info.objv_tracker.write_version = objv_tracker.write_version; - - return 0; -} - -int RGWMetadataHandlerPut_BucketInstance::put_checked(const DoutPrefixProvider *dpp) -{ - RGWBucketInstanceMetadataObject *orig_obj = static_cast(old_obj); - - RGWBucketInfo *orig_info = (orig_obj ? &orig_obj->get_bucket_info() : nullptr); - - auto& info = obj->get_bucket_info(); - auto mtime = obj->get_mtime(); - auto pattrs = obj->get_pattrs(); - - RGWSI_Bucket_BI_Ctx ctx(op->ctx()); - - return bihandler->svc.bucket->store_bucket_instance_info(ctx, - entry, - info, - orig_info, - false, - mtime, - pattrs, - y, - dpp); -} - -int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp) -{ - RGWBucketCompleteInfo& bci = obj->get_bci(); - - objv_tracker = bci.info.objv_tracker; - - int ret = bihandler->svc.bi->init_index(dpp, bci.info, bci.info.layout.current_index); - if (ret < 0) { - return ret; - } - - /* update lifecyle policy */ - { - std::unique_ptr bucket; - ret = bihandler->store->get_bucket(nullptr, bci.info, &bucket); - if (ret < 0) { - ldpp_dout(dpp, 0) << __func__ << " failed to get_bucket(...) for " - << bci.info.bucket.name - << dendl; - return ret; - } - - auto lc = bihandler->store->get_rgwlc(); - - auto lc_it = bci.attrs.find(RGW_ATTR_LC); - if (lc_it != bci.attrs.end()) { - ldpp_dout(dpp, 20) << "set lc config for " << bci.info.bucket.name << dendl; - ret = lc->set_bucket_config(bucket.get(), bci.attrs, nullptr); - if (ret < 0) { - ldpp_dout(dpp, 0) << __func__ << " failed to set lc config for " - << bci.info.bucket.name - << dendl; - return ret; - } - - } else { - ldpp_dout(dpp, 20) << "remove lc config for " << bci.info.bucket.name << dendl; - ret = lc->remove_bucket_config(bucket.get(), bci.attrs, false /* cannot merge attrs */); - if (ret < 0) { - ldpp_dout(dpp, 0) << __func__ << " failed to remove lc config for " - << bci.info.bucket.name - << dendl; - return ret; - } - } - } /* update lc */ - - return STATUS_APPLIED; -} - -class RGWArchiveBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandler { -public: - RGWArchiveBucketInstanceMetadataHandler(rgw::sal::Store* store) - : RGWBucketInstanceMetadataHandler(store) {} - - // N.B. replication of lifecycle policy relies on logic in RGWBucketInstanceMetadataHandler::do_put(...), override with caution - - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) override { - ldpp_dout(dpp, 0) << "SKIP: bucket instance removal is not allowed on archive zone: bucket.instance:" << entry << dendl; - return 0; - } -}; - -RGWBucketCtl::RGWBucketCtl(RGWSI_Zone *zone_svc, - RGWSI_Bucket *bucket_svc, - RGWSI_Bucket_Sync *bucket_sync_svc, - RGWSI_BucketIndex *bi_svc, - RGWSI_User* user_svc) - : cct(zone_svc->ctx()) -{ - svc.zone = zone_svc; - svc.bucket = bucket_svc; - svc.bucket_sync = bucket_sync_svc; - svc.bi = bi_svc; - svc.user = user_svc; -} - -void RGWBucketCtl::init(RGWUserCtl *user_ctl, - RGWBucketMetadataHandler *_bm_handler, - RGWBucketInstanceMetadataHandler *_bmi_handler, - RGWDataChangesLog *datalog, - const DoutPrefixProvider *dpp) -{ - ctl.user = user_ctl; - - bm_handler = _bm_handler; - bmi_handler = _bmi_handler; - - bucket_be_handler = bm_handler->get_be_handler(); - bi_be_handler = bmi_handler->get_be_handler(); - - datalog->set_bucket_filter( - [this](const rgw_bucket& bucket, optional_yield y, const DoutPrefixProvider *dpp) { - return bucket_exports_data(bucket, y, dpp); - }); -} - -int RGWBucketCtl::call(std::function f) { - return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ep_ctx) { - return bmi_handler->call([&](RGWSI_Bucket_BI_Ctx& bi_ctx) { - RGWSI_Bucket_X_Ctx ctx{ep_ctx, bi_ctx}; - return f(ctx); - }); - }); -} - -int RGWBucketCtl::read_bucket_entrypoint_info(const rgw_bucket& bucket, - RGWBucketEntryPoint *info, - optional_yield y, const DoutPrefixProvider *dpp, - const Bucket::GetParams& params) -{ - return bm_handler->call(params.bectx_params, [&](RGWSI_Bucket_EP_Ctx& ctx) { - return svc.bucket->read_bucket_entrypoint_info(ctx, - RGWSI_Bucket::get_entrypoint_meta_key(bucket), - info, - params.objv_tracker, - params.mtime, - params.attrs, - y, - dpp, - params.cache_info, - params.refresh_version); - }); -} - -int RGWBucketCtl::store_bucket_entrypoint_info(const rgw_bucket& bucket, - RGWBucketEntryPoint& info, - optional_yield y, - const DoutPrefixProvider *dpp, - const Bucket::PutParams& params) -{ - return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { - return svc.bucket->store_bucket_entrypoint_info(ctx, - RGWSI_Bucket::get_entrypoint_meta_key(bucket), - info, - params.exclusive, - params.mtime, - params.attrs, - params.objv_tracker, - y, - dpp); - }); -} - -int RGWBucketCtl::remove_bucket_entrypoint_info(const rgw_bucket& bucket, - optional_yield y, - const DoutPrefixProvider *dpp, - const Bucket::RemoveParams& params) -{ - return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { - return svc.bucket->remove_bucket_entrypoint_info(ctx, - RGWSI_Bucket::get_entrypoint_meta_key(bucket), - params.objv_tracker, - y, - dpp); - }); -} - -int RGWBucketCtl::read_bucket_instance_info(const rgw_bucket& bucket, - RGWBucketInfo *info, - optional_yield y, - const DoutPrefixProvider *dpp, - const BucketInstance::GetParams& params) -{ - int ret = bmi_handler->call(params.bectx_params, [&](RGWSI_Bucket_BI_Ctx& ctx) { - return svc.bucket->read_bucket_instance_info(ctx, - RGWSI_Bucket::get_bi_meta_key(bucket), - info, - params.mtime, - params.attrs, - y, - dpp, - params.cache_info, - params.refresh_version); - }); - - if (ret < 0) { - return ret; - } - - if (params.objv_tracker) { - *params.objv_tracker = info->objv_tracker; - } - - return 0; -} - -int RGWBucketCtl::read_bucket_info(const rgw_bucket& bucket, - RGWBucketInfo *info, - optional_yield y, - const DoutPrefixProvider *dpp, - const BucketInstance::GetParams& params, - RGWObjVersionTracker *ep_objv_tracker) -{ - const rgw_bucket *b = &bucket; - - std::optional ep; - - if (b->bucket_id.empty()) { - ep.emplace(); - - int r = read_bucket_entrypoint_info(*b, &(*ep), y, dpp, RGWBucketCtl::Bucket::GetParams() - .set_bectx_params(params.bectx_params) - .set_objv_tracker(ep_objv_tracker)); - if (r < 0) { - return r; - } - - b = &ep->bucket; - } - - int ret = bmi_handler->call(params.bectx_params, [&](RGWSI_Bucket_BI_Ctx& ctx) { - return svc.bucket->read_bucket_instance_info(ctx, - RGWSI_Bucket::get_bi_meta_key(*b), - info, - params.mtime, - params.attrs, - y, dpp, - params.cache_info, - params.refresh_version); - }); - - if (ret < 0) { - return ret; - } - - if (params.objv_tracker) { - *params.objv_tracker = info->objv_tracker; - } - - return 0; -} - -int RGWBucketCtl::do_store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, - const rgw_bucket& bucket, - RGWBucketInfo& info, - optional_yield y, - const DoutPrefixProvider *dpp, - const BucketInstance::PutParams& params) -{ - if (params.objv_tracker) { - info.objv_tracker = *params.objv_tracker; - } - - return svc.bucket->store_bucket_instance_info(ctx, - RGWSI_Bucket::get_bi_meta_key(bucket), - info, - params.orig_info, - params.exclusive, - params.mtime, - params.attrs, - y, - dpp); -} - -int RGWBucketCtl::store_bucket_instance_info(const rgw_bucket& bucket, - RGWBucketInfo& info, - optional_yield y, - const DoutPrefixProvider *dpp, - const BucketInstance::PutParams& params) -{ - return bmi_handler->call([&](RGWSI_Bucket_BI_Ctx& ctx) { - return do_store_bucket_instance_info(ctx, bucket, info, y, dpp, params); - }); -} - -int RGWBucketCtl::remove_bucket_instance_info(const rgw_bucket& bucket, - RGWBucketInfo& info, - optional_yield y, - const DoutPrefixProvider *dpp, - const BucketInstance::RemoveParams& params) -{ - if (params.objv_tracker) { - info.objv_tracker = *params.objv_tracker; - } - - return bmi_handler->call([&](RGWSI_Bucket_BI_Ctx& ctx) { - return svc.bucket->remove_bucket_instance_info(ctx, - RGWSI_Bucket::get_bi_meta_key(bucket), - info, - &info.objv_tracker, - y, - dpp); - }); -} - -int RGWBucketCtl::do_store_linked_bucket_info(RGWSI_Bucket_X_Ctx& ctx, - RGWBucketInfo& info, - RGWBucketInfo *orig_info, - bool exclusive, real_time mtime, - obj_version *pep_objv, - map *pattrs, - bool create_entry_point, - optional_yield y, const DoutPrefixProvider *dpp) -{ - bool create_head = !info.has_instance_obj || create_entry_point; - - int ret = svc.bucket->store_bucket_instance_info(ctx.bi, - RGWSI_Bucket::get_bi_meta_key(info.bucket), - info, - orig_info, - exclusive, - mtime, pattrs, - y, dpp); - if (ret < 0) { - return ret; - } - - if (!create_head) - return 0; /* done! */ - - RGWBucketEntryPoint entry_point; - entry_point.bucket = info.bucket; - entry_point.owner = info.owner; - entry_point.creation_time = info.creation_time; - entry_point.linked = true; - RGWObjVersionTracker ot; - if (pep_objv && !pep_objv->tag.empty()) { - ot.write_version = *pep_objv; + int pos = bucket.find(':'); + if (pos >= 0) { + /* + * N.B.: We allow ":bucket" syntax with explicit empty tenant in order + * to refer to the legacy tenant, in case users in new named tenants + * want to access old global buckets. + */ + tenant_name = bucket.substr(0, pos); + bucket_name = bucket.substr(pos + 1); } else { - ot.generate_new_write_ver(cct); - if (pep_objv) { - *pep_objv = ot.write_version; - } - } - ret = svc.bucket->store_bucket_entrypoint_info(ctx.ep, - RGWSI_Bucket::get_entrypoint_meta_key(info.bucket), - entry_point, - exclusive, - mtime, - pattrs, - &ot, - y, - dpp); - if (ret < 0) - return ret; - - return 0; -} -int RGWBucketCtl::convert_old_bucket_info(RGWSI_Bucket_X_Ctx& ctx, - const rgw_bucket& bucket, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - RGWBucketEntryPoint entry_point; - real_time ep_mtime; - RGWObjVersionTracker ot; - map attrs; - RGWBucketInfo info; - auto cct = svc.bucket->ctx(); - - ldpp_dout(dpp, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket << dendl; - - int ret = svc.bucket->read_bucket_entrypoint_info(ctx.ep, - RGWSI_Bucket::get_entrypoint_meta_key(bucket), - &entry_point, &ot, &ep_mtime, &attrs, y, dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: get_bucket_entrypoint_info() returned " << ret << " bucket=" << bucket << dendl; - return ret; - } - - if (!entry_point.has_bucket_info) { - /* already converted! */ - return 0; - } - - info = entry_point.old_bucket_info; - - ot.generate_new_write_ver(cct); - - ret = do_store_linked_bucket_info(ctx, info, nullptr, false, ep_mtime, &ot.write_version, &attrs, true, y, dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret << dendl; - return ret; - } - - return 0; -} - -int RGWBucketCtl::set_bucket_instance_attrs(RGWBucketInfo& bucket_info, - map& attrs, - RGWObjVersionTracker *objv_tracker, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - return call([&](RGWSI_Bucket_X_Ctx& ctx) { - rgw_bucket& bucket = bucket_info.bucket; - - if (!bucket_info.has_instance_obj) { - /* an old bucket object, need to convert it */ - int ret = convert_old_bucket_info(ctx, bucket, y, dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed converting old bucket info: " << ret << dendl; - return ret; - } - } - - return do_store_bucket_instance_info(ctx.bi, - bucket, - bucket_info, - y, - dpp, - BucketInstance::PutParams().set_attrs(&attrs) - .set_objv_tracker(objv_tracker) - .set_orig_info(&bucket_info)); - }); -} - - -int RGWBucketCtl::link_bucket(const rgw_user& user_id, - const rgw_bucket& bucket, - ceph::real_time creation_time, - optional_yield y, - const DoutPrefixProvider *dpp, - bool update_entrypoint, - rgw_ep_info *pinfo) -{ - return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { - return do_link_bucket(ctx, user_id, bucket, creation_time, - update_entrypoint, pinfo, y, dpp); - }); -} - -int RGWBucketCtl::do_link_bucket(RGWSI_Bucket_EP_Ctx& ctx, - const rgw_user& user_id, - const rgw_bucket& bucket, - ceph::real_time creation_time, - bool update_entrypoint, - rgw_ep_info *pinfo, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - int ret; - - RGWBucketEntryPoint ep; - RGWObjVersionTracker ot; - RGWObjVersionTracker& rot = (pinfo) ? pinfo->ep_objv : ot; - map attrs, *pattrs = nullptr; - string meta_key; - - if (update_entrypoint) { - meta_key = RGWSI_Bucket::get_entrypoint_meta_key(bucket); - if (pinfo) { - ep = pinfo->ep; - pattrs = &pinfo->attrs; - } else { - ret = svc.bucket->read_bucket_entrypoint_info(ctx, - meta_key, - &ep, &rot, - nullptr, &attrs, - y, dpp); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: store->get_bucket_entrypoint_info() returned: " - << cpp_strerror(-ret) << dendl; - } - pattrs = &attrs; - } - } - - ret = svc.user->add_bucket(dpp, user_id, bucket, creation_time, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: error adding bucket to user directory:" - << " user=" << user_id - << " bucket=" << bucket - << " err=" << cpp_strerror(-ret) - << dendl; - goto done_err; - } - - if (!update_entrypoint) - return 0; - - ep.linked = true; - ep.owner = user_id; - ep.bucket = bucket; - ret = svc.bucket->store_bucket_entrypoint_info( - ctx, meta_key, ep, false, real_time(), pattrs, &rot, y, dpp); - if (ret < 0) - goto done_err; - - return 0; - -done_err: - int r = do_unlink_bucket(ctx, user_id, bucket, true, y, dpp); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed unlinking bucket on error cleanup: " - << cpp_strerror(-r) << dendl; - } - return ret; -} - -int RGWBucketCtl::unlink_bucket(const rgw_user& user_id, const rgw_bucket& bucket, optional_yield y, const DoutPrefixProvider *dpp, bool update_entrypoint) -{ - return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { - return do_unlink_bucket(ctx, user_id, bucket, update_entrypoint, y, dpp); - }); -} - -int RGWBucketCtl::do_unlink_bucket(RGWSI_Bucket_EP_Ctx& ctx, - const rgw_user& user_id, - const rgw_bucket& bucket, - bool update_entrypoint, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - int ret = svc.user->remove_bucket(dpp, user_id, bucket, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: error removing bucket from directory: " - << cpp_strerror(-ret)<< dendl; - } - - if (!update_entrypoint) - return 0; - - RGWBucketEntryPoint ep; - RGWObjVersionTracker ot; - map attrs; - string meta_key = RGWSI_Bucket::get_entrypoint_meta_key(bucket); - ret = svc.bucket->read_bucket_entrypoint_info(ctx, meta_key, &ep, &ot, nullptr, &attrs, y, dpp); - if (ret == -ENOENT) - return 0; - if (ret < 0) - return ret; - - if (!ep.linked) - return 0; - - if (ep.owner != user_id) { - ldpp_dout(dpp, 0) << "bucket entry point user mismatch, can't unlink bucket: " << ep.owner << " != " << user_id << dendl; - return -EINVAL; - } - - ep.linked = false; - return svc.bucket->store_bucket_entrypoint_info(ctx, meta_key, ep, false, real_time(), &attrs, &ot, y, dpp); -} - -// TODO: remove RGWRados dependency for bucket listing -int RGWBucketCtl::chown(rgw::sal::Store* store, rgw::sal::Bucket* bucket, - const rgw_user& user_id, const std::string& display_name, - const std::string& marker, optional_yield y, const DoutPrefixProvider *dpp) -{ - map common_prefixes; - - rgw::sal::Bucket::ListParams params; - rgw::sal::Bucket::ListResults results; - - params.list_versions = true; - params.allow_unordered = true; - params.marker = marker; - - int count = 0; - int max_entries = 1000; - - //Loop through objects and update object acls to point to bucket owner - - do { - RGWObjectCtx obj_ctx(store); - results.objs.clear(); - int ret = bucket->list(dpp, params, max_entries, results, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: list objects failed: " << cpp_strerror(-ret) << dendl; - return ret; - } - - params.marker = results.next_marker; - count += results.objs.size(); - - for (const auto& obj : results.objs) { - std::unique_ptr r_obj = bucket->get_object(obj.key); - - ret = r_obj->get_obj_attrs(y, dpp); - if (ret < 0){ - ldpp_dout(dpp, 0) << "ERROR: failed to read object " << obj.key.name << cpp_strerror(-ret) << dendl; - continue; - } - const auto& aiter = r_obj->get_attrs().find(RGW_ATTR_ACL); - if (aiter == r_obj->get_attrs().end()) { - ldpp_dout(dpp, 0) << "ERROR: no acls found for object " << obj.key.name << " .Continuing with next object." << dendl; - continue; - } else { - bufferlist& bl = aiter->second; - RGWAccessControlPolicy policy(store->ctx()); - ACLOwner owner; - try { - decode(policy, bl); - owner = policy.get_owner(); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "ERROR: decode policy failed" << err.what() - << dendl; - return -EIO; - } - - //Get the ACL from the policy - RGWAccessControlList& acl = policy.get_acl(); - - //Remove grant that is set to old owner - acl.remove_canon_user_grant(owner.get_id()); - - //Create a grant and add grant - ACLGrant grant; - grant.set_canon(user_id, display_name, RGW_PERM_FULL_CONTROL); - acl.add_grant(&grant); - - //Update the ACL owner to the new user - owner.set_id(user_id); - owner.set_name(display_name); - policy.set_owner(owner); - - bl.clear(); - encode(policy, bl); - - r_obj->set_atomic(); - map attrs; - attrs[RGW_ATTR_ACL] = bl; - ret = r_obj->set_obj_attrs(dpp, &attrs, nullptr, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: modify attr failed " << cpp_strerror(-ret) << dendl; - return ret; - } - } - } - cerr << count << " objects processed in " << bucket - << ". Next marker " << params.marker.name << std::endl; - } while(results.is_truncated); - return 0; -} - -int RGWBucketCtl::read_bucket_stats(const rgw_bucket& bucket, - RGWBucketEnt *result, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - return call([&](RGWSI_Bucket_X_Ctx& ctx) { - return svc.bucket->read_bucket_stats(ctx, bucket, result, y, dpp); - }); -} - -int RGWBucketCtl::read_buckets_stats(map& m, - optional_yield y, const DoutPrefixProvider *dpp) -{ - return call([&](RGWSI_Bucket_X_Ctx& ctx) { - return svc.bucket->read_buckets_stats(ctx, m, y, dpp); - }); -} - -int RGWBucketCtl::sync_user_stats(const DoutPrefixProvider *dpp, - const rgw_user& user_id, - const RGWBucketInfo& bucket_info, - optional_yield y, - RGWBucketEnt* pent) -{ - RGWBucketEnt ent; - if (!pent) { - pent = &ent; - } - int r = svc.bi->read_stats(dpp, bucket_info, pent, null_yield); - if (r < 0) { - ldpp_dout(dpp, 20) << __func__ << "(): failed to read bucket stats (r=" << r << ")" << dendl; - return r; - } - - return svc.user->flush_bucket_stats(dpp, user_id, *pent, y); -} - -int RGWBucketCtl::get_sync_policy_handler(std::optional zone, - std::optional bucket, - RGWBucketSyncPolicyHandlerRef *phandler, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - int r = call([&](RGWSI_Bucket_X_Ctx& ctx) { - return svc.bucket_sync->get_policy_handler(ctx, zone, bucket, phandler, y, dpp); - }); - if (r < 0) { - ldpp_dout(dpp, 20) << __func__ << "(): failed to get policy handler for bucket=" << bucket << " (r=" << r << ")" << dendl; - return r; - } - return 0; -} - -int RGWBucketCtl::bucket_exports_data(const rgw_bucket& bucket, - optional_yield y, - const DoutPrefixProvider *dpp) -{ - - RGWBucketSyncPolicyHandlerRef handler; - - int r = get_sync_policy_handler(std::nullopt, bucket, &handler, y, dpp); - if (r < 0) { - return r; - } - - return handler->bucket_exports_data(); -} - -int RGWBucketCtl::bucket_imports_data(const rgw_bucket& bucket, - optional_yield y, const DoutPrefixProvider *dpp) -{ - - RGWBucketSyncPolicyHandlerRef handler; - - int r = get_sync_policy_handler(std::nullopt, bucket, &handler, y, dpp); - if (r < 0) { - return r; - } - - return handler->bucket_imports_data(); -} - -RGWBucketMetadataHandlerBase* RGWBucketMetaHandlerAllocator::alloc() -{ - return new RGWBucketMetadataHandler(); -} - -RGWBucketInstanceMetadataHandlerBase* RGWBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Store* store) -{ - return new RGWBucketInstanceMetadataHandler(store); -} - -RGWBucketMetadataHandlerBase* RGWArchiveBucketMetaHandlerAllocator::alloc() -{ - return new RGWArchiveBucketMetadataHandler(); -} - -RGWBucketInstanceMetadataHandlerBase* RGWArchiveBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Store* store) -{ - return new RGWArchiveBucketInstanceMetadataHandler(store); -} - - -void RGWBucketEntryPoint::generate_test_instances(list& o) -{ - RGWBucketEntryPoint *bp = new RGWBucketEntryPoint(); - init_bucket(&bp->bucket, "tenant", "bucket", "pool", ".index.pool", "marker", "10"); - bp->owner = "owner"; - bp->creation_time = ceph::real_clock::from_ceph_timespec({ceph_le32(2), ceph_le32(3)}); - - o.push_back(bp); - o.push_back(new RGWBucketEntryPoint); -} - -void RGWBucketEntryPoint::dump(Formatter *f) const -{ - encode_json("bucket", bucket, f); - encode_json("owner", owner, f); - utime_t ut(creation_time); - encode_json("creation_time", ut, f); - encode_json("linked", linked, f); - encode_json("has_bucket_info", has_bucket_info, f); - if (has_bucket_info) { - encode_json("old_bucket_info", old_bucket_info, f); - } -} - -void RGWBucketEntryPoint::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("bucket", bucket, obj); - JSONDecoder::decode_json("owner", owner, obj); - utime_t ut; - JSONDecoder::decode_json("creation_time", ut, obj); - creation_time = ut.to_real_time(); - JSONDecoder::decode_json("linked", linked, obj); - JSONDecoder::decode_json("has_bucket_info", has_bucket_info, obj); - if (has_bucket_info) { - JSONDecoder::decode_json("old_bucket_info", old_bucket_info, obj); + tenant_name = auth_tenant; + bucket_name = bucket; } } diff --git a/src/rgw/rgw_common.cc b/src/rgw/rgw_common.cc index 43d4a043ae57..422874b4f4d9 100644 --- a/src/rgw/rgw_common.cc +++ b/src/rgw/rgw_common.cc @@ -168,6 +168,8 @@ rgw_http_errors rgw_http_iam_errors({ using namespace std; using namespace ceph::crypto; +thread_local bool is_asio_thread = false; + rgw_err:: rgw_err() { @@ -3057,3 +3059,13 @@ rgw_global_init(const std::map *defaults, // Finish global init, indicating we already ran pre-init return global_init(defaults, args, module_type, code_env, flags, false); } + +void RGWObjVersionTracker::generate_new_write_ver(CephContext *cct) +{ + write_version.ver = 1; +#define TAG_LEN 24 + + write_version.tag.clear(); + append_rand_alpha(cct, write_version.tag, write_version.tag, TAG_LEN); +} + diff --git a/src/rgw/rgw_lua.cc b/src/rgw/rgw_lua.cc index 76b67381c0a2..30d29731604c 100644 --- a/src/rgw/rgw_lua.cc +++ b/src/rgw/rgw_lua.cc @@ -1,7 +1,8 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + #include #include "services/svc_zone.h" -#include "services/svc_sys_obj.h" -#include "common/dout.h" #include "rgw_lua_utils.h" #include "rgw_sal_rados.h" #include "rgw_lua.h" diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 41a4713ea6de..7fd25ae75879 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -1,52 +1,18 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp -#include -#include "common/ceph_json.h" -#include "common/errno.h" #include "rgw_metadata.h" -#include "rgw_coroutine.h" -#include "cls/version/cls_version_types.h" -#include "rgw_zone.h" -#include "rgw_tools.h" #include "rgw_mdlog.h" -#include "rgw_sal.h" -#include "rgw_cr_rados.h" -#include "services/svc_zone.h" #include "services/svc_meta.h" -#include "services/svc_meta_be.h" #include "services/svc_meta_be_sobj.h" -#include "services/svc_cls.h" - -#include "include/ceph_assert.h" - -#include #define dout_subsys ceph_subsys_rgw using namespace std; -const std::string RGWMetadataLogHistory::oid = "meta.history"; - -struct obj_version; - -void encode_json(const char *name, const obj_version& v, Formatter *f) -{ - f->open_object_section(name); - f->dump_string("tag", v.tag); - f->dump_unsigned("ver", v.ver); - f->close_section(); -} - -void decode_json_obj(obj_version& v, JSONObj *obj) -{ - JSONDecoder::decode_json("tag", v.tag, obj); - JSONDecoder::decode_json("ver", v.ver, obj); -} - void LogStatusDump::dump(Formatter *f) const { string s; switch (status) { @@ -72,6 +38,20 @@ void LogStatusDump::dump(Formatter *f) const { encode_json("status", s, f); } +void encode_json(const char *name, const obj_version& v, Formatter *f) +{ + f->open_object_section(name); + f->dump_string("tag", v.tag); + f->dump_unsigned("ver", v.ver); + f->close_section(); +} + +void decode_json_obj(obj_version& v, JSONObj *obj) +{ + JSONDecoder::decode_json("tag", v.tag, obj); + JSONDecoder::decode_json("ver", v.ver, obj); +} + void RGWMetadataLogData::encode(bufferlist& bl) const { ENCODE_START(1, 1, bl); encode(read_version, bl); @@ -121,208 +101,194 @@ void RGWMetadataLogData::decode_json(JSONObj *obj) { JSONDecoder::decode_json("status", status, obj); } -void rgw_shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id) -{ - uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); - char buf[16]; - if (shard_id) { - *shard_id = val % max_shards; - } - snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); - name = prefix + buf; -} - -void rgw_shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name) +RGWMetadataHandler_GenericMetaBE::Put::Put(RGWMetadataHandler_GenericMetaBE *_handler, + RGWSI_MetaBackend_Handler::Op *_op, + string& _entry, RGWMetadataObject *_obj, + RGWObjVersionTracker& _objv_tracker, + optional_yield _y, + RGWMDLogSyncType _type, bool _from_remote_zone): + handler(_handler), op(_op), + entry(_entry), obj(_obj), + objv_tracker(_objv_tracker), + apply_type(_type), + y(_y), + from_remote_zone(_from_remote_zone) { - uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); - val ^= ceph_str_hash_linux(section.c_str(), section.size()); - char buf[16]; - snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); - name = prefix + buf; } -void rgw_shard_name(const string& prefix, unsigned shard_id, string& name) +int RGWMetadataHandler_GenericMetaBE::do_put_operate(Put *put_op, const DoutPrefixProvider *dpp) { - char buf[16]; - snprintf(buf, sizeof(buf), "%u", shard_id); - name = prefix + buf; -} + int r = put_op->put_pre(dpp); + if (r != 0) { /* r can also be STATUS_NO_APPLY */ + return r; + } -int RGWMetadataLog::add_entry(const DoutPrefixProvider *dpp, const string& hash_key, const string& section, const string& key, bufferlist& bl) { - if (!svc.zone->need_to_log_metadata()) - return 0; + r = put_op->put(dpp); + if (r != 0) { + return r; + } - string oid; - int shard_id; + r = put_op->put_post(dpp); + if (r != 0) { /* e.g., -error or STATUS_APPLIED */ + return r; + } - rgw_shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, &shard_id); - mark_modified(shard_id); - real_time now = real_clock::now(); - return svc.cls->timelog.add(dpp, oid, now, section, key, bl, null_yield); + return 0; } -int RGWMetadataLog::get_shard_id(const string& hash_key, int *shard_id) +int RGWMetadataHandler_GenericMetaBE::get(string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) { - string oid; - - rgw_shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, shard_id); - return 0; + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return do_get(op, entry, obj, y, dpp); + }); } -int RGWMetadataLog::store_entries_in_shard(const DoutPrefixProvider *dpp, list& entries, int shard_id, librados::AioCompletion *completion) +int RGWMetadataHandler_GenericMetaBE::put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp, RGWMDLogSyncType type, bool from_remote_zone) { - string oid; - - mark_modified(shard_id); - rgw_shard_name(prefix, shard_id, oid); - return svc.cls->timelog.add(dpp, oid, entries, completion, false, null_yield); + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return do_put(op, entry, obj, objv_tracker, y, dpp, type, from_remote_zone); + }); } -void RGWMetadataLog::init_list_entries(int shard_id, const real_time& from_time, const real_time& end_time, - const string& marker, void **handle) +int RGWMetadataHandler_GenericMetaBE::remove(string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) { - LogListCtx *ctx = new LogListCtx(); - - ctx->cur_shard = shard_id; - ctx->from_time = from_time; - ctx->end_time = end_time; - ctx->marker = marker; - - get_shard_oid(ctx->cur_shard, ctx->cur_oid); - - *handle = (void *)ctx; + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return do_remove(op, entry, objv_tracker, y, dpp); + }); } -void RGWMetadataLog::complete_list_entries(void *handle) { - LogListCtx *ctx = static_cast(handle); - delete ctx; +int RGWMetadataHandler_GenericMetaBE::mutate(const string& entry, + const ceph::real_time& mtime, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + RGWMDLogStatus op_type, + std::function f) +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + RGWSI_MetaBackend::MutateParams params(mtime, op_type); + return op->mutate(entry, + params, + objv_tracker, + y, + f, + dpp); + }); } -int RGWMetadataLog::list_entries(const DoutPrefixProvider *dpp, void *handle, - int max_entries, - list& entries, - string *last_marker, - bool *truncated) { - LogListCtx *ctx = static_cast(handle); +int RGWMetadataHandler_GenericMetaBE::get_shard_id(const string& entry, int *shard_id) +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return op->get_shard_id(entry, shard_id); + }); +} - if (!max_entries) { - *truncated = false; - return 0; - } +int RGWMetadataHandler_GenericMetaBE::list_keys_init(const DoutPrefixProvider *dpp, const string& marker, void **phandle) +{ + auto op = std::make_unique(be_handler); - std::string next_marker; - int ret = svc.cls->timelog.list(dpp, ctx->cur_oid, ctx->from_time, ctx->end_time, - max_entries, entries, ctx->marker, - &next_marker, truncated, null_yield); - if ((ret < 0) && (ret != -ENOENT)) + int ret = op->list_init(dpp, marker); + if (ret < 0) { return ret; - - ctx->marker = std::move(next_marker); - if (last_marker) { - *last_marker = ctx->marker; } - if (ret == -ENOENT) - *truncated = false; + *phandle = (void *)op.release(); return 0; } -int RGWMetadataLog::get_info(const DoutPrefixProvider *dpp, int shard_id, RGWMetadataLogInfo *info) +int RGWMetadataHandler_GenericMetaBE::list_keys_next(const DoutPrefixProvider *dpp, void *handle, int max, list& keys, bool *truncated) { - string oid; - get_shard_oid(shard_id, oid); - - cls_log_header header; + auto op = static_cast(handle); - int ret = svc.cls->timelog.info(dpp, oid, &header, null_yield); - if ((ret < 0) && (ret != -ENOENT)) + int ret = op->list_next(dpp, max, &keys, truncated); + if (ret < 0 && ret != -ENOENT) { return ret; - - info->marker = header.max_marker; - info->last_update = header.max_time.to_real_time(); + } + if (ret == -ENOENT) { + if (truncated) { + *truncated = false; + } + return 0; + } return 0; } -static void _mdlog_info_completion(librados::completion_t cb, void *arg) +void RGWMetadataHandler_GenericMetaBE::list_keys_complete(void *handle) { - auto infoc = static_cast(arg); - infoc->finish(cb); - infoc->put(); // drop the ref from get_info_async() + auto op = static_cast(handle); + delete op; } -RGWMetadataLogInfoCompletion::RGWMetadataLogInfoCompletion(info_callback_t cb) - : completion(librados::Rados::aio_create_completion((void *)this, - _mdlog_info_completion)), - callback(cb) +string RGWMetadataHandler_GenericMetaBE::get_marker(void *handle) { -} + auto op = static_cast(handle); + string marker; + int r = op->list_get_marker(&marker); + if (r < 0) { + ldout(cct, 0) << "ERROR: " << __func__ << "(): list_get_marker() returned: r=" << r << dendl; + /* not much else to do */ + } -RGWMetadataLogInfoCompletion::~RGWMetadataLogInfoCompletion() -{ - completion->release(); + return marker; } -int RGWMetadataLog::get_info_async(const DoutPrefixProvider *dpp, int shard_id, RGWMetadataLogInfoCompletion *completion) -{ - string oid; - get_shard_oid(shard_id, oid); - - completion->get(); // hold a ref until the completion fires - - return svc.cls->timelog.info_async(dpp, completion->get_io_obj(), oid, - &completion->get_header(), - completion->get_completion()); +RGWMetadataHandlerPut_SObj::RGWMetadataHandlerPut_SObj(RGWMetadataHandler_GenericMetaBE *handler, + RGWSI_MetaBackend_Handler::Op *op, + string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, + RGWMDLogSyncType type, bool from_remote_zone) : Put(handler, op, entry, obj, objv_tracker, y, type, from_remote_zone) { } -int RGWMetadataLog::trim(const DoutPrefixProvider *dpp, int shard_id, const real_time& from_time, const real_time& end_time, - const string& start_marker, const string& end_marker) +int RGWMetadataHandlerPut_SObj::put_pre(const DoutPrefixProvider *dpp) { - string oid; - get_shard_oid(shard_id, oid); + int ret = get(&old_obj, dpp); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + exists = (ret != -ENOENT); - return svc.cls->timelog.trim(dpp, oid, from_time, end_time, start_marker, - end_marker, nullptr, null_yield); -} - -int RGWMetadataLog::lock_exclusive(const DoutPrefixProvider *dpp, int shard_id, timespan duration, string& zone_id, string& owner_id) { - string oid; - get_shard_oid(shard_id, oid); + oo.reset(old_obj); - return svc.cls->lock.lock_exclusive(dpp, svc.zone->get_zone_params().log_pool, oid, duration, zone_id, owner_id); -} + auto old_ver = (!old_obj ? obj_version() : old_obj->get_version()); + auto old_mtime = (!old_obj ? ceph::real_time() : old_obj->get_mtime()); + + // are we actually going to perform this put, or is it too old? + if (!handler->check_versions(exists, old_ver, old_mtime, + objv_tracker.write_version, obj->get_mtime(), + apply_type)) { + return STATUS_NO_APPLY; + } -int RGWMetadataLog::unlock(const DoutPrefixProvider *dpp, int shard_id, string& zone_id, string& owner_id) { - string oid; - get_shard_oid(shard_id, oid); + objv_tracker.read_version = old_ver; /* maintain the obj version we just read */ - return svc.cls->lock.unlock(dpp, svc.zone->get_zone_params().log_pool, oid, zone_id, owner_id); + return 0; } -void RGWMetadataLog::mark_modified(int shard_id) +int RGWMetadataHandlerPut_SObj::put(const DoutPrefixProvider *dpp) { - lock.get_read(); - if (modified_shards.find(shard_id) != modified_shards.end()) { - lock.unlock(); - return; + int ret = put_check(dpp); + if (ret != 0) { + return ret; } - lock.unlock(); - std::unique_lock wl{lock}; - modified_shards.insert(shard_id); + return put_checked(dpp); } -void RGWMetadataLog::read_clear_modified(set &modified) +int RGWMetadataHandlerPut_SObj::put_checked(const DoutPrefixProvider *dpp) { - std::unique_lock wl{lock}; - modified.swap(modified_shards); - modified_shards.clear(); -} + RGWSI_MBSObj_PutParams params(obj->get_pattrs(), obj->get_mtime()); -obj_version& RGWMetadataObject::get_version() -{ - return objv; + encode_obj(¶ms.bl); + + int ret = op->put(entry, params, &objv_tracker, y, dpp); + if (ret < 0) { + return ret; + } + + return 0; } class RGWMetadataTopHandler : public RGWMetadataHandler { @@ -413,211 +379,28 @@ public: } }; -RGWMetadataManager::RGWMetadataManager(RGWSI_Meta *_meta_svc) - : cct(_meta_svc->ctx()), meta_svc(_meta_svc) -{ - md_top_handler.reset(new RGWMetadataTopHandler(meta_svc, this)); -} - -RGWMetadataManager::~RGWMetadataManager() -{ -} +RGWMetadataHandlerPut_SObj::~RGWMetadataHandlerPut_SObj() {} int RGWMetadataHandler::attach(RGWMetadataManager *manager) { return manager->register_handler(this); } -RGWMetadataHandler_GenericMetaBE::Put::Put(RGWMetadataHandler_GenericMetaBE *_handler, - RGWSI_MetaBackend_Handler::Op *_op, - string& _entry, RGWMetadataObject *_obj, - RGWObjVersionTracker& _objv_tracker, - optional_yield _y, - RGWMDLogSyncType _type, bool _from_remote_zone): - handler(_handler), op(_op), - entry(_entry), obj(_obj), - objv_tracker(_objv_tracker), - apply_type(_type), - y(_y), - from_remote_zone(_from_remote_zone) -{ -} - -RGWMetadataHandlerPut_SObj::RGWMetadataHandlerPut_SObj(RGWMetadataHandler_GenericMetaBE *handler, RGWSI_MetaBackend_Handler::Op *op, - string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, - RGWMDLogSyncType type, bool from_remote_zone) : Put(handler, op, entry, obj, objv_tracker, y, type, from_remote_zone) { -} - -RGWMetadataHandlerPut_SObj::~RGWMetadataHandlerPut_SObj() { -} - -int RGWMetadataHandlerPut_SObj::put_pre(const DoutPrefixProvider *dpp) -{ - int ret = get(&old_obj, dpp); - if (ret < 0 && ret != -ENOENT) { - return ret; - } - exists = (ret != -ENOENT); - - oo.reset(old_obj); - - auto old_ver = (!old_obj ? obj_version() : old_obj->get_version()); - auto old_mtime = (!old_obj ? ceph::real_time() : old_obj->get_mtime()); - - // are we actually going to perform this put, or is it too old? - if (!handler->check_versions(exists, old_ver, old_mtime, - objv_tracker.write_version, obj->get_mtime(), - apply_type)) { - return STATUS_NO_APPLY; - } - - objv_tracker.read_version = old_ver; /* maintain the obj version we just read */ - - return 0; -} - -int RGWMetadataHandlerPut_SObj::put(const DoutPrefixProvider *dpp) -{ - int ret = put_check(dpp); - if (ret != 0) { - return ret; - } - - return put_checked(dpp); -} - -int RGWMetadataHandlerPut_SObj::put_checked(const DoutPrefixProvider *dpp) -{ - RGWSI_MBSObj_PutParams params(obj->get_pattrs(), obj->get_mtime()); +RGWMetadataHandler::~RGWMetadataHandler() {} - encode_obj(¶ms.bl); - - int ret = op->put(entry, params, &objv_tracker, y, dpp); - if (ret < 0) { - return ret; - } - - return 0; -} - -int RGWMetadataHandler_GenericMetaBE::do_put_operate(Put *put_op, const DoutPrefixProvider *dpp) -{ - int r = put_op->put_pre(dpp); - if (r != 0) { /* r can also be STATUS_NO_APPLY */ - return r; - } - - r = put_op->put(dpp); - if (r != 0) { - return r; - } - - r = put_op->put_post(dpp); - if (r != 0) { /* e.g., -error or STATUS_APPLIED */ - return r; - } - - return 0; -} - -int RGWMetadataHandler_GenericMetaBE::get(string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return do_get(op, entry, obj, y, dpp); - }); -} - -int RGWMetadataHandler_GenericMetaBE::put(string& entry, RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp, RGWMDLogSyncType type, bool from_remote_zone) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return do_put(op, entry, obj, objv_tracker, y, dpp, type, from_remote_zone); - }); -} - -int RGWMetadataHandler_GenericMetaBE::remove(string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return do_remove(op, entry, objv_tracker, y, dpp); - }); -} - -int RGWMetadataHandler_GenericMetaBE::mutate(const string& entry, - const ceph::real_time& mtime, - RGWObjVersionTracker *objv_tracker, - optional_yield y, - const DoutPrefixProvider *dpp, - RGWMDLogStatus op_type, - std::function f) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - RGWSI_MetaBackend::MutateParams params(mtime, op_type); - return op->mutate(entry, - params, - objv_tracker, - y, - f, - dpp); - }); -} - -int RGWMetadataHandler_GenericMetaBE::get_shard_id(const string& entry, int *shard_id) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return op->get_shard_id(entry, shard_id); - }); -} - -int RGWMetadataHandler_GenericMetaBE::list_keys_init(const DoutPrefixProvider *dpp, const string& marker, void **phandle) -{ - auto op = std::make_unique(be_handler); - - int ret = op->list_init(dpp, marker); - if (ret < 0) { - return ret; - } - - *phandle = (void *)op.release(); - - return 0; -} - -int RGWMetadataHandler_GenericMetaBE::list_keys_next(const DoutPrefixProvider *dpp, void *handle, int max, list& keys, bool *truncated) +obj_version& RGWMetadataObject::get_version() { - auto op = static_cast(handle); - - int ret = op->list_next(dpp, max, &keys, truncated); - if (ret < 0 && ret != -ENOENT) { - return ret; - } - if (ret == -ENOENT) { - if (truncated) { - *truncated = false; - } - return 0; - } - - return 0; + return objv; } -void RGWMetadataHandler_GenericMetaBE::list_keys_complete(void *handle) +RGWMetadataManager::RGWMetadataManager(RGWSI_Meta *_meta_svc) + : cct(_meta_svc->ctx()), meta_svc(_meta_svc) { - auto op = static_cast(handle); - delete op; + md_top_handler.reset(new RGWMetadataTopHandler(meta_svc, this)); } -string RGWMetadataHandler_GenericMetaBE::get_marker(void *handle) +RGWMetadataManager::~RGWMetadataManager() { - auto op = static_cast(handle); - string marker; - int r = op->list_get_marker(&marker); - if (r < 0) { - ldout(cct, 0) << "ERROR: " << __func__ << "(): list_get_marker() returned: r=" << r << dendl; - /* not much else to do */ - } - - return marker; } int RGWMetadataManager::register_handler(RGWMetadataHandler *handler) @@ -706,7 +489,7 @@ int RGWMetadataManager::get(string& metadata_key, Formatter *f, optional_yield y } int RGWMetadataManager::put(string& metadata_key, bufferlist& bl, - optional_yield y, + optional_yield y, const DoutPrefixProvider *dpp, RGWMDLogSyncType sync_type, bool from_remote_zone, @@ -743,7 +526,6 @@ int RGWMetadataManager::put(string& metadata_key, bufferlist& bl, if (!jo) { return -EINVAL; } - RGWMetadataObject *obj = handler->get_meta_obj(jo, *objv, mtime.to_real_time()); if (!obj) { return -EINVAL; @@ -784,7 +566,7 @@ int RGWMetadataManager::remove(string& metadata_key, optional_yield y, const Dou int RGWMetadataManager::mutate(const string& metadata_key, const ceph::real_time& mtime, RGWObjVersionTracker *objv_tracker, - optional_yield y, + optional_yield y, const DoutPrefixProvider *dpp, RGWMDLogStatus op_type, std::function f) @@ -899,18 +681,3 @@ void RGWMetadataManager::get_sections(list& sections) } } -void RGWMetadataLogInfo::dump(Formatter *f) const -{ - encode_json("marker", marker, f); - utime_t ut(last_update); - encode_json("last_update", ut, f); -} - -void RGWMetadataLogInfo::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("marker", marker, obj); - utime_t ut; - JSONDecoder::decode_json("last_update", ut, obj); - last_update = ut.to_real_time(); -} - diff --git a/src/rgw/rgw_multipart_meta_filter.cc b/src/rgw/rgw_multipart_meta_filter.cc new file mode 100644 index 000000000000..c616cd480f75 --- /dev/null +++ b/src/rgw/rgw_multipart_meta_filter.cc @@ -0,0 +1,32 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "svc_tier_rados.h" + +using namespace std; + +const std::string MP_META_SUFFIX = ".meta"; + +bool MultipartMetaFilter::filter(const string& name, string& key) { + // the length of the suffix so we can skip past it + static const size_t MP_META_SUFFIX_LEN = MP_META_SUFFIX.length(); + + size_t len = name.size(); + + // make sure there's room for suffix plus at least one more + // character + if (len <= MP_META_SUFFIX_LEN) + return false; + + size_t pos = name.find(MP_META_SUFFIX, len - MP_META_SUFFIX_LEN); + if (pos == string::npos) + return false; + + pos = name.rfind('.', pos - 1); + if (pos == string::npos) + return false; + + key = name.substr(0, pos); + + return true; +} diff --git a/src/rgw/rgw_obj_manifest.cc b/src/rgw/rgw_obj_manifest.cc index 61eeb83bbe93..1d1c3b5cf23e 100644 --- a/src/rgw/rgw_obj_manifest.cc +++ b/src/rgw/rgw_obj_manifest.cc @@ -3,201 +3,10 @@ #include "rgw_obj_manifest.h" -#include "services/svc_zone.h" -#include "services/svc_tier_rados.h" #include "rgw_rados.h" // RGW_OBJ_NS_SHADOW and RGW_OBJ_NS_MULTIPART -#include "rgw_bucket.h" - -#define dout_context g_ceph_context -#define dout_subsys ceph_subsys_rgw using namespace std; -int RGWObjManifest::generator::create_next(uint64_t ofs) -{ - if (ofs < last_ofs) /* only going forward */ - return -EINVAL; - - uint64_t max_head_size = manifest->get_max_head_size(); - - if (ofs < max_head_size) { - manifest->set_head_size(ofs); - } - - if (ofs >= max_head_size) { - manifest->set_head_size(max_head_size); - cur_stripe = (ofs - max_head_size) / rule.stripe_max_size; - cur_stripe_size = rule.stripe_max_size; - - if (cur_part_id == 0 && max_head_size > 0) { - cur_stripe++; - } - } - - last_ofs = ofs; - manifest->set_obj_size(ofs); - - manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj); - - return 0; -} - -int RGWObjManifest::append(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, - const RGWZoneParams& zone_params) -{ - if (explicit_objs || m.explicit_objs) { - return append_explicit(dpp, m, zonegroup, zone_params); - } - - if (rules.empty()) { - *this = m; - return 0; - } - - string override_prefix; - - if (prefix.empty()) { - prefix = m.prefix; - } - - if (prefix != m.prefix) { - override_prefix = m.prefix; - } - - map::iterator miter = m.rules.begin(); - if (miter == m.rules.end()) { - return append_explicit(dpp, m, zonegroup, zone_params); - } - - for (; miter != m.rules.end(); ++miter) { - map::reverse_iterator last_rule = rules.rbegin(); - - RGWObjManifestRule& rule = last_rule->second; - - if (rule.part_size == 0) { - rule.part_size = obj_size - rule.start_ofs; - } - - RGWObjManifestRule& next_rule = miter->second; - if (!next_rule.part_size) { - next_rule.part_size = m.obj_size - next_rule.start_ofs; - } - - string rule_prefix = prefix; - if (!rule.override_prefix.empty()) { - rule_prefix = rule.override_prefix; - } - - string next_rule_prefix = m.prefix; - if (!next_rule.override_prefix.empty()) { - next_rule_prefix = next_rule.override_prefix; - } - - if (rule.part_size != next_rule.part_size || - rule.stripe_max_size != next_rule.stripe_max_size || - rule_prefix != next_rule_prefix) { - if (next_rule_prefix != prefix) { - append_rules(m, miter, &next_rule_prefix); - } else { - append_rules(m, miter, NULL); - } - break; - } - - uint64_t expected_part_num = rule.start_part_num + 1; - if (rule.part_size > 0) { - expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size; - } - - if (expected_part_num != next_rule.start_part_num) { - append_rules(m, miter, NULL); - break; - } - } - - set_obj_size(obj_size + m.obj_size); - - return 0; -} - -void RGWObjManifest::append_rules(RGWObjManifest& m, map::iterator& miter, - string *override_prefix) -{ - for (; miter != m.rules.end(); ++miter) { - RGWObjManifestRule rule = miter->second; - rule.start_ofs += obj_size; - if (override_prefix) - rule.override_prefix = *override_prefix; - rules[rule.start_ofs] = rule; - } -} - -void RGWObjManifest::convert_to_explicit(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) -{ - if (explicit_objs) { - return; - } - obj_iterator iter = obj_begin(dpp); - - while (iter != obj_end(dpp)) { - RGWObjManifestPart& part = objs[iter.get_stripe_ofs()]; - const rgw_obj_select& os = iter.get_location(); - const rgw_raw_obj& raw_loc = os.get_raw_obj(zonegroup, zone_params); - part.loc_ofs = 0; - - uint64_t ofs = iter.get_stripe_ofs(); - - if (ofs == 0) { - part.loc = obj; - } else { - RGWSI_Tier_RADOS::raw_obj_to_obj(tail_placement.bucket, raw_loc, &part.loc); - } - ++iter; - uint64_t next_ofs = iter.get_stripe_ofs(); - - part.size = next_ofs - ofs; - } - - explicit_objs = true; - rules.clear(); - prefix.clear(); -} - -int RGWObjManifest::append_explicit(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) -{ - if (!explicit_objs) { - convert_to_explicit(dpp, zonegroup, zone_params); - } - if (!m.explicit_objs) { - m.convert_to_explicit(dpp, zonegroup, zone_params); - } - map::iterator iter; - uint64_t base = obj_size; - for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) { - RGWObjManifestPart& part = iter->second; - objs[base + iter->first] = part; - } - obj_size += m.obj_size; - - return 0; -} - -bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule) -{ - if (rules.empty()) { - return false; - } - - map::iterator iter = rules.upper_bound(ofs); - if (iter != rules.begin()) { - --iter; - } - - *rule = iter->second; - - return true; -} - void RGWObjManifest::obj_iterator::operator++() { if (manifest->explicit_objs) { @@ -290,59 +99,6 @@ void RGWObjManifest::obj_iterator::operator++() update_location(); } -int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m, - const rgw_placement_rule& head_placement_rule, - const rgw_placement_rule *tail_placement_rule, - const rgw_bucket& _b, const rgw_obj& _obj) -{ - manifest = _m; - - if (!tail_placement_rule) { - manifest->set_tail_placement(head_placement_rule, _b); - } else { - rgw_placement_rule new_tail_rule = *tail_placement_rule; - new_tail_rule.inherit_from(head_placement_rule); - manifest->set_tail_placement(new_tail_rule, _b); - } - - manifest->set_head(head_placement_rule, _obj, 0); - last_ofs = 0; - - if (manifest->get_prefix().empty()) { - char buf[33]; - gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1); - - string oid_prefix = "."; - oid_prefix.append(buf); - oid_prefix.append("_"); - - manifest->set_prefix(oid_prefix); - } - - bool found = manifest->get_rule(0, &rule); - if (!found) { - derr << "ERROR: manifest->get_rule() could not find rule" << dendl; - return -EIO; - } - - uint64_t head_size = manifest->get_head_size(); - - if (head_size > 0) { - cur_stripe_size = head_size; - } else { - cur_stripe_size = rule.stripe_max_size; - } - - cur_part_id = rule.start_part_num; - - manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj); - - // Normal object which not generated through copy operation - manifest->set_tail_instance(_obj.key.instance); - - return 0; -} - void RGWObjManifest::obj_iterator::seek(uint64_t o) { ofs = o; @@ -416,6 +172,20 @@ void RGWObjManifest::obj_iterator::seek(uint64_t o) update_location(); } +void RGWObjManifest::obj_iterator::update_explicit_pos() +{ + ofs = explicit_iter->first; + stripe_ofs = ofs; + + auto next_iter = explicit_iter; + ++next_iter; + if (next_iter != manifest->objs.end()) { + stripe_size = next_iter->first - ofs; + } else { + stripe_size = manifest->obj_size - ofs; + } +} + void RGWObjManifest::obj_iterator::update_location() { if (manifest->explicit_objs) { @@ -436,20 +206,6 @@ void RGWObjManifest::obj_iterator::update_location() manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, &cur_override_prefix, &location); } -void RGWObjManifest::obj_iterator::update_explicit_pos() -{ - ofs = explicit_iter->first; - stripe_ofs = ofs; - - auto next_iter = explicit_iter; - ++next_iter; - if (next_iter != manifest->objs.end()) { - stripe_size = next_iter->first - ofs; - } else { - stripe_size = manifest->obj_size - ofs; - } -} - void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_stripe, uint64_t ofs, string *override_prefix, rgw_obj_select *location) const { @@ -502,108 +258,3 @@ void RGWObjManifest::get_implicit_location(uint64_t cur_part_id, uint64_t cur_st *location = loc; } -void RGWObjManifestPart::generate_test_instances(std::list& o) -{ - o.push_back(new RGWObjManifestPart); - - RGWObjManifestPart *p = new RGWObjManifestPart; - rgw_bucket b; - init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12"); - - p->loc = rgw_obj(b, "object"); - p->loc_ofs = 512 * 1024; - p->size = 128 * 1024; - o.push_back(p); -} - -void RGWObjManifest::generate_test_instances(std::list& o) -{ - RGWObjManifest *m = new RGWObjManifest; - map objs; - uint64_t total_size = 0; - for (int i = 0; i<10; i++) { - RGWObjManifestPart p; - rgw_bucket b; - init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12"); - p.loc = rgw_obj(b, "object"); - p.loc_ofs = 0; - p.size = 512 * 1024; - total_size += p.size; - objs[total_size] = p; - } - m->set_explicit(total_size, objs); - o.push_back(m); - o.push_back(new RGWObjManifest); -} - -void RGWObjManifestPart::dump(Formatter *f) const -{ - f->open_object_section("loc"); - loc.dump(f); - f->close_section(); - f->dump_unsigned("loc_ofs", loc_ofs); - f->dump_unsigned("size", size); -} - -void RGWObjManifest::obj_iterator::dump(Formatter *f) const -{ - f->dump_unsigned("part_ofs", part_ofs); - f->dump_unsigned("stripe_ofs", stripe_ofs); - f->dump_unsigned("ofs", ofs); - f->dump_unsigned("stripe_size", stripe_size); - f->dump_int("cur_part_id", cur_part_id); - f->dump_int("cur_stripe", cur_stripe); - f->dump_string("cur_override_prefix", cur_override_prefix); - f->dump_object("location", location); -} - -void RGWObjManifest::dump(Formatter *f) const -{ - map::const_iterator iter = objs.begin(); - f->open_array_section("objs"); - for (; iter != objs.end(); ++iter) { - f->dump_unsigned("ofs", iter->first); - f->open_object_section("part"); - iter->second.dump(f); - f->close_section(); - } - f->close_section(); - f->dump_unsigned("obj_size", obj_size); - ::encode_json("explicit_objs", explicit_objs, f); - ::encode_json("head_size", head_size, f); - ::encode_json("max_head_size", max_head_size, f); - ::encode_json("prefix", prefix, f); - ::encode_json("rules", rules, f); - ::encode_json("tail_instance", tail_instance, f); - ::encode_json("tail_placement", tail_placement, f); - - // nullptr being passed into iterators since there - // is no cct and we aren't doing anything with these - // iterators that would write do the log - f->dump_object("begin_iter", obj_begin(nullptr)); - f->dump_object("end_iter", obj_end(nullptr)); -} - -void RGWObjManifestRule::dump(Formatter *f) const -{ - encode_json("start_part_num", start_part_num, f); - encode_json("start_ofs", start_ofs, f); - encode_json("part_size", part_size, f); - encode_json("stripe_max_size", stripe_max_size, f); - encode_json("override_prefix", override_prefix, f); -} - -void rgw_obj_select::dump(Formatter *f) const -{ - f->dump_string("placement_rule", placement_rule.to_str()); - f->dump_object("obj", obj); - f->dump_object("raw_obj", raw_obj); - f->dump_bool("is_raw", is_raw); -} - -void RGWObjTier::dump(Formatter *f) const -{ - encode_json("name", name, f); - encode_json("tier_placement", tier_placement, f); - encode_json("is_multipart_upload", is_multipart_upload, f); -} diff --git a/src/rgw/rgw_period.cc b/src/rgw/rgw_period.cc new file mode 100644 index 000000000000..1e7de60ea6fa --- /dev/null +++ b/src/rgw/rgw_period.cc @@ -0,0 +1,350 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_sync.h" + +using namespace std; +using namespace rgw_zone_defaults; + +std::string period_latest_epoch_info_oid = ".latest_epoch"; +std::string period_info_oid_prefix = "periods."; + +#define FIRST_EPOCH 1 + +int RGWPeriod::init(const DoutPrefixProvider *dpp, + CephContext *_cct, RGWSI_SysObj *_sysobj_svc, + optional_yield y, bool setup_obj) +{ + cct = _cct; + sysobj_svc = _sysobj_svc; + + if (!setup_obj) + return 0; + + if (id.empty()) { + RGWRealm realm(realm_id, realm_name); + int ret = realm.init(dpp, cct, sysobj_svc, y); + if (ret < 0) { + ldpp_dout(dpp, 4) << "RGWPeriod::init failed to init realm " << realm_name << " id " << realm_id << " : " << + cpp_strerror(-ret) << dendl; + return ret; + } + id = realm.get_current_period(); + realm_id = realm.get_id(); + } + + if (!epoch) { + int ret = use_latest_epoch(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed to use_latest_epoch period id " << id << " realm " << realm_name << " id " << realm_id + << " : " << cpp_strerror(-ret) << dendl; + return ret; + } + } + + return read_info(dpp, y); +} + +int RGWPeriod::init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, + const string& period_realm_id, optional_yield y, + const string& period_realm_name, bool setup_obj) +{ + cct = _cct; + sysobj_svc = _sysobj_svc; + + realm_id = period_realm_id; + realm_name = period_realm_name; + + if (!setup_obj) + return 0; + + return init(dpp, _cct, _sysobj_svc, y, setup_obj); +} + +const string& RGWPeriod::get_latest_epoch_oid() const +{ + if (cct->_conf->rgw_period_latest_epoch_info_oid.empty()) { + return period_latest_epoch_info_oid; + } + return cct->_conf->rgw_period_latest_epoch_info_oid; +} + +const string& RGWPeriod::get_info_oid_prefix() const +{ + return period_info_oid_prefix; +} + +const string RGWPeriod::get_period_oid_prefix() const +{ + return get_info_oid_prefix() + id; +} + +const string RGWPeriod::get_period_oid() const +{ + std::ostringstream oss; + oss << get_period_oid_prefix(); + // skip the epoch for the staging period + if (id != get_staging_id(realm_id)) + oss << "." << epoch; + return oss.str(); +} + +bool RGWPeriod::find_zone(const DoutPrefixProvider *dpp, + const rgw_zone_id& zid, + RGWZoneGroup *pzonegroup, + optional_yield y) const +{ + RGWZoneGroup zg; + RGWZone zone; + + bool found = period_map.find_zone_by_id(zid, &zg, &zone); + if (found) { + *pzonegroup = zg; + } + + return found; +} + +rgw_pool RGWPeriod::get_pool(CephContext *cct) const +{ + if (cct->_conf->rgw_period_root_pool.empty()) { + return rgw_pool(RGW_DEFAULT_PERIOD_ROOT_POOL); + } + return rgw_pool(cct->_conf->rgw_period_root_pool); +} + +int RGWPeriod::set_latest_epoch(const DoutPrefixProvider *dpp, + optional_yield y, + epoch_t epoch, bool exclusive, + RGWObjVersionTracker *objv) +{ + string oid = get_period_oid_prefix() + get_latest_epoch_oid(); + + rgw_pool pool(get_pool(cct)); + bufferlist bl; + + RGWPeriodLatestEpochInfo info; + info.epoch = epoch; + + using ceph::encode; + encode(info, bl); + + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); + return sysobj.wop() + .set_exclusive(exclusive) + .write(dpp, bl, y); +} + +int RGWPeriod::read_info(const DoutPrefixProvider *dpp, optional_yield y) +{ + rgw_pool pool(get_pool(cct)); + + bufferlist bl; + + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, get_period_oid()}); + int ret = sysobj.rop().read(dpp, &bl, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed reading obj info from " << pool << ":" << get_period_oid() << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + + try { + using ceph::decode; + auto iter = bl.cbegin(); + decode(*this, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << get_period_oid() << dendl; + return -EIO; + } + + return 0; +} + +int RGWPeriod::store_info(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) +{ + rgw_pool pool(get_pool(cct)); + + string oid = get_period_oid(); + bufferlist bl; + using ceph::encode; + encode(*this, bl); + + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); + return sysobj.wop() + .set_exclusive(exclusive) + .write(dpp, bl, y); +} + +int RGWPeriod::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +{ + int ret; + + /* create unique id */ + uuid_d new_uuid; + char uuid_str[37]; + new_uuid.generate_random(); + new_uuid.print(uuid_str); + id = uuid_str; + + epoch = FIRST_EPOCH; + + period_map.id = id; + + ret = store_info(dpp, exclusive, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: storing info for " << id << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + + ret = set_latest_epoch(dpp, y, epoch); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: setting latest epoch " << id << ": " << cpp_strerror(-ret) << dendl; + } + + return ret; +} + +int RGWPeriod::reflect(const DoutPrefixProvider *dpp, optional_yield y) +{ + for (auto& iter : period_map.zonegroups) { + RGWZoneGroup& zg = iter.second; + zg.reinit_instance(cct, sysobj_svc); + int r = zg.write(dpp, false, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to store zonegroup info for zonegroup=" << iter.first << ": " << cpp_strerror(-r) << dendl; + return r; + } + if (zg.is_master_zonegroup()) { + // set master as default if no default exists + r = zg.set_as_default(dpp, y, true); + if (r == 0) { + ldpp_dout(dpp, 1) << "Set the period's master zonegroup " << zg.get_id() + << " as the default" << dendl; + } + } + } + + int r = period_config.write(dpp, sysobj_svc, realm_id, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to store period config: " + << cpp_strerror(-r) << dendl; + return r; + } + return 0; +} + +void RGWPeriod::dump(Formatter *f) const +{ + encode_json("id", id, f); + encode_json("epoch", epoch , f); + encode_json("predecessor_uuid", predecessor_uuid, f); + encode_json("sync_status", sync_status, f); + encode_json("period_map", period_map, f); + encode_json("master_zonegroup", master_zonegroup, f); + encode_json("master_zone", master_zone, f); + encode_json("period_config", period_config, f); + encode_json("realm_id", realm_id, f); + encode_json("realm_name", realm_name, f); + encode_json("realm_epoch", realm_epoch, f); +} + +void RGWPeriod::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("id", id, obj); + JSONDecoder::decode_json("epoch", epoch, obj); + JSONDecoder::decode_json("predecessor_uuid", predecessor_uuid, obj); + JSONDecoder::decode_json("sync_status", sync_status, obj); + JSONDecoder::decode_json("period_map", period_map, obj); + JSONDecoder::decode_json("master_zonegroup", master_zonegroup, obj); + JSONDecoder::decode_json("master_zone", master_zone, obj); + JSONDecoder::decode_json("period_config", period_config, obj); + JSONDecoder::decode_json("realm_id", realm_id, obj); + JSONDecoder::decode_json("realm_name", realm_name, obj); + JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); +} + +int RGWPeriod::update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, optional_yield y) +{ + static constexpr int MAX_RETRIES = 20; + + for (int i = 0; i < MAX_RETRIES; i++) { + RGWPeriodLatestEpochInfo info; + RGWObjVersionTracker objv; + bool exclusive = false; + + // read existing epoch + int r = read_latest_epoch(dpp, info, y, &objv); + if (r == -ENOENT) { + // use an exclusive create to set the epoch atomically + exclusive = true; + ldpp_dout(dpp, 20) << "creating initial latest_epoch=" << epoch + << " for period=" << id << dendl; + } else if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to read latest_epoch" << dendl; + return r; + } else if (epoch <= info.epoch) { + r = -EEXIST; // fail with EEXIST if epoch is not newer + ldpp_dout(dpp, 10) << "found existing latest_epoch " << info.epoch + << " >= given epoch " << epoch << ", returning r=" << r << dendl; + return r; + } else { + ldpp_dout(dpp, 20) << "updating latest_epoch from " << info.epoch + << " -> " << epoch << " on period=" << id << dendl; + } + + r = set_latest_epoch(dpp, y, epoch, exclusive, &objv); + if (r == -EEXIST) { + continue; // exclusive create raced with another update, retry + } else if (r == -ECANCELED) { + continue; // write raced with a conflicting version, retry + } + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to write latest_epoch" << dendl; + return r; + } + return 0; // return success + } + + return -ECANCELED; // fail after max retries +} + +int RGWPeriod::read_latest_epoch(const DoutPrefixProvider *dpp, + RGWPeriodLatestEpochInfo& info, + optional_yield y, + RGWObjVersionTracker *objv) +{ + string oid = get_period_oid_prefix() + get_latest_epoch_oid(); + + rgw_pool pool(get_pool(cct)); + bufferlist bl; + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); + int ret = sysobj.rop().read(dpp, &bl, y); + if (ret < 0) { + ldpp_dout(dpp, 1) << "error read_lastest_epoch " << pool << ":" << oid << dendl; + return ret; + } + try { + auto iter = bl.cbegin(); + using ceph::decode; + decode(info, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "error decoding data from " << pool << ":" << oid << dendl; + return -EIO; + } + + return 0; +} + +int RGWPeriod::use_latest_epoch(const DoutPrefixProvider *dpp, optional_yield y) +{ + RGWPeriodLatestEpochInfo info; + int ret = read_latest_epoch(dpp, info, y); + if (ret < 0) { + return ret; + } + + epoch = info.epoch; + + return 0; +} + diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 63992e9bc1ee..1d05cdda6f57 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -116,47 +116,6 @@ static string default_storage_extra_pool_suffix = "rgw.buckets.non-ec"; static RGWObjCategory main_category = RGWObjCategory::Main; #define RGW_USAGE_OBJ_PREFIX "usage." - -// returns true on success, false on failure -static bool rgw_get_obj_data_pool(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params, - const rgw_placement_rule& head_placement_rule, - const rgw_obj& obj, rgw_pool *pool) -{ - if (!zone_params.get_head_data_pool(head_placement_rule, obj, pool)) { - RGWZonePlacementInfo placement; - if (!zone_params.get_placement(zonegroup.default_placement.name, &placement)) { - return false; - } - - if (!obj.in_extra_data) { - *pool = placement.get_data_pool(zonegroup.default_placement.storage_class); - } else { - *pool = placement.get_data_extra_pool(); - } - } - - return true; -} - -static bool rgw_obj_to_raw(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params, - const rgw_placement_rule& head_placement_rule, - const rgw_obj& obj, rgw_raw_obj *raw_obj) -{ - get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc); - - return rgw_get_obj_data_pool(zonegroup, zone_params, head_placement_rule, obj, &raw_obj->pool); -} - -rgw_raw_obj rgw_obj_select::get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const -{ - if (!is_raw) { - rgw_raw_obj r; - rgw_obj_to_raw(zonegroup, zone_params, placement_rule, obj, &r); - return r; - } - return raw_obj; -} - rgw_raw_obj rgw_obj_select::get_raw_obj(rgw::sal::RadosStore* store) const { if (!is_raw) { @@ -263,15 +222,6 @@ void RGWObjectCtx::invalidate(const rgw_obj& obj) { } } -void RGWObjVersionTracker::generate_new_write_ver(CephContext* cct) -{ - static constexpr auto TAG_LEN = 24; - write_version.ver = 1; - - write_version.tag.clear(); - append_rand_alpha(cct, write_version.tag, write_version.tag, TAG_LEN); -} - class RGWMetaNotifierManager : public RGWCoroutinesManager { RGWRados* store; RGWHTTPManager http_manager; @@ -2389,12 +2339,6 @@ int RGWRados::create_bucket(const RGWUserInfo& owner, rgw_bucket& bucket, return -ENOENT; } -// returns true on success, false on failure -bool RGWRados::get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool) -{ - return rgw_get_obj_data_pool(svc.zone->get_zonegroup(), svc.zone->get_zone_params(), placement_rule, obj, pool); -} - bool RGWRados::obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_raw_obj *raw_obj) { get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc); diff --git a/src/rgw/rgw_realm.cc b/src/rgw/rgw_realm.cc new file mode 100644 index 000000000000..8dd6d6f50b99 --- /dev/null +++ b/src/rgw/rgw_realm.cc @@ -0,0 +1,265 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include + +#include "common/errno.h" + +#include "rgw_zone.h" +#include "rgw_realm_watcher.h" +#include "rgw_meta_sync_status.h" +#include "rgw_sal_config.h" +#include "rgw_string.h" +#include "rgw_sync.h" + +#include "services/svc_zone.h" +#include "services/svc_sys_obj.h" + +#include "common/ceph_json.h" +#include "common/Formatter.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +namespace rgw_zone_defaults { + +std::string realm_info_oid_prefix = "realms."; +std::string realm_names_oid_prefix = "realms_names."; +std::string default_realm_info_oid = "default.realm"; +std::string RGW_DEFAULT_REALM_ROOT_POOL = "rgw.root"; + +} + +using namespace std; +using namespace rgw_zone_defaults; + +RGWRealm::~RGWRealm() {} + +RGWRemoteMetaLog::~RGWRemoteMetaLog() +{ + delete error_logger; +} + +string RGWRealm::get_predefined_id(CephContext *cct) const { + return cct->_conf.get_val("rgw_realm_id"); +} + +const string& RGWRealm::get_predefined_name(CephContext *cct) const { + return cct->_conf->rgw_realm; +} + +int RGWRealm::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +{ + int ret = RGWSystemMetaObj::create(dpp, y, exclusive); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR creating new realm object " << name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + // create the control object for watch/notify + ret = create_control(dpp, exclusive, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR creating control for new realm " << name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + RGWPeriod period; + if (current_period.empty()) { + /* create new period for the realm */ + ret = period.init(dpp, cct, sysobj_svc, id, y, name, false); + if (ret < 0 ) { + return ret; + } + ret = period.create(dpp, y, true); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: creating new period for realm " << name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + } else { + period = RGWPeriod(current_period, 0); + int ret = period.init(dpp, cct, sysobj_svc, id, y, name); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to init period " << current_period << dendl; + return ret; + } + } + ret = set_current_period(dpp, period, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed set current period " << current_period << dendl; + return ret; + } + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + ret = set_as_default(dpp, y, true); + if (ret < 0 && ret != -EEXIST) { + ldpp_dout(dpp, 0) << "WARNING: failed to set realm as default realm, ret=" << ret << dendl; + } + + return 0; +} + +int RGWRealm::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) +{ + int ret = RGWSystemMetaObj::delete_obj(dpp, y); + if (ret < 0) { + return ret; + } + return delete_control(dpp, y); +} + +int RGWRealm::create_control(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) +{ + auto pool = rgw_pool{get_pool(cct)}; + auto oid = get_control_oid(); + bufferlist bl; + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); + return sysobj.wop() + .set_exclusive(exclusive) + .write(dpp, bl, y); +} + +int RGWRealm::delete_control(const DoutPrefixProvider *dpp, optional_yield y) +{ + auto pool = rgw_pool{get_pool(cct)}; + auto obj = rgw_raw_obj{pool, get_control_oid()}; + auto sysobj = sysobj_svc->get_obj(obj); + return sysobj.wop().remove(dpp, y); +} + +rgw_pool RGWRealm::get_pool(CephContext *cct) const +{ + if (cct->_conf->rgw_realm_root_pool.empty()) { + return rgw_pool(RGW_DEFAULT_REALM_ROOT_POOL); + } + return rgw_pool(cct->_conf->rgw_realm_root_pool); +} + +const string RGWRealm::get_default_oid(bool old_format) const +{ + if (cct->_conf->rgw_default_realm_info_oid.empty()) { + return default_realm_info_oid; + } + return cct->_conf->rgw_default_realm_info_oid; +} + +const string& RGWRealm::get_names_oid_prefix() const +{ + return realm_names_oid_prefix; +} + +const string& RGWRealm::get_info_oid_prefix(bool old_format) const +{ + return realm_info_oid_prefix; +} + +int RGWRealm::set_current_period(const DoutPrefixProvider *dpp, RGWPeriod& period, optional_yield y) +{ + // update realm epoch to match the period's + if (epoch > period.get_realm_epoch()) { + ldpp_dout(dpp, 0) << "ERROR: set_current_period with old realm epoch " + << period.get_realm_epoch() << ", current epoch=" << epoch << dendl; + return -EINVAL; + } + if (epoch == period.get_realm_epoch() && current_period != period.get_id()) { + ldpp_dout(dpp, 0) << "ERROR: set_current_period with same realm epoch " + << period.get_realm_epoch() << ", but different period id " + << period.get_id() << " != " << current_period << dendl; + return -EINVAL; + } + + epoch = period.get_realm_epoch(); + current_period = period.get_id(); + + int ret = update(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: period update: " << cpp_strerror(-ret) << dendl; + return ret; + } + + ret = period.reflect(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: period.reflect(): " << cpp_strerror(-ret) << dendl; + return ret; + } + + return 0; +} + +string RGWRealm::get_control_oid() const +{ + return get_info_oid_prefix() + id + ".control"; +} + +int RGWRealm::notify_zone(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y) +{ + rgw_pool pool{get_pool(cct)}; + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, get_control_oid()}); + int ret = sysobj.wn().notify(dpp, bl, 0, nullptr, y); + if (ret < 0) { + return ret; + } + return 0; +} + +int RGWRealm::notify_new_period(const DoutPrefixProvider *dpp, const RGWPeriod& period, optional_yield y) +{ + bufferlist bl; + using ceph::encode; + // push the period to dependent zonegroups/zones + encode(RGWRealmNotify::ZonesNeedPeriod, bl); + encode(period, bl); + // reload the gateway with the new period + encode(RGWRealmNotify::Reload, bl); + + return notify_zone(dpp, bl, y); +} + + +int RGWRealm::find_zone(const DoutPrefixProvider *dpp, + const rgw_zone_id& zid, + RGWPeriod *pperiod, + RGWZoneGroup *pzonegroup, + bool *pfound, + optional_yield y) const +{ + auto& found = *pfound; + + found = false; + + string period_id; + epoch_t epoch = 0; + + RGWPeriod period(period_id, epoch); + int r = period.init(dpp, cct, sysobj_svc, get_id(), y, get_name()); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: period init failed: " << cpp_strerror(-r) << " ... skipping" << dendl; + return r; + } + + found = period.find_zone(dpp, zid, pzonegroup, y); + if (found) { + *pperiod = period; + } + return 0; +} + +void RGWRealm::generate_test_instances(list &o) +{ + RGWRealm *z = new RGWRealm; + o.push_back(z); + o.push_back(new RGWRealm); +} + +void RGWRealm::dump(Formatter *f) const +{ + RGWSystemMetaObj::dump(f); + encode_json("current_period", current_period, f); + encode_json("epoch", epoch, f); +} + + +void RGWRealm::decode_json(JSONObj *obj) +{ + RGWSystemMetaObj::decode_json(obj); + JSONDecoder::decode_json("current_period", current_period, obj); + JSONDecoder::decode_json("epoch", epoch, obj); +} + diff --git a/src/rgw/rgw_sync.cc b/src/rgw/rgw_sync.cc index 9a93c07395e3..b41d9c672142 100644 --- a/src/rgw/rgw_sync.cc +++ b/src/rgw/rgw_sync.cc @@ -1,319 +1,20 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp -#include - -#include "common/RefCountedObj.h" -#include "common/WorkQueue.h" -#include "common/Throttle.h" -#include "common/admin_socket.h" -#include "common/errno.h" - -#include "common/ceph_json.h" -#include "common/Formatter.h" - -#include "rgw_common.h" -#include "rgw_zone.h" #include "rgw_sync.h" -#include "rgw_metadata.h" -#include "rgw_mdlog_types.h" -#include "rgw_rest_conn.h" -#include "rgw_tools.h" -#include "rgw_cr_rados.h" -#include "rgw_cr_rest.h" -#include "rgw_http_client.h" -#include "rgw_sync_trace.h" - -#include "cls/lock/cls_lock_client.h" - -#include "services/svc_zone.h" -#include "services/svc_mdlog.h" -#include "services/svc_meta.h" -#include "services/svc_cls.h" - -#include #define dout_subsys ceph_subsys_rgw -#undef dout_prefix -#define dout_prefix (*_dout << "meta sync: ") - using namespace std; -static string mdlog_sync_status_oid = "mdlog.sync-status"; -static string mdlog_sync_status_shard_prefix = "mdlog.sync-status.shard"; -static string mdlog_sync_full_sync_index_prefix = "meta.full-sync.index"; - -RGWContinuousLeaseCR::~RGWContinuousLeaseCR() {} - -RGWSyncErrorLogger::RGWSyncErrorLogger(rgw::sal::RadosStore* _store, const string &oid_prefix, int _num_shards) : store(_store), num_shards(_num_shards) { - for (int i = 0; i < num_shards; i++) { - oids.push_back(get_shard_oid(oid_prefix, i)); - } -} -string RGWSyncErrorLogger::get_shard_oid(const string& oid_prefix, int shard_id) { - char buf[oid_prefix.size() + 16]; - snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), shard_id); - return string(buf); -} - -RGWCoroutine *RGWSyncErrorLogger::log_error_cr(const DoutPrefixProvider *dpp, const string& source_zone, const string& section, const string& name, uint32_t error_code, const string& message) { - cls_log_entry entry; - - rgw_sync_error_info info(source_zone, error_code, message); - bufferlist bl; - encode(info, bl); - store->svc()->cls->timelog.prepare_entry(entry, real_clock::now(), section, name, bl); - - uint32_t shard_id = ++counter % num_shards; - - - return new RGWRadosTimelogAddCR(dpp, store, oids[shard_id], entry); -} - -void RGWSyncBackoff::update_wait_time() -{ - if (cur_wait == 0) { - cur_wait = 1; - } else { - cur_wait = (cur_wait << 1); - } - if (cur_wait >= max_secs) { - cur_wait = max_secs; - } -} - -void RGWSyncBackoff::backoff_sleep() -{ - update_wait_time(); - sleep(cur_wait); -} - -void RGWSyncBackoff::backoff(RGWCoroutine *op) -{ - update_wait_time(); - op->wait(utime_t(cur_wait, 0)); -} - -int RGWBackoffControlCR::operate(const DoutPrefixProvider *dpp) { - reenter(this) { - // retry the operation until it succeeds - while (true) { - yield { - std::lock_guard l{lock}; - cr = alloc_cr(); - cr->get(); - call(cr); - } - { - std::lock_guard l{lock}; - cr->put(); - cr = NULL; - } - if (retcode >= 0) { - break; - } - if (retcode != -EBUSY && retcode != -EAGAIN) { - ldout(cct, 0) << "ERROR: RGWBackoffControlCR called coroutine returned " << retcode << dendl; - if (exit_on_error) { - return set_cr_error(retcode); - } - } - if (reset_backoff) { - backoff.reset(); - } - yield backoff.backoff(this); - } - - // run an optional finisher - yield call(alloc_finisher_cr()); - if (retcode < 0) { - ldout(cct, 0) << "ERROR: call to finisher_cr() failed: retcode=" << retcode << dendl; - return set_cr_error(retcode); - } - return set_cr_done(); - } - return 0; -} - -void rgw_mdlog_info::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("num_objects", num_shards, obj); - JSONDecoder::decode_json("period", period, obj); - JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); -} - -void rgw_mdlog_entry::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("id", id, obj); - JSONDecoder::decode_json("section", section, obj); - JSONDecoder::decode_json("name", name, obj); - utime_t ut; - JSONDecoder::decode_json("timestamp", ut, obj); - timestamp = ut.to_real_time(); - JSONDecoder::decode_json("data", log_data, obj); -} - -void rgw_mdlog_shard_data::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("marker", marker, obj); - JSONDecoder::decode_json("truncated", truncated, obj); - JSONDecoder::decode_json("entries", entries, obj); -}; - -int RGWShardCollectCR::operate(const DoutPrefixProvider *dpp) { - reenter(this) { - while (spawn_next()) { - current_running++; - - if (current_running >= max_concurrent) { - int child_ret; - yield wait_for_child(); - if (collect_next(&child_ret)) { - current_running--; - child_ret = handle_result(child_ret); - if (child_ret < 0) { - status = child_ret; - } - } - } - } - while (current_running > 0) { - int child_ret; - yield wait_for_child(); - if (collect_next(&child_ret)) { - current_running--; - child_ret = handle_result(child_ret); - if (child_ret < 0) { - status = child_ret; - } - } - } - if (status < 0) { - return set_cr_error(status); - } - return set_cr_done(); - } - return 0; -} - -class RGWReadRemoteMDLogInfoCR : public RGWShardCollectCR { - RGWMetaSyncEnv *sync_env; - - const std::string& period; - int num_shards; - map *mdlog_info; - - int shard_id; -#define READ_MDLOG_MAX_CONCURRENT 10 - - int handle_result(int r) override { - if (r == -ENOENT) { // ENOENT is not a fatal error - return 0; - } - if (r < 0) { - ldout(cct, 4) << "failed to fetch mdlog status: " << cpp_strerror(r) << dendl; - } - return r; - } -public: - RGWReadRemoteMDLogInfoCR(RGWMetaSyncEnv *_sync_env, - const std::string& period, int _num_shards, - map *_mdlog_info) : RGWShardCollectCR(_sync_env->cct, READ_MDLOG_MAX_CONCURRENT), - sync_env(_sync_env), - period(period), num_shards(_num_shards), - mdlog_info(_mdlog_info), shard_id(0) {} - bool spawn_next() override; -}; - -class RGWListRemoteMDLogCR : public RGWShardCollectCR { - RGWMetaSyncEnv *sync_env; - - const std::string& period; - map shards; - int max_entries_per_shard; - map *result; - - map::iterator iter; -#define READ_MDLOG_MAX_CONCURRENT 10 - - int handle_result(int r) override { - if (r == -ENOENT) { // ENOENT is not a fatal error - return 0; - } - if (r < 0) { - ldout(cct, 4) << "failed to list remote mdlog shard: " << cpp_strerror(r) << dendl; - } - return r; - } -public: - RGWListRemoteMDLogCR(RGWMetaSyncEnv *_sync_env, - const std::string& period, map& _shards, - int _max_entries_per_shard, - map *_result) : RGWShardCollectCR(_sync_env->cct, READ_MDLOG_MAX_CONCURRENT), - sync_env(_sync_env), period(period), - max_entries_per_shard(_max_entries_per_shard), - result(_result) { - shards.swap(_shards); - iter = shards.begin(); - } - bool spawn_next() override; -}; - -int RGWRemoteMetaLog::read_log_info(const DoutPrefixProvider *dpp, rgw_mdlog_info *log_info) -{ - rgw_http_param_pair pairs[] = { { "type", "metadata" }, - { NULL, NULL } }; - - int ret = conn->get_json_resource(dpp, "/admin/log", pairs, null_yield, *log_info); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog info" << dendl; - return ret; - } - - ldpp_dout(dpp, 20) << "remote mdlog, num_shards=" << log_info->num_shards << dendl; - - return 0; -} - -int RGWRemoteMetaLog::read_master_log_shards_info(const DoutPrefixProvider *dpp, const string &master_period, map *shards_info) -{ - if (store->svc()->zone->is_meta_master()) { - return 0; - } - - rgw_mdlog_info log_info; - int ret = read_log_info(dpp, &log_info); - if (ret < 0) { - return ret; - } - - return run(dpp, new RGWReadRemoteMDLogInfoCR(&sync_env, master_period, log_info.num_shards, shards_info)); -} - -int RGWRemoteMetaLog::read_master_log_shards_next(const DoutPrefixProvider *dpp, const string& period, map shard_markers, map *result) +std::ostream& RGWMetaSyncStatusManager::gen_prefix(std::ostream& out) const { - if (store->svc()->zone->is_meta_master()) { - return 0; - } - - return run(dpp, new RGWListRemoteMDLogCR(&sync_env, period, shard_markers, 1, result)); + return out << "meta sync: "; } -int RGWRemoteMetaLog::init() +unsigned RGWMetaSyncStatusManager::get_subsys() const { - conn = store->svc()->zone->get_master_conn(); - - int ret = http_manager.start(); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl; - return ret; - } - - error_logger = new RGWSyncErrorLogger(store, RGW_SYNC_ERROR_LOG_SHARD_PREFIX, ERROR_LOGGER_SHARDS); - - init_sync_env(&sync_env); - - tn = sync_env.sync_tracer->add_node(sync_env.sync_tracer->root_node, "meta"); - - return 0; + return dout_subsys; } void RGWRemoteMetaLog::finish() @@ -321,2284 +22,3 @@ void RGWRemoteMetaLog::finish() going_down = true; stop(); } - -#define CLONE_MAX_ENTRIES 100 - -int RGWMetaSyncStatusManager::init(const DoutPrefixProvider *dpp) -{ - if (store->svc()->zone->is_meta_master()) { - return 0; - } - - if (!store->svc()->zone->get_master_conn()) { - ldpp_dout(dpp, -1) << "no REST connection to master zone" << dendl; - return -EIO; - } - - int r = rgw_init_ioctx(dpp, store->getRados()->get_rados_handle(), store->svc()->zone->get_zone_params().log_pool, ioctx, true); - if (r < 0) { - ldpp_dout(dpp, -1) << "ERROR: failed to open log pool (" << store->svc()->zone->get_zone_params().log_pool << " ret=" << r << dendl; - return r; - } - - r = master_log.init(); - if (r < 0) { - ldpp_dout(dpp, -1) << "ERROR: failed to init remote log, r=" << r << dendl; - return r; - } - - RGWMetaSyncEnv& sync_env = master_log.get_sync_env(); - - rgw_meta_sync_status sync_status; - r = read_sync_status(dpp, &sync_status); - if (r < 0 && r != -ENOENT) { - ldpp_dout(dpp, -1) << "ERROR: failed to read sync status, r=" << r << dendl; - return r; - } - - int num_shards = sync_status.sync_info.num_shards; - - for (int i = 0; i < num_shards; i++) { - shard_objs[i] = rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env.shard_obj_name(i)); - } - - std::unique_lock wl{ts_to_shard_lock}; - for (int i = 0; i < num_shards; i++) { - clone_markers.push_back(string()); - utime_shard ut; - ut.shard_id = i; - ts_to_shard[ut] = i; - } - - return 0; -} - -unsigned RGWMetaSyncStatusManager::get_subsys() const -{ - return dout_subsys; -} - -std::ostream& RGWMetaSyncStatusManager::gen_prefix(std::ostream& out) const -{ - return out << "meta sync: "; -} - -void RGWMetaSyncEnv::init(const DoutPrefixProvider *_dpp, CephContext *_cct, rgw::sal::RadosStore* _store, RGWRESTConn *_conn, - RGWAsyncRadosProcessor *_async_rados, RGWHTTPManager *_http_manager, - RGWSyncErrorLogger *_error_logger, RGWSyncTraceManager *_sync_tracer) { - dpp = _dpp; - cct = _cct; - store = _store; - conn = _conn; - async_rados = _async_rados; - http_manager = _http_manager; - error_logger = _error_logger; - sync_tracer = _sync_tracer; -} - -string RGWMetaSyncEnv::status_oid() -{ - return mdlog_sync_status_oid; -} - -string RGWMetaSyncEnv::shard_obj_name(int shard_id) -{ - char buf[mdlog_sync_status_shard_prefix.size() + 16]; - snprintf(buf, sizeof(buf), "%s.%d", mdlog_sync_status_shard_prefix.c_str(), shard_id); - - return string(buf); -} - -class RGWAsyncReadMDLogEntries : public RGWAsyncRadosRequest { - const DoutPrefixProvider *dpp; - rgw::sal::RadosStore* store; - RGWMetadataLog *mdlog; - int shard_id; - int max_entries; - -protected: - int _send_request(const DoutPrefixProvider *dpp) override { - real_time from_time; - real_time end_time; - - void *handle; - - mdlog->init_list_entries(shard_id, from_time, end_time, marker, &handle); - - int ret = mdlog->list_entries(dpp, handle, max_entries, entries, &marker, &truncated); - - mdlog->complete_list_entries(handle); - - return ret; - } -public: - string marker; - list entries; - bool truncated; - - RGWAsyncReadMDLogEntries(const DoutPrefixProvider *dpp, RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store, - RGWMetadataLog* mdlog, int _shard_id, - std::string _marker, int _max_entries) - : RGWAsyncRadosRequest(caller, cn), dpp(dpp), store(_store), mdlog(mdlog), - shard_id(_shard_id), max_entries(_max_entries), marker(std::move(_marker)) {} -}; - -class RGWReadMDLogEntriesCR : public RGWSimpleCoroutine { - RGWMetaSyncEnv *sync_env; - RGWMetadataLog *const mdlog; - int shard_id; - string marker; - string *pmarker; - int max_entries; - list *entries; - bool *truncated; - - RGWAsyncReadMDLogEntries *req{nullptr}; - -public: - RGWReadMDLogEntriesCR(RGWMetaSyncEnv *_sync_env, RGWMetadataLog* mdlog, - int _shard_id, string*_marker, int _max_entries, - list *_entries, bool *_truncated) - : RGWSimpleCoroutine(_sync_env->cct), sync_env(_sync_env), mdlog(mdlog), - shard_id(_shard_id), pmarker(_marker), max_entries(_max_entries), - entries(_entries), truncated(_truncated) {} - - ~RGWReadMDLogEntriesCR() override { - if (req) { - req->finish(); - } - } - - int send_request(const DoutPrefixProvider *dpp) override { - marker = *pmarker; - req = new RGWAsyncReadMDLogEntries(dpp, this, stack->create_completion_notifier(), - sync_env->store, mdlog, shard_id, marker, - max_entries); - sync_env->async_rados->queue(req); - return 0; - } - - int request_complete() override { - *pmarker = std::move(req->marker); - *entries = std::move(req->entries); - *truncated = req->truncated; - return req->get_ret_status(); - } -}; - - -class RGWReadRemoteMDLogShardInfoCR : public RGWCoroutine { - RGWMetaSyncEnv *env; - RGWRESTReadResource *http_op; - - const std::string& period; - int shard_id; - RGWMetadataLogInfo *shard_info; - -public: - RGWReadRemoteMDLogShardInfoCR(RGWMetaSyncEnv *env, const std::string& period, - int _shard_id, RGWMetadataLogInfo *_shard_info) - : RGWCoroutine(env->store->ctx()), env(env), http_op(NULL), - period(period), shard_id(_shard_id), shard_info(_shard_info) {} - - int operate(const DoutPrefixProvider *dpp) override { - auto store = env->store; - RGWRESTConn *conn = store->svc()->zone->get_master_conn(); - reenter(this) { - yield { - char buf[16]; - snprintf(buf, sizeof(buf), "%d", shard_id); - rgw_http_param_pair pairs[] = { { "type" , "metadata" }, - { "id", buf }, - { "period", period.c_str() }, - { "info" , NULL }, - { NULL, NULL } }; - - string p = "/admin/log/"; - - http_op = new RGWRESTReadResource(conn, p, pairs, NULL, - env->http_manager); - - init_new_io(http_op); - - int ret = http_op->aio_read(dpp); - if (ret < 0) { - ldpp_dout(env->dpp, 0) << "ERROR: failed to read from " << p << dendl; - log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; - http_op->put(); - return set_cr_error(ret); - } - - return io_block(0); - } - yield { - int ret = http_op->wait(shard_info, null_yield); - http_op->put(); - if (ret < 0) { - return set_cr_error(ret); - } - return set_cr_done(); - } - } - return 0; - } -}; - -RGWCoroutine* create_read_remote_mdlog_shard_info_cr(RGWMetaSyncEnv *env, - const std::string& period, - int shard_id, - RGWMetadataLogInfo* info) -{ - return new RGWReadRemoteMDLogShardInfoCR(env, period, shard_id, info); -} - -class RGWListRemoteMDLogShardCR : public RGWSimpleCoroutine { - RGWMetaSyncEnv *sync_env; - RGWRESTReadResource *http_op; - - const std::string& period; - int shard_id; - string marker; - uint32_t max_entries; - rgw_mdlog_shard_data *result; - -public: - RGWListRemoteMDLogShardCR(RGWMetaSyncEnv *env, const std::string& period, - int _shard_id, const string& _marker, uint32_t _max_entries, - rgw_mdlog_shard_data *_result) - : RGWSimpleCoroutine(env->store->ctx()), sync_env(env), http_op(NULL), - period(period), shard_id(_shard_id), marker(_marker), max_entries(_max_entries), result(_result) {} - - int send_request(const DoutPrefixProvider *dpp) override { - RGWRESTConn *conn = sync_env->conn; - - char buf[32]; - snprintf(buf, sizeof(buf), "%d", shard_id); - - char max_entries_buf[32]; - snprintf(max_entries_buf, sizeof(max_entries_buf), "%d", (int)max_entries); - - const char *marker_key = (marker.empty() ? "" : "marker"); - - rgw_http_param_pair pairs[] = { { "type", "metadata" }, - { "id", buf }, - { "period", period.c_str() }, - { "max-entries", max_entries_buf }, - { marker_key, marker.c_str() }, - { NULL, NULL } }; - - string p = "/admin/log/"; - - http_op = new RGWRESTReadResource(conn, p, pairs, NULL, sync_env->http_manager); - init_new_io(http_op); - - int ret = http_op->aio_read(dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to read from " << p << dendl; - log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; - http_op->put(); - return ret; - } - - return 0; - } - - int request_complete() override { - int ret = http_op->wait(result, null_yield); - http_op->put(); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to list remote mdlog shard, ret=" << ret << dendl; - return ret; - } - return 0; - } -}; - -RGWCoroutine* create_list_remote_mdlog_shard_cr(RGWMetaSyncEnv *env, - const std::string& period, - int shard_id, - const std::string& marker, - uint32_t max_entries, - rgw_mdlog_shard_data *result) -{ - return new RGWListRemoteMDLogShardCR(env, period, shard_id, marker, - max_entries, result); -} - -bool RGWReadRemoteMDLogInfoCR::spawn_next() { - if (shard_id >= num_shards) { - return false; - } - spawn(new RGWReadRemoteMDLogShardInfoCR(sync_env, period, shard_id, &(*mdlog_info)[shard_id]), false); - shard_id++; - return true; -} - -bool RGWListRemoteMDLogCR::spawn_next() { - if (iter == shards.end()) { - return false; - } - - spawn(new RGWListRemoteMDLogShardCR(sync_env, period, iter->first, iter->second, max_entries_per_shard, &(*result)[iter->first]), false); - ++iter; - return true; -} - -class RGWInitSyncStatusCoroutine : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - - rgw_meta_sync_info status; - vector shards_info; - boost::intrusive_ptr lease_cr; - boost::intrusive_ptr lease_stack; -public: - RGWInitSyncStatusCoroutine(RGWMetaSyncEnv *_sync_env, - const rgw_meta_sync_info &status) - : RGWCoroutine(_sync_env->store->ctx()), sync_env(_sync_env), - status(status), shards_info(status.num_shards), - lease_cr(nullptr), lease_stack(nullptr) {} - - ~RGWInitSyncStatusCoroutine() override { - if (lease_cr) { - lease_cr->abort(); - } - } - - int operate(const DoutPrefixProvider *dpp) override { - int ret; - reenter(this) { - yield { - set_status("acquiring sync lock"); - uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; - string lock_name = "sync_lock"; - rgw::sal::RadosStore* store = sync_env->store; - lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, store, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), - lock_name, lock_duration, this)); - lease_stack.reset(spawn(lease_cr.get(), false)); - } - while (!lease_cr->is_locked()) { - if (lease_cr->is_done()) { - ldpp_dout(dpp, 5) << "failed to take lease" << dendl; - set_status("lease lock failed, early abort"); - return set_cr_error(lease_cr->get_ret_status()); - } - set_sleeping(true); - yield; - } - yield { - set_status("writing sync status"); - rgw::sal::RadosStore* store = sync_env->store; - call(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, store->svc()->sysobj, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), - status)); - } - - if (retcode < 0) { - set_status("failed to write sync status"); - ldpp_dout(dpp, 0) << "ERROR: failed to write sync status, retcode=" << retcode << dendl; - yield lease_cr->go_down(); - return set_cr_error(retcode); - } - /* fetch current position in logs */ - set_status("fetching remote log position"); - yield { - for (int i = 0; i < (int)status.num_shards; i++) { - spawn(new RGWReadRemoteMDLogShardInfoCR(sync_env, status.period, i, - &shards_info[i]), false); - } - } - - drain_all_but_stack(lease_stack.get()); /* the lease cr still needs to run */ - - yield { - set_status("updating sync status"); - for (int i = 0; i < (int)status.num_shards; i++) { - rgw_meta_sync_marker marker; - RGWMetadataLogInfo& info = shards_info[i]; - marker.next_step_marker = info.marker; - marker.timestamp = info.last_update; - rgw::sal::RadosStore* store = sync_env->store; - spawn(new RGWSimpleRadosWriteCR(dpp, - sync_env->async_rados, - store->svc()->sysobj, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->shard_obj_name(i)), - marker), true); - } - } - yield { - set_status("changing sync state: build full sync maps"); - status.state = rgw_meta_sync_info::StateBuildingFullSyncMaps; - rgw::sal::RadosStore* store = sync_env->store; - call(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, store->svc()->sysobj, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), - status)); - } - set_status("drop lock lease"); - yield lease_cr->go_down(); - while (collect(&ret, NULL)) { - if (ret < 0) { - return set_cr_error(ret); - } - yield; - } - drain_all(); - return set_cr_done(); - } - return 0; - } -}; - -class RGWReadSyncStatusMarkersCR : public RGWShardCollectCR { - static constexpr int MAX_CONCURRENT_SHARDS = 16; - - RGWMetaSyncEnv *env; - const int num_shards; - int shard_id{0}; - map& markers; - - int handle_result(int r) override { - if (r == -ENOENT) { // ENOENT is not a fatal error - return 0; - } - if (r < 0) { - ldout(cct, 4) << "failed to read metadata sync markers: " - << cpp_strerror(r) << dendl; - } - return r; - } - public: - RGWReadSyncStatusMarkersCR(RGWMetaSyncEnv *env, int num_shards, - map& markers) - : RGWShardCollectCR(env->cct, MAX_CONCURRENT_SHARDS), - env(env), num_shards(num_shards), markers(markers) - {} - bool spawn_next() override; -}; - -bool RGWReadSyncStatusMarkersCR::spawn_next() -{ - if (shard_id >= num_shards) { - return false; - } - using CR = RGWSimpleRadosReadCR; - rgw_raw_obj obj{env->store->svc()->zone->get_zone_params().log_pool, - env->shard_obj_name(shard_id)}; - spawn(new CR(env->dpp, env->async_rados, env->store->svc()->sysobj, obj, &markers[shard_id]), false); - shard_id++; - return true; -} - -class RGWReadSyncStatusCoroutine : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - rgw_meta_sync_status *sync_status; - -public: - RGWReadSyncStatusCoroutine(RGWMetaSyncEnv *_sync_env, - rgw_meta_sync_status *_status) - : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), sync_status(_status) - {} - int operate(const DoutPrefixProvider *dpp) override; -}; - -int RGWReadSyncStatusCoroutine::operate(const DoutPrefixProvider *dpp) -{ - reenter(this) { - // read sync info - using ReadInfoCR = RGWSimpleRadosReadCR; - yield { - bool empty_on_enoent = false; // fail on ENOENT - rgw_raw_obj obj{sync_env->store->svc()->zone->get_zone_params().log_pool, - sync_env->status_oid()}; - call(new ReadInfoCR(dpp, sync_env->async_rados, sync_env->store->svc()->sysobj, obj, - &sync_status->sync_info, empty_on_enoent)); - } - if (retcode < 0) { - ldpp_dout(dpp, 4) << "failed to read sync status info with " - << cpp_strerror(retcode) << dendl; - return set_cr_error(retcode); - } - // read shard markers - using ReadMarkersCR = RGWReadSyncStatusMarkersCR; - yield call(new ReadMarkersCR(sync_env, sync_status->sync_info.num_shards, - sync_status->sync_markers)); - if (retcode < 0) { - ldpp_dout(dpp, 4) << "failed to read sync status markers with " - << cpp_strerror(retcode) << dendl; - return set_cr_error(retcode); - } - return set_cr_done(); - } - return 0; -} - -class RGWFetchAllMetaCR : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - - int num_shards; - - - int ret_status; - - list sections; - list::iterator sections_iter; - - struct meta_list_result { - list keys; - string marker; - uint64_t count{0}; - bool truncated{false}; - - void decode_json(JSONObj *obj) { - JSONDecoder::decode_json("keys", keys, obj); - JSONDecoder::decode_json("marker", marker, obj); - JSONDecoder::decode_json("count", count, obj); - JSONDecoder::decode_json("truncated", truncated, obj); - } - } result; - list::iterator iter; - - std::unique_ptr entries_index; - - boost::intrusive_ptr lease_cr; - boost::intrusive_ptr lease_stack; - bool lost_lock; - bool failed; - - string marker; - - map& markers; - - RGWSyncTraceNodeRef tn; - -public: - RGWFetchAllMetaCR(RGWMetaSyncEnv *_sync_env, int _num_shards, - map& _markers, - RGWSyncTraceNodeRef& _tn_parent) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), - num_shards(_num_shards), - ret_status(0), lease_cr(nullptr), lease_stack(nullptr), - lost_lock(false), failed(false), markers(_markers) { - tn = sync_env->sync_tracer->add_node(_tn_parent, "fetch_all_meta"); - } - - ~RGWFetchAllMetaCR() override { - } - - void append_section_from_set(set& all_sections, const string& name) { - set::iterator iter = all_sections.find(name); - if (iter != all_sections.end()) { - sections.emplace_back(std::move(*iter)); - all_sections.erase(iter); - } - } - /* - * meta sync should go in the following order: user, bucket.instance, bucket - * then whatever other sections exist (if any) - */ - void rearrange_sections() { - set all_sections; - std::move(sections.begin(), sections.end(), - std::inserter(all_sections, all_sections.end())); - sections.clear(); - - append_section_from_set(all_sections, "user"); - append_section_from_set(all_sections, "bucket.instance"); - append_section_from_set(all_sections, "bucket"); - append_section_from_set(all_sections, "roles"); - - std::move(all_sections.begin(), all_sections.end(), - std::back_inserter(sections)); - } - - int operate(const DoutPrefixProvider *dpp) override { - RGWRESTConn *conn = sync_env->conn; - - reenter(this) { - yield { - set_status(string("acquiring lock (") + sync_env->status_oid() + ")"); - uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; - string lock_name = "sync_lock"; - lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, - sync_env->store, - rgw_raw_obj(sync_env->store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), - lock_name, lock_duration, this)); - lease_stack.reset(spawn(lease_cr.get(), false)); - } - while (!lease_cr->is_locked()) { - if (lease_cr->is_done()) { - ldpp_dout(dpp, 5) << "failed to take lease" << dendl; - set_status("lease lock failed, early abort"); - return set_cr_error(lease_cr->get_ret_status()); - } - set_sleeping(true); - yield; - } - entries_index.reset(new RGWShardedOmapCRManager(sync_env->async_rados, sync_env->store, this, num_shards, - sync_env->store->svc()->zone->get_zone_params().log_pool, - mdlog_sync_full_sync_index_prefix)); - yield { - call(new RGWReadRESTResourceCR >(cct, conn, sync_env->http_manager, - "/admin/metadata", NULL, §ions)); - } - if (get_ret_status() < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to fetch metadata sections" << dendl; - yield entries_index->finish(); - yield lease_cr->go_down(); - drain_all(); - return set_cr_error(get_ret_status()); - } - rearrange_sections(); - sections_iter = sections.begin(); - for (; sections_iter != sections.end(); ++sections_iter) { - do { - yield { -#define META_FULL_SYNC_CHUNK_SIZE "1000" - string entrypoint = string("/admin/metadata/") + *sections_iter; - rgw_http_param_pair pairs[] = { { "max-entries", META_FULL_SYNC_CHUNK_SIZE }, - { "marker", result.marker.c_str() }, - { NULL, NULL } }; - result.keys.clear(); - call(new RGWReadRESTResourceCR(cct, conn, sync_env->http_manager, - entrypoint, pairs, &result)); - } - ret_status = get_ret_status(); - if (ret_status == -ENOENT) { - set_retcode(0); /* reset coroutine status so that we don't return it */ - ret_status = 0; - } - if (ret_status < 0) { - tn->log(0, SSTR("ERROR: failed to fetch metadata section: " << *sections_iter)); - yield entries_index->finish(); - yield lease_cr->go_down(); - drain_all(); - return set_cr_error(ret_status); - } - iter = result.keys.begin(); - for (; iter != result.keys.end(); ++iter) { - if (!lease_cr->is_locked()) { - lost_lock = true; - tn->log(1, "lease is lost, abort"); - break; - } - yield; // allow entries_index consumer to make progress - - tn->log(20, SSTR("list metadata: section=" << *sections_iter << " key=" << *iter)); - string s = *sections_iter + ":" + *iter; - int shard_id; - rgw::sal::RadosStore* store = sync_env->store; - int ret = store->ctl()->meta.mgr->get_shard_id(*sections_iter, *iter, &shard_id); - if (ret < 0) { - tn->log(0, SSTR("ERROR: could not determine shard id for " << *sections_iter << ":" << *iter)); - ret_status = ret; - break; - } - if (!entries_index->append(s, shard_id)) { - break; - } - } - } while (result.truncated); - } - yield { - if (!entries_index->finish()) { - failed = true; - } - } - if (!failed) { - for (map::iterator iter = markers.begin(); iter != markers.end(); ++iter) { - int shard_id = (int)iter->first; - rgw_meta_sync_marker& marker = iter->second; - marker.total_entries = entries_index->get_total_entries(shard_id); - spawn(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, sync_env->store->svc()->sysobj, - rgw_raw_obj(sync_env->store->svc()->zone->get_zone_params().log_pool, sync_env->shard_obj_name(shard_id)), - marker), true); - } - } - - drain_all_but_stack(lease_stack.get()); /* the lease cr still needs to run */ - - yield lease_cr->go_down(); - - int ret; - while (collect(&ret, NULL)) { - if (ret < 0) { - return set_cr_error(ret); - } - yield; - } - drain_all(); - if (failed) { - yield return set_cr_error(-EIO); - } - if (lost_lock) { - yield return set_cr_error(-EBUSY); - } - - if (ret_status < 0) { - yield return set_cr_error(ret_status); - } - - yield return set_cr_done(); - } - return 0; - } -}; - -static string full_sync_index_shard_oid(int shard_id) -{ - char buf[mdlog_sync_full_sync_index_prefix.size() + 16]; - snprintf(buf, sizeof(buf), "%s.%d", mdlog_sync_full_sync_index_prefix.c_str(), shard_id); - return string(buf); -} - -class RGWReadRemoteMetadataCR : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - - RGWRESTReadResource *http_op; - - string section; - string key; - - bufferlist *pbl; - - RGWSyncTraceNodeRef tn; - -public: - RGWReadRemoteMetadataCR(RGWMetaSyncEnv *_sync_env, - const string& _section, const string& _key, bufferlist *_pbl, - const RGWSyncTraceNodeRef& _tn_parent) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), - http_op(NULL), - section(_section), - key(_key), - pbl(_pbl) { - tn = sync_env->sync_tracer->add_node(_tn_parent, "read_remote_meta", - section + ":" + key); - } - - int operate(const DoutPrefixProvider *dpp) override { - RGWRESTConn *conn = sync_env->conn; - reenter(this) { - yield { - string key_encode; - url_encode(key, key_encode); - rgw_http_param_pair pairs[] = { { "key" , key.c_str()}, - { NULL, NULL } }; - - string p = string("/admin/metadata/") + section + "/" + key_encode; - - http_op = new RGWRESTReadResource(conn, p, pairs, NULL, sync_env->http_manager); - - init_new_io(http_op); - - int ret = http_op->aio_read(dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog data" << dendl; - log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; - http_op->put(); - return set_cr_error(ret); - } - - return io_block(0); - } - yield { - int ret = http_op->wait(pbl, null_yield); - http_op->put(); - if (ret < 0) { - return set_cr_error(ret); - } - return set_cr_done(); - } - } - return 0; - } -}; - -class RGWAsyncMetaStoreEntry : public RGWAsyncRadosRequest { - rgw::sal::RadosStore* store; - string raw_key; - bufferlist bl; - const DoutPrefixProvider *dpp; -protected: - int _send_request(const DoutPrefixProvider *dpp) override { - int ret = store->ctl()->meta.mgr->put(raw_key, bl, null_yield, dpp, RGWMDLogSyncType::APPLY_ALWAYS, true); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: can't store key: " << raw_key << " ret=" << ret << dendl; - return ret; - } - return 0; - } -public: - RGWAsyncMetaStoreEntry(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store, - const string& _raw_key, - bufferlist& _bl, - const DoutPrefixProvider *dpp) : RGWAsyncRadosRequest(caller, cn), store(_store), - raw_key(_raw_key), bl(_bl), dpp(dpp) {} -}; - - -class RGWMetaStoreEntryCR : public RGWSimpleCoroutine { - RGWMetaSyncEnv *sync_env; - string raw_key; - bufferlist bl; - - RGWAsyncMetaStoreEntry *req; - -public: - RGWMetaStoreEntryCR(RGWMetaSyncEnv *_sync_env, - const string& _raw_key, - bufferlist& _bl) : RGWSimpleCoroutine(_sync_env->cct), sync_env(_sync_env), - raw_key(_raw_key), bl(_bl), req(NULL) { - } - - ~RGWMetaStoreEntryCR() override { - if (req) { - req->finish(); - } - } - - int send_request(const DoutPrefixProvider *dpp) override { - req = new RGWAsyncMetaStoreEntry(this, stack->create_completion_notifier(), - sync_env->store, raw_key, bl, dpp); - sync_env->async_rados->queue(req); - return 0; - } - - int request_complete() override { - return req->get_ret_status(); - } -}; - -class RGWAsyncMetaRemoveEntry : public RGWAsyncRadosRequest { - rgw::sal::RadosStore* store; - string raw_key; - const DoutPrefixProvider *dpp; -protected: - int _send_request(const DoutPrefixProvider *dpp) override { - int ret = store->ctl()->meta.mgr->remove(raw_key, null_yield, dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: can't remove key: " << raw_key << " ret=" << ret << dendl; - return ret; - } - return 0; - } -public: - RGWAsyncMetaRemoveEntry(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store, - const string& _raw_key, const DoutPrefixProvider *dpp) : RGWAsyncRadosRequest(caller, cn), store(_store), - raw_key(_raw_key), dpp(dpp) {} -}; - - -class RGWMetaRemoveEntryCR : public RGWSimpleCoroutine { - RGWMetaSyncEnv *sync_env; - string raw_key; - - RGWAsyncMetaRemoveEntry *req; - -public: - RGWMetaRemoveEntryCR(RGWMetaSyncEnv *_sync_env, - const string& _raw_key) : RGWSimpleCoroutine(_sync_env->cct), sync_env(_sync_env), - raw_key(_raw_key), req(NULL) { - } - - ~RGWMetaRemoveEntryCR() override { - if (req) { - req->finish(); - } - } - - int send_request(const DoutPrefixProvider *dpp) override { - req = new RGWAsyncMetaRemoveEntry(this, stack->create_completion_notifier(), - sync_env->store, raw_key, dpp); - sync_env->async_rados->queue(req); - return 0; - } - - int request_complete() override { - int r = req->get_ret_status(); - if (r == -ENOENT) { - r = 0; - } - return r; - } -}; - -#define META_SYNC_UPDATE_MARKER_WINDOW 10 - - -int RGWLastCallerWinsCR::operate(const DoutPrefixProvider *dpp) { - RGWCoroutine *call_cr; - reenter(this) { - while (cr) { - call_cr = cr; - cr = nullptr; - yield call(call_cr); - /* cr might have been modified at this point */ - } - return set_cr_done(); - } - return 0; -} - -class RGWMetaSyncShardMarkerTrack : public RGWSyncShardMarkerTrack { - RGWMetaSyncEnv *sync_env; - - string marker_oid; - rgw_meta_sync_marker sync_marker; - - RGWSyncTraceNodeRef tn; - -public: - RGWMetaSyncShardMarkerTrack(RGWMetaSyncEnv *_sync_env, - const string& _marker_oid, - const rgw_meta_sync_marker& _marker, - RGWSyncTraceNodeRef& _tn) : RGWSyncShardMarkerTrack(META_SYNC_UPDATE_MARKER_WINDOW), - sync_env(_sync_env), - marker_oid(_marker_oid), - sync_marker(_marker), - tn(_tn){} - - RGWCoroutine *store_marker(const string& new_marker, uint64_t index_pos, const real_time& timestamp) override { - sync_marker.marker = new_marker; - if (index_pos > 0) { - sync_marker.pos = index_pos; - } - - if (!real_clock::is_zero(timestamp)) { - sync_marker.timestamp = timestamp; - } - - ldpp_dout(sync_env->dpp, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << " realm_epoch=" << sync_marker.realm_epoch << dendl; - tn->log(20, SSTR("new marker=" << new_marker)); - rgw::sal::RadosStore* store = sync_env->store; - return new RGWSimpleRadosWriteCR(sync_env->dpp, sync_env->async_rados, - store->svc()->sysobj, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, marker_oid), - sync_marker); - } - - RGWOrderCallCR *allocate_order_control_cr() override { - return new RGWLastCallerWinsCR(sync_env->cct); - } -}; - -RGWMetaSyncSingleEntryCR::RGWMetaSyncSingleEntryCR(RGWMetaSyncEnv *_sync_env, - const string& _raw_key, const string& _entry_marker, - const RGWMDLogStatus& _op_status, - RGWMetaSyncShardMarkerTrack *_marker_tracker, const RGWSyncTraceNodeRef& _tn_parent) : RGWCoroutine(_sync_env->cct), - sync_env(_sync_env), - raw_key(_raw_key), entry_marker(_entry_marker), - op_status(_op_status), - pos(0), sync_status(0), - marker_tracker(_marker_tracker), tries(0) { - error_injection = (sync_env->cct->_conf->rgw_sync_meta_inject_err_probability > 0); - tn = sync_env->sync_tracer->add_node(_tn_parent, "entry", raw_key); -} - -int RGWMetaSyncSingleEntryCR::operate(const DoutPrefixProvider *dpp) { - reenter(this) { -#define NUM_TRANSIENT_ERROR_RETRIES 10 - - if (error_injection && - rand() % 10000 < cct->_conf->rgw_sync_meta_inject_err_probability * 10000.0) { - return set_cr_error(-EIO); - } - - if (op_status != MDLOG_STATUS_COMPLETE) { - tn->log(20, "skipping pending operation"); - yield call(marker_tracker->finish(entry_marker)); - if (retcode < 0) { - return set_cr_error(retcode); - } - return set_cr_done(); - } - tn->set_flag(RGW_SNS_FLAG_ACTIVE); - for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) { - yield { - pos = raw_key.find(':'); - section = raw_key.substr(0, pos); - key = raw_key.substr(pos + 1); - tn->log(10, SSTR("fetching remote metadata entry" << (tries == 0 ? "" : " (retry)"))); - call(new RGWReadRemoteMetadataCR(sync_env, section, key, &md_bl, tn)); - } - - sync_status = retcode; - - if (sync_status == -ENOENT) { - break; - } - - if (sync_status < 0) { - if (tries < NUM_TRANSIENT_ERROR_RETRIES - 1) { - ldpp_dout(dpp, 20) << *this << ": failed to fetch remote metadata entry: " << section << ":" << key << ", will retry" << dendl; - continue; - } - - tn->log(10, SSTR("failed to read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status)); - log_error() << "failed to read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << std::endl; - yield call(sync_env->error_logger->log_error_cr(dpp, sync_env->conn->get_remote_id(), section, key, -sync_status, - string("failed to read remote metadata entry: ") + cpp_strerror(-sync_status))); - return set_cr_error(sync_status); - } - - break; - } - - retcode = 0; - for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) { - if (sync_status != -ENOENT) { - tn->log(10, SSTR("storing local metadata entry: " << section << ":" << key)); - yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl)); - } else { - tn->log(10, SSTR("removing local metadata entry:" << section << ":" << key)); - yield call(new RGWMetaRemoveEntryCR(sync_env, raw_key)); - if (retcode == -ENOENT) { - retcode = 0; - break; - } - } - if ((retcode < 0) && (tries < NUM_TRANSIENT_ERROR_RETRIES - 1)) { - ldpp_dout(dpp, 20) << *this << ": failed to store metadata entry: " << section << ":" << key << ", got retcode=" << retcode << ", will retry" << dendl; - continue; - } - break; - } - - sync_status = retcode; - - if (sync_status == 0 && marker_tracker) { - /* update marker */ - yield call(marker_tracker->finish(entry_marker)); - sync_status = retcode; - } - if (sync_status < 0) { - tn->log(10, SSTR("failed, status=" << sync_status)); - return set_cr_error(sync_status); - } - tn->log(10, "success"); - return set_cr_done(); - } - return 0; -} - -class RGWCloneMetaLogCoroutine : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - RGWMetadataLog *mdlog; - - const std::string& period; - int shard_id; - string marker; - bool truncated = false; - string *new_marker; - - int max_entries = CLONE_MAX_ENTRIES; - - RGWRESTReadResource *http_op = nullptr; - boost::intrusive_ptr completion; - - RGWMetadataLogInfo shard_info; - rgw_mdlog_shard_data data; - -public: - RGWCloneMetaLogCoroutine(RGWMetaSyncEnv *_sync_env, RGWMetadataLog* mdlog, - const std::string& period, int _id, - const string& _marker, string *_new_marker) - : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), mdlog(mdlog), - period(period), shard_id(_id), marker(_marker), new_marker(_new_marker) { - if (new_marker) { - *new_marker = marker; - } - } - ~RGWCloneMetaLogCoroutine() override { - if (http_op) { - http_op->put(); - } - if (completion) { - completion->cancel(); - } - } - - int operate(const DoutPrefixProvider *dpp) override; - - int state_init(); - int state_read_shard_status(); - int state_read_shard_status_complete(); - int state_send_rest_request(const DoutPrefixProvider *dpp); - int state_receive_rest_response(); - int state_store_mdlog_entries(); - int state_store_mdlog_entries_complete(); -}; - -class RGWMetaSyncShardCR : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - - const rgw_pool& pool; - const std::string& period; //< currently syncing period id - const epoch_t realm_epoch; //< realm_epoch of period - RGWMetadataLog* mdlog; //< log of syncing period - uint32_t shard_id; - rgw_meta_sync_marker& sync_marker; - boost::optional temp_marker; //< for pending updates - string marker; - string max_marker; - const std::string& period_marker; //< max marker stored in next period - - RGWRadosGetOmapKeysCR::ResultPtr omapkeys; - std::set entries; - std::set::iterator iter; - - string oid; - - RGWMetaSyncShardMarkerTrack *marker_tracker = nullptr; - - list log_entries; - list::iterator log_iter; - bool truncated = false; - - string mdlog_marker; - string raw_key; - rgw_mdlog_entry mdlog_entry; - - ceph::mutex inc_lock = ceph::make_mutex("RGWMetaSyncShardCR::inc_lock"); - ceph::condition_variable inc_cond; - - boost::asio::coroutine incremental_cr; - boost::asio::coroutine full_cr; - - boost::intrusive_ptr lease_cr; - boost::intrusive_ptr lease_stack; - - bool lost_lock = false; - - bool *reset_backoff; - - // hold a reference to the cr stack while it's in the map - using StackRef = boost::intrusive_ptr; - map stack_to_pos; - map pos_to_prev; - - bool can_adjust_marker = false; - bool done_with_period = false; - - int total_entries = 0; - - RGWSyncTraceNodeRef tn; -public: - RGWMetaSyncShardCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, - const std::string& period, epoch_t realm_epoch, - RGWMetadataLog* mdlog, uint32_t _shard_id, - rgw_meta_sync_marker& _marker, - const std::string& period_marker, bool *_reset_backoff, - RGWSyncTraceNodeRef& _tn) - : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), pool(_pool), - period(period), realm_epoch(realm_epoch), mdlog(mdlog), - shard_id(_shard_id), sync_marker(_marker), - period_marker(period_marker), - reset_backoff(_reset_backoff), tn(_tn) { - *reset_backoff = false; - } - - ~RGWMetaSyncShardCR() override { - delete marker_tracker; - if (lease_cr) { - lease_cr->abort(); - } - } - - void set_marker_tracker(RGWMetaSyncShardMarkerTrack *mt) { - delete marker_tracker; - marker_tracker = mt; - } - - int operate(const DoutPrefixProvider *dpp) override { - int r; - while (true) { - switch (sync_marker.state) { - case rgw_meta_sync_marker::FullSync: - r = full_sync(); - if (r < 0) { - ldpp_dout(dpp, 10) << "sync: full_sync: shard_id=" << shard_id << " r=" << r << dendl; - return set_cr_error(r); - } - return 0; - case rgw_meta_sync_marker::IncrementalSync: - r = incremental_sync(); - if (r < 0) { - ldpp_dout(dpp, 10) << "sync: incremental_sync: shard_id=" << shard_id << " r=" << r << dendl; - return set_cr_error(r); - } - return 0; - } - } - /* unreachable */ - return 0; - } - - void collect_children() - { - int child_ret; - RGWCoroutinesStack *child; - while (collect_next(&child_ret, &child)) { - auto iter = stack_to_pos.find(child); - if (iter == stack_to_pos.end()) { - /* some other stack that we don't care about */ - continue; - } - - string& pos = iter->second; - - if (child_ret < 0) { - ldpp_dout(sync_env->dpp, 0) << *this << ": child operation stack=" << child << " entry=" << pos << " returned " << child_ret << dendl; - // on any error code from RGWMetaSyncSingleEntryCR, we do not advance - // the sync status marker past this entry, and set - // can_adjust_marker=false to exit out of RGWMetaSyncShardCR. - // RGWMetaSyncShardControlCR will rerun RGWMetaSyncShardCR from the - // previous marker and retry - can_adjust_marker = false; - } - - map::iterator prev_iter = pos_to_prev.find(pos); - ceph_assert(prev_iter != pos_to_prev.end()); - - if (pos_to_prev.size() == 1) { - if (can_adjust_marker) { - sync_marker.marker = pos; - } - pos_to_prev.erase(prev_iter); - } else { - ceph_assert(pos_to_prev.size() > 1); - pos_to_prev.erase(prev_iter); - prev_iter = pos_to_prev.begin(); - if (can_adjust_marker) { - sync_marker.marker = prev_iter->second; - } - } - - ldpp_dout(sync_env->dpp, 4) << *this << ": adjusting marker pos=" << sync_marker.marker << dendl; - stack_to_pos.erase(iter); - } - } - - int full_sync() { -#define OMAP_GET_MAX_ENTRIES 100 - int max_entries = OMAP_GET_MAX_ENTRIES; - reenter(&full_cr) { - set_status("full_sync"); - tn->log(10, "start full sync"); - oid = full_sync_index_shard_oid(shard_id); - can_adjust_marker = true; - /* grab lock */ - yield { - uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; - string lock_name = "sync_lock"; - rgw::sal::RadosStore* store = sync_env->store; - lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, store, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - lock_name, lock_duration, this)); - lease_stack.reset(spawn(lease_cr.get(), false)); - lost_lock = false; - } - while (!lease_cr->is_locked()) { - if (lease_cr->is_done()) { - drain_all(); - tn->log(5, "failed to take lease"); - return lease_cr->get_ret_status(); - } - set_sleeping(true); - yield; - } - tn->log(10, "took lease"); - - /* lock succeeded, a retry now should avoid previous backoff status */ - *reset_backoff = true; - - /* prepare marker tracker */ - set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env, - sync_env->shard_obj_name(shard_id), - sync_marker, tn)); - - marker = sync_marker.marker; - - total_entries = sync_marker.pos; - - /* sync! */ - do { - if (!lease_cr->is_locked()) { - tn->log(1, "lease is lost, abort"); - lost_lock = true; - break; - } - omapkeys = std::make_shared(); - yield call(new RGWRadosGetOmapKeysCR(sync_env->store, rgw_raw_obj(pool, oid), - marker, max_entries, omapkeys)); - if (retcode < 0) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: " << __func__ << "(): RGWRadosGetOmapKeysCR() returned ret=" << retcode << dendl; - tn->log(0, SSTR("ERROR: failed to list omap keys, status=" << retcode)); - yield lease_cr->go_down(); - drain_all(); - return retcode; - } - entries = std::move(omapkeys->entries); - tn->log(20, SSTR("retrieved " << entries.size() << " entries to sync")); - if (entries.size() > 0) { - tn->set_flag(RGW_SNS_FLAG_ACTIVE); /* actually have entries to sync */ - } - iter = entries.begin(); - for (; iter != entries.end(); ++iter) { - marker = *iter; - tn->log(20, SSTR("full sync: " << marker)); - total_entries++; - if (!marker_tracker->start(marker, total_entries, real_time())) { - tn->log(0, SSTR("ERROR: cannot start syncing " << marker << ". Duplicate entry?")); - } else { - // fetch remote and write locally - yield { - RGWCoroutinesStack *stack = spawn(new RGWMetaSyncSingleEntryCR(sync_env, marker, marker, MDLOG_STATUS_COMPLETE, marker_tracker, tn), false); - // stack_to_pos holds a reference to the stack - stack_to_pos[stack] = marker; - pos_to_prev[marker] = marker; - } - // limit spawn window - while (num_spawned() > static_cast(cct->_conf->rgw_meta_sync_spawn_window)) { - yield wait_for_child(); - collect_children(); - } - } - } - collect_children(); - } while (omapkeys->more && can_adjust_marker); - - tn->unset_flag(RGW_SNS_FLAG_ACTIVE); /* actually have entries to sync */ - - while (num_spawned() > 1) { - yield wait_for_child(); - collect_children(); - } - - if (!lost_lock) { - /* update marker to reflect we're done with full sync */ - if (can_adjust_marker) { - // apply updates to a temporary marker, or operate() will send us - // to incremental_sync() after we yield - temp_marker = sync_marker; - temp_marker->state = rgw_meta_sync_marker::IncrementalSync; - temp_marker->marker = std::move(temp_marker->next_step_marker); - temp_marker->next_step_marker.clear(); - temp_marker->realm_epoch = realm_epoch; - ldpp_dout(sync_env->dpp, 4) << *this << ": saving marker pos=" << temp_marker->marker << " realm_epoch=" << realm_epoch << dendl; - - using WriteMarkerCR = RGWSimpleRadosWriteCR; - yield call(new WriteMarkerCR(sync_env->dpp, sync_env->async_rados, sync_env->store->svc()->sysobj, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - *temp_marker)); - } - - if (retcode < 0) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to set sync marker: retcode=" << retcode << dendl; - yield lease_cr->go_down(); - drain_all(); - return retcode; - } - // clean up full sync index - yield { - auto oid = full_sync_index_shard_oid(shard_id); - call(new RGWRadosRemoveCR(sync_env->store, {pool, oid})); - } - } - - /* - * if we reached here, it means that lost_lock is true, otherwise the state - * change in the previous block will prevent us from reaching here - */ - - yield lease_cr->go_down(); - - lease_cr.reset(); - - drain_all(); - - if (!can_adjust_marker) { - return -EAGAIN; - } - - if (lost_lock) { - return -EBUSY; - } - - tn->log(10, "full sync complete"); - - // apply the sync marker update - ceph_assert(temp_marker); - sync_marker = std::move(*temp_marker); - temp_marker = boost::none; - // must not yield after this point! - } - return 0; - } - - - int incremental_sync() { - reenter(&incremental_cr) { - set_status("incremental_sync"); - tn->log(10, "start incremental sync"); - can_adjust_marker = true; - /* grab lock */ - if (!lease_cr) { /* could have had a lease_cr lock from previous state */ - yield { - uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; - string lock_name = "sync_lock"; - rgw::sal::RadosStore* store = sync_env->store; - lease_cr.reset( new RGWContinuousLeaseCR(sync_env->async_rados, store, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - lock_name, lock_duration, this)); - lease_stack.reset(spawn(lease_cr.get(), false)); - lost_lock = false; - } - while (!lease_cr->is_locked()) { - if (lease_cr->is_done()) { - drain_all(); - tn->log(5, "failed to take lease"); - return lease_cr->get_ret_status(); - } - set_sleeping(true); - yield; - } - } - tn->log(10, "took lease"); - // if the period has advanced, we can't use the existing marker - if (sync_marker.realm_epoch < realm_epoch) { - ldpp_dout(sync_env->dpp, 4) << "clearing marker=" << sync_marker.marker - << " from old realm_epoch=" << sync_marker.realm_epoch - << " (now " << realm_epoch << ')' << dendl; - sync_marker.realm_epoch = realm_epoch; - sync_marker.marker.clear(); - } - mdlog_marker = sync_marker.marker; - set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env, - sync_env->shard_obj_name(shard_id), - sync_marker, tn)); - - /* - * mdlog_marker: the remote sync marker positiion - * sync_marker: the local sync marker position - * max_marker: the max mdlog position that we fetched - * marker: the current position we try to sync - * period_marker: the last marker before the next period begins (optional) - */ - marker = max_marker = sync_marker.marker; - /* inc sync */ - do { - if (!lease_cr->is_locked()) { - lost_lock = true; - tn->log(1, "lease is lost, abort"); - break; - } -#define INCREMENTAL_MAX_ENTRIES 100 - ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << " truncated=" << truncated << dendl; - if (!period_marker.empty() && period_marker <= mdlog_marker) { - tn->log(10, SSTR("finished syncing current period: mdlog_marker=" << mdlog_marker << " sync_marker=" << sync_marker.marker << " period_marker=" << period_marker)); - done_with_period = true; - break; - } - if (mdlog_marker <= max_marker || !truncated) { - /* we're at the tip, try to bring more entries */ - ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " syncing mdlog for shard_id=" << shard_id << dendl; - yield call(new RGWCloneMetaLogCoroutine(sync_env, mdlog, - period, shard_id, - mdlog_marker, &mdlog_marker)); - } - if (retcode < 0) { - tn->log(10, SSTR(*this << ": failed to fetch more log entries, retcode=" << retcode)); - yield lease_cr->go_down(); - drain_all(); - *reset_backoff = false; // back off and try again later - return retcode; - } - truncated = true; - *reset_backoff = true; /* if we got to this point, all systems function */ - if (mdlog_marker > max_marker) { - tn->set_flag(RGW_SNS_FLAG_ACTIVE); /* actually have entries to sync */ - tn->log(20, SSTR("mdlog_marker=" << mdlog_marker << " sync_marker=" << sync_marker.marker)); - marker = max_marker; - yield call(new RGWReadMDLogEntriesCR(sync_env, mdlog, shard_id, - &max_marker, INCREMENTAL_MAX_ENTRIES, - &log_entries, &truncated)); - if (retcode < 0) { - tn->log(10, SSTR("failed to list mdlog entries, retcode=" << retcode)); - yield lease_cr->go_down(); - drain_all(); - *reset_backoff = false; // back off and try again later - return retcode; - } - for (log_iter = log_entries.begin(); log_iter != log_entries.end() && !done_with_period; ++log_iter) { - if (!period_marker.empty() && period_marker <= log_iter->id) { - done_with_period = true; - if (period_marker < log_iter->id) { - tn->log(10, SSTR("found key=" << log_iter->id - << " past period_marker=" << period_marker)); - break; - } - ldpp_dout(sync_env->dpp, 10) << "found key at period_marker=" << period_marker << dendl; - // sync this entry, then return control to RGWMetaSyncCR - } - if (!mdlog_entry.convert_from(*log_iter)) { - tn->log(0, SSTR("ERROR: failed to convert mdlog entry, shard_id=" << shard_id << " log_entry: " << log_iter->id << ":" << log_iter->section << ":" << log_iter->name << ":" << log_iter->timestamp << " ... skipping entry")); - continue; - } - tn->log(20, SSTR("log_entry: " << log_iter->id << ":" << log_iter->section << ":" << log_iter->name << ":" << log_iter->timestamp)); - if (!marker_tracker->start(log_iter->id, 0, log_iter->timestamp.to_real_time())) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: cannot start syncing " << log_iter->id << ". Duplicate entry?" << dendl; - } else { - raw_key = log_iter->section + ":" + log_iter->name; - yield { - RGWCoroutinesStack *stack = spawn(new RGWMetaSyncSingleEntryCR(sync_env, raw_key, log_iter->id, mdlog_entry.log_data.status, marker_tracker, tn), false); - ceph_assert(stack); - // stack_to_pos holds a reference to the stack - stack_to_pos[stack] = log_iter->id; - pos_to_prev[log_iter->id] = marker; - } - // limit spawn window - while (num_spawned() > static_cast(cct->_conf->rgw_meta_sync_spawn_window)) { - yield wait_for_child(); - collect_children(); - } - } - marker = log_iter->id; - } - } - collect_children(); - ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " max_marker=" << max_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl; - if (done_with_period) { - // return control to RGWMetaSyncCR and advance to the next period - tn->log(10, SSTR(*this << ": done with period")); - break; - } - if (mdlog_marker == max_marker && can_adjust_marker) { - tn->unset_flag(RGW_SNS_FLAG_ACTIVE); - yield wait(utime_t(cct->_conf->rgw_meta_sync_poll_interval, 0)); - } - } while (can_adjust_marker); - - tn->unset_flag(RGW_SNS_FLAG_ACTIVE); - - while (num_spawned() > 1) { - yield wait_for_child(); - collect_children(); - } - - yield lease_cr->go_down(); - - drain_all(); - - if (lost_lock) { - return -EBUSY; - } - - if (!can_adjust_marker) { - return -EAGAIN; - } - - return set_cr_done(); - } - /* TODO */ - return 0; - } -}; - -class RGWMetaSyncShardControlCR : public RGWBackoffControlCR -{ - RGWMetaSyncEnv *sync_env; - - const rgw_pool& pool; - const std::string& period; - epoch_t realm_epoch; - RGWMetadataLog* mdlog; - uint32_t shard_id; - rgw_meta_sync_marker sync_marker; - const std::string period_marker; - - RGWSyncTraceNodeRef tn; - - static constexpr bool exit_on_error = false; // retry on all errors -public: - RGWMetaSyncShardControlCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, - const std::string& period, epoch_t realm_epoch, - RGWMetadataLog* mdlog, uint32_t _shard_id, - const rgw_meta_sync_marker& _marker, - std::string&& period_marker, - RGWSyncTraceNodeRef& _tn_parent) - : RGWBackoffControlCR(_sync_env->cct, exit_on_error), sync_env(_sync_env), - pool(_pool), period(period), realm_epoch(realm_epoch), mdlog(mdlog), - shard_id(_shard_id), sync_marker(_marker), - period_marker(std::move(period_marker)) { - tn = sync_env->sync_tracer->add_node(_tn_parent, "shard", - std::to_string(shard_id)); - } - - RGWCoroutine *alloc_cr() override { - return new RGWMetaSyncShardCR(sync_env, pool, period, realm_epoch, mdlog, - shard_id, sync_marker, period_marker, backoff_ptr(), tn); - } - - RGWCoroutine *alloc_finisher_cr() override { - rgw::sal::RadosStore* store = sync_env->store; - return new RGWSimpleRadosReadCR(sync_env->dpp, sync_env->async_rados, store->svc()->sysobj, - rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), - &sync_marker); - } -}; - -class RGWMetaSyncCR : public RGWCoroutine { - RGWMetaSyncEnv *sync_env; - const rgw_pool& pool; - RGWPeriodHistory::Cursor cursor; //< sync position in period history - RGWPeriodHistory::Cursor next; //< next period in history - rgw_meta_sync_status sync_status; - RGWSyncTraceNodeRef tn; - - std::mutex mutex; //< protect access to shard_crs - - // TODO: it should be enough to hold a reference on the stack only, as calling - // RGWCoroutinesStack::wakeup() doesn't refer to the RGWCoroutine if it has - // already completed - using ControlCRRef = boost::intrusive_ptr; - using StackRef = boost::intrusive_ptr; - using RefPair = std::pair; - map shard_crs; - int ret{0}; - -public: - RGWMetaSyncCR(RGWMetaSyncEnv *_sync_env, const RGWPeriodHistory::Cursor &cursor, - const rgw_meta_sync_status& _sync_status, RGWSyncTraceNodeRef& _tn) - : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), - pool(sync_env->store->svc()->zone->get_zone_params().log_pool), - cursor(cursor), sync_status(_sync_status), tn(_tn) {} - - ~RGWMetaSyncCR() { - } - - int operate(const DoutPrefixProvider *dpp) override { - reenter(this) { - // loop through one period at a time - tn->log(1, "start"); - for (;;) { - if (cursor == sync_env->store->svc()->mdlog->get_period_history()->get_current()) { - next = RGWPeriodHistory::Cursor{}; - if (cursor) { - ldpp_dout(dpp, 10) << "RGWMetaSyncCR on current period=" - << cursor.get_period().get_id() << dendl; - } else { - ldpp_dout(dpp, 10) << "RGWMetaSyncCR with no period" << dendl; - } - } else { - next = cursor; - next.next(); - ldpp_dout(dpp, 10) << "RGWMetaSyncCR on period=" - << cursor.get_period().get_id() << ", next=" - << next.get_period().get_id() << dendl; - } - - yield { - // get the mdlog for the current period (may be empty) - auto& period_id = sync_status.sync_info.period; - auto realm_epoch = sync_status.sync_info.realm_epoch; - auto mdlog = sync_env->store->svc()->mdlog->get_log(period_id); - - tn->log(1, SSTR("realm epoch=" << realm_epoch << " period id=" << period_id)); - - // prevent wakeup() from accessing shard_crs while we're spawning them - std::lock_guard lock(mutex); - - // sync this period on each shard - for (const auto& m : sync_status.sync_markers) { - uint32_t shard_id = m.first; - auto& marker = m.second; - - std::string period_marker; - if (next) { - // read the maximum marker from the next period's sync status - period_marker = next.get_period().get_sync_status()[shard_id]; - if (period_marker.empty()) { - // no metadata changes have occurred on this shard, skip it - ldpp_dout(dpp, 10) << "RGWMetaSyncCR: skipping shard " << shard_id - << " with empty period marker" << dendl; - continue; - } - } - - using ShardCR = RGWMetaSyncShardControlCR; - auto cr = new ShardCR(sync_env, pool, period_id, realm_epoch, - mdlog, shard_id, marker, - std::move(period_marker), tn); - auto stack = spawn(cr, false); - shard_crs[shard_id] = RefPair{cr, stack}; - } - } - // wait for each shard to complete - while (ret == 0 && num_spawned() > 0) { - yield wait_for_child(); - collect(&ret, nullptr); - } - drain_all(); - { - // drop shard cr refs under lock - std::lock_guard lock(mutex); - shard_crs.clear(); - } - if (ret < 0) { - return set_cr_error(ret); - } - // advance to the next period - ceph_assert(next); - cursor = next; - - // write the updated sync info - sync_status.sync_info.period = cursor.get_period().get_id(); - sync_status.sync_info.realm_epoch = cursor.get_epoch(); - yield call(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, - sync_env->store->svc()->sysobj, - rgw_raw_obj(pool, sync_env->status_oid()), - sync_status.sync_info)); - } - } - return 0; - } - - void wakeup(int shard_id) { - std::lock_guard lock(mutex); - auto iter = shard_crs.find(shard_id); - if (iter == shard_crs.end()) { - return; - } - iter->second.first->wakeup(); - } -}; - -void RGWRemoteMetaLog::init_sync_env(RGWMetaSyncEnv *env) { - env->dpp = dpp; - env->cct = store->ctx(); - env->store = store; - env->conn = conn; - env->async_rados = async_rados; - env->http_manager = &http_manager; - env->error_logger = error_logger; - env->sync_tracer = store->getRados()->get_sync_tracer(); -} - -int RGWRemoteMetaLog::read_sync_status(const DoutPrefixProvider *dpp, rgw_meta_sync_status *sync_status) -{ - if (store->svc()->zone->is_meta_master()) { - return 0; - } - // cannot run concurrently with run_sync(), so run in a separate manager - RGWCoroutinesManager crs(store->ctx(), store->getRados()->get_cr_registry()); - RGWHTTPManager http_manager(store->ctx(), crs.get_completion_mgr()); - int ret = http_manager.start(); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl; - return ret; - } - RGWMetaSyncEnv sync_env_local = sync_env; - sync_env_local.http_manager = &http_manager; - tn->log(20, "read sync status"); - ret = crs.run(dpp, new RGWReadSyncStatusCoroutine(&sync_env_local, sync_status)); - http_manager.stop(); - return ret; -} - -int RGWRemoteMetaLog::init_sync_status(const DoutPrefixProvider *dpp) -{ - if (store->svc()->zone->is_meta_master()) { - return 0; - } - - rgw_mdlog_info mdlog_info; - int r = read_log_info(dpp, &mdlog_info); - if (r < 0) { - ldpp_dout(dpp, -1) << "ERROR: fail to fetch master log info (r=" << r << ")" << dendl; - return r; - } - - rgw_meta_sync_info sync_info; - sync_info.num_shards = mdlog_info.num_shards; - auto cursor = store->svc()->mdlog->get_period_history()->get_current(); - if (cursor) { - sync_info.period = cursor.get_period().get_id(); - sync_info.realm_epoch = cursor.get_epoch(); - } - - return run(dpp, new RGWInitSyncStatusCoroutine(&sync_env, sync_info)); -} - -int RGWRemoteMetaLog::store_sync_info(const DoutPrefixProvider *dpp, const rgw_meta_sync_info& sync_info) -{ - tn->log(20, "store sync info"); - return run(dpp, new RGWSimpleRadosWriteCR(dpp, async_rados, store->svc()->sysobj, - rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env.status_oid()), - sync_info)); -} - -// return a cursor to the period at our sync position -static RGWPeriodHistory::Cursor get_period_at(const DoutPrefixProvider *dpp, - rgw::sal::RadosStore* store, - const rgw_meta_sync_info& info, - optional_yield y) -{ - if (info.period.empty()) { - // return an empty cursor with error=0 - return RGWPeriodHistory::Cursor{}; - } - - // look for an existing period in our history - auto cursor = store->svc()->mdlog->get_period_history()->lookup(info.realm_epoch); - if (cursor) { - // verify that the period ids match - auto& existing = cursor.get_period().get_id(); - if (existing != info.period) { - ldpp_dout(dpp, -1) << "ERROR: sync status period=" << info.period - << " does not match period=" << existing - << " in history at realm epoch=" << info.realm_epoch << dendl; - return RGWPeriodHistory::Cursor{-EEXIST}; - } - return cursor; - } - - // read the period from rados or pull it from the master - RGWPeriod period; - int r = store->svc()->mdlog->pull_period(dpp, info.period, period, y); - if (r < 0) { - ldpp_dout(dpp, -1) << "ERROR: failed to read period id " - << info.period << ": " << cpp_strerror(r) << dendl; - return RGWPeriodHistory::Cursor{r}; - } - // attach the period to our history - cursor = store->svc()->mdlog->get_period_history()->attach(dpp, std::move(period), y); - if (!cursor) { - r = cursor.get_error(); - ldpp_dout(dpp, -1) << "ERROR: failed to read period history back to " - << info.period << ": " << cpp_strerror(r) << dendl; - } - return cursor; -} - -int RGWRemoteMetaLog::run_sync(const DoutPrefixProvider *dpp, optional_yield y) -{ - if (store->svc()->zone->is_meta_master()) { - return 0; - } - - int r = 0; - - // get shard count and oldest log period from master - rgw_mdlog_info mdlog_info; - for (;;) { - if (going_down) { - ldpp_dout(dpp, 1) << __func__ << "(): going down" << dendl; - return 0; - } - r = read_log_info(dpp, &mdlog_info); - if (r == -EIO || r == -ENOENT) { - // keep retrying if master isn't alive or hasn't initialized the log - ldpp_dout(dpp, 10) << __func__ << "(): waiting for master.." << dendl; - backoff.backoff_sleep(); - continue; - } - backoff.reset(); - if (r < 0) { - ldpp_dout(dpp, -1) << "ERROR: fail to fetch master log info (r=" << r << ")" << dendl; - return r; - } - break; - } - - rgw_meta_sync_status sync_status; - do { - if (going_down) { - ldpp_dout(dpp, 1) << __func__ << "(): going down" << dendl; - return 0; - } - r = run(dpp, new RGWReadSyncStatusCoroutine(&sync_env, &sync_status)); - if (r < 0 && r != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: failed to fetch sync status r=" << r << dendl; - return r; - } - - if (!mdlog_info.period.empty()) { - // restart sync if the remote has a period, but: - // a) our status does not, or - // b) our sync period comes before the remote's oldest log period - if (sync_status.sync_info.period.empty() || - sync_status.sync_info.realm_epoch < mdlog_info.realm_epoch) { - sync_status.sync_info.state = rgw_meta_sync_info::StateInit; - string reason; - if (sync_status.sync_info.period.empty()) { - reason = "period is empty"; - } else { - reason = SSTR("sync_info realm epoch is behind: " << sync_status.sync_info.realm_epoch << " < " << mdlog_info.realm_epoch); - } - tn->log(1, "initialize sync (reason: " + reason + ")"); - ldpp_dout(dpp, 1) << "epoch=" << sync_status.sync_info.realm_epoch - << " in sync status comes before remote's oldest mdlog epoch=" - << mdlog_info.realm_epoch << ", restarting sync" << dendl; - } - } - - if (sync_status.sync_info.state == rgw_meta_sync_info::StateInit) { - ldpp_dout(dpp, 20) << __func__ << "(): init" << dendl; - sync_status.sync_info.num_shards = mdlog_info.num_shards; - auto cursor = store->svc()->mdlog->get_period_history()->get_current(); - if (cursor) { - // run full sync, then start incremental from the current period/epoch - sync_status.sync_info.period = cursor.get_period().get_id(); - sync_status.sync_info.realm_epoch = cursor.get_epoch(); - } - r = run(dpp, new RGWInitSyncStatusCoroutine(&sync_env, sync_status.sync_info)); - if (r == -EBUSY) { - backoff.backoff_sleep(); - continue; - } - backoff.reset(); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to init sync status r=" << r << dendl; - return r; - } - } - } while (sync_status.sync_info.state == rgw_meta_sync_info::StateInit); - - auto num_shards = sync_status.sync_info.num_shards; - if (num_shards != mdlog_info.num_shards) { - ldpp_dout(dpp, -1) << "ERROR: can't sync, mismatch between num shards, master num_shards=" << mdlog_info.num_shards << " local num_shards=" << num_shards << dendl; - return -EINVAL; - } - - RGWPeriodHistory::Cursor cursor; - do { - r = run(dpp, new RGWReadSyncStatusCoroutine(&sync_env, &sync_status)); - if (r < 0 && r != -ENOENT) { - tn->log(0, SSTR("ERROR: failed to fetch sync status r=" << r)); - return r; - } - - switch ((rgw_meta_sync_info::SyncState)sync_status.sync_info.state) { - case rgw_meta_sync_info::StateBuildingFullSyncMaps: - tn->log(20, "building full sync maps"); - r = run(dpp, new RGWFetchAllMetaCR(&sync_env, num_shards, sync_status.sync_markers, tn)); - if (r == -EBUSY || r == -EIO) { - backoff.backoff_sleep(); - continue; - } - backoff.reset(); - if (r < 0) { - tn->log(0, SSTR("ERROR: failed to fetch all metadata keys (r=" << r << ")")); - return r; - } - - sync_status.sync_info.state = rgw_meta_sync_info::StateSync; - r = store_sync_info(dpp, sync_status.sync_info); - if (r < 0) { - tn->log(0, SSTR("ERROR: failed to update sync status (r=" << r << ")")); - return r; - } - /* fall through */ - case rgw_meta_sync_info::StateSync: - tn->log(20, "sync"); - // find our position in the period history (if any) - cursor = get_period_at(dpp, store, sync_status.sync_info, y); - r = cursor.get_error(); - if (r < 0) { - return r; - } - meta_sync_cr = new RGWMetaSyncCR(&sync_env, cursor, sync_status, tn); - r = run(dpp, meta_sync_cr); - if (r < 0) { - tn->log(0, "ERROR: failed to fetch all metadata keys"); - return r; - } - break; - default: - tn->log(0, "ERROR: bad sync state!"); - return -EIO; - } - } while (!going_down); - - return 0; -} - -void RGWRemoteMetaLog::wakeup(int shard_id) -{ - if (!meta_sync_cr) { - return; - } - meta_sync_cr->wakeup(shard_id); -} - -int RGWCloneMetaLogCoroutine::operate(const DoutPrefixProvider *dpp) -{ - reenter(this) { - do { - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": init request" << dendl; - return state_init(); - } - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": reading shard status" << dendl; - return state_read_shard_status(); - } - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": reading shard status complete" << dendl; - return state_read_shard_status_complete(); - } - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": sending rest request" << dendl; - return state_send_rest_request(dpp); - } - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": receiving rest response" << dendl; - return state_receive_rest_response(); - } - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": storing mdlog entries" << dendl; - return state_store_mdlog_entries(); - } - } while (truncated); - yield { - ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": storing mdlog entries complete" << dendl; - return state_store_mdlog_entries_complete(); - } - } - - return 0; -} - -int RGWCloneMetaLogCoroutine::state_init() -{ - data = rgw_mdlog_shard_data(); - - return 0; -} - -int RGWCloneMetaLogCoroutine::state_read_shard_status() -{ - const bool add_ref = false; // default constructs with refs=1 - - completion.reset(new RGWMetadataLogInfoCompletion( - [this](int ret, const cls_log_header& header) { - if (ret < 0) { - if (ret != -ENOENT) { - ldpp_dout(sync_env->dpp, 1) << "ERROR: failed to read mdlog info with " - << cpp_strerror(ret) << dendl; - } - } else { - shard_info.marker = header.max_marker; - shard_info.last_update = header.max_time.to_real_time(); - } - // wake up parent stack - io_complete(); - }), add_ref); - - int ret = mdlog->get_info_async(sync_env->dpp, shard_id, completion.get()); - if (ret < 0) { - ldpp_dout(sync_env->dpp, 0) << "ERROR: mdlog->get_info_async() returned ret=" << ret << dendl; - return set_cr_error(ret); - } - - return io_block(0); -} - -int RGWCloneMetaLogCoroutine::state_read_shard_status_complete() -{ - completion.reset(); - - ldpp_dout(sync_env->dpp, 20) << "shard_id=" << shard_id << " marker=" << shard_info.marker << " last_update=" << shard_info.last_update << dendl; - - marker = shard_info.marker; - - return 0; -} - -int RGWCloneMetaLogCoroutine::state_send_rest_request(const DoutPrefixProvider *dpp) -{ - RGWRESTConn *conn = sync_env->conn; - - char buf[32]; - snprintf(buf, sizeof(buf), "%d", shard_id); - - char max_entries_buf[32]; - snprintf(max_entries_buf, sizeof(max_entries_buf), "%d", max_entries); - - const char *marker_key = (marker.empty() ? "" : "marker"); - - rgw_http_param_pair pairs[] = { { "type", "metadata" }, - { "id", buf }, - { "period", period.c_str() }, - { "max-entries", max_entries_buf }, - { marker_key, marker.c_str() }, - { NULL, NULL } }; - - http_op = new RGWRESTReadResource(conn, "/admin/log", pairs, NULL, sync_env->http_manager); - - init_new_io(http_op); - - int ret = http_op->aio_read(dpp); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog data" << dendl; - log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; - http_op->put(); - http_op = NULL; - return set_cr_error(ret); - } - - return io_block(0); -} - -int RGWCloneMetaLogCoroutine::state_receive_rest_response() -{ - int ret = http_op->wait(&data, null_yield); - if (ret < 0) { - error_stream << "http operation failed: " << http_op->to_str() << " status=" << http_op->get_http_status() << std::endl; - ldpp_dout(sync_env->dpp, 5) << "failed to wait for op, ret=" << ret << dendl; - http_op->put(); - http_op = NULL; - return set_cr_error(ret); - } - http_op->put(); - http_op = NULL; - - ldpp_dout(sync_env->dpp, 20) << "remote mdlog, shard_id=" << shard_id << " num of shard entries: " << data.entries.size() << dendl; - - truncated = ((int)data.entries.size() == max_entries); - - if (data.entries.empty()) { - if (new_marker) { - *new_marker = marker; - } - return set_cr_done(); - } - - if (new_marker) { - *new_marker = data.entries.back().id; - } - - return 0; -} - - -int RGWCloneMetaLogCoroutine::state_store_mdlog_entries() -{ - list dest_entries; - - vector::iterator iter; - for (iter = data.entries.begin(); iter != data.entries.end(); ++iter) { - rgw_mdlog_entry& entry = *iter; - ldpp_dout(sync_env->dpp, 20) << "entry: name=" << entry.name << dendl; - - cls_log_entry dest_entry; - dest_entry.id = entry.id; - dest_entry.section = entry.section; - dest_entry.name = entry.name; - dest_entry.timestamp = utime_t(entry.timestamp); - - encode(entry.log_data, dest_entry.data); - - dest_entries.push_back(dest_entry); - - marker = entry.id; - } - - RGWAioCompletionNotifier *cn = stack->create_completion_notifier(); - - int ret = mdlog->store_entries_in_shard(sync_env->dpp, dest_entries, shard_id, cn->completion()); - if (ret < 0) { - cn->put(); - ldpp_dout(sync_env->dpp, 10) << "failed to store md log entries shard_id=" << shard_id << " ret=" << ret << dendl; - return set_cr_error(ret); - } - return io_block(0); -} - -int RGWCloneMetaLogCoroutine::state_store_mdlog_entries_complete() -{ - return set_cr_done(); -} - -void rgw_meta_sync_info::decode_json(JSONObj *obj) -{ - string s; - JSONDecoder::decode_json("status", s, obj); - if (s == "init") { - state = StateInit; - } else if (s == "building-full-sync-maps") { - state = StateBuildingFullSyncMaps; - } else if (s == "sync") { - state = StateSync; - } - JSONDecoder::decode_json("num_shards", num_shards, obj); - JSONDecoder::decode_json("period", period, obj); - JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); -} - -void rgw_meta_sync_info::dump(Formatter *f) const -{ - string s; - switch ((SyncState)state) { - case StateInit: - s = "init"; - break; - case StateBuildingFullSyncMaps: - s = "building-full-sync-maps"; - break; - case StateSync: - s = "sync"; - break; - default: - s = "unknown"; - break; - } - encode_json("status", s, f); - encode_json("num_shards", num_shards, f); - encode_json("period", period, f); - encode_json("realm_epoch", realm_epoch, f); -} - - -void rgw_meta_sync_marker::decode_json(JSONObj *obj) -{ - int s; - JSONDecoder::decode_json("state", s, obj); - state = s; - JSONDecoder::decode_json("marker", marker, obj); - JSONDecoder::decode_json("next_step_marker", next_step_marker, obj); - JSONDecoder::decode_json("total_entries", total_entries, obj); - JSONDecoder::decode_json("pos", pos, obj); - utime_t ut; - JSONDecoder::decode_json("timestamp", ut, obj); - timestamp = ut.to_real_time(); - JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); -} - -void rgw_meta_sync_marker::dump(Formatter *f) const -{ - encode_json("state", (int)state, f); - encode_json("marker", marker, f); - encode_json("next_step_marker", next_step_marker, f); - encode_json("total_entries", total_entries, f); - encode_json("pos", pos, f); - encode_json("timestamp", utime_t(timestamp), f); - encode_json("realm_epoch", realm_epoch, f); -} - -void rgw_meta_sync_status::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("info", sync_info, obj); - JSONDecoder::decode_json("markers", sync_markers, obj); -} - -void rgw_meta_sync_status::dump(Formatter *f) const { - encode_json("info", sync_info, f); - encode_json("markers", sync_markers, f); -} - -void rgw_sync_error_info::dump(Formatter *f) const { - encode_json("source_zone", source_zone, f); - encode_json("error_code", error_code, f); - encode_json("message", message, f); -} - diff --git a/src/rgw/rgw_tools.cc b/src/rgw/rgw_tools.cc index 81d6b88837c2..7e6513cde4dd 100644 --- a/src/rgw/rgw_tools.cc +++ b/src/rgw/rgw_tools.cc @@ -4,29 +4,8 @@ #include #include "common/errno.h" -#include "common/safe_io.h" -#include "librados/librados_asio.h" -#include "common/async/yield_context.h" -#include "include/types.h" -#include "include/stringify.h" - -#include "librados/AioCompletionImpl.h" - -#include "rgw_common.h" #include "rgw_tools.h" -#include "rgw_acl_s3.h" -#include "rgw_op.h" -#include "rgw_putobj_processor.h" -#include "rgw_aio_throttle.h" -#include "rgw_compression.h" -#include "rgw_zone.h" -#include "rgw_sal_rados.h" -#include "osd/osd_types.h" - -#include "services/svc_sys_obj.h" -#include "services/svc_zone.h" -#include "services/svc_zone_utils.h" #define dout_subsys ceph_subsys_rgw #define dout_context g_ceph_context @@ -37,215 +16,6 @@ using namespace std; static std::map* ext_mime_map; -int rgw_init_ioctx(const DoutPrefixProvider *dpp, - librados::Rados *rados, const rgw_pool& pool, - librados::IoCtx& ioctx, bool create, - bool mostly_omap) -{ - int r = rados->ioctx_create(pool.name.c_str(), ioctx); - if (r == -ENOENT && create) { - r = rados->pool_create(pool.name.c_str()); - if (r == -ERANGE) { - ldpp_dout(dpp, 0) - << __func__ - << " ERROR: librados::Rados::pool_create returned " << cpp_strerror(-r) - << " (this can be due to a pool or placement group misconfiguration, e.g." - << " pg_num < pgp_num or mon_max_pg_per_osd exceeded)" - << dendl; - } - if (r < 0 && r != -EEXIST) { - return r; - } - - r = rados->ioctx_create(pool.name.c_str(), ioctx); - if (r < 0) { - return r; - } - - r = ioctx.application_enable(pg_pool_t::APPLICATION_NAME_RGW, false); - if (r < 0 && r != -EOPNOTSUPP) { - return r; - } - - if (mostly_omap) { - // set pg_autoscale_bias - bufferlist inbl; - float bias = g_conf().get_val("rgw_rados_pool_autoscale_bias"); - int r = rados->mon_command( - "{\"prefix\": \"osd pool set\", \"pool\": \"" + - pool.name + "\", \"var\": \"pg_autoscale_bias\", \"val\": \"" + - stringify(bias) + "\"}", - inbl, NULL, NULL); - if (r < 0) { - ldpp_dout(dpp, 10) << __func__ << " warning: failed to set pg_autoscale_bias on " - << pool.name << dendl; - } - // set recovery_priority - int p = g_conf().get_val("rgw_rados_pool_recovery_priority"); - r = rados->mon_command( - "{\"prefix\": \"osd pool set\", \"pool\": \"" + - pool.name + "\", \"var\": \"recovery_priority\": \"" + - stringify(p) + "\"}", - inbl, NULL, NULL); - if (r < 0) { - ldpp_dout(dpp, 10) << __func__ << " warning: failed to set recovery_priority on " - << pool.name << dendl; - } - } - } else if (r < 0) { - return r; - } - if (!pool.ns.empty()) { - ioctx.set_namespace(pool.ns); - } - return 0; -} - -map* no_change_attrs() { - static map no_change; - return &no_change; -} - -int rgw_put_system_obj(const DoutPrefixProvider *dpp, RGWSI_SysObj* svc_sysobj, - const rgw_pool& pool, const string& oid, bufferlist& data, bool exclusive, - RGWObjVersionTracker *objv_tracker, real_time set_mtime, optional_yield y, map *pattrs) -{ - map no_attrs; - if (!pattrs) { - pattrs = &no_attrs; - } - - rgw_raw_obj obj(pool, oid); - - auto sysobj = svc_sysobj->get_obj(obj); - int ret; - - if (pattrs != no_change_attrs()) { - ret = sysobj.wop() - .set_objv_tracker(objv_tracker) - .set_exclusive(exclusive) - .set_mtime(set_mtime) - .set_attrs(*pattrs) - .write(dpp, data, y); - } else { - ret = sysobj.wop() - .set_objv_tracker(objv_tracker) - .set_exclusive(exclusive) - .set_mtime(set_mtime) - .write_data(dpp, data, y); - } - - return ret; -} - -int rgw_stat_system_obj(const DoutPrefixProvider *dpp, RGWSI_SysObj* svc_sysobj, - const rgw_pool& pool, const std::string& key, - RGWObjVersionTracker *objv_tracker, - real_time *pmtime, optional_yield y, - std::map *pattrs) -{ - rgw_raw_obj obj(pool, key); - auto sysobj = svc_sysobj->get_obj(obj); - return sysobj.rop() - .set_attrs(pattrs) - .set_last_mod(pmtime) - .stat(y, dpp); -} - - -int rgw_get_system_obj(RGWSI_SysObj* svc_sysobj, const rgw_pool& pool, const string& key, bufferlist& bl, - RGWObjVersionTracker *objv_tracker, real_time *pmtime, optional_yield y, - const DoutPrefixProvider *dpp, map *pattrs, - rgw_cache_entry_info *cache_info, - boost::optional refresh_version, bool raw_attrs) -{ - const rgw_raw_obj obj(pool, key); - auto sysobj = svc_sysobj->get_obj(obj); - auto rop = sysobj.rop(); - return rop.set_attrs(pattrs) - .set_last_mod(pmtime) - .set_objv_tracker(objv_tracker) - .set_raw_attrs(raw_attrs) - .set_cache_info(cache_info) - .set_refresh_version(refresh_version) - .read(dpp, &bl, y); -} - -int rgw_delete_system_obj(const DoutPrefixProvider *dpp, - RGWSI_SysObj *sysobj_svc, const rgw_pool& pool, const string& oid, - RGWObjVersionTracker *objv_tracker, optional_yield y) -{ - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - rgw_raw_obj obj(pool, oid); - return sysobj.wop() - .set_objv_tracker(objv_tracker) - .remove(dpp, y); -} - -thread_local bool is_asio_thread = false; - -int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid, - librados::ObjectReadOperation *op, bufferlist* pbl, - optional_yield y, int flags) -{ - // given a yield_context, call async_operate() to yield the coroutine instead - // of blocking - if (y) { - auto& context = y.get_io_context(); - auto& yield = y.get_yield_context(); - boost::system::error_code ec; - auto bl = librados::async_operate( - context, ioctx, oid, op, flags, yield[ec]); - if (pbl) { - *pbl = std::move(bl); - } - return -ec.value(); - } - // work on asio threads should be asynchronous, so warn when they block - if (is_asio_thread) { - ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; - } - return ioctx.operate(oid, op, nullptr, flags); -} - -int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid, - librados::ObjectWriteOperation *op, optional_yield y, - int flags) -{ - if (y) { - auto& context = y.get_io_context(); - auto& yield = y.get_yield_context(); - boost::system::error_code ec; - librados::async_operate(context, ioctx, oid, op, flags, yield[ec]); - return -ec.value(); - } - if (is_asio_thread) { - ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; - } - return ioctx.operate(oid, op, flags); -} - -int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid, - bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl, - optional_yield y) -{ - if (y) { - auto& context = y.get_io_context(); - auto& yield = y.get_yield_context(); - boost::system::error_code ec; - auto reply = librados::async_notify(context, ioctx, oid, - bl, timeout_ms, yield[ec]); - if (pbl) { - *pbl = std::move(reply); - } - return -ec.value(); - } - if (is_asio_thread) { - ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; - } - return ioctx.notify2(oid, bl, timeout_ms, pbl); -} - void parse_mime_map_line(const char *start, const char *end) { char line[end - start + 1]; @@ -339,189 +109,6 @@ const char *rgw_find_mime_by_ext(string& ext) return iter->second.c_str(); } -void rgw_filter_attrset(map& unfiltered_attrset, const string& check_prefix, - map *attrset) -{ - attrset->clear(); - map::iterator iter; - for (iter = unfiltered_attrset.lower_bound(check_prefix); - iter != unfiltered_attrset.end(); ++iter) { - if (!boost::algorithm::starts_with(iter->first, check_prefix)) - break; - (*attrset)[iter->first] = iter->second; - } -} - -RGWDataAccess::RGWDataAccess(rgw::sal::Store* _store) : store(_store) -{ -} - - -int RGWDataAccess::Bucket::finish_init() -{ - auto iter = attrs.find(RGW_ATTR_ACL); - if (iter == attrs.end()) { - return 0; - } - - bufferlist::const_iterator bliter = iter->second.begin(); - try { - policy.decode(bliter); - } catch (buffer::error& err) { - return -EIO; - } - - return 0; -} - -int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y) -{ - std::unique_ptr bucket; - int ret = sd->store->get_bucket(dpp, nullptr, tenant, name, &bucket, y); - if (ret < 0) { - return ret; - } - - bucket_info = bucket->get_info(); - mtime = bucket->get_modification_time(); - attrs = bucket->get_attrs(); - - return finish_init(); -} - -int RGWDataAccess::Bucket::init(const RGWBucketInfo& _bucket_info, - const map& _attrs) -{ - bucket_info = _bucket_info; - attrs = _attrs; - - return finish_init(); -} - -int RGWDataAccess::Bucket::get_object(const rgw_obj_key& key, - ObjectRef *obj) { - obj->reset(new Object(sd, shared_from_this(), key)); - return 0; -} - -int RGWDataAccess::Object::put(bufferlist& data, - map& attrs, - const DoutPrefixProvider *dpp, - optional_yield y) -{ - rgw::sal::Store* store = sd->store; - CephContext *cct = store->ctx(); - - string tag; - append_rand_alpha(cct, tag, tag, 32); - - RGWBucketInfo& bucket_info = bucket->bucket_info; - - rgw::BlockingAioThrottle aio(store->ctx()->_conf->rgw_put_obj_min_window_size); - - std::unique_ptr b; - store->get_bucket(NULL, bucket_info, &b); - std::unique_ptr obj = b->get_object(key); - - auto& owner = bucket->policy.get_owner(); - - string req_id = store->zone_unique_id(store->get_new_req_id()); - - std::unique_ptr processor; - processor = store->get_atomic_writer(dpp, y, std::move(obj), - owner.get_id(), - nullptr, olh_epoch, req_id); - - int ret = processor->prepare(y); - if (ret < 0) - return ret; - - rgw::sal::DataProcessor *filter = processor.get(); - - CompressorRef plugin; - boost::optional compressor; - - const auto& compression_type = store->get_compression_type(bucket_info.placement_rule); - if (compression_type != "none") { - plugin = Compressor::create(store->ctx(), compression_type); - if (!plugin) { - ldpp_dout(dpp, 1) << "Cannot load plugin for compression type " - << compression_type << dendl; - } else { - compressor.emplace(store->ctx(), plugin, filter); - filter = &*compressor; - } - } - - off_t ofs = 0; - auto obj_size = data.length(); - - RGWMD5Etag etag_calc; - - do { - size_t read_len = std::min(data.length(), (unsigned int)cct->_conf->rgw_max_chunk_size); - - bufferlist bl; - - data.splice(0, read_len, &bl); - etag_calc.update(bl); - - ret = filter->process(std::move(bl), ofs); - if (ret < 0) - return ret; - - ofs += read_len; - } while (data.length() > 0); - - ret = filter->process({}, ofs); - if (ret < 0) { - return ret; - } - bool has_etag_attr = false; - auto iter = attrs.find(RGW_ATTR_ETAG); - if (iter != attrs.end()) { - bufferlist& bl = iter->second; - etag = bl.to_str(); - has_etag_attr = true; - } - - if (!aclbl) { - RGWAccessControlPolicy_S3 policy(cct); - - policy.create_canned(bucket->policy.get_owner(), bucket->policy.get_owner(), string()); /* default private policy */ - - policy.encode(aclbl.emplace()); - } - - if (etag.empty()) { - etag_calc.finish(&etag); - } - - if (!has_etag_attr) { - bufferlist etagbl; - etagbl.append(etag); - attrs[RGW_ATTR_ETAG] = etagbl; - } - attrs[RGW_ATTR_ACL] = *aclbl; - - string *puser_data = nullptr; - if (user_data) { - puser_data = &(*user_data); - } - - return processor->complete(obj_size, etag, - &mtime, mtime, - attrs, delete_at, - nullptr, nullptr, - puser_data, - nullptr, nullptr, y); -} - -void RGWDataAccess::Object::set_policy(const RGWAccessControlPolicy& policy) -{ - policy.encode(aclbl.emplace()); -} - int rgw_tools_init(const DoutPrefixProvider *dpp, CephContext *cct) { ext_mime_map = new std::map; @@ -535,9 +122,3 @@ void rgw_tools_cleanup() delete ext_mime_map; ext_mime_map = nullptr; } - -void rgw_complete_aio_completion(librados::AioCompletion* c, int r) { - auto pc = c->pc; - librados::CB_AioCompleteAndSafe cb(pc); - cb(r); -} diff --git a/src/rgw/rgw_user.cc b/src/rgw/rgw_user.cc index 69b785c6ba79..954c7fd246d5 100644 --- a/src/rgw/rgw_user.cc +++ b/src/rgw/rgw_user.cc @@ -1,51 +1,18 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp -#include - -#include -#include -#include - -#include "common/errno.h" -#include "common/Formatter.h" -#include "common/ceph_json.h" #include "rgw_sal_rados.h" -#include "rgw_zone.h" -#include "rgw_acl.h" #include "include/types.h" #include "rgw_user.h" -#include "rgw_string.h" // until everything is moved from rgw_common #include "rgw_common.h" -#include "rgw_bucket.h" -#include "rgw_quota.h" - -#include "services/svc_zone.h" -#include "services/svc_sys_obj.h" -#include "services/svc_sys_obj_cache.h" -#include "services/svc_user.h" -#include "services/svc_meta.h" - #define dout_subsys ceph_subsys_rgw using namespace std; -extern void op_type_to_str(uint32_t mask, char *buf, int len); - -/** - * Get the anonymous (ie, unauthenticated) user info. - */ -void rgw_get_anon_user(RGWUserInfo& info) -{ - info.user_id = RGW_USER_ANON_ID; - info.display_name.clear(); - info.access_keys.clear(); -} - int rgw_user_sync_all_stats(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::User* user, optional_yield y) { @@ -136,34 +103,6 @@ int rgw_user_get_all_buckets_stats(const DoutPrefixProvider *dpp, return 0; } -static string key_type_to_str(int key_type) { - switch (key_type) { - case KEY_TYPE_SWIFT: - return "swift"; - break; - - default: - return "s3"; - break; - } -} - -static bool char_is_unreserved_url(char c) -{ - if (isalnum(c)) - return true; - - switch (c) { - case '-': - case '.': - case '_': - case '~': - return true; - default: - return false; - } -} - int rgw_validate_tenant_name(const string& t) { struct tench { @@ -176,2723 +115,13 @@ int rgw_validate_tenant_name(const string& t) return (it == t.end())? 0: -ERR_INVALID_TENANT_NAME; } -static bool validate_access_key(string& key) -{ - const char *p = key.c_str(); - while (*p) { - if (!char_is_unreserved_url(*p)) - return false; - p++; - } - return true; -} - -static void set_err_msg(std::string *sink, std::string msg) -{ - if (sink && !msg.empty()) - *sink = msg; -} - -/* - * Dump either the full user info or a subset to a formatter. - * - * NOTE: It is the caller's responsibility to ensure that the - * formatter is flushed at the correct time. - */ - -static void dump_subusers_info(Formatter *f, RGWUserInfo &info) -{ - map::iterator uiter; - - f->open_array_section("subusers"); - for (uiter = info.subusers.begin(); uiter != info.subusers.end(); ++uiter) { - RGWSubUser& u = uiter->second; - f->open_object_section("user"); - string s; - info.user_id.to_str(s); - f->dump_format("id", "%s:%s", s.c_str(), u.name.c_str()); - char buf[256]; - rgw_perm_to_str(u.perm_mask, buf, sizeof(buf)); - f->dump_string("permissions", buf); - f->close_section(); - } - f->close_section(); -} - -static void dump_access_keys_info(Formatter *f, RGWUserInfo &info) -{ - map::iterator kiter; - f->open_array_section("keys"); - for (kiter = info.access_keys.begin(); kiter != info.access_keys.end(); ++kiter) { - RGWAccessKey& k = kiter->second; - const char *sep = (k.subuser.empty() ? "" : ":"); - const char *subuser = (k.subuser.empty() ? "" : k.subuser.c_str()); - f->open_object_section("key"); - string s; - info.user_id.to_str(s); - f->dump_format("user", "%s%s%s", s.c_str(), sep, subuser); - f->dump_string("access_key", k.id); - f->dump_string("secret_key", k.key); - f->close_section(); - } - f->close_section(); -} - -static void dump_swift_keys_info(Formatter *f, RGWUserInfo &info) -{ - map::iterator kiter; - f->open_array_section("swift_keys"); - for (kiter = info.swift_keys.begin(); kiter != info.swift_keys.end(); ++kiter) { - RGWAccessKey& k = kiter->second; - const char *sep = (k.subuser.empty() ? "" : ":"); - const char *subuser = (k.subuser.empty() ? "" : k.subuser.c_str()); - f->open_object_section("key"); - string s; - info.user_id.to_str(s); - f->dump_format("user", "%s%s%s", s.c_str(), sep, subuser); - f->dump_string("secret_key", k.key); - f->close_section(); - } - f->close_section(); -} - -static void dump_user_info(Formatter *f, RGWUserInfo &info, - RGWStorageStats *stats = NULL) -{ - f->open_object_section("user_info"); - encode_json("tenant", info.user_id.tenant, f); - encode_json("user_id", info.user_id.id, f); - encode_json("display_name", info.display_name, f); - encode_json("email", info.user_email, f); - encode_json("suspended", (int)info.suspended, f); - encode_json("max_buckets", (int)info.max_buckets, f); - - dump_subusers_info(f, info); - dump_access_keys_info(f, info); - dump_swift_keys_info(f, info); - - encode_json("caps", info.caps, f); - - char buf[256]; - op_type_to_str(info.op_mask, buf, sizeof(buf)); - encode_json("op_mask", (const char *)buf, f); - encode_json("system", (bool)info.system, f); - encode_json("admin", (bool)info.admin, f); - encode_json("default_placement", info.default_placement.name, f); - encode_json("default_storage_class", info.default_placement.storage_class, f); - encode_json("placement_tags", info.placement_tags, f); - encode_json("bucket_quota", info.quota.bucket_quota, f); - encode_json("user_quota", info.quota.user_quota, f); - encode_json("temp_url_keys", info.temp_url_keys, f); - - string user_source_type; - switch ((RGWIdentityType)info.type) { - case TYPE_RGW: - user_source_type = "rgw"; - break; - case TYPE_KEYSTONE: - user_source_type = "keystone"; - break; - case TYPE_LDAP: - user_source_type = "ldap"; - break; - case TYPE_NONE: - user_source_type = "none"; - break; - default: - user_source_type = "none"; - break; - } - encode_json("type", user_source_type, f); - encode_json("mfa_ids", info.mfa_ids, f); - if (stats) { - encode_json("stats", *stats, f); - } - f->close_section(); -} - -static int user_add_helper(RGWUserAdminOpState& op_state, std::string *err_msg) -{ - int ret = 0; - const rgw_user& uid = op_state.get_user_id(); - std::string user_email = op_state.get_user_email(); - std::string display_name = op_state.get_display_name(); - - // fail if the user exists already - if (op_state.has_existing_user()) { - if (op_state.found_by_email) { - set_err_msg(err_msg, "email: " + user_email + - " is the email address of an existing user"); - ret = -ERR_EMAIL_EXIST; - } else if (op_state.found_by_key) { - set_err_msg(err_msg, "duplicate key provided"); - ret = -ERR_KEY_EXIST; - } else { - set_err_msg(err_msg, "user: " + uid.to_str() + " exists"); - ret = -EEXIST; - } - return ret; - } - - // fail if the user_info has already been populated - if (op_state.is_populated()) { - set_err_msg(err_msg, "cannot overwrite already populated user"); - return -EEXIST; - } - - // fail if the display name was not included - if (display_name.empty()) { - set_err_msg(err_msg, "no display name specified"); - return -EINVAL; - } - - return ret; -} - -RGWAccessKeyPool::RGWAccessKeyPool(RGWUser* usr) -{ - if (!usr) { - return; - } - - user = usr; - - store = user->get_store(); -} - -int RGWAccessKeyPool::init(RGWUserAdminOpState& op_state) -{ - if (!op_state.is_initialized()) { - keys_allowed = false; - return -EINVAL; - } - - const rgw_user& uid = op_state.get_user_id(); - if (uid.compare(RGW_USER_ANON_ID) == 0) { - keys_allowed = false; - return -EINVAL; - } - - swift_keys = op_state.get_swift_keys(); - access_keys = op_state.get_access_keys(); - - keys_allowed = true; - - return 0; -} - -RGWUserAdminOpState::RGWUserAdminOpState(rgw::sal::Store* store) -{ - user = store->get_user(rgw_user(RGW_USER_ANON_ID)); -} - -void RGWUserAdminOpState::set_user_id(const rgw_user& id) -{ - if (id.empty()) - return; - - user->get_info().user_id = id; -} - -void RGWUserAdminOpState::set_subuser(std::string& _subuser) -{ - if (_subuser.empty()) - return; - - size_t pos = _subuser.find(":"); - if (pos != string::npos) { - rgw_user tmp_id; - tmp_id.from_str(_subuser.substr(0, pos)); - if (tmp_id.tenant.empty()) { - user->get_info().user_id.id = tmp_id.id; - } else { - user->get_info().user_id = tmp_id; - } - subuser = _subuser.substr(pos+1); - } else { - subuser = _subuser; - } - - subuser_specified = true; -} - -void RGWUserAdminOpState::set_user_info(RGWUserInfo& user_info) -{ - user->get_info() = user_info; -} - -void RGWUserAdminOpState::set_user_version_tracker(RGWObjVersionTracker& objv_tracker) -{ - user->get_version_tracker() = objv_tracker; -} - -const rgw_user& RGWUserAdminOpState::get_user_id() -{ - return user->get_id(); -} - -RGWUserInfo& RGWUserAdminOpState::get_user_info() -{ - return user->get_info(); -} - -map* RGWUserAdminOpState::get_swift_keys() -{ - return &user->get_info().swift_keys; -} - -map* RGWUserAdminOpState::get_access_keys() -{ - return &user->get_info().access_keys; -} - -map* RGWUserAdminOpState::get_subusers() -{ - return &user->get_info().subusers; -} - -RGWUserCaps *RGWUserAdminOpState::get_caps_obj() -{ - return &user->get_info().caps; -} - -std::string RGWUserAdminOpState::build_default_swift_kid() -{ - if (user->get_id().empty() || subuser.empty()) - return ""; - - std::string kid; - user->get_id().to_str(kid); - kid.append(":"); - kid.append(subuser); - - return kid; -} - -std::string RGWUserAdminOpState::generate_subuser() { - if (user->get_id().empty()) - return ""; - - std::string generated_subuser; - user->get_id().to_str(generated_subuser); - std::string rand_suffix; - - int sub_buf_size = RAND_SUBUSER_LEN + 1; - char sub_buf[RAND_SUBUSER_LEN + 1]; - - gen_rand_alphanumeric_upper(g_ceph_context, sub_buf, sub_buf_size); - - rand_suffix = sub_buf; - if (rand_suffix.empty()) - return ""; - - generated_subuser.append(rand_suffix); - subuser = generated_subuser; - - return generated_subuser; -} - -/* - * Do a fairly exhaustive search for an existing key matching the parameters - * given. Also handles the case where no key type was specified and updates - * the operation state if needed. +/** + * Get the anonymous (ie, unauthenticated) user info. */ - -bool RGWAccessKeyPool::check_existing_key(RGWUserAdminOpState& op_state) -{ - bool existing_key = false; - - int key_type = op_state.get_key_type(); - std::string kid = op_state.get_access_key(); - std::map::iterator kiter; - std::string swift_kid = op_state.build_default_swift_kid(); - - RGWUserInfo dup_info; - - if (kid.empty() && swift_kid.empty()) - return false; - - switch (key_type) { - case KEY_TYPE_SWIFT: - kiter = swift_keys->find(swift_kid); - - existing_key = (kiter != swift_keys->end()); - if (existing_key) - op_state.set_access_key(swift_kid); - - break; - case KEY_TYPE_S3: - kiter = access_keys->find(kid); - existing_key = (kiter != access_keys->end()); - - break; - default: - kiter = access_keys->find(kid); - - existing_key = (kiter != access_keys->end()); - if (existing_key) { - op_state.set_key_type(KEY_TYPE_S3); - break; - } - - kiter = swift_keys->find(kid); - - existing_key = (kiter != swift_keys->end()); - if (existing_key) { - op_state.set_key_type(KEY_TYPE_SWIFT); - break; - } - - // handle the case where the access key was not provided in user:key format - if (swift_kid.empty()) - return false; - - kiter = swift_keys->find(swift_kid); - - existing_key = (kiter != swift_keys->end()); - if (existing_key) { - op_state.set_access_key(swift_kid); - op_state.set_key_type(KEY_TYPE_SWIFT); - } - } - - op_state.set_existing_key(existing_key); - - return existing_key; -} - -int RGWAccessKeyPool::check_op(RGWUserAdminOpState& op_state, - std::string *err_msg) -{ - RGWUserInfo dup_info; - - if (!op_state.is_populated()) { - set_err_msg(err_msg, "user info was not populated"); - return -EINVAL; - } - - if (!keys_allowed) { - set_err_msg(err_msg, "keys not allowed for this user"); - return -EACCES; - } - - int32_t key_type = op_state.get_key_type(); - - // if a key type wasn't specified - if (key_type < 0) { - if (op_state.has_subuser()) { - key_type = KEY_TYPE_SWIFT; - } else { - key_type = KEY_TYPE_S3; - } - } - - op_state.set_key_type(key_type); - - /* see if the access key was specified */ - if (key_type == KEY_TYPE_S3 && !op_state.will_gen_access() && - op_state.get_access_key().empty()) { - set_err_msg(err_msg, "empty access key"); - return -ERR_INVALID_ACCESS_KEY; - } - - // don't check for secret key because we may be doing a removal - - if (check_existing_key(op_state)) { - op_state.set_access_key_exist(); - } - return 0; -} - -// Generate a new random key -int RGWAccessKeyPool::generate_key(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, - optional_yield y, std::string *err_msg) -{ - std::string id; - std::string key; - - std::pair key_pair; - RGWAccessKey new_key; - std::unique_ptr duplicate_check; - - int key_type = op_state.get_key_type(); - bool gen_access = op_state.will_gen_access(); - bool gen_secret = op_state.will_gen_secret(); - - if (!keys_allowed) { - set_err_msg(err_msg, "access keys not allowed for this user"); - return -EACCES; - } - - if (op_state.has_existing_key()) { - set_err_msg(err_msg, "cannot create existing key"); - return -ERR_KEY_EXIST; - } - - if (!gen_access) { - id = op_state.get_access_key(); - } - - if (!id.empty()) { - switch (key_type) { - case KEY_TYPE_SWIFT: - if (store->get_user_by_swift(dpp, id, y, &duplicate_check) >= 0) { - set_err_msg(err_msg, "existing swift key in RGW system:" + id); - return -ERR_KEY_EXIST; - } - break; - case KEY_TYPE_S3: - if (store->get_user_by_access_key(dpp, id, y, &duplicate_check) >= 0) { - set_err_msg(err_msg, "existing S3 key in RGW system:" + id); - return -ERR_KEY_EXIST; - } - } - } - - //key's subuser - if (op_state.has_subuser()) { - //create user and subuser at the same time, user's s3 key should not be set this - if (!op_state.key_type_setbycontext || (key_type == KEY_TYPE_SWIFT)) { - new_key.subuser = op_state.get_subuser(); - } - } - - //Secret key - if (!gen_secret) { - if (op_state.get_secret_key().empty()) { - set_err_msg(err_msg, "empty secret key"); - return -ERR_INVALID_SECRET_KEY; - } - - key = op_state.get_secret_key(); - } else { - char secret_key_buf[SECRET_KEY_LEN + 1]; - gen_rand_alphanumeric_plain(g_ceph_context, secret_key_buf, sizeof(secret_key_buf)); - key = secret_key_buf; - } - - // Generate the access key - if (key_type == KEY_TYPE_S3 && gen_access) { - char public_id_buf[PUBLIC_ID_LEN + 1]; - - do { - int id_buf_size = sizeof(public_id_buf); - gen_rand_alphanumeric_upper(g_ceph_context, public_id_buf, id_buf_size); - id = public_id_buf; - if (!validate_access_key(id)) - continue; - - } while (!store->get_user_by_access_key(dpp, id, y, &duplicate_check)); - } - - if (key_type == KEY_TYPE_SWIFT) { - id = op_state.build_default_swift_kid(); - if (id.empty()) { - set_err_msg(err_msg, "empty swift access key"); - return -ERR_INVALID_ACCESS_KEY; - } - - // check that the access key doesn't exist - if (store->get_user_by_swift(dpp, id, y, &duplicate_check) >= 0) { - set_err_msg(err_msg, "cannot create existing swift key"); - return -ERR_KEY_EXIST; - } - } - - // finally create the new key - new_key.id = id; - new_key.key = key; - - key_pair.first = id; - key_pair.second = new_key; - - if (key_type == KEY_TYPE_S3) { - access_keys->insert(key_pair); - } else if (key_type == KEY_TYPE_SWIFT) { - swift_keys->insert(key_pair); - } - - return 0; -} - -// modify an existing key -int RGWAccessKeyPool::modify_key(RGWUserAdminOpState& op_state, std::string *err_msg) -{ - std::string id; - std::string key = op_state.get_secret_key(); - int key_type = op_state.get_key_type(); - - RGWAccessKey modify_key; - - pair key_pair; - map::iterator kiter; - - switch (key_type) { - case KEY_TYPE_S3: - id = op_state.get_access_key(); - if (id.empty()) { - set_err_msg(err_msg, "no access key specified"); - return -ERR_INVALID_ACCESS_KEY; - } - break; - case KEY_TYPE_SWIFT: - id = op_state.build_default_swift_kid(); - if (id.empty()) { - set_err_msg(err_msg, "no subuser specified"); - return -EINVAL; - } - break; - default: - set_err_msg(err_msg, "invalid key type"); - return -ERR_INVALID_KEY_TYPE; - } - - if (!op_state.has_existing_key()) { - set_err_msg(err_msg, "key does not exist"); - return -ERR_INVALID_ACCESS_KEY; - } - - key_pair.first = id; - - if (key_type == KEY_TYPE_SWIFT) { - modify_key.id = id; - modify_key.subuser = op_state.get_subuser(); - } else if (key_type == KEY_TYPE_S3) { - kiter = access_keys->find(id); - if (kiter != access_keys->end()) { - modify_key = kiter->second; - } - } - - if (op_state.will_gen_secret()) { - char secret_key_buf[SECRET_KEY_LEN + 1]; - int key_buf_size = sizeof(secret_key_buf); - gen_rand_alphanumeric_plain(g_ceph_context, secret_key_buf, key_buf_size); - key = secret_key_buf; - } - - if (key.empty()) { - set_err_msg(err_msg, "empty secret key"); - return -ERR_INVALID_SECRET_KEY; - } - - // update the access key with the new secret key - modify_key.key = key; - - key_pair.second = modify_key; - - - if (key_type == KEY_TYPE_S3) { - (*access_keys)[id] = modify_key; - } else if (key_type == KEY_TYPE_SWIFT) { - (*swift_keys)[id] = modify_key; - } - - return 0; -} - -int RGWAccessKeyPool::execute_add(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, - std::string *err_msg, bool defer_user_update, - optional_yield y) +void rgw_get_anon_user(RGWUserInfo& info) { - int ret = 0; - - std::string subprocess_msg; - int key_op = GENERATE_KEY; - - // set the op - if (op_state.has_existing_key()) - key_op = MODIFY_KEY; - - switch (key_op) { - case GENERATE_KEY: - ret = generate_key(dpp, op_state, y, &subprocess_msg); - break; - case MODIFY_KEY: - ret = modify_key(op_state, &subprocess_msg); - break; - } - - if (ret < 0) { - set_err_msg(err_msg, subprocess_msg); - return ret; - } - - // store the updated info - if (!defer_user_update) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -int RGWAccessKeyPool::add(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, optional_yield y, - std::string *err_msg) -{ - return add(dpp, op_state, err_msg, false, y); -} - -int RGWAccessKeyPool::add(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, std::string *err_msg, - bool defer_user_update, optional_yield y) -{ - int ret; - std::string subprocess_msg; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); - return ret; - } - - ret = execute_add(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to add access key, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWAccessKeyPool::execute_remove(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, - std::string *err_msg, - bool defer_user_update, - optional_yield y) -{ - int ret = 0; - - int key_type = op_state.get_key_type(); - std::string id = op_state.get_access_key(); - map::iterator kiter; - map *keys_map; - - if (!op_state.has_existing_key()) { - set_err_msg(err_msg, "unable to find access key, with key type: " + - key_type_to_str(key_type)); - return -ERR_INVALID_ACCESS_KEY; - } - - if (key_type == KEY_TYPE_S3) { - keys_map = access_keys; - } else if (key_type == KEY_TYPE_SWIFT) { - keys_map = swift_keys; - } else { - keys_map = NULL; - set_err_msg(err_msg, "invalid access key"); - return -ERR_INVALID_ACCESS_KEY; - } - - kiter = keys_map->find(id); - if (kiter == keys_map->end()) { - set_err_msg(err_msg, "key not found"); - return -ERR_INVALID_ACCESS_KEY; - } - - keys_map->erase(kiter); - - if (!defer_user_update) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -int RGWAccessKeyPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, - std::string *err_msg) -{ - return remove(dpp, op_state, err_msg, false, y); -} - -int RGWAccessKeyPool::remove(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, - std::string *err_msg, bool defer_user_update, - optional_yield y) -{ - int ret; - - std::string subprocess_msg; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); - return ret; - } - - ret = execute_remove(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to remove access key, " + subprocess_msg); - return ret; - } - - return 0; -} - -// remove all keys associated with a subuser -int RGWAccessKeyPool::remove_subuser_keys(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, - std::string *err_msg, - bool defer_user_update, - optional_yield y) -{ - int ret = 0; - - if (!op_state.is_populated()) { - set_err_msg(err_msg, "user info was not populated"); - return -EINVAL; - } - - if (!op_state.has_subuser()) { - set_err_msg(err_msg, "no subuser specified"); - return -EINVAL; - } - - std::string swift_kid = op_state.build_default_swift_kid(); - if (swift_kid.empty()) { - set_err_msg(err_msg, "empty swift access key"); - return -EINVAL; - } - - map::iterator kiter; - map *keys_map; - - // a subuser can have at most one swift key - keys_map = swift_keys; - kiter = keys_map->find(swift_kid); - if (kiter != keys_map->end()) { - keys_map->erase(kiter); - } - - // a subuser may have multiple s3 key pairs - std::string subuser_str = op_state.get_subuser(); - keys_map = access_keys; - RGWUserInfo user_info = op_state.get_user_info(); - auto user_kiter = user_info.access_keys.begin(); - for (; user_kiter != user_info.access_keys.end(); ++user_kiter) { - if (user_kiter->second.subuser == subuser_str) { - kiter = keys_map->find(user_kiter->first); - if (kiter != keys_map->end()) { - keys_map->erase(kiter); - } - } - } - - if (!defer_user_update) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -RGWSubUserPool::RGWSubUserPool(RGWUser *usr) -{ - if (!usr) { - return; - } - - user = usr; - - subusers_allowed = true; - store = user->get_store(); -} - -int RGWSubUserPool::init(RGWUserAdminOpState& op_state) -{ - if (!op_state.is_initialized()) { - subusers_allowed = false; - return -EINVAL; - } - - const rgw_user& uid = op_state.get_user_id(); - if (uid.compare(RGW_USER_ANON_ID) == 0) { - subusers_allowed = false; - return -EACCES; - } - - subuser_map = op_state.get_subusers(); - if (subuser_map == NULL) { - subusers_allowed = false; - return -EINVAL; - } - - subusers_allowed = true; - - return 0; -} - -bool RGWSubUserPool::exists(std::string subuser) -{ - if (subuser.empty()) - return false; - - if (!subuser_map) - return false; - - if (subuser_map->count(subuser)) - return true; - - return false; -} - -int RGWSubUserPool::check_op(RGWUserAdminOpState& op_state, - std::string *err_msg) -{ - bool existing = false; - std::string subuser = op_state.get_subuser(); - - if (!op_state.is_populated()) { - set_err_msg(err_msg, "user info was not populated"); - return -EINVAL; - } - - if (!subusers_allowed) { - set_err_msg(err_msg, "subusers not allowed for this user"); - return -EACCES; - } - - if (subuser.empty() && !op_state.will_gen_subuser()) { - set_err_msg(err_msg, "empty subuser name"); - return -EINVAL; - } - - if (op_state.get_subuser_perm() == RGW_PERM_INVALID) { - set_err_msg(err_msg, "invalid subuser access"); - return -EINVAL; - } - - //set key type when it not set or set by context - if ((op_state.get_key_type() < 0) || op_state.key_type_setbycontext) { - op_state.set_key_type(KEY_TYPE_SWIFT); - op_state.key_type_setbycontext = true; - } - - // check if the subuser exists - if (!subuser.empty()) - existing = exists(subuser); - - op_state.set_existing_subuser(existing); - - return 0; -} - -int RGWSubUserPool::execute_add(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, - std::string *err_msg, bool defer_user_update, - optional_yield y) -{ - int ret = 0; - std::string subprocess_msg; - - RGWSubUser subuser; - std::pair subuser_pair; - std::string subuser_str = op_state.get_subuser(); - - subuser_pair.first = subuser_str; - - // assumes key should be created - if (op_state.has_key_op()) { - ret = user->keys.add(dpp, op_state, &subprocess_msg, true, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to create subuser key, " + subprocess_msg); - return ret; - } - } - - // create the subuser - subuser.name = subuser_str; - - if (op_state.has_subuser_perm()) - subuser.perm_mask = op_state.get_subuser_perm(); - - // insert the subuser into user info - subuser_pair.second = subuser; - subuser_map->insert(subuser_pair); - - // attempt to save the subuser - if (!defer_user_update) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -int RGWSubUserPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, - std::string *err_msg) -{ - return add(dpp, op_state, err_msg, false, y); -} - -int RGWSubUserPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update, optional_yield y) -{ - std::string subprocess_msg; - int ret; - int32_t key_type = op_state.get_key_type(); - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); - return ret; - } - - if (op_state.get_access_key_exist()) { - set_err_msg(err_msg, "cannot create existing key"); - return -ERR_KEY_EXIST; - } - - if (key_type == KEY_TYPE_S3 && op_state.get_access_key().empty()) { - op_state.set_gen_access(); - } - - if (op_state.get_secret_key().empty()) { - op_state.set_gen_secret(); - } - - ret = execute_add(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to create subuser, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWSubUserPool::execute_remove(const DoutPrefixProvider *dpp, - RGWUserAdminOpState& op_state, - std::string *err_msg, bool defer_user_update, - optional_yield y) -{ - int ret = 0; - std::string subprocess_msg; - - std::string subuser_str = op_state.get_subuser(); - - map::iterator siter; - siter = subuser_map->find(subuser_str); - if (siter == subuser_map->end()){ - set_err_msg(err_msg, "subuser not found: " + subuser_str); - return -ERR_NO_SUCH_SUBUSER; - } - if (!op_state.has_existing_subuser()) { - set_err_msg(err_msg, "subuser not found: " + subuser_str); - return -ERR_NO_SUCH_SUBUSER; - } - - // always purge all associate keys - user->keys.remove_subuser_keys(dpp, op_state, &subprocess_msg, true, y); - - // remove the subuser from the user info - subuser_map->erase(siter); - - // attempt to save the subuser - if (!defer_user_update) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -int RGWSubUserPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, - std::string *err_msg) -{ - return remove(dpp, op_state, err_msg, false, y); -} - -int RGWSubUserPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, - bool defer_user_update, optional_yield y) -{ - std::string subprocess_msg; - int ret; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); - return ret; - } - - ret = execute_remove(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to remove subuser, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWSubUserPool::execute_modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update, optional_yield y) -{ - int ret = 0; - std::string subprocess_msg; - std::map::iterator siter; - std::pair subuser_pair; - - std::string subuser_str = op_state.get_subuser(); - RGWSubUser subuser; - - if (!op_state.has_existing_subuser()) { - set_err_msg(err_msg, "subuser does not exist"); - return -ERR_NO_SUCH_SUBUSER; - } - - subuser_pair.first = subuser_str; - - siter = subuser_map->find(subuser_str); - subuser = siter->second; - - if (op_state.has_key_op()) { - ret = user->keys.add(dpp, op_state, &subprocess_msg, true, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to create subuser keys, " + subprocess_msg); - return ret; - } - } - - if (op_state.has_subuser_perm()) - subuser.perm_mask = op_state.get_subuser_perm(); - - subuser_pair.second = subuser; - - subuser_map->erase(siter); - subuser_map->insert(subuser_pair); - - // attempt to save the subuser - if (!defer_user_update) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -int RGWSubUserPool::modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) -{ - return RGWSubUserPool::modify(dpp, op_state, y, err_msg, false); -} - -int RGWSubUserPool::modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg, bool defer_user_update) -{ - std::string subprocess_msg; - int ret; - - RGWSubUser subuser; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); - return ret; - } - - ret = execute_modify(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to modify subuser, " + subprocess_msg); - return ret; - } - - return 0; -} - -RGWUserCapPool::RGWUserCapPool(RGWUser *usr) -{ - if (!usr) { - return; - } - user = usr; - caps_allowed = true; -} - -int RGWUserCapPool::init(RGWUserAdminOpState& op_state) -{ - if (!op_state.is_initialized()) { - caps_allowed = false; - return -EINVAL; - } - - const rgw_user& uid = op_state.get_user_id(); - if (uid.compare(RGW_USER_ANON_ID) == 0) { - caps_allowed = false; - return -EACCES; - } - - caps = op_state.get_caps_obj(); - if (!caps) { - caps_allowed = false; - return -ERR_INVALID_CAP; - } - - caps_allowed = true; - - return 0; -} - -int RGWUserCapPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, - std::string *err_msg) -{ - return add(dpp, op_state, err_msg, false, y); -} - -int RGWUserCapPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, - bool defer_save, optional_yield y) -{ - int ret = 0; - std::string caps_str = op_state.get_caps(); - - if (!op_state.is_populated()) { - set_err_msg(err_msg, "user info was not populated"); - return -EINVAL; - } - - if (!caps_allowed) { - set_err_msg(err_msg, "caps not allowed for this user"); - return -EACCES; - } - - if (caps_str.empty()) { - set_err_msg(err_msg, "empty user caps"); - return -ERR_INVALID_CAP; - } - - int r = caps->add_from_string(caps_str); - if (r < 0) { - set_err_msg(err_msg, "unable to add caps: " + caps_str); - return r; - } - - if (!defer_save) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -int RGWUserCapPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, - std::string *err_msg) -{ - return remove(dpp, op_state, err_msg, false, y); -} - -int RGWUserCapPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, - bool defer_save, optional_yield y) -{ - int ret = 0; - - std::string caps_str = op_state.get_caps(); - - if (!op_state.is_populated()) { - set_err_msg(err_msg, "user info was not populated"); - return -EINVAL; - } - - if (!caps_allowed) { - set_err_msg(err_msg, "caps not allowed for this user"); - return -EACCES; - } - - if (caps_str.empty()) { - set_err_msg(err_msg, "empty user caps"); - return -ERR_INVALID_CAP; - } - - int r = caps->remove_from_string(caps_str); - if (r < 0) { - set_err_msg(err_msg, "unable to remove caps: " + caps_str); - return r; - } - - if (!defer_save) - ret = user->update(dpp, op_state, err_msg, y); - - if (ret < 0) - return ret; - - return 0; -} - -RGWUser::RGWUser() : caps(this), keys(this), subusers(this) -{ - init_default(); -} - -int RGWUser::init(const DoutPrefixProvider *dpp, rgw::sal::Store* storage, - RGWUserAdminOpState& op_state, optional_yield y) -{ - init_default(); - int ret = init_storage(storage); - if (ret < 0) - return ret; - - ret = init(dpp, op_state, y); - if (ret < 0) - return ret; - - return 0; -} - -void RGWUser::init_default() -{ - // use anonymous user info as a placeholder - rgw_get_anon_user(old_info); - user_id = RGW_USER_ANON_ID; - - clear_populated(); -} - -int RGWUser::init_storage(rgw::sal::Store* storage) -{ - if (!storage) { - return -EINVAL; - } - - store = storage; - - clear_populated(); - - /* API wrappers */ - keys = RGWAccessKeyPool(this); - caps = RGWUserCapPool(this); - subusers = RGWSubUserPool(this); - - return 0; -} - -int RGWUser::init(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y) -{ - bool found = false; - std::string swift_user; - user_id = op_state.get_user_id(); - std::string user_email = op_state.get_user_email(); - std::string access_key = op_state.get_access_key(); - std::string subuser = op_state.get_subuser(); - - int key_type = op_state.get_key_type(); - if (key_type == KEY_TYPE_SWIFT) { - swift_user = op_state.get_access_key(); - access_key.clear(); - } - - std::unique_ptr user; - - clear_populated(); - - if (user_id.empty() && !subuser.empty()) { - size_t pos = subuser.find(':'); - if (pos != string::npos) { - user_id = subuser.substr(0, pos); - op_state.set_user_id(user_id); - } - } - - if (!user_id.empty() && (user_id.compare(RGW_USER_ANON_ID) != 0)) { - user = store->get_user(user_id); - found = (user->load_user(dpp, y) >= 0); - op_state.found_by_uid = found; - } - if (store->ctx()->_conf.get_val("rgw_user_unique_email")) { - if (!user_email.empty() && !found) { - found = (store->get_user_by_email(dpp, user_email, y, &user) >= 0); - op_state.found_by_email = found; - } - } - if (!swift_user.empty() && !found) { - found = (store->get_user_by_swift(dpp, swift_user, y, &user) >= 0); - op_state.found_by_key = found; - } - if (!access_key.empty() && !found) { - found = (store->get_user_by_access_key(dpp, access_key, y, &user) >= 0); - op_state.found_by_key = found; - } - - op_state.set_existing_user(found); - if (found) { - op_state.set_user_info(user->get_info()); - op_state.set_populated(); - op_state.objv = user->get_version_tracker(); - op_state.set_user_version_tracker(user->get_version_tracker()); - - old_info = user->get_info(); - set_populated(); - } - - if (user_id.empty()) { - user_id = user->get_id(); - } - op_state.set_initialized(); - - // this may have been called by a helper object - int ret = init_members(op_state); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUser::init_members(RGWUserAdminOpState& op_state) -{ - int ret = 0; - - ret = keys.init(op_state); - if (ret < 0) - return ret; - - ret = subusers.init(op_state); - if (ret < 0) - return ret; - - ret = caps.init(op_state); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUser::update(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, - optional_yield y) -{ - int ret; - std::string subprocess_msg; - rgw::sal::User* user = op_state.get_user(); - - if (!store) { - set_err_msg(err_msg, "couldn't initialize storage"); - return -EINVAL; - } - - RGWUserInfo *pold_info = (is_populated() ? &old_info : nullptr); - - ret = user->store_user(dpp, y, false, pold_info); - op_state.objv = user->get_version_tracker(); - op_state.set_user_version_tracker(user->get_version_tracker()); - - if (ret < 0) { - set_err_msg(err_msg, "unable to store user info"); - return ret; - } - - old_info = user->get_info(); - set_populated(); - - return 0; -} - -int RGWUser::check_op(RGWUserAdminOpState& op_state, std::string *err_msg) -{ - int ret = 0; - const rgw_user& uid = op_state.get_user_id(); - - if (uid.compare(RGW_USER_ANON_ID) == 0) { - set_err_msg(err_msg, "unable to perform operations on the anonymous user"); - return -EINVAL; - } - - if (is_populated() && user_id.compare(uid) != 0) { - set_err_msg(err_msg, "user id mismatch, operation id: " + uid.to_str() - + " does not match: " + user_id.to_str()); - - return -EINVAL; - } - - ret = rgw_validate_tenant_name(uid.tenant); - if (ret) { - set_err_msg(err_msg, - "invalid tenant only alphanumeric and _ characters are allowed"); - return ret; - } - - //set key type when it not set or set by context - if ((op_state.get_key_type() < 0) || op_state.key_type_setbycontext) { - op_state.set_key_type(KEY_TYPE_S3); - op_state.key_type_setbycontext = true; - } - - return 0; -} - -// update swift_keys with new user id -static void rename_swift_keys(const rgw_user& user, - std::map& keys) -{ - std::string user_id; - user.to_str(user_id); - - auto modify_keys = std::move(keys); - for ([[maybe_unused]] auto& [k, key] : modify_keys) { - std::string id = user_id + ":" + key.subuser; - key.id = id; - keys[id] = std::move(key); - } -} - -int RGWUser::execute_rename(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, optional_yield y) -{ - int ret; - bool populated = op_state.is_populated(); - - if (!op_state.has_existing_user() && !populated) { - set_err_msg(err_msg, "user not found"); - return -ENOENT; - } - - if (!populated) { - ret = init(dpp, op_state, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to retrieve user info"); - return ret; - } - } - - std::unique_ptr old_user = store->get_user(op_state.get_user_info().user_id); - std::unique_ptr new_user = store->get_user(op_state.get_new_uid()); - if (old_user->get_tenant() != new_user->get_tenant()) { - set_err_msg(err_msg, "users have to be under the same tenant namespace " - + old_user->get_tenant() + " != " + new_user->get_tenant()); - return -EINVAL; - } - - // create a stub user and write only the uid index and buckets object - std::unique_ptr user; - user = store->get_user(new_user->get_id()); - - const bool exclusive = !op_state.get_overwrite_new_user(); // overwrite if requested - - ret = user->store_user(dpp, y, exclusive); - if (ret == -EEXIST) { - set_err_msg(err_msg, "user name given by --new-uid already exists"); - return ret; - } - if (ret < 0) { - set_err_msg(err_msg, "unable to store new user info"); - return ret; - } - - RGWAccessControlPolicy policy_instance; - policy_instance.create_default(new_user->get_id(), old_user->get_display_name()); - - //unlink and link buckets to new user - string marker; - CephContext *cct = store->ctx(); - size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; - rgw::sal::BucketList buckets; - - do { - ret = old_user->list_buckets(dpp, marker, "", max_buckets, false, buckets, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to list user buckets"); - return ret; - } - - auto& m = buckets.get_buckets(); - - for (auto it = m.begin(); it != m.end(); ++it) { - auto& bucket = it->second; - marker = it->first; - - ret = bucket->load_bucket(dpp, y); - if (ret < 0) { - set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name()); - return ret; - } - - ret = bucket->set_acl(dpp, policy_instance, y); - if (ret < 0) { - set_err_msg(err_msg, "failed to set acl on bucket " + bucket->get_name()); - return ret; - } - - ret = bucket->chown(dpp, new_user.get(), old_user.get(), y); - if (ret < 0) { - set_err_msg(err_msg, "failed to run bucket chown" + cpp_strerror(-ret)); - return ret; - } - } - - } while (buckets.is_truncated()); - - // update the 'stub user' with all of the other fields and rewrite all of the - // associated index objects - RGWUserInfo& user_info = op_state.get_user_info(); - user_info.user_id = new_user->get_id(); - op_state.objv = user->get_version_tracker(); - op_state.set_user_version_tracker(user->get_version_tracker()); - - rename_swift_keys(new_user->get_id(), user_info.swift_keys); - - return update(dpp, op_state, err_msg, y); -} - -int RGWUser::execute_add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, - optional_yield y) -{ - const rgw_user& uid = op_state.get_user_id(); - std::string user_email = op_state.get_user_email(); - std::string display_name = op_state.get_display_name(); - - // set the user info - RGWUserInfo user_info; - user_id = uid; - user_info.user_id = user_id; - user_info.display_name = display_name; - user_info.type = TYPE_RGW; - - if (!user_email.empty()) - user_info.user_email = user_email; - - CephContext *cct = store->ctx(); - if (op_state.max_buckets_specified) { - user_info.max_buckets = op_state.get_max_buckets(); - } else { - user_info.max_buckets = - cct->_conf.get_val("rgw_user_max_buckets"); - } - - user_info.suspended = op_state.get_suspension_status(); - user_info.admin = op_state.admin; - user_info.system = op_state.system; - - if (op_state.op_mask_specified) - user_info.op_mask = op_state.get_op_mask(); - - if (op_state.has_bucket_quota()) { - user_info.quota.bucket_quota = op_state.get_bucket_quota(); - } else { - rgw_apply_default_bucket_quota(user_info.quota.bucket_quota, cct->_conf); - } - - if (op_state.temp_url_key_specified) { - map::iterator iter; - for (iter = op_state.temp_url_keys.begin(); - iter != op_state.temp_url_keys.end(); ++iter) { - user_info.temp_url_keys[iter->first] = iter->second; - } - } - - if (op_state.has_user_quota()) { - user_info.quota.user_quota = op_state.get_user_quota(); - } else { - rgw_apply_default_user_quota(user_info.quota.user_quota, cct->_conf); - } - - if (op_state.default_placement_specified) { - user_info.default_placement = op_state.default_placement; - } - - if (op_state.placement_tags_specified) { - user_info.placement_tags = op_state.placement_tags; - } - - // update the request - op_state.set_user_info(user_info); - op_state.set_populated(); - - // update the helper objects - int ret = init_members(op_state); - if (ret < 0) { - set_err_msg(err_msg, "unable to initialize user"); - return ret; - } - - // see if we need to add an access key - std::string subprocess_msg; - bool defer_user_update = true; - if (op_state.has_key_op()) { - ret = keys.add(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to create access key, " + subprocess_msg); - return ret; - } - } - - // see if we need to add some caps - if (op_state.has_caps_op()) { - ret = caps.add(dpp, op_state, &subprocess_msg, defer_user_update, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to add user capabilities, " + subprocess_msg); - return ret; - } - } - - ret = update(dpp, op_state, err_msg, y); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUser::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) -{ - std::string subprocess_msg; - int ret = user_add_helper(op_state, &subprocess_msg); - if (ret != 0) { - set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); - return ret; - } - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); - return ret; - } - - ret = execute_add(dpp, op_state, &subprocess_msg, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to create user, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWUser::rename(RGWUserAdminOpState& op_state, optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg) -{ - std::string subprocess_msg; - int ret; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); - return ret; - } - - ret = execute_rename(dpp, op_state, &subprocess_msg, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to rename user, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWUser::execute_remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, optional_yield y) -{ - int ret; - - bool purge_data = op_state.will_purge_data(); - rgw::sal::User* user = op_state.get_user(); - - if (!op_state.has_existing_user()) { - set_err_msg(err_msg, "user does not exist"); - return -ENOENT; - } - - rgw::sal::BucketList buckets; - string marker; - CephContext *cct = store->ctx(); - size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; - do { - ret = user->list_buckets(dpp, marker, string(), max_buckets, false, buckets, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to read user bucket info"); - return ret; - } - - auto& m = buckets.get_buckets(); - if (!m.empty() && !purge_data) { - set_err_msg(err_msg, "must specify purge data to remove user with buckets"); - return -EEXIST; // change to code that maps to 409: conflict - } - - for (auto it = m.begin(); it != m.end(); ++it) { - ret = it->second->remove_bucket(dpp, true, false, nullptr, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to delete user data"); - return ret; - } - - marker = it->first; - } - - } while (buckets.is_truncated()); - - ret = user->remove_user(dpp, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to remove user from RADOS"); - return ret; - } - - op_state.clear_populated(); - clear_populated(); - - return 0; -} - -int RGWUser::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) -{ - std::string subprocess_msg; - int ret; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); - return ret; - } - - ret = execute_remove(dpp, op_state, &subprocess_msg, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to remove user, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWUser::execute_modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, optional_yield y) -{ - bool populated = op_state.is_populated(); - int ret = 0; - std::string subprocess_msg; - std::string op_email = op_state.get_user_email(); - std::string display_name = op_state.get_display_name(); - - RGWUserInfo user_info; - std::unique_ptr duplicate_check; - - // ensure that the user info has been populated or is populate-able - if (!op_state.has_existing_user() && !populated) { - set_err_msg(err_msg, "user not found"); - return -ENOENT; - } - - // if the user hasn't already been populated...attempt to - if (!populated) { - ret = init(dpp, op_state, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to retrieve user info"); - return ret; - } - } - - // ensure that we can modify the user's attributes - if (user_id.compare(RGW_USER_ANON_ID) == 0) { - set_err_msg(err_msg, "unable to modify anonymous user's info"); - return -EACCES; - } - - user_info = old_info; - - std::string old_email = old_info.user_email; - if (!op_email.empty()) { - // make sure we are not adding a duplicate email - if (old_email != op_email) { - ret = store->get_user_by_email(dpp, op_email, y, &duplicate_check); - if (ret >= 0 && duplicate_check->get_id().compare(user_id) != 0) { - set_err_msg(err_msg, "cannot add duplicate email"); - return -ERR_EMAIL_EXIST; - } - } - user_info.user_email = op_email; - } else if (op_email.empty() && op_state.user_email_specified) { - ldpp_dout(dpp, 10) << "removing email index: " << user_info.user_email << dendl; - /* will be physically removed later when calling update() */ - user_info.user_email.clear(); - } - - // update the remaining user info - if (!display_name.empty()) - user_info.display_name = display_name; - - if (op_state.max_buckets_specified) - user_info.max_buckets = op_state.get_max_buckets(); - - if (op_state.admin_specified) - user_info.admin = op_state.admin; - - if (op_state.system_specified) - user_info.system = op_state.system; - - if (op_state.temp_url_key_specified) { - map::iterator iter; - for (iter = op_state.temp_url_keys.begin(); - iter != op_state.temp_url_keys.end(); ++iter) { - user_info.temp_url_keys[iter->first] = iter->second; - } - } - - if (op_state.op_mask_specified) - user_info.op_mask = op_state.get_op_mask(); - - if (op_state.has_bucket_quota()) - user_info.quota.bucket_quota = op_state.get_bucket_quota(); - - if (op_state.has_user_quota()) - user_info.quota.user_quota = op_state.get_user_quota(); - - if (op_state.has_suspension_op()) { - __u8 suspended = op_state.get_suspension_status(); - user_info.suspended = suspended; - - rgw::sal::BucketList buckets; - - if (user_id.empty()) { - set_err_msg(err_msg, "empty user id passed...aborting"); - return -EINVAL; - } - - string marker; - CephContext *cct = store->ctx(); - size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; - std::unique_ptr user = store->get_user(user_id); - do { - ret = user->list_buckets(dpp, marker, string(), max_buckets, false, buckets, y); - if (ret < 0) { - set_err_msg(err_msg, "could not get buckets for uid: " + user_id.to_str()); - return ret; - } - - auto& m = buckets.get_buckets(); - - vector bucket_names; - for (auto iter = m.begin(); iter != m.end(); ++iter) { - auto& bucket = iter->second; - bucket_names.push_back(bucket->get_key()); - - marker = iter->first; - } - - ret = store->set_buckets_enabled(dpp, bucket_names, !suspended); - if (ret < 0) { - set_err_msg(err_msg, "failed to modify bucket"); - return ret; - } - - } while (buckets.is_truncated()); - } - - if (op_state.mfa_ids_specified) { - user_info.mfa_ids = op_state.mfa_ids; - } - - if (op_state.default_placement_specified) { - user_info.default_placement = op_state.default_placement; - } - - if (op_state.placement_tags_specified) { - user_info.placement_tags = op_state.placement_tags; - } - - op_state.set_user_info(user_info); - - // if we're supposed to modify keys, do so - if (op_state.has_key_op()) { - ret = keys.add(dpp, op_state, &subprocess_msg, true, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to create or modify keys, " + subprocess_msg); - return ret; - } - } - - ret = update(dpp, op_state, err_msg, y); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUser::modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) -{ - std::string subprocess_msg; - int ret; - - ret = check_op(op_state, &subprocess_msg); - if (ret < 0) { - set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); - return ret; - } - - ret = execute_modify(dpp, op_state, &subprocess_msg, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to modify user, " + subprocess_msg); - return ret; - } - - return 0; -} - -int RGWUser::info(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, RGWUserInfo& fetched_info, - optional_yield y, std::string *err_msg) -{ - int ret = init(dpp, op_state, y); - if (ret < 0) { - set_err_msg(err_msg, "unable to fetch user info"); - return ret; - } - - fetched_info = op_state.get_user_info(); - - return 0; -} - -int RGWUser::info(RGWUserInfo& fetched_info, std::string *err_msg) -{ - if (!is_populated()) { - set_err_msg(err_msg, "no user info saved"); - return -EINVAL; - } - - fetched_info = old_info; - - return 0; -} - -int RGWUser::list(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher) -{ - Formatter *formatter = flusher.get_formatter(); - void *handle = nullptr; - std::string metadata_key = "user"; - if (op_state.max_entries > 1000) { - op_state.max_entries = 1000; - } - - int ret = store->meta_list_keys_init(dpp, metadata_key, op_state.marker, &handle); - if (ret < 0) { - return ret; - } - - bool truncated = false; - uint64_t count = 0; - uint64_t left = 0; - flusher.start(0); - - // open the result object section - formatter->open_object_section("result"); - - // open the user id list array section - formatter->open_array_section("keys"); - do { - std::list keys; - left = op_state.max_entries - count; - ret = store->meta_list_keys_next(dpp, handle, left, keys, &truncated); - if (ret < 0 && ret != -ENOENT) { - return ret; - } if (ret != -ENOENT) { - for (std::list::iterator iter = keys.begin(); iter != keys.end(); ++iter) { - formatter->dump_string("key", *iter); - ++count; - } - } - } while (truncated && left > 0); - // close user id list section - formatter->close_section(); - - formatter->dump_bool("truncated", truncated); - formatter->dump_int("count", count); - if (truncated) { - formatter->dump_string("marker", store->meta_get_marker(handle)); - } - - // close result object section - formatter->close_section(); - - store->meta_list_keys_complete(handle); - - flusher.flush(); - return 0; -} - -int RGWUserAdminOp_User::list(const DoutPrefixProvider *dpp, rgw::sal::Store* store, RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher) -{ - RGWUser user; - - int ret = user.init_storage(store); - if (ret < 0) - return ret; - - ret = user.list(dpp, op_state, flusher); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUserAdminOp_User::info(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - std::unique_ptr ruser; - - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - ruser = store->get_user(info.user_id); - - if (op_state.sync_stats) { - ret = rgw_user_sync_all_stats(dpp, store, ruser.get(), y); - if (ret < 0) { - return ret; - } - } - - RGWStorageStats stats; - RGWStorageStats *arg_stats = NULL; - if (op_state.fetch_stats) { - int ret = ruser->read_stats(dpp, y, &stats); - if (ret < 0 && ret != -ENOENT) { - return ret; - } - - arg_stats = &stats; - } - - if (formatter) { - flusher.start(0); - - dump_user_info(formatter, info, arg_stats); - flusher.flush(); - } - - return 0; -} - -int RGWUserAdminOp_User::create(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.add(dpp, op_state, y, NULL); - if (ret < 0) { - if (ret == -EEXIST) - ret = -ERR_USER_EXIST; - return ret; - } - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - dump_user_info(formatter, info); - flusher.flush(); - } - - return 0; -} - -int RGWUserAdminOp_User::modify(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - Formatter *formatter = flusher.get_formatter(); - - ret = user.modify(dpp, op_state, y, NULL); - if (ret < 0) { - if (ret == -ENOENT) - ret = -ERR_NO_SUCH_USER; - return ret; - } - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - dump_user_info(formatter, info); - flusher.flush(); - } - - return 0; -} - -int RGWUserAdminOp_User::remove(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - - ret = user.remove(dpp, op_state, y, NULL); - - if (ret == -ENOENT) - ret = -ERR_NO_SUCH_USER; - return ret; -} - -int RGWUserAdminOp_Subuser::create(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.subusers.add(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - dump_subusers_info(formatter, info); - flusher.flush(); - } - - return 0; -} - -int RGWUserAdminOp_Subuser::modify(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.subusers.modify(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - dump_subusers_info(formatter, info); - flusher.flush(); - } - - return 0; -} - -int RGWUserAdminOp_Subuser::remove(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - ret = user.subusers.remove(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUserAdminOp_Key::create(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.keys.add(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - int key_type = op_state.get_key_type(); - - if (key_type == KEY_TYPE_SWIFT) - dump_swift_keys_info(formatter, info); - - else if (key_type == KEY_TYPE_S3) - dump_access_keys_info(formatter, info); - - flusher.flush(); - } - - return 0; -} - -int RGWUserAdminOp_Key::remove(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, - optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - - ret = user.keys.remove(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - return 0; -} - -int RGWUserAdminOp_Caps::add(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.caps.add(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - info.caps.dump(formatter); - flusher.flush(); - } - - return 0; -} - - -int RGWUserAdminOp_Caps::remove(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWUserAdminOpState& op_state, - RGWFormatterFlusher& flusher, optional_yield y) -{ - RGWUserInfo info; - RGWUser user; - int ret = user.init(dpp, store, op_state, y); - if (ret < 0) - return ret; - - if (!op_state.has_existing_user()) - return -ERR_NO_SUCH_USER; - - Formatter *formatter = flusher.get_formatter(); - - ret = user.caps.remove(dpp, op_state, y, NULL); - if (ret < 0) - return ret; - - ret = user.info(info, NULL); - if (ret < 0) - return ret; - - if (formatter) { - flusher.start(0); - - info.caps.dump(formatter); - flusher.flush(); - } - - return 0; -} - -class RGWUserMetadataHandler : public RGWMetadataHandler_GenericMetaBE { -public: - struct Svc { - RGWSI_User *user{nullptr}; - } svc; - - RGWUserMetadataHandler(RGWSI_User *user_svc) { - base_init(user_svc->ctx(), user_svc->get_be_handler()); - svc.user = user_svc; - } - - string get_type() override { return "user"; } - - int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { - RGWUserCompleteInfo uci; - RGWObjVersionTracker objv_tracker; - real_time mtime; - - rgw_user user = RGWSI_User::user_from_meta_key(entry); - - int ret = svc.user->read_user_info(op->ctx(), user, &uci.info, &objv_tracker, - &mtime, nullptr, &uci.attrs, - y, dpp); - if (ret < 0) { - return ret; - } - - RGWUserMetadataObject *mdo = new RGWUserMetadataObject(uci, objv_tracker.read_version, mtime); - *obj = mdo; - - return 0; - } - - RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) override { - RGWUserCompleteInfo uci; - - try { - decode_json_obj(uci, jo); - } catch (JSONDecoder::err& e) { - return nullptr; - } - - return new RGWUserMetadataObject(uci, objv, mtime); - } - - int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) override; - - int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp) override { - RGWUserInfo info; - - rgw_user user = RGWSI_User::user_from_meta_key(entry); - - int ret = svc.user->read_user_info(op->ctx(), user, &info, nullptr, - nullptr, nullptr, nullptr, - y, dpp); - if (ret < 0) { - return ret; - } - - return svc.user->remove_user_info(op->ctx(), info, &objv_tracker, - y, dpp); - } -}; - -class RGWMetadataHandlerPut_User : public RGWMetadataHandlerPut_SObj -{ - RGWUserMetadataHandler *uhandler; - RGWUserMetadataObject *uobj; -public: - RGWMetadataHandlerPut_User(RGWUserMetadataHandler *_handler, - RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, - optional_yield y, - RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type, from_remote_zone), - uhandler(_handler) { - uobj = static_cast(obj); - } - - int put_checked(const DoutPrefixProvider *dpp) override; -}; - -int RGWUserMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, - RGWMetadataObject *obj, - RGWObjVersionTracker& objv_tracker, - optional_yield y, const DoutPrefixProvider *dpp, - RGWMDLogSyncType type, bool from_remote_zone) -{ - RGWMetadataHandlerPut_User put_op(this, op, entry, obj, objv_tracker, y, type, from_remote_zone); - return do_put_operate(&put_op, dpp); -} - -int RGWMetadataHandlerPut_User::put_checked(const DoutPrefixProvider *dpp) -{ - RGWUserMetadataObject *orig_obj = static_cast(old_obj); - RGWUserCompleteInfo& uci = uobj->get_uci(); - - map *pattrs{nullptr}; - if (uci.has_attrs) { - pattrs = &uci.attrs; - } - - RGWUserInfo *pold_info = (orig_obj ? &orig_obj->get_uci().info : nullptr); - - auto mtime = obj->get_mtime(); - - int ret = uhandler->svc.user->store_user_info(op->ctx(), uci.info, pold_info, - &objv_tracker, mtime, - false, pattrs, y, dpp); - if (ret < 0) { - return ret; - } - - return STATUS_APPLIED; -} - - -RGWUserCtl::RGWUserCtl(RGWSI_Zone *zone_svc, - RGWSI_User *user_svc, - RGWUserMetadataHandler *_umhandler) : umhandler(_umhandler) { - svc.zone = zone_svc; - svc.user = user_svc; - be_handler = umhandler->get_be_handler(); -} - -template -class optional_default -{ - const std::optional& opt; - std::optional def; - const T *p; -public: - optional_default(const std::optional& _o) : opt(_o) { - if (opt) { - p = &(*opt); - } else { - def = T(); - p = &(*def); - } - } - - const T *operator->() { - return p; - } - - const T& operator*() { - return *p; - } -}; - -int RGWUserCtl::get_info_by_uid(const DoutPrefixProvider *dpp, - const rgw_user& uid, - RGWUserInfo *info, - optional_yield y, - const GetParams& params) - -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->read_user_info(op->ctx(), - uid, - info, - params.objv_tracker, - params.mtime, - params.cache_info, - params.attrs, - y, - dpp); - }); -} - -int RGWUserCtl::get_info_by_email(const DoutPrefixProvider *dpp, - const string& email, - RGWUserInfo *info, - optional_yield y, - const GetParams& params) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->get_user_info_by_email(op->ctx(), email, - info, - params.objv_tracker, - params.mtime, - y, - dpp); - }); -} - -int RGWUserCtl::get_info_by_swift(const DoutPrefixProvider *dpp, - const string& swift_name, - RGWUserInfo *info, - optional_yield y, - const GetParams& params) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->get_user_info_by_swift(op->ctx(), swift_name, - info, - params.objv_tracker, - params.mtime, - y, - dpp); - }); -} - -int RGWUserCtl::get_info_by_access_key(const DoutPrefixProvider *dpp, - const string& access_key, - RGWUserInfo *info, - optional_yield y, - const GetParams& params) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->get_user_info_by_access_key(op->ctx(), access_key, - info, - params.objv_tracker, - params.mtime, - y, - dpp); - }); -} - -int RGWUserCtl::get_attrs_by_uid(const DoutPrefixProvider *dpp, - const rgw_user& user_id, - map *pattrs, - optional_yield y, - RGWObjVersionTracker *objv_tracker) -{ - RGWUserInfo user_info; - - return get_info_by_uid(dpp, user_id, &user_info, y, RGWUserCtl::GetParams() - .set_attrs(pattrs) - .set_objv_tracker(objv_tracker)); -} - -int RGWUserCtl::store_info(const DoutPrefixProvider *dpp, - const RGWUserInfo& info, optional_yield y, - const PutParams& params) -{ - string key = RGWSI_User::get_meta_key(info.user_id); - - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->store_user_info(op->ctx(), info, - params.old_info, - params.objv_tracker, - params.mtime, - params.exclusive, - params.attrs, - y, - dpp); - }); -} - -int RGWUserCtl::remove_info(const DoutPrefixProvider *dpp, - const RGWUserInfo& info, optional_yield y, - const RemoveParams& params) - -{ - string key = RGWSI_User::get_meta_key(info.user_id); - - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->remove_user_info(op->ctx(), info, - params.objv_tracker, - y, dpp); - }); -} - -int RGWUserCtl::list_buckets(const DoutPrefixProvider *dpp, - const rgw_user& user, - const string& marker, - const string& end_marker, - uint64_t max, - bool need_stats, - RGWUserBuckets *buckets, - bool *is_truncated, - optional_yield y, - uint64_t default_max) -{ - if (!max) { - max = default_max; - } - - int ret = svc.user->list_buckets(dpp, user, marker, end_marker, - max, buckets, is_truncated, y); - if (ret < 0) { - return ret; - } - if (need_stats) { - map& m = buckets->get_buckets(); - ret = ctl.bucket->read_buckets_stats(m, y, dpp); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: could not get stats for buckets" << dendl; - return ret; - } - } - return 0; -} - -int RGWUserCtl::read_stats(const DoutPrefixProvider *dpp, - const rgw_user& user, RGWStorageStats *stats, - optional_yield y, - ceph::real_time *last_stats_sync, - ceph::real_time *last_stats_update) -{ - return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { - return svc.user->read_stats(dpp, op->ctx(), user, stats, - last_stats_sync, last_stats_update, y); - }); -} - -RGWMetadataHandler *RGWUserMetaHandlerAllocator::alloc(RGWSI_User *user_svc) { - return new RGWUserMetadataHandler(user_svc); -} - -void rgw_user::dump(Formatter *f) const -{ - ::encode_json("user", *this, f); + info.user_id = RGW_USER_ANON_ID; + info.display_name.clear(); + info.access_keys.clear(); } diff --git a/src/rgw/rgw_zone.cc b/src/rgw/rgw_zone.cc index 2aed28032955..0f4f7d1104d8 100644 --- a/src/rgw/rgw_zone.cc +++ b/src/rgw/rgw_zone.cc @@ -6,44 +6,33 @@ #include "common/errno.h" #include "rgw_zone.h" -#include "rgw_realm_watcher.h" -#include "rgw_meta_sync_status.h" #include "rgw_sal_config.h" -#include "rgw_string.h" #include "rgw_sync.h" #include "services/svc_zone.h" -#include "services/svc_sys_obj.h" -#include "common/ceph_json.h" -#include "common/Formatter.h" #define dout_context g_ceph_context #define dout_subsys ceph_subsys_rgw namespace rgw_zone_defaults { -std::string zone_info_oid_prefix = "zone_info."; +static std::string default_bucket_index_pool_suffix = "rgw.buckets.index"; +static std::string default_storage_extra_pool_suffix = "rgw.buckets.non-ec"; +static std::string zone_info_oid_prefix = "zone_info."; + std::string zone_names_oid_prefix = "zone_names."; std::string region_info_oid_prefix = "region_info."; -std::string realm_names_oid_prefix = "realms_names."; std::string zone_group_info_oid_prefix = "zonegroup_info."; -std::string realm_info_oid_prefix = "realms."; std::string default_region_info_oid = "default.region"; std::string default_zone_group_info_oid = "default.zonegroup"; -std::string period_info_oid_prefix = "periods."; -std::string period_latest_epoch_info_oid = ".latest_epoch"; std::string region_map_oid = "region_map"; -std::string default_realm_info_oid = "default.realm"; std::string default_zonegroup_name = "default"; std::string default_zone_name = "default"; std::string zonegroup_names_oid_prefix = "zonegroups_names."; std::string RGW_DEFAULT_ZONE_ROOT_POOL = "rgw.root"; std::string RGW_DEFAULT_ZONEGROUP_ROOT_POOL = "rgw.root"; -std::string RGW_DEFAULT_REALM_ROOT_POOL = "rgw.root"; std::string RGW_DEFAULT_PERIOD_ROOT_POOL = "rgw.root"; -std::string default_bucket_index_pool_suffix = "rgw.buckets.index"; -std::string default_storage_extra_pool_suffix = "rgw.buckets.non-ec"; std::string avail_pools = ".pools.avail"; std::string default_storage_pool_suffix = "rgw.buckets.data"; @@ -52,20 +41,6 @@ std::string default_storage_pool_suffix = "rgw.buckets.data"; using namespace std; using namespace rgw_zone_defaults; -RGWMetaSyncStatusManager::~RGWMetaSyncStatusManager(){} - -#define FIRST_EPOCH 1 - -struct RGWAccessKey; - -/// Generate a random uuid for realm/period/zonegroup/zone ids -static std::string gen_random_uuid() -{ - uuid_d uuid; - uuid.generate_random(); - return uuid.to_string(); -} - void encode_json_plain(const char *name, const RGWAccessKey& val, Formatter *f) { f->open_object_section(name); @@ -73,97 +48,105 @@ void encode_json_plain(const char *name, const RGWAccessKey& val, Formatter *f) f->close_section(); } -void RGWDefaultZoneGroupInfo::dump(Formatter *f) const { - encode_json("default_zonegroup", default_zonegroup, f); +static void decode_zones(map& zones, JSONObj *o) +{ + RGWZone z; + z.decode_json(o); + zones[z.id] = z; } -void RGWDefaultZoneGroupInfo::decode_json(JSONObj *obj) { - - JSONDecoder::decode_json("default_zonegroup", default_zonegroup, obj); - /* backward compatability with region */ - if (default_zonegroup.empty()) { - JSONDecoder::decode_json("default_region", default_zonegroup, obj); - } +static void decode_placement_targets(map& targets, JSONObj *o) +{ + RGWZoneGroupPlacementTarget t; + t.decode_json(o); + targets[t.name] = t; } -rgw_pool RGWZoneGroup::get_pool(CephContext *cct_) const +void RGWZone::generate_test_instances(list &o) { - if (cct_->_conf->rgw_zonegroup_root_pool.empty()) { - return rgw_pool(RGW_DEFAULT_ZONEGROUP_ROOT_POOL); - } + RGWZone *z = new RGWZone; + o.push_back(z); + o.push_back(new RGWZone); +} - return rgw_pool(cct_->_conf->rgw_zonegroup_root_pool); +void RGWZone::dump(Formatter *f) const +{ + encode_json("id", id, f); + encode_json("name", name, f); + encode_json("endpoints", endpoints, f); + encode_json("log_meta", log_meta, f); + encode_json("log_data", log_data, f); + encode_json("bucket_index_max_shards", bucket_index_max_shards, f); + encode_json("read_only", read_only, f); + encode_json("tier_type", tier_type, f); + encode_json("sync_from_all", sync_from_all, f); + encode_json("sync_from", sync_from, f); + encode_json("redirect_zone", redirect_zone, f); + encode_json("supported_features", supported_features, f); } -int RGWZoneGroup::create_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) +void RGWZone::decode_json(JSONObj *obj) { - name = default_zonegroup_name; - api_name = default_zonegroup_name; - is_master = true; + JSONDecoder::decode_json("id", id, obj); + JSONDecoder::decode_json("name", name, obj); + if (id.empty()) { + id = name; + } + JSONDecoder::decode_json("endpoints", endpoints, obj); + JSONDecoder::decode_json("log_meta", log_meta, obj); + JSONDecoder::decode_json("log_data", log_data, obj); + JSONDecoder::decode_json("bucket_index_max_shards", bucket_index_max_shards, obj); + JSONDecoder::decode_json("read_only", read_only, obj); + JSONDecoder::decode_json("tier_type", tier_type, obj); + JSONDecoder::decode_json("sync_from_all", sync_from_all, true, obj); + JSONDecoder::decode_json("sync_from", sync_from, obj); + JSONDecoder::decode_json("redirect_zone", redirect_zone, obj); + JSONDecoder::decode_json("supported_features", supported_features, obj); +} - RGWZoneGroupPlacementTarget placement_target; - placement_target.name = "default-placement"; - placement_targets[placement_target.name] = placement_target; - default_placement.name = "default-placement"; +int RGWSystemMetaObj::init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, + optional_yield y, + bool setup_obj, bool old_format) +{ + reinit_instance(_cct, _sysobj_svc); - RGWZoneParams zone_params(default_zone_name); + if (!setup_obj) + return 0; - int r = zone_params.init(dpp, cct, sysobj_svc, y, false); - if (r < 0) { - ldpp_dout(dpp, 0) << "create_default: error initializing zone params: " << cpp_strerror(-r) << dendl; - return r; + if (old_format && id.empty()) { + id = name; } - r = zone_params.create_default(dpp, y); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "create_default: error in create_default zone params: " << cpp_strerror(-r) << dendl; - return r; - } else if (r == -EEXIST) { - ldpp_dout(dpp, 10) << "zone_params::create_default() returned -EEXIST, we raced with another default zone_params creation" << dendl; - zone_params.clear_id(); - r = zone_params.init(dpp, cct, sysobj_svc, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "create_default: error in init existing zone params: " << cpp_strerror(-r) << dendl; - return r; - } - ldpp_dout(dpp, 20) << "zone_params::create_default() " << zone_params.get_name() << " id " << zone_params.get_id() - << dendl; - } - - RGWZone& default_zone = zones[zone_params.get_id()]; - default_zone.name = zone_params.get_name(); - default_zone.id = zone_params.get_id(); - master_zone = default_zone.id; - - // enable all supported features - enabled_features.insert(rgw::zone_features::supported.begin(), - rgw::zone_features::supported.end()); - default_zone.supported_features = enabled_features; - - r = create(dpp, y); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "error storing zone group info: " << cpp_strerror(-r) << dendl; - return r; + if (id.empty()) { + id = get_predefined_id(cct); } - if (r == -EEXIST) { - ldpp_dout(dpp, 10) << "create_default() returned -EEXIST, we raced with another zonegroup creation" << dendl; - id.clear(); - r = init(dpp, cct, sysobj_svc, y); - if (r < 0) { - return r; + if (id.empty()) { + int r; + if (name.empty()) { + name = get_predefined_name(cct); + } + if (name.empty()) { + r = use_default(dpp, y, old_format); + if (r < 0) { + return r; + } + } else if (!old_format) { + r = read_id(dpp, name, id, y); + if (r < 0) { + if (r != -ENOENT) { + ldpp_dout(dpp, 0) << "error in read_id for object name: " << name << " : " << cpp_strerror(-r) << dendl; + } + return r; + } } } - if (old_format) { - name = id; - } - - post_process_params(dpp, y); - - return 0; + return read_info(dpp, id, y, old_format); } +RGWZoneGroup::~RGWZoneGroup() {} + const string RGWZoneGroup::get_default_oid(bool old_region_format) const { if (old_region_format) { @@ -205,174 +188,13 @@ const string& RGWZoneGroup::get_predefined_name(CephContext *cct) const { return cct->_conf->rgw_zonegroup; } -int RGWZoneGroup::equals(const string& other_zonegroup) const -{ - if (is_master && other_zonegroup.empty()) - return true; - - return (id == other_zonegroup); -} - -int RGWZoneGroup::add_zone(const DoutPrefixProvider *dpp, - const RGWZoneParams& zone_params, bool *is_master, bool *read_only, - const list& endpoints, const string *ptier_type, - bool *psync_from_all, list& sync_from, list& sync_from_rm, - string *predirect_zone, std::optional bucket_index_max_shards, - RGWSyncModulesManager *sync_mgr, - const rgw::zone_features::set& enable_features, - const rgw::zone_features::set& disable_features, - optional_yield y) -{ - auto& zone_id = zone_params.get_id(); - auto& zone_name = zone_params.get_name(); - - // check for duplicate zone name on insert - if (!zones.count(zone_id)) { - for (const auto& zone : zones) { - if (zone.second.name == zone_name) { - ldpp_dout(dpp, 0) << "ERROR: found existing zone name " << zone_name - << " (" << zone.first << ") in zonegroup " << get_name() << dendl; - return -EEXIST; - } - } - } - - if (is_master) { - if (*is_master) { - if (!master_zone.empty() && master_zone != zone_id) { - ldpp_dout(dpp, 0) << "NOTICE: overriding master zone: " << master_zone << dendl; - } - master_zone = zone_id; - } else if (master_zone == zone_id) { - master_zone.clear(); - } - } - - RGWZone& zone = zones[zone_id]; - zone.name = zone_name; - zone.id = zone_id; - if (!endpoints.empty()) { - zone.endpoints = endpoints; - } - if (read_only) { - zone.read_only = *read_only; - } - if (ptier_type) { - zone.tier_type = *ptier_type; - if (!sync_mgr->get_module(*ptier_type, nullptr)) { - ldpp_dout(dpp, 0) << "ERROR: could not found sync module: " << *ptier_type - << ", valid sync modules: " - << sync_mgr->get_registered_module_names() - << dendl; - return -ENOENT; - } - } - - if (psync_from_all) { - zone.sync_from_all = *psync_from_all; - } - - if (predirect_zone) { - zone.redirect_zone = *predirect_zone; - } - - if (bucket_index_max_shards) { - zone.bucket_index_max_shards = *bucket_index_max_shards; - } - - for (auto add : sync_from) { - zone.sync_from.insert(add); - } - - for (auto rm : sync_from_rm) { - zone.sync_from.erase(rm); - } - - zone.supported_features.insert(enable_features.begin(), - enable_features.end()); - - for (const auto& feature : disable_features) { - if (enabled_features.contains(feature)) { - lderr(cct) << "ERROR: Cannot disable zone feature \"" << feature - << "\" until it's been disabled in zonegroup " << name << dendl; - return -EINVAL; - } - auto i = zone.supported_features.find(feature); - if (i == zone.supported_features.end()) { - ldout(cct, 1) << "WARNING: zone feature \"" << feature - << "\" was not enabled in zone " << zone.name << dendl; - continue; - } - zone.supported_features.erase(i); - } - - post_process_params(dpp, y); - - return update(dpp,y); -} - - -int RGWZoneGroup::rename_zone(const DoutPrefixProvider *dpp, - const RGWZoneParams& zone_params, - optional_yield y) -{ - RGWZone& zone = zones[zone_params.get_id()]; - zone.name = zone_params.get_name(); - - return update(dpp, y); -} - -void RGWZoneGroup::post_process_params(const DoutPrefixProvider *dpp, optional_yield y) -{ - bool log_data = zones.size() > 1; - - if (master_zone.empty()) { - auto iter = zones.begin(); - if (iter != zones.end()) { - master_zone = iter->first; - } - } - - for (auto& item : zones) { - RGWZone& zone = item.second; - zone.log_data = log_data; - - RGWZoneParams zone_params(zone.id, zone.name); - int ret = zone_params.init(dpp, cct, sysobj_svc, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: could not read zone params for zone id=" << zone.id << " name=" << zone.name << dendl; - continue; - } - - for (auto& pitem : zone_params.placement_pools) { - const string& placement_name = pitem.first; - if (placement_targets.find(placement_name) == placement_targets.end()) { - RGWZoneGroupPlacementTarget placement_target; - placement_target.name = placement_name; - placement_targets[placement_name] = placement_target; - } - } - } - - if (default_placement.empty() && !placement_targets.empty()) { - default_placement.init(placement_targets.begin()->first, RGW_STORAGE_CLASS_STANDARD); - } -} - -int RGWZoneGroup::remove_zone(const DoutPrefixProvider *dpp, const std::string& zone_id, optional_yield y) +rgw_pool RGWZoneGroup::get_pool(CephContext *cct_) const { - auto iter = zones.find(zone_id); - if (iter == zones.end()) { - ldpp_dout(dpp, 0) << "zone id " << zone_id << " is not a part of zonegroup " - << name << dendl; - return -ENOENT; + if (cct_->_conf->rgw_zonegroup_root_pool.empty()) { + return rgw_pool(RGW_DEFAULT_ZONEGROUP_ROOT_POOL); } - zones.erase(iter); - - post_process_params(dpp, y); - - return update(dpp, y); + return rgw_pool(cct_->_conf->rgw_zonegroup_root_pool); } int RGWZoneGroup::read_default_id(const DoutPrefixProvider *dpp, string& default_id, optional_yield y, @@ -392,20 +214,9 @@ int RGWZoneGroup::read_default_id(const DoutPrefixProvider *dpp, string& default return RGWSystemMetaObj::read_default_id(dpp, default_id, y, old_format); } -int RGWZoneGroup::set_as_default(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +int RGWSystemMetaObj::use_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) { - if (realm_id.empty()) { - /* try using default realm */ - RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); - if (ret < 0) { - ldpp_dout(dpp, 10) << "could not read realm id: " << cpp_strerror(-ret) << dendl; - return -EINVAL; - } - realm_id = realm.get_id(); - } - - return RGWSystemMetaObj::set_as_default(dpp, y, exclusive); + return read_default_id(dpp, id, y, old_format); } void RGWSystemMetaObj::reinit_instance(CephContext *_cct, RGWSI_SysObj *_sysobj_svc) @@ -415,3099 +226,1141 @@ void RGWSystemMetaObj::reinit_instance(CephContext *_cct, RGWSI_SysObj *_sysobj_ zone_svc = _sysobj_svc->get_zone_svc(); } -int RGWSystemMetaObj::init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, - optional_yield y, - bool setup_obj, bool old_format) +int RGWSystemMetaObj::read_info(const DoutPrefixProvider *dpp, const string& obj_id, optional_yield y, + bool old_format) { - reinit_instance(_cct, _sysobj_svc); + rgw_pool pool(get_pool(cct)); - if (!setup_obj) - return 0; + bufferlist bl; - if (old_format && id.empty()) { - id = name; + string oid = get_info_oid_prefix(old_format) + obj_id; + + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); + int ret = sysobj.rop().read(dpp, &bl, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed reading obj info from " << pool << ":" << oid << ": " << cpp_strerror(-ret) << dendl; + return ret; } + using ceph::decode; - if (id.empty()) { - id = get_predefined_id(cct); + try { + auto iter = bl.cbegin(); + decode(*this, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << oid << dendl; + return -EIO; } + return 0; +} + +void RGWZoneGroup::decode_json(JSONObj *obj) +{ + RGWSystemMetaObj::decode_json(obj); if (id.empty()) { - int r; - if (name.empty()) { - name = get_predefined_name(cct); - } - if (name.empty()) { - r = use_default(dpp, y, old_format); - if (r < 0) { - return r; - } - } else if (!old_format) { - r = read_id(dpp, name, id, y); - if (r < 0) { - if (r != -ENOENT) { - ldpp_dout(dpp, 0) << "error in read_id for object name: " << name << " : " << cpp_strerror(-r) << dendl; - } - return r; - } - } + derr << "old format " << dendl; + JSONDecoder::decode_json("name", name, obj); + id = name; } - - return read_info(dpp, id, y, old_format); + JSONDecoder::decode_json("api_name", api_name, obj); + JSONDecoder::decode_json("is_master", is_master, obj); + JSONDecoder::decode_json("endpoints", endpoints, obj); + JSONDecoder::decode_json("hostnames", hostnames, obj); + JSONDecoder::decode_json("hostnames_s3website", hostnames_s3website, obj); + JSONDecoder::decode_json("master_zone", master_zone, obj); + JSONDecoder::decode_json("zones", zones, decode_zones, obj); + JSONDecoder::decode_json("placement_targets", placement_targets, decode_placement_targets, obj); + string pr; + JSONDecoder::decode_json("default_placement", pr, obj); + default_placement.from_str(pr); + JSONDecoder::decode_json("realm_id", realm_id, obj); + JSONDecoder::decode_json("sync_policy", sync_policy, obj); + JSONDecoder::decode_json("enabled_features", enabled_features, obj); } -void RGWDefaultSystemMetaObjInfo::dump(Formatter *f) const { - encode_json("default_id", default_id, f); +RGWZoneParams::~RGWZoneParams() {} + +void RGWZoneParams::decode_json(JSONObj *obj) +{ + RGWSystemMetaObj::decode_json(obj); + JSONDecoder::decode_json("domain_root", domain_root, obj); + JSONDecoder::decode_json("control_pool", control_pool, obj); + JSONDecoder::decode_json("gc_pool", gc_pool, obj); + JSONDecoder::decode_json("lc_pool", lc_pool, obj); + JSONDecoder::decode_json("log_pool", log_pool, obj); + JSONDecoder::decode_json("intent_log_pool", intent_log_pool, obj); + JSONDecoder::decode_json("roles_pool", roles_pool, obj); + JSONDecoder::decode_json("reshard_pool", reshard_pool, obj); + JSONDecoder::decode_json("usage_log_pool", usage_log_pool, obj); + JSONDecoder::decode_json("user_keys_pool", user_keys_pool, obj); + JSONDecoder::decode_json("user_email_pool", user_email_pool, obj); + JSONDecoder::decode_json("user_swift_pool", user_swift_pool, obj); + JSONDecoder::decode_json("user_uid_pool", user_uid_pool, obj); + JSONDecoder::decode_json("otp_pool", otp_pool, obj); + JSONDecoder::decode_json("system_key", system_key, obj); + JSONDecoder::decode_json("placement_pools", placement_pools, obj); + JSONDecoder::decode_json("tier_config", tier_config, obj); + JSONDecoder::decode_json("realm_id", realm_id, obj); + JSONDecoder::decode_json("notif_pool", notif_pool, obj); + } -void RGWDefaultSystemMetaObjInfo::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("default_id", default_id, obj); +void RGWZoneParams::dump(Formatter *f) const +{ + RGWSystemMetaObj::dump(f); + encode_json("domain_root", domain_root, f); + encode_json("control_pool", control_pool, f); + encode_json("gc_pool", gc_pool, f); + encode_json("lc_pool", lc_pool, f); + encode_json("log_pool", log_pool, f); + encode_json("intent_log_pool", intent_log_pool, f); + encode_json("usage_log_pool", usage_log_pool, f); + encode_json("roles_pool", roles_pool, f); + encode_json("reshard_pool", reshard_pool, f); + encode_json("user_keys_pool", user_keys_pool, f); + encode_json("user_email_pool", user_email_pool, f); + encode_json("user_swift_pool", user_swift_pool, f); + encode_json("user_uid_pool", user_uid_pool, f); + encode_json("otp_pool", otp_pool, f); + encode_json_plain("system_key", system_key, f); + encode_json("placement_pools", placement_pools, f); + encode_json("tier_config", tier_config, f); + encode_json("realm_id", realm_id, f); + encode_json("notif_pool", notif_pool, f); } -int RGWSystemMetaObj::read_default(const DoutPrefixProvider *dpp, - RGWDefaultSystemMetaObjInfo& default_info, - const string& oid, optional_yield y) +int RGWZoneParams::init(const DoutPrefixProvider *dpp, + CephContext *cct, RGWSI_SysObj *sysobj_svc, + optional_yield y, bool setup_obj, bool old_format) { - using ceph::decode; - auto pool = get_pool(cct); - bufferlist bl; + if (name.empty()) { + name = cct->_conf->rgw_zone; + } - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - int ret = sysobj.rop().read(dpp, &bl, y); - if (ret < 0) - return ret; + return RGWSystemMetaObj::init(dpp, cct, sysobj_svc, y, setup_obj, old_format); +} - try { - auto iter = bl.cbegin(); - decode(default_info, iter); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "error decoding data from " << pool << ":" << oid << dendl; - return -EIO; +rgw_pool RGWZoneParams::get_pool(CephContext *cct) const +{ + if (cct->_conf->rgw_zone_root_pool.empty()) { + return rgw_pool(RGW_DEFAULT_ZONE_ROOT_POOL); } - return 0; + return rgw_pool(cct->_conf->rgw_zone_root_pool); } -int RGWSystemMetaObj::read_default_id(const DoutPrefixProvider *dpp, string& default_id, optional_yield y, - bool old_format) +const string RGWZoneParams::get_default_oid(bool old_format) const { - RGWDefaultSystemMetaObjInfo default_info; - - int ret = read_default(dpp, default_info, get_default_oid(old_format), y); - if (ret < 0) { - return ret; + if (old_format) { + return cct->_conf->rgw_default_zone_info_oid; } - default_id = default_info.default_id; - - return 0; + return cct->_conf->rgw_default_zone_info_oid + "." + realm_id; } -int RGWSystemMetaObj::use_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) +const string& RGWZoneParams::get_names_oid_prefix() const { - return read_default_id(dpp, id, y, old_format); + return zone_names_oid_prefix; } -int RGWSystemMetaObj::set_as_default(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +const string& RGWZoneParams::get_info_oid_prefix(bool old_format) const { - using ceph::encode; - string oid = get_default_oid(); - - rgw_pool pool(get_pool(cct)); - bufferlist bl; - - RGWDefaultSystemMetaObjInfo default_info; - default_info.default_id = id; - - encode(default_info, bl); + return zone_info_oid_prefix; +} - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - int ret = sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); - if (ret < 0) - return ret; +string RGWZoneParams::get_predefined_id(CephContext *cct) const { + return cct->_conf.get_val("rgw_zone_id"); +} - return 0; +const string& RGWZoneParams::get_predefined_name(CephContext *cct) const { + return cct->_conf->rgw_zone; } -int RGWSystemMetaObj::read_id(const DoutPrefixProvider *dpp, const string& obj_name, string& object_id, - optional_yield y) +int RGWZoneParams::read_default_id(const DoutPrefixProvider *dpp, string& default_id, optional_yield y, + bool old_format) { - using ceph::decode; - rgw_pool pool(get_pool(cct)); - bufferlist bl; - - string oid = get_names_oid_prefix() + obj_name; - - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - int ret = sysobj.rop().read(dpp, &bl, y); - if (ret < 0) { - return ret; + if (realm_id.empty()) { + /* try using default realm */ + RGWRealm realm; + int ret = realm.init(dpp, cct, sysobj_svc, y); + //no default realm exist + if (ret < 0) { + return read_id(dpp, default_zone_name, default_id, y); + } + realm_id = realm.get_id(); } - RGWNameToId nameToId; - try { - auto iter = bl.cbegin(); - decode(nameToId, iter); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << oid << dendl; - return -EIO; - } - object_id = nameToId.obj_id; - return 0; + return RGWSystemMetaObj::read_default_id(dpp, default_id, y, old_format); } -int RGWSystemMetaObj::delete_obj(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) -{ - rgw_pool pool(get_pool(cct)); - /* check to see if obj is the default */ - RGWDefaultSystemMetaObjInfo default_info; - int ret = read_default(dpp, default_info, get_default_oid(old_format), y); - if (ret < 0 && ret != -ENOENT) - return ret; - if (default_info.default_id == id || (old_format && default_info.default_id == name)) { - string oid = get_default_oid(old_format); - rgw_raw_obj default_named_obj(pool, oid); - auto sysobj = sysobj_svc->get_obj(default_named_obj); - ret = sysobj.wop().remove(dpp, y); +int RGWZoneParams::set_as_default(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +{ + if (realm_id.empty()) { + /* try using default realm */ + RGWRealm realm; + int ret = realm.init(dpp, cct, sysobj_svc, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "Error delete default obj name " << name << ": " << cpp_strerror(-ret) << dendl; - return ret; + ldpp_dout(dpp, 10) << "could not read realm id: " << cpp_strerror(-ret) << dendl; + return -EINVAL; } + realm_id = realm.get_id(); } - if (!old_format) { - string oid = get_names_oid_prefix() + name; - rgw_raw_obj object_name(pool, oid); - auto sysobj = sysobj_svc->get_obj(object_name); - ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "Error delete obj name " << name << ": " << cpp_strerror(-ret) << dendl; - return ret; - } + + return RGWSystemMetaObj::set_as_default(dpp, y, exclusive); +} + +int RGWZoneParams::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +{ + /* check for old pools config */ + rgw_raw_obj obj(domain_root, avail_pools); + auto sysobj = sysobj_svc->get_obj(obj); + int r = sysobj.rop().stat(y, dpp); + if (r < 0) { + ldpp_dout(dpp, 10) << "couldn't find old data placement pools config, setting up new ones for the zone" << dendl; + /* a new system, let's set new placement info */ + RGWZonePlacementInfo default_placement; + default_placement.index_pool = name + "." + default_bucket_index_pool_suffix; + rgw_pool pool = name + "." + default_storage_pool_suffix; + default_placement.storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); + default_placement.data_extra_pool = name + "." + default_storage_extra_pool_suffix; + placement_pools["default-placement"] = default_placement; } - string oid = get_info_oid_prefix(old_format); - if (old_format) { - oid += name; - } else { - oid += id; + r = fix_pool_names(dpp, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: fix_pool_names returned r=" << r << dendl; + return r; } - rgw_raw_obj object_id(pool, oid); - auto sysobj = sysobj_svc->get_obj(object_id); - ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "Error delete object id " << id << ": " << cpp_strerror(-ret) << dendl; + r = RGWSystemMetaObj::create(dpp, y, exclusive); + if (r < 0) { + return r; } - return ret; + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + r = set_as_default(dpp, y, true); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 10) << "WARNING: failed to set zone as default, r=" << r << dendl; + } + + return 0; } -int RGWSystemMetaObj::store_name(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) +rgw_pool fix_zone_pool_dup(const set& pools, + const string& default_prefix, + const string& default_suffix, + const rgw_pool& suggested_pool) { - rgw_pool pool(get_pool(cct)); - string oid = get_names_oid_prefix() + name; + string suggested_name = suggested_pool.to_str(); - RGWNameToId nameToId; - nameToId.obj_id = id; + string prefix = default_prefix; + string suffix = default_suffix; - bufferlist bl; - using ceph::encode; - encode(nameToId, bl); - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); + if (!suggested_pool.empty()) { + prefix = suggested_name.substr(0, suggested_name.find(".")); + suffix = suggested_name.substr(prefix.length()); + } + + rgw_pool pool(prefix + suffix); + + while (pools.count(pool)) { + pool = prefix + "_" + std::to_string(std::rand()) + suffix; + } + return pool; } -int RGWSystemMetaObj::rename(const DoutPrefixProvider *dpp, const string& new_name, optional_yield y) +void add_zone_pools(const RGWZoneParams& info, + std::set& pools) { - string new_id; - int ret = read_id(dpp, new_name, new_id, y); - if (!ret) { - return -EEXIST; + pools.insert(info.domain_root); + pools.insert(info.control_pool); + pools.insert(info.gc_pool); + pools.insert(info.log_pool); + pools.insert(info.intent_log_pool); + pools.insert(info.usage_log_pool); + pools.insert(info.user_keys_pool); + pools.insert(info.user_email_pool); + pools.insert(info.user_swift_pool); + pools.insert(info.user_uid_pool); + pools.insert(info.otp_pool); + pools.insert(info.roles_pool); + pools.insert(info.reshard_pool); + pools.insert(info.oidc_pool); + pools.insert(info.notif_pool); + + for (const auto& [pname, placement] : info.placement_pools) { + pools.insert(placement.index_pool); + for (const auto& [sname, sc] : placement.storage_classes.get_all()) { + if (sc.data_pool) { + pools.insert(sc.data_pool.get()); + } + } + pools.insert(placement.data_extra_pool); } - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "Error read_id " << new_name << ": " << cpp_strerror(-ret) << dendl; - return ret; +} + +namespace rgw { + +int get_zones_pool_set(const DoutPrefixProvider *dpp, + optional_yield y, + rgw::sal::ConfigStore* cfgstore, + std::string_view my_zone_id, + std::set& pools) +{ + std::array zone_names; + rgw::sal::ListResult listing; + do { + int r = cfgstore->list_zone_names(dpp, y, listing.next, + zone_names, listing); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to list zones with " << cpp_strerror(r) << dendl; + return r; + } + + for (const auto& name : listing.entries) { + RGWZoneParams info; + r = cfgstore->read_zone_by_name(dpp, y, name, info, nullptr); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to load zone " << name + << " with " << cpp_strerror(r) << dendl; + return r; + } + if (info.get_id() != my_zone_id) { + add_zone_pools(info, pools); + } + } + } while (!listing.next.empty()); + + return 0; +} + +} + +static int get_zones_pool_set(const DoutPrefixProvider *dpp, + CephContext* cct, + RGWSI_SysObj* sysobj_svc, + const list& zone_names, + const string& my_zone_id, + set& pool_names, + optional_yield y) +{ + for (const auto& name : zone_names) { + RGWZoneParams zone(name); + int r = zone.init(dpp, cct, sysobj_svc, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "Error: failed to load zone " << name + << " with " << cpp_strerror(-r) << dendl; + return r; + } + if (zone.get_id() != my_zone_id) { + add_zone_pools(zone, pool_names); + } } - string old_name = name; - name = new_name; - ret = update(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "Error storing new obj info " << new_name << ": " << cpp_strerror(-ret) << dendl; - return ret; + return 0; +} + +int RGWZoneParams::fix_pool_names(const DoutPrefixProvider *dpp, optional_yield y) +{ + + list zones; + int r = zone_svc->list_zones(dpp, zones); + if (r < 0) { + ldpp_dout(dpp, 10) << "WARNING: store->list_zones() returned r=" << r << dendl; } - ret = store_name(dpp, true, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "Error storing new name " << new_name << ": " << cpp_strerror(-ret) << dendl; - return ret; + + set pools; + r = get_zones_pool_set(dpp, cct, sysobj_svc, zones, id, pools, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "Error: get_zones_pool_names" << r << dendl; + return r; } - /* delete old name */ - rgw_pool pool(get_pool(cct)); - string oid = get_names_oid_prefix() + old_name; - rgw_raw_obj old_name_obj(pool, oid); - auto sysobj = sysobj_svc->get_obj(old_name_obj); - ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "Error delete old obj name " << old_name << ": " << cpp_strerror(-ret) << dendl; - return ret; + + domain_root = fix_zone_pool_dup(pools, name, ".rgw.meta:root", domain_root); + control_pool = fix_zone_pool_dup(pools, name, ".rgw.control", control_pool); + gc_pool = fix_zone_pool_dup(pools, name ,".rgw.log:gc", gc_pool); + lc_pool = fix_zone_pool_dup(pools, name ,".rgw.log:lc", lc_pool); + log_pool = fix_zone_pool_dup(pools, name, ".rgw.log", log_pool); + intent_log_pool = fix_zone_pool_dup(pools, name, ".rgw.log:intent", intent_log_pool); + usage_log_pool = fix_zone_pool_dup(pools, name, ".rgw.log:usage", usage_log_pool); + user_keys_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.keys", user_keys_pool); + user_email_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.email", user_email_pool); + user_swift_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.swift", user_swift_pool); + user_uid_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.uid", user_uid_pool); + roles_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:roles", roles_pool); + reshard_pool = fix_zone_pool_dup(pools, name, ".rgw.log:reshard", reshard_pool); + otp_pool = fix_zone_pool_dup(pools, name, ".rgw.otp", otp_pool); + oidc_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:oidc", oidc_pool); + notif_pool = fix_zone_pool_dup(pools, name ,".rgw.log:notif", notif_pool); + + for(auto& iter : placement_pools) { + iter.second.index_pool = fix_zone_pool_dup(pools, name, "." + default_bucket_index_pool_suffix, + iter.second.index_pool); + for (auto& pi : iter.second.storage_classes.get_all()) { + if (pi.second.data_pool) { + rgw_pool& pool = pi.second.data_pool.get(); + pool = fix_zone_pool_dup(pools, name, "." + default_storage_pool_suffix, + pool); + } + } + iter.second.data_extra_pool= fix_zone_pool_dup(pools, name, "." + default_storage_extra_pool_suffix, + iter.second.data_extra_pool); } - return ret; + return 0; } -int RGWSystemMetaObj::read_info(const DoutPrefixProvider *dpp, const string& obj_id, optional_yield y, - bool old_format) +int RGWPeriodConfig::read(const DoutPrefixProvider *dpp, RGWSI_SysObj *sysobj_svc, const std::string& realm_id, + optional_yield y) { - rgw_pool pool(get_pool(cct)); - + const auto& pool = get_pool(sysobj_svc->ctx()); + const auto& oid = get_oid(realm_id); bufferlist bl; - string oid = get_info_oid_prefix(old_format) + obj_id; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); int ret = sysobj.rop().read(dpp, &bl, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "failed reading obj info from " << pool << ":" << oid << ": " << cpp_strerror(-ret) << dendl; return ret; } using ceph::decode; - try { auto iter = bl.cbegin(); decode(*this, iter); } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << oid << dendl; return -EIO; } - return 0; } -int RGWSystemMetaObj::read(const DoutPrefixProvider *dpp, optional_yield y) +int RGWPeriodConfig::write(const DoutPrefixProvider *dpp, + RGWSI_SysObj *sysobj_svc, + const std::string& realm_id, optional_yield y) { - int ret = read_id(dpp, name, id, y); - if (ret < 0) { - return ret; - } - - return read_info(dpp, id, y); + const auto& pool = get_pool(sysobj_svc->ctx()); + const auto& oid = get_oid(realm_id); + bufferlist bl; + using ceph::encode; + encode(*this, bl); + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); + return sysobj.wop() + .set_exclusive(false) + .write(dpp, bl, y); } -int RGWSystemMetaObj::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +void RGWPeriodConfig::decode_json(JSONObj *obj) { - int ret; + JSONDecoder::decode_json("bucket_quota", quota.bucket_quota, obj); + JSONDecoder::decode_json("user_quota", quota.user_quota, obj); + JSONDecoder::decode_json("user_ratelimit", user_ratelimit, obj); + JSONDecoder::decode_json("bucket_ratelimit", bucket_ratelimit, obj); + JSONDecoder::decode_json("anonymous_ratelimit", anon_ratelimit, obj); +} - /* check to see the name is not used */ - ret = read_id(dpp, name, id, y); - if (exclusive && ret == 0) { - ldpp_dout(dpp, 10) << "ERROR: name " << name << " already in use for obj id " << id << dendl; - return -EEXIST; - } else if ( ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "failed reading obj id " << id << ": " << cpp_strerror(-ret) << dendl; - return ret; - } +void RGWPeriodConfig::dump(Formatter *f) const +{ + encode_json("bucket_quota", quota.bucket_quota, f); + encode_json("user_quota", quota.user_quota, f); + encode_json("user_ratelimit", user_ratelimit, f); + encode_json("bucket_ratelimit", bucket_ratelimit, f); + encode_json("anonymous_ratelimit", anon_ratelimit, f); +} - if (id.empty()) { - /* create unique id */ - uuid_d new_uuid; - char uuid_str[37]; - new_uuid.generate_random(); - new_uuid.print(uuid_str); - id = uuid_str; +std::string RGWPeriodConfig::get_oid(const std::string& realm_id) +{ + if (realm_id.empty()) { + return "period_config.default"; } + return "period_config." + realm_id; +} - ret = store_info(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: storing info for " << id << ": " << cpp_strerror(-ret) << dendl; - return ret; +rgw_pool RGWPeriodConfig::get_pool(CephContext *cct) +{ + const auto& pool_name = cct->_conf->rgw_period_root_pool; + if (pool_name.empty()) { + return {RGW_DEFAULT_PERIOD_ROOT_POOL}; } - - return store_name(dpp, exclusive, y); + return {pool_name}; } -int RGWSystemMetaObj::store_info(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) +int RGWSystemMetaObj::delete_obj(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) { rgw_pool pool(get_pool(cct)); - string oid = get_info_oid_prefix() + id; - - bufferlist bl; - using ceph::encode; - encode(*this, bl); - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); -} - -int RGWSystemMetaObj::write(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) -{ - int ret = store_info(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 20) << __func__ << "(): store_info() returned ret=" << ret << dendl; - return ret; - } - ret = store_name(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 20) << __func__ << "(): store_name() returned ret=" << ret << dendl; - return ret; - } - return 0; -} - - -RGWRealm::~RGWRealm() {} - -RGWRemoteMetaLog::~RGWRemoteMetaLog() -{ - delete error_logger; -} - -string RGWRealm::get_predefined_id(CephContext *cct) const { - return cct->_conf.get_val("rgw_realm_id"); -} - -const string& RGWRealm::get_predefined_name(CephContext *cct) const { - return cct->_conf->rgw_realm; -} - -int RGWRealm::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) -{ - int ret = RGWSystemMetaObj::create(dpp, y, exclusive); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR creating new realm object " << name << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - // create the control object for watch/notify - ret = create_control(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR creating control for new realm " << name << ": " << cpp_strerror(-ret) << dendl; + /* check to see if obj is the default */ + RGWDefaultSystemMetaObjInfo default_info; + int ret = read_default(dpp, default_info, get_default_oid(old_format), y); + if (ret < 0 && ret != -ENOENT) return ret; - } - RGWPeriod period; - if (current_period.empty()) { - /* create new period for the realm */ - ret = period.init(dpp, cct, sysobj_svc, id, y, name, false); - if (ret < 0 ) { - return ret; - } - ret = period.create(dpp, y, true); + if (default_info.default_id == id || (old_format && default_info.default_id == name)) { + string oid = get_default_oid(old_format); + rgw_raw_obj default_named_obj(pool, oid); + auto sysobj = sysobj_svc->get_obj(default_named_obj); + ret = sysobj.wop().remove(dpp, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: creating new period for realm " << name << ": " << cpp_strerror(-ret) << dendl; + ldpp_dout(dpp, 0) << "Error delete default obj name " << name << ": " << cpp_strerror(-ret) << dendl; return ret; } - } else { - period = RGWPeriod(current_period, 0); - int ret = period.init(dpp, cct, sysobj_svc, id, y, name); + } + if (!old_format) { + string oid = get_names_oid_prefix() + name; + rgw_raw_obj object_name(pool, oid); + auto sysobj = sysobj_svc->get_obj(object_name); + ret = sysobj.wop().remove(dpp, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to init period " << current_period << dendl; + ldpp_dout(dpp, 0) << "Error delete obj name " << name << ": " << cpp_strerror(-ret) << dendl; return ret; } } - ret = set_current_period(dpp, period, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed set current period " << current_period << dendl; - return ret; - } - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - ret = set_as_default(dpp, y, true); - if (ret < 0 && ret != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set realm as default realm, ret=" << ret << dendl; - } - return 0; -} + string oid = get_info_oid_prefix(old_format); + if (old_format) { + oid += name; + } else { + oid += id; + } -int RGWRealm::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) -{ - int ret = RGWSystemMetaObj::delete_obj(dpp, y); + rgw_raw_obj object_id(pool, oid); + auto sysobj = sysobj_svc->get_obj(object_id); + ret = sysobj.wop().remove(dpp, y); if (ret < 0) { - return ret; + ldpp_dout(dpp, 0) << "Error delete object id " << id << ": " << cpp_strerror(-ret) << dendl; } - return delete_control(dpp, y); + + return ret; } -int RGWRealm::create_control(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) +void RGWZoneGroup::dump(Formatter *f) const { - auto pool = rgw_pool{get_pool(cct)}; - auto oid = get_control_oid(); - bufferlist bl; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); + RGWSystemMetaObj::dump(f); + encode_json("api_name", api_name, f); + encode_json("is_master", is_master, f); + encode_json("endpoints", endpoints, f); + encode_json("hostnames", hostnames, f); + encode_json("hostnames_s3website", hostnames_s3website, f); + encode_json("master_zone", master_zone, f); + encode_json_map("zones", zones, f); /* more friendly representation */ + encode_json_map("placement_targets", placement_targets, f); /* more friendly representation */ + encode_json("default_placement", default_placement, f); + encode_json("realm_id", realm_id, f); + encode_json("sync_policy", sync_policy, f); + encode_json("enabled_features", enabled_features, f); } -int RGWRealm::delete_control(const DoutPrefixProvider *dpp, optional_yield y) +void RGWZoneGroupPlacementTarget::decode_json(JSONObj *obj) { - auto pool = rgw_pool{get_pool(cct)}; - auto obj = rgw_raw_obj{pool, get_control_oid()}; - auto sysobj = sysobj_svc->get_obj(obj); - return sysobj.wop().remove(dpp, y); + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("tags", tags, obj); + JSONDecoder::decode_json("storage_classes", storage_classes, obj); + if (storage_classes.empty()) { + storage_classes.insert(RGW_STORAGE_CLASS_STANDARD); + } + if (!tier_targets.empty()) { + JSONDecoder::decode_json("tier_targets", tier_targets, obj); + } } -rgw_pool RGWRealm::get_pool(CephContext *cct) const +void RGWZonePlacementInfo::dump(Formatter *f) const { - if (cct->_conf->rgw_realm_root_pool.empty()) { - return rgw_pool(RGW_DEFAULT_REALM_ROOT_POOL); - } - return rgw_pool(cct->_conf->rgw_realm_root_pool); + encode_json("index_pool", index_pool, f); + encode_json("storage_classes", storage_classes, f); + encode_json("data_extra_pool", data_extra_pool, f); + encode_json("index_type", (uint32_t)index_type, f); + + /* no real need for backward compatibility of compression_type and data_pool in here, + * rather not clutter the output */ } -const string RGWRealm::get_default_oid(bool old_format) const +void RGWZonePlacementInfo::decode_json(JSONObj *obj) { - if (cct->_conf->rgw_default_realm_info_oid.empty()) { - return default_realm_info_oid; + JSONDecoder::decode_json("index_pool", index_pool, obj); + JSONDecoder::decode_json("storage_classes", storage_classes, obj); + JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj); + uint32_t it; + JSONDecoder::decode_json("index_type", it, obj); + index_type = (rgw::BucketIndexType)it; + + /* backward compatibility, these are now defined in storage_classes */ + string standard_compression_type; + string *pcompression = nullptr; + if (JSONDecoder::decode_json("compression", standard_compression_type, obj)) { + pcompression = &standard_compression_type; + } + rgw_pool standard_data_pool; + rgw_pool *ppool = nullptr; + if (JSONDecoder::decode_json("data_pool", standard_data_pool, obj)) { + ppool = &standard_data_pool; + } + if (ppool || pcompression) { + storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, ppool, pcompression); } - return cct->_conf->rgw_default_realm_info_oid; } -const string& RGWRealm::get_names_oid_prefix() const +void RGWSystemMetaObj::dump(Formatter *f) const { - return realm_names_oid_prefix; + encode_json("id", id , f); + encode_json("name", name , f); } -const string& RGWRealm::get_info_oid_prefix(bool old_format) const +void RGWSystemMetaObj::decode_json(JSONObj *obj) { - return realm_info_oid_prefix; + JSONDecoder::decode_json("id", id, obj); + JSONDecoder::decode_json("name", name, obj); } -int RGWRealm::set_current_period(const DoutPrefixProvider *dpp, RGWPeriod& period, optional_yield y) +int RGWSystemMetaObj::read_default(const DoutPrefixProvider *dpp, + RGWDefaultSystemMetaObjInfo& default_info, + const string& oid, optional_yield y) { - // update realm epoch to match the period's - if (epoch > period.get_realm_epoch()) { - ldpp_dout(dpp, 0) << "ERROR: set_current_period with old realm epoch " - << period.get_realm_epoch() << ", current epoch=" << epoch << dendl; - return -EINVAL; - } - if (epoch == period.get_realm_epoch() && current_period != period.get_id()) { - ldpp_dout(dpp, 0) << "ERROR: set_current_period with same realm epoch " - << period.get_realm_epoch() << ", but different period id " - << period.get_id() << " != " << current_period << dendl; - return -EINVAL; - } - - epoch = period.get_realm_epoch(); - current_period = period.get_id(); + using ceph::decode; + auto pool = get_pool(cct); + bufferlist bl; - int ret = update(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: period update: " << cpp_strerror(-ret) << dendl; + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); + int ret = sysobj.rop().read(dpp, &bl, y); + if (ret < 0) return ret; - } - ret = period.reflect(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: period.reflect(): " << cpp_strerror(-ret) << dendl; - return ret; + try { + auto iter = bl.cbegin(); + decode(default_info, iter); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "error decoding data from " << pool << ":" << oid << dendl; + return -EIO; } return 0; } -string RGWRealm::get_control_oid() const +void RGWZoneGroupPlacementTarget::dump(Formatter *f) const { - return get_info_oid_prefix() + id + ".control"; + encode_json("name", name, f); + encode_json("tags", tags, f); + encode_json("storage_classes", storage_classes, f); + if (!tier_targets.empty()) { + encode_json("tier_targets", tier_targets, f); + } } -int RGWRealm::notify_zone(const DoutPrefixProvider *dpp, bufferlist& bl, optional_yield y) +void RGWZoneGroupPlacementTier::decode_json(JSONObj *obj) { - rgw_pool pool{get_pool(cct)}; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, get_control_oid()}); - int ret = sysobj.wn().notify(dpp, bl, 0, nullptr, y); - if (ret < 0) { - return ret; + JSONDecoder::decode_json("tier_type", tier_type, obj); + JSONDecoder::decode_json("storage_class", storage_class, obj); + JSONDecoder::decode_json("retain_head_object", retain_head_object, obj); + + if (tier_type == "cloud-s3") { + JSONDecoder::decode_json("s3", t.s3, obj); } - return 0; } -int RGWRealm::notify_new_period(const DoutPrefixProvider *dpp, const RGWPeriod& period, optional_yield y) +void RGWZoneStorageClasses::dump(Formatter *f) const { - bufferlist bl; - using ceph::encode; - // push the period to dependent zonegroups/zones - encode(RGWRealmNotify::ZonesNeedPeriod, bl); - encode(period, bl); - // reload the gateway with the new period - encode(RGWRealmNotify::Reload, bl); - - return notify_zone(dpp, bl, y); + for (auto& i : m) { + encode_json(i.first.c_str(), i.second, f); + } } - -int RGWRealm::find_zone(const DoutPrefixProvider *dpp, - const rgw_zone_id& zid, - RGWPeriod *pperiod, - RGWZoneGroup *pzonegroup, - bool *pfound, - optional_yield y) const +void RGWZoneStorageClasses::decode_json(JSONObj *obj) { - auto& found = *pfound; - - found = false; + JSONFormattable f; + decode_json_obj(f, obj); - string period_id; - epoch_t epoch = 0; + for (auto& field : f.object()) { + JSONObj *field_obj = obj->find_obj(field.first); + assert(field_obj); - RGWPeriod period(period_id, epoch); - int r = period.init(dpp, cct, sysobj_svc, get_id(), y, get_name()); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: period init failed: " << cpp_strerror(-r) << " ... skipping" << dendl; - return r; + decode_json_obj(m[field.first], field_obj); } + standard_class = &m[RGW_STORAGE_CLASS_STANDARD]; +} + +void RGWZoneGroupPlacementTier::dump(Formatter *f) const +{ + encode_json("tier_type", tier_type, f); + encode_json("storage_class", storage_class, f); + encode_json("retain_head_object", retain_head_object, f); - found = period.find_zone(dpp, zid, pzonegroup, y); - if (found) { - *pperiod = period; + if (tier_type == "cloud-s3") { + encode_json("s3", t.s3, f); } - return 0; } -std::string RGWPeriodConfig::get_oid(const std::string& realm_id) +void RGWZoneGroupPlacementTierS3::decode_json(JSONObj *obj) { - if (realm_id.empty()) { - return "period_config.default"; + JSONDecoder::decode_json("endpoint", endpoint, obj); + JSONDecoder::decode_json("access_key", key.id, obj); + JSONDecoder::decode_json("secret", key.key, obj); + JSONDecoder::decode_json("region", region, obj); + string s; + JSONDecoder::decode_json("host_style", s, obj); + if (s != "virtual") { + host_style = PathStyle; + } else { + host_style = VirtualStyle; } - return "period_config." + realm_id; + JSONDecoder::decode_json("target_storage_class", target_storage_class, obj); + JSONDecoder::decode_json("target_path", target_path, obj); + JSONDecoder::decode_json("acl_mappings", acl_mappings, obj); + JSONDecoder::decode_json("multipart_sync_threshold", multipart_sync_threshold, obj); + JSONDecoder::decode_json("multipart_min_part_size", multipart_min_part_size, obj); } -rgw_pool RGWPeriodConfig::get_pool(CephContext *cct) +void RGWZoneStorageClass::dump(Formatter *f) const { - const auto& pool_name = cct->_conf->rgw_period_root_pool; - if (pool_name.empty()) { - return {RGW_DEFAULT_PERIOD_ROOT_POOL}; + if (data_pool) { + encode_json("data_pool", data_pool.get(), f); + } + if (compression_type) { + encode_json("compression_type", compression_type.get(), f); } - return {pool_name}; } -int RGWPeriodConfig::read(const DoutPrefixProvider *dpp, RGWSI_SysObj *sysobj_svc, const std::string& realm_id, - optional_yield y) +void RGWZoneStorageClass::decode_json(JSONObj *obj) { - const auto& pool = get_pool(sysobj_svc->ctx()); - const auto& oid = get_oid(realm_id); - bufferlist bl; + JSONDecoder::decode_json("data_pool", data_pool, obj); + JSONDecoder::decode_json("compression_type", compression_type, obj); +} - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - int ret = sysobj.rop().read(dpp, &bl, y); - if (ret < 0) { - return ret; - } - using ceph::decode; - try { - auto iter = bl.cbegin(); - decode(*this, iter); - } catch (buffer::error& err) { - return -EIO; +void RGWTierACLMapping::decode_json(JSONObj *obj) +{ + string s; + JSONDecoder::decode_json("type", s, obj); + if (s == "email") { + type = ACL_TYPE_EMAIL_USER; + } else if (s == "uri") { + type = ACL_TYPE_GROUP; + } else { + type = ACL_TYPE_CANON_USER; } - return 0; -} -int RGWPeriodConfig::write(const DoutPrefixProvider *dpp, - RGWSI_SysObj *sysobj_svc, - const std::string& realm_id, optional_yield y) -{ - const auto& pool = get_pool(sysobj_svc->ctx()); - const auto& oid = get_oid(realm_id); - bufferlist bl; - using ceph::encode; - encode(*this, bl); - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - return sysobj.wop() - .set_exclusive(false) - .write(dpp, bl, y); + JSONDecoder::decode_json("source_id", source_id, obj); + JSONDecoder::decode_json("dest_id", dest_id, obj); } -int RGWPeriod::init(const DoutPrefixProvider *dpp, CephContext *_cct, RGWSI_SysObj *_sysobj_svc, - const string& period_realm_id, optional_yield y, - const string& period_realm_name, bool setup_obj) +void RGWZoneGroupPlacementTierS3::dump(Formatter *f) const { - cct = _cct; - sysobj_svc = _sysobj_svc; - - realm_id = period_realm_id; - realm_name = period_realm_name; - - if (!setup_obj) - return 0; - - return init(dpp, _cct, _sysobj_svc, y, setup_obj); + encode_json("endpoint", endpoint, f); + encode_json("access_key", key.id, f); + encode_json("secret", key.key, f); + encode_json("region", region, f); + string s = (host_style == PathStyle ? "path" : "virtual"); + encode_json("host_style", s, f); + encode_json("target_storage_class", target_storage_class, f); + encode_json("target_path", target_path, f); + encode_json("acl_mappings", acl_mappings, f); + encode_json("multipart_sync_threshold", multipart_sync_threshold, f); + encode_json("multipart_min_part_size", multipart_min_part_size, f); } - -int RGWPeriod::init(const DoutPrefixProvider *dpp, - CephContext *_cct, RGWSI_SysObj *_sysobj_svc, - optional_yield y, bool setup_obj) +void RGWTierACLMapping::dump(Formatter *f) const { - cct = _cct; - sysobj_svc = _sysobj_svc; - - if (!setup_obj) - return 0; - - if (id.empty()) { - RGWRealm realm(realm_id, realm_name); - int ret = realm.init(dpp, cct, sysobj_svc, y); - if (ret < 0) { - ldpp_dout(dpp, 4) << "RGWPeriod::init failed to init realm " << realm_name << " id " << realm_id << " : " << - cpp_strerror(-ret) << dendl; - return ret; - } - id = realm.get_current_period(); - realm_id = realm.get_id(); - } - - if (!epoch) { - int ret = use_latest_epoch(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "failed to use_latest_epoch period id " << id << " realm " << realm_name << " id " << realm_id - << " : " << cpp_strerror(-ret) << dendl; - return ret; - } + string s; + switch (type) { + case ACL_TYPE_EMAIL_USER: + s = "email"; + break; + case ACL_TYPE_GROUP: + s = "uri"; + break; + default: + s = "id"; + break; } - - return read_info(dpp, y); + encode_json("type", s, f); + encode_json("source_id", source_id, f); + encode_json("dest_id", dest_id, f); } - -int RGWPeriod::get_zonegroup(RGWZoneGroup& zonegroup, - const string& zonegroup_id) const +void RGWPeriodMap::dump(Formatter *f) const { - map::const_iterator iter; - if (!zonegroup_id.empty()) { - iter = period_map.zonegroups.find(zonegroup_id); - } else { - iter = period_map.zonegroups.find("default"); - } - if (iter != period_map.zonegroups.end()) { - zonegroup = iter->second; - return 0; - } - - return -ENOENT; + encode_json("id", id, f); + encode_json_map("zonegroups", zonegroups, f); + encode_json("short_zone_ids", short_zone_ids, f); } -bool RGWPeriod::find_zone(const DoutPrefixProvider *dpp, - const rgw_zone_id& zid, - RGWZoneGroup *pzonegroup, - optional_yield y) const +static void decode_zonegroups(map& zonegroups, JSONObj *o) { RGWZoneGroup zg; - RGWZone zone; - - bool found = period_map.find_zone_by_id(zid, &zg, &zone); - if (found) { - *pzonegroup = zg; - } - - return found; + zg.decode_json(o); + zonegroups[zg.get_id()] = zg; } -const string& RGWPeriod::get_latest_epoch_oid() const +void RGWPeriodMap::decode_json(JSONObj *obj) { - if (cct->_conf->rgw_period_latest_epoch_info_oid.empty()) { - return period_latest_epoch_info_oid; + JSONDecoder::decode_json("id", id, obj); + JSONDecoder::decode_json("zonegroups", zonegroups, decode_zonegroups, obj); + /* backward compatability with region */ + if (zonegroups.empty()) { + JSONDecoder::decode_json("regions", zonegroups, obj); + } + /* backward compatability with region */ + if (master_zonegroup.empty()) { + JSONDecoder::decode_json("master_region", master_zonegroup, obj); } - return cct->_conf->rgw_period_latest_epoch_info_oid; + JSONDecoder::decode_json("short_zone_ids", short_zone_ids, obj); } -const string& RGWPeriod::get_info_oid_prefix() const -{ - return period_info_oid_prefix; -} +void RGWPeriodMap::decode(bufferlist::const_iterator& bl) { + DECODE_START(2, bl); + decode(id, bl); + decode(zonegroups, bl); + decode(master_zonegroup, bl); + if (struct_v >= 2) { + decode(short_zone_ids, bl); + } + DECODE_FINISH(bl); -const string RGWPeriod::get_period_oid_prefix() const -{ - return get_info_oid_prefix() + id; + zonegroups_by_api.clear(); + for (map::iterator iter = zonegroups.begin(); + iter != zonegroups.end(); ++iter) { + RGWZoneGroup& zonegroup = iter->second; + zonegroups_by_api[zonegroup.api_name] = zonegroup; + if (zonegroup.is_master_zonegroup()) { + master_zonegroup = zonegroup.get_id(); + } + } } -const string RGWPeriod::get_period_oid() const +void RGWPeriodMap::encode(bufferlist& bl) const { - std::ostringstream oss; - oss << get_period_oid_prefix(); - // skip the epoch for the staging period - if (id != get_staging_id(realm_id)) - oss << "." << epoch; - return oss.str(); + ENCODE_START(2, 1, bl); + encode(id, bl); + encode(zonegroups, bl); + encode(master_zonegroup, bl); + encode(short_zone_ids, bl); + ENCODE_FINISH(bl); } -int RGWPeriod::read_latest_epoch(const DoutPrefixProvider *dpp, - RGWPeriodLatestEpochInfo& info, - optional_yield y, - RGWObjVersionTracker *objv) +int RGWSystemMetaObj::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) { - string oid = get_period_oid_prefix() + get_latest_epoch_oid(); + int ret; - rgw_pool pool(get_pool(cct)); - bufferlist bl; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); - int ret = sysobj.rop().read(dpp, &bl, y); - if (ret < 0) { - ldpp_dout(dpp, 1) << "error read_lastest_epoch " << pool << ":" << oid << dendl; + /* check to see the name is not used */ + ret = read_id(dpp, name, id, y); + if (exclusive && ret == 0) { + ldpp_dout(dpp, 10) << "ERROR: name " << name << " already in use for obj id " << id << dendl; + return -EEXIST; + } else if ( ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "failed reading obj id " << id << ": " << cpp_strerror(-ret) << dendl; return ret; } - try { - auto iter = bl.cbegin(); - using ceph::decode; - decode(info, iter); - } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "error decoding data from " << pool << ":" << oid << dendl; - return -EIO; - } - - return 0; -} -int RGWPeriod::get_latest_epoch(const DoutPrefixProvider *dpp, epoch_t& latest_epoch, optional_yield y) -{ - RGWPeriodLatestEpochInfo info; + if (id.empty()) { + /* create unique id */ + uuid_d new_uuid; + char uuid_str[37]; + new_uuid.generate_random(); + new_uuid.print(uuid_str); + id = uuid_str; + } - int ret = read_latest_epoch(dpp, info, y); + ret = store_info(dpp, exclusive, y); if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: storing info for " << id << ": " << cpp_strerror(-ret) << dendl; return ret; } - latest_epoch = info.epoch; - - return 0; + return store_name(dpp, exclusive, y); } -int RGWPeriod::use_latest_epoch(const DoutPrefixProvider *dpp, optional_yield y) +int RGWSystemMetaObj::read_default_id(const DoutPrefixProvider *dpp, string& default_id, optional_yield y, + bool old_format) { - RGWPeriodLatestEpochInfo info; - int ret = read_latest_epoch(dpp, info, y); + RGWDefaultSystemMetaObjInfo default_info; + + int ret = read_default(dpp, default_info, get_default_oid(old_format), y); if (ret < 0) { return ret; } - epoch = info.epoch; + default_id = default_info.default_id; return 0; } -int RGWPeriod::set_latest_epoch(const DoutPrefixProvider *dpp, - optional_yield y, - epoch_t epoch, bool exclusive, - RGWObjVersionTracker *objv) +int RGWSystemMetaObj::set_as_default(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) { - string oid = get_period_oid_prefix() + get_latest_epoch_oid(); + using ceph::encode; + string oid = get_default_oid(); rgw_pool pool(get_pool(cct)); bufferlist bl; - RGWPeriodLatestEpochInfo info; - info.epoch = epoch; + RGWDefaultSystemMetaObjInfo default_info; + default_info.default_id = id; - using ceph::encode; - encode(info, bl); + encode(default_info, bl); auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); - return sysobj.wop() - .set_exclusive(exclusive) - .write(dpp, bl, y); -} - -int RGWPeriod::update_latest_epoch(const DoutPrefixProvider *dpp, epoch_t epoch, optional_yield y) -{ - static constexpr int MAX_RETRIES = 20; - - for (int i = 0; i < MAX_RETRIES; i++) { - RGWPeriodLatestEpochInfo info; - RGWObjVersionTracker objv; - bool exclusive = false; - - // read existing epoch - int r = read_latest_epoch(dpp, info, y, &objv); - if (r == -ENOENT) { - // use an exclusive create to set the epoch atomically - exclusive = true; - ldpp_dout(dpp, 20) << "creating initial latest_epoch=" << epoch - << " for period=" << id << dendl; - } else if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to read latest_epoch" << dendl; - return r; - } else if (epoch <= info.epoch) { - r = -EEXIST; // fail with EEXIST if epoch is not newer - ldpp_dout(dpp, 10) << "found existing latest_epoch " << info.epoch - << " >= given epoch " << epoch << ", returning r=" << r << dendl; - return r; - } else { - ldpp_dout(dpp, 20) << "updating latest_epoch from " << info.epoch - << " -> " << epoch << " on period=" << id << dendl; - } - - r = set_latest_epoch(dpp, y, epoch, exclusive, &objv); - if (r == -EEXIST) { - continue; // exclusive create raced with another update, retry - } else if (r == -ECANCELED) { - continue; // write raced with a conflicting version, retry - } - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to write latest_epoch" << dendl; - return r; - } - return 0; // return success - } + int ret = sysobj.wop() + .set_exclusive(exclusive) + .write(dpp, bl, y); + if (ret < 0) + return ret; - return -ECANCELED; // fail after max retries + return 0; } -int RGWPeriod::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) +int RGWSystemMetaObj::store_info(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) { rgw_pool pool(get_pool(cct)); - // delete the object for each period epoch - for (epoch_t e = 1; e <= epoch; e++) { - RGWPeriod p{get_id(), e}; - rgw_raw_obj oid{pool, p.get_period_oid()}; - auto sysobj = sysobj_svc->get_obj(oid); - int ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid - << ": " << cpp_strerror(-ret) << dendl; - } - } + string oid = get_info_oid_prefix() + id; - // delete the .latest_epoch object - rgw_raw_obj oid{pool, get_period_oid_prefix() + get_latest_epoch_oid()}; - auto sysobj = sysobj_svc->get_obj(oid); - int ret = sysobj.wop().remove(dpp, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid - << ": " << cpp_strerror(-ret) << dendl; - } - return ret; + bufferlist bl; + using ceph::encode; + encode(*this, bl); + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); + return sysobj.wop() + .set_exclusive(exclusive) + .write(dpp, bl, y); } -int RGWPeriod::read_info(const DoutPrefixProvider *dpp, optional_yield y) +int RGWSystemMetaObj::read_id(const DoutPrefixProvider *dpp, const string& obj_name, string& object_id, + optional_yield y) { + using ceph::decode; rgw_pool pool(get_pool(cct)); - bufferlist bl; - auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, get_period_oid()}); + string oid = get_names_oid_prefix() + obj_name; + + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); int ret = sysobj.rop().read(dpp, &bl, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "failed reading obj info from " << pool << ":" << get_period_oid() << ": " << cpp_strerror(-ret) << dendl; return ret; } + RGWNameToId nameToId; try { - using ceph::decode; auto iter = bl.cbegin(); - decode(*this, iter); + decode(nameToId, iter); } catch (buffer::error& err) { - ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << get_period_oid() << dendl; + ldpp_dout(dpp, 0) << "ERROR: failed to decode obj from " << pool << ":" << oid << dendl; return -EIO; } - + object_id = nameToId.obj_id; return 0; } -int RGWPeriod::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) +int RGWSystemMetaObj::store_name(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) { - int ret; - - /* create unique id */ - uuid_d new_uuid; - char uuid_str[37]; - new_uuid.generate_random(); - new_uuid.print(uuid_str); - id = uuid_str; - - epoch = FIRST_EPOCH; + rgw_pool pool(get_pool(cct)); + string oid = get_names_oid_prefix() + name; - period_map.id = id; + RGWNameToId nameToId; + nameToId.obj_id = id; - ret = store_info(dpp, exclusive, y); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: storing info for " << id << ": " << cpp_strerror(-ret) << dendl; - return ret; - } - - ret = set_latest_epoch(dpp, y, epoch); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: setting latest epoch " << id << ": " << cpp_strerror(-ret) << dendl; - } - - return ret; -} - -int RGWPeriod::store_info(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) -{ - rgw_pool pool(get_pool(cct)); - - string oid = get_period_oid(); bufferlist bl; using ceph::encode; - encode(*this, bl); - + encode(nameToId, bl); auto sysobj = sysobj_svc->get_obj(rgw_raw_obj(pool, oid)); return sysobj.wop() .set_exclusive(exclusive) .write(dpp, bl, y); } -rgw_pool RGWPeriod::get_pool(CephContext *cct) const +bool RGWPeriodMap::find_zone_by_id(const rgw_zone_id& zone_id, + RGWZoneGroup *zonegroup, + RGWZone *zone) const { - if (cct->_conf->rgw_period_root_pool.empty()) { - return rgw_pool(RGW_DEFAULT_PERIOD_ROOT_POOL); - } - return rgw_pool(cct->_conf->rgw_period_root_pool); -} + for (auto& iter : zonegroups) { + auto& zg = iter.second; -int RGWPeriod::add_zonegroup(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, optional_yield y) -{ - if (zonegroup.realm_id != realm_id) { - return 0; - } - int ret = period_map.update(zonegroup, cct); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: updating period map: " << cpp_strerror(-ret) << dendl; - return ret; + auto ziter = zg.zones.find(zone_id); + if (ziter != zg.zones.end()) { + *zonegroup = zg; + *zone = ziter->second; + return true; + } } - return store_info(dpp, false, y); + return false; } -int RGWPeriod::update(const DoutPrefixProvider *dpp, optional_yield y) +int RGWZoneGroup::set_as_default(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) { - auto zone_svc = sysobj_svc->get_zone_svc(); - ldpp_dout(dpp, 20) << __func__ << " realm " << realm_id << " period " << get_id() << dendl; - list zonegroups; - int ret = zone_svc->list_zonegroups(dpp, zonegroups); - if (ret < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to list zonegroups: " << cpp_strerror(-ret) << dendl; - return ret; - } - - // clear zone short ids of removed zones. period_map.update() will add the - // remaining zones back - period_map.short_zone_ids.clear(); - - for (auto& iter : zonegroups) { - RGWZoneGroup zg(string(), iter); - ret = zg.init(dpp, cct, sysobj_svc, y); + if (realm_id.empty()) { + /* try using default realm */ + RGWRealm realm; + int ret = realm.init(dpp, cct, sysobj_svc, y); if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: zg.init() failed: " << cpp_strerror(-ret) << dendl; - continue; - } - - if (zg.realm_id != realm_id) { - ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() << " zone realm id " << zg.realm_id << ", not on our realm " << realm_id << dendl; - continue; - } - - if (zg.master_zone.empty()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; - return -EINVAL; - } - - if (zg.zones.find(zg.master_zone) == zg.zones.end()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() - << " has a non existent master zone "<< dendl; + ldpp_dout(dpp, 10) << "could not read realm id: " << cpp_strerror(-ret) << dendl; return -EINVAL; } + realm_id = realm.get_id(); + } - if (zg.is_master_zonegroup()) { - master_zonegroup = zg.get_id(); - master_zone = zg.master_zone; - } + return RGWSystemMetaObj::set_as_default(dpp, y, exclusive); +} - int ret = period_map.update(zg, cct); - if (ret < 0) { - return ret; - } +int RGWSystemMetaObj::write(const DoutPrefixProvider *dpp, bool exclusive, optional_yield y) +{ + int ret = store_info(dpp, exclusive, y); + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << "(): store_info() returned ret=" << ret << dendl; + return ret; } - - ret = period_config.read(dpp, sysobj_svc, realm_id, y); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " - << cpp_strerror(ret) << dendl; + ret = store_name(dpp, exclusive, y); + if (ret < 0) { + ldpp_dout(dpp, 20) << __func__ << "(): store_name() returned ret=" << ret << dendl; return ret; } return 0; } -int RGWPeriod::reflect(const DoutPrefixProvider *dpp, optional_yield y) +namespace rgw { + +int init_zone_pool_names(const DoutPrefixProvider *dpp, optional_yield y, + const std::set& pools, RGWZoneParams& info) { - for (auto& iter : period_map.zonegroups) { - RGWZoneGroup& zg = iter.second; - zg.reinit_instance(cct, sysobj_svc); - int r = zg.write(dpp, false, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to store zonegroup info for zonegroup=" << iter.first << ": " << cpp_strerror(-r) << dendl; - return r; - } - if (zg.is_master_zonegroup()) { - // set master as default if no default exists - r = zg.set_as_default(dpp, y, true); - if (r == 0) { - ldpp_dout(dpp, 1) << "Set the period's master zonegroup " << zg.get_id() - << " as the default" << dendl; + info.domain_root = fix_zone_pool_dup(pools, info.name, ".rgw.meta:root", info.domain_root); + info.control_pool = fix_zone_pool_dup(pools, info.name, ".rgw.control", info.control_pool); + info.gc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:gc", info.gc_pool); + info.lc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:lc", info.lc_pool); + info.log_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log", info.log_pool); + info.intent_log_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:intent", info.intent_log_pool); + info.usage_log_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:usage", info.usage_log_pool); + info.user_keys_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.keys", info.user_keys_pool); + info.user_email_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.email", info.user_email_pool); + info.user_swift_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.swift", info.user_swift_pool); + info.user_uid_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.uid", info.user_uid_pool); + info.roles_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:roles", info.roles_pool); + info.reshard_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:reshard", info.reshard_pool); + info.otp_pool = fix_zone_pool_dup(pools, info.name, ".rgw.otp", info.otp_pool); + info.oidc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:oidc", info.oidc_pool); + info.notif_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:notif", info.notif_pool); + + for (auto& [pname, placement] : info.placement_pools) { + placement.index_pool = fix_zone_pool_dup(pools, info.name, "." + default_bucket_index_pool_suffix, placement.index_pool); + placement.data_extra_pool= fix_zone_pool_dup(pools, info.name, "." + default_storage_extra_pool_suffix, placement.data_extra_pool); + for (auto& [sname, sc] : placement.storage_classes.get_all()) { + if (sc.data_pool) { + sc.data_pool = fix_zone_pool_dup(pools, info.name, "." + default_storage_pool_suffix, *sc.data_pool); } } } - int r = period_config.write(dpp, sysobj_svc, realm_id, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to store period config: " - << cpp_strerror(-r) << dendl; - return r; - } return 0; } -void RGWPeriod::fork() +int add_zone_to_group(const DoutPrefixProvider* dpp, RGWZoneGroup& zonegroup, + const RGWZoneParams& zone_params, + const bool *pis_master, const bool *pread_only, + const std::list& endpoints, + const std::string *ptier_type, + const bool *psync_from_all, + const std::list& sync_from, + const std::list& sync_from_rm, + const std::string *predirect_zone, + std::optional bucket_index_max_shards, + const rgw::zone_features::set& enable_features, + const rgw::zone_features::set& disable_features) { - ldout(cct, 20) << __func__ << " realm " << realm_id << " period " << id << dendl; - predecessor_uuid = id; - id = get_staging_id(realm_id); - period_map.reset(); - realm_epoch++; -} + const std::string& zone_id = zone_params.id; + const std::string& zone_name = zone_params.name; -static int read_sync_status(const DoutPrefixProvider *dpp, rgw::sal::RadosStore* store, rgw_meta_sync_status *sync_status) -{ - // initialize a sync status manager to read the status - RGWMetaSyncStatusManager mgr(store, store->svc()->rados->get_async_processor()); - int r = mgr.init(dpp); - if (r < 0) { - return r; + if (zone_id.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a zone id" << dendl; + return -EINVAL; } - r = mgr.read_sync_status(dpp, sync_status); - mgr.stop(); - return r; -} - -int RGWPeriod::update_sync_status(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, /* for now */ - const RGWPeriod ¤t_period, - std::ostream& error_stream, - bool force_if_stale) -{ - rgw_meta_sync_status status; - int r = read_sync_status(dpp, static_cast(store), &status); - if (r < 0) { - ldpp_dout(dpp, 0) << "period failed to read sync status: " - << cpp_strerror(-r) << dendl; - return r; + if (zone_name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl; + return -EINVAL; } - std::vector markers; - - const auto current_epoch = current_period.get_realm_epoch(); - if (current_epoch != status.sync_info.realm_epoch) { - // no sync status markers for the current period - ceph_assert(current_epoch > status.sync_info.realm_epoch); - const int behind = current_epoch - status.sync_info.realm_epoch; - if (!force_if_stale && current_epoch > 1) { - error_stream << "ERROR: This zone is " << behind << " period(s) behind " - "the current master zone in metadata sync. If this zone is promoted " - "to master, any metadata changes during that time are likely to " - "be lost.\n" - "Waiting for this zone to catch up on metadata sync (see " - "'radosgw-admin sync status') is recommended.\n" - "To promote this zone to master anyway, add the flag " - "--yes-i-really-mean-it." << std::endl; - return -EINVAL; - } - // empty sync status markers - other zones will skip this period during - // incremental metadata sync - markers.resize(status.sync_info.num_shards); - } else { - markers.reserve(status.sync_info.num_shards); - for (auto& i : status.sync_markers) { - auto& marker = i.second; - // filter out markers from other periods - if (marker.realm_epoch != current_epoch) { - marker.marker.clear(); + // check for duplicate zone name on insert + if (!zonegroup.zones.count(zone_id)) { + for (const auto& [id, zone] : zonegroup.zones) { + if (zone.name == zone_name) { + ldpp_dout(dpp, 0) << "ERROR: found existing zone name " << zone_name + << " (" << id << ") in zonegroup " << zonegroup.name << dendl; + return -EEXIST; } - markers.emplace_back(std::move(marker.marker)); } } - std::swap(sync_status, markers); - return 0; -} + if (pis_master) { + rgw_zone_id& master_zone = zonegroup.master_zone; + if (*pis_master) { + if (!master_zone.empty() && master_zone != zone_id) { + ldpp_dout(dpp, 0) << "NOTICE: overriding master zone: " + << master_zone << dendl; + } + master_zone = zone_id; + } else if (master_zone == zone_id) { + master_zone.clear(); + } + } -int RGWPeriod::commit(const DoutPrefixProvider *dpp, - rgw::sal::Store* store, - RGWRealm& realm, const RGWPeriod& current_period, - std::ostream& error_stream, optional_yield y, - bool force_if_stale) -{ - auto zone_svc = sysobj_svc->get_zone_svc(); - ldpp_dout(dpp, 20) << __func__ << " realm " << realm.get_id() << " period " << current_period.get_id() << dendl; - // gateway must be in the master zone to commit - if (master_zone != zone_svc->get_zone_params().get_id()) { - error_stream << "Cannot commit period on zone " - << zone_svc->get_zone_params().get_id() << ", it must be sent to " - "the period's master zone " << master_zone << '.' << std::endl; - return -EINVAL; + // make sure the zone's placement targets are named in the zonegroup + for (const auto& [name, placement] : zone_params.placement_pools) { + auto target = RGWZoneGroupPlacementTarget{.name = name}; + zonegroup.placement_targets.emplace(name, std::move(target)); } - // period predecessor must match current period - if (predecessor_uuid != current_period.get_id()) { - error_stream << "Period predecessor " << predecessor_uuid - << " does not match current period " << current_period.get_id() - << ". Use 'period pull' to get the latest period from the master, " - "reapply your changes, and try again." << std::endl; - return -EINVAL; + + RGWZone& zone = zonegroup.zones[zone_params.id]; + zone.id = zone_params.id; + zone.name = zone_params.name; + if (!endpoints.empty()) { + zone.endpoints = endpoints; } - // realm epoch must be 1 greater than current period - if (realm_epoch != current_period.get_realm_epoch() + 1) { - error_stream << "Period's realm epoch " << realm_epoch - << " does not come directly after current realm epoch " - << current_period.get_realm_epoch() << ". Use 'realm pull' to get the " - "latest realm and period from the master zone, reapply your changes, " - "and try again." << std::endl; - return -EINVAL; + if (pread_only) { + zone.read_only = *pread_only; } - // did the master zone change? - if (master_zone != current_period.get_master_zone()) { - // store the current metadata sync status in the period - int r = update_sync_status(dpp, store, current_period, error_stream, force_if_stale); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update metadata sync status: " - << cpp_strerror(-r) << dendl; - return r; - } - // create an object with a new period id - r = create(dpp, y, true); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; - return r; - } - // set as current period - r = realm.set_current_period(dpp, *this, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update realm's current period: " - << cpp_strerror(-r) << dendl; - return r; - } - ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period " - << id << dendl; - realm.notify_new_period(dpp, *this, y); - return 0; + if (ptier_type) { + zone.tier_type = *ptier_type; } - // period must be based on current epoch - if (epoch != current_period.get_epoch()) { - error_stream << "Period epoch " << epoch << " does not match " - "predecessor epoch " << current_period.get_epoch() - << ". Use 'period pull' to get the latest epoch from the master zone, " - "reapply your changes, and try again." << std::endl; - return -EINVAL; + if (psync_from_all) { + zone.sync_from_all = *psync_from_all; } - // set period as next epoch - set_id(current_period.get_id()); - set_epoch(current_period.get_epoch() + 1); - set_predecessor(current_period.get_predecessor()); - realm_epoch = current_period.get_realm_epoch(); - // write the period to rados - int r = store_info(dpp, false, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(-r) << dendl; - return r; + if (predirect_zone) { + zone.redirect_zone = *predirect_zone; } - // set as latest epoch - r = update_latest_epoch(dpp, epoch, y); - if (r == -EEXIST) { - // already have this epoch (or a more recent one) - return 0; + if (bucket_index_max_shards) { + zone.bucket_index_max_shards = *bucket_index_max_shards; } - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to set latest epoch: " << cpp_strerror(-r) << dendl; - return r; + + // add/remove sync_from + for (auto add : sync_from) { + zone.sync_from.insert(add); } - r = reflect(dpp, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(-r) << dendl; - return r; + + for (const auto& rm : sync_from_rm) { + auto i = zone.sync_from.find(rm); + if (i == zone.sync_from.end()) { + ldpp_dout(dpp, 1) << "WARNING: zone \"" << rm + << "\" was not in sync_from" << dendl; + continue; + } + zone.sync_from.erase(i); } - ldpp_dout(dpp, 4) << "Committed new epoch " << epoch - << " for period " << id << dendl; - realm.notify_new_period(dpp, *this, y); - return 0; -} -int RGWZoneParams::create_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) -{ - name = default_zone_name; + // add/remove supported features + zone.supported_features.insert(enable_features.begin(), + enable_features.end()); - int r = create(dpp, y); - if (r < 0) { - return r; + for (const auto& feature : disable_features) { + if (zonegroup.enabled_features.contains(feature)) { + ldpp_dout(dpp, -1) << "ERROR: Cannot disable zone feature \"" << feature + << "\" until it's been disabled in zonegroup " << zonegroup.name << dendl; + return -EINVAL; + } + auto i = zone.supported_features.find(feature); + if (i == zone.supported_features.end()) { + ldpp_dout(dpp, 1) << "WARNING: zone feature \"" << feature + << "\" was not enabled in zone " << zone.name << dendl; + continue; + } + zone.supported_features.erase(i); } - if (old_format) { - name = id; + const bool log_data = zonegroup.zones.size() > 1; + for (auto& [id, zone] : zonegroup.zones) { + zone.log_data = log_data; } - return r; + return 0; } -void RGWZoneParams::dump(Formatter *f) const -{ - RGWSystemMetaObj::dump(f); - encode_json("domain_root", domain_root, f); - encode_json("control_pool", control_pool, f); - encode_json("gc_pool", gc_pool, f); - encode_json("lc_pool", lc_pool, f); - encode_json("log_pool", log_pool, f); - encode_json("intent_log_pool", intent_log_pool, f); - encode_json("usage_log_pool", usage_log_pool, f); - encode_json("roles_pool", roles_pool, f); - encode_json("reshard_pool", reshard_pool, f); - encode_json("user_keys_pool", user_keys_pool, f); - encode_json("user_email_pool", user_email_pool, f); - encode_json("user_swift_pool", user_swift_pool, f); - encode_json("user_uid_pool", user_uid_pool, f); - encode_json("otp_pool", otp_pool, f); - encode_json_plain("system_key", system_key, f); - encode_json("placement_pools", placement_pools, f); - encode_json("tier_config", tier_config, f); - encode_json("realm_id", realm_id, f); - encode_json("notif_pool", notif_pool, f); -} - -namespace { - -void add_zone_pools(const RGWZoneParams& info, - std::set& pools) -{ - pools.insert(info.domain_root); - pools.insert(info.control_pool); - pools.insert(info.gc_pool); - pools.insert(info.log_pool); - pools.insert(info.intent_log_pool); - pools.insert(info.usage_log_pool); - pools.insert(info.user_keys_pool); - pools.insert(info.user_email_pool); - pools.insert(info.user_swift_pool); - pools.insert(info.user_uid_pool); - pools.insert(info.otp_pool); - pools.insert(info.roles_pool); - pools.insert(info.reshard_pool); - pools.insert(info.oidc_pool); - pools.insert(info.notif_pool); - - for (const auto& [pname, placement] : info.placement_pools) { - pools.insert(placement.index_pool); - for (const auto& [sname, sc] : placement.storage_classes.get_all()) { - if (sc.data_pool) { - pools.insert(sc.data_pool.get()); - } - } - pools.insert(placement.data_extra_pool); - } -} - -int get_zones_pool_set(const DoutPrefixProvider *dpp, - CephContext* cct, - RGWSI_SysObj* sysobj_svc, - const list& zone_names, - const string& my_zone_id, - set& pool_names, - optional_yield y) -{ - for (const auto& name : zone_names) { - RGWZoneParams zone(name); - int r = zone.init(dpp, cct, sysobj_svc, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "Error: failed to load zone " << name - << " with " << cpp_strerror(-r) << dendl; - return r; - } - if (zone.get_id() != my_zone_id) { - add_zone_pools(zone, pool_names); - } - } - return 0; -} - -rgw_pool fix_zone_pool_dup(const set& pools, - const string& default_prefix, - const string& default_suffix, - const rgw_pool& suggested_pool) -{ - string suggested_name = suggested_pool.to_str(); - - string prefix = default_prefix; - string suffix = default_suffix; - - if (!suggested_pool.empty()) { - prefix = suggested_name.substr(0, suggested_name.find(".")); - suffix = suggested_name.substr(prefix.length()); - } - - rgw_pool pool(prefix + suffix); - - while (pools.count(pool)) { - pool = prefix + "_" + std::to_string(std::rand()) + suffix; - } - return pool; -} - -} // anonymous namespace - -int RGWZoneParams::fix_pool_names(const DoutPrefixProvider *dpp, optional_yield y) -{ - - list zones; - int r = zone_svc->list_zones(dpp, zones); - if (r < 0) { - ldpp_dout(dpp, 10) << "WARNING: store->list_zones() returned r=" << r << dendl; - } - - set pools; - r = get_zones_pool_set(dpp, cct, sysobj_svc, zones, id, pools, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "Error: get_zones_pool_names" << r << dendl; - return r; - } - - domain_root = fix_zone_pool_dup(pools, name, ".rgw.meta:root", domain_root); - control_pool = fix_zone_pool_dup(pools, name, ".rgw.control", control_pool); - gc_pool = fix_zone_pool_dup(pools, name ,".rgw.log:gc", gc_pool); - lc_pool = fix_zone_pool_dup(pools, name ,".rgw.log:lc", lc_pool); - log_pool = fix_zone_pool_dup(pools, name, ".rgw.log", log_pool); - intent_log_pool = fix_zone_pool_dup(pools, name, ".rgw.log:intent", intent_log_pool); - usage_log_pool = fix_zone_pool_dup(pools, name, ".rgw.log:usage", usage_log_pool); - user_keys_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.keys", user_keys_pool); - user_email_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.email", user_email_pool); - user_swift_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.swift", user_swift_pool); - user_uid_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:users.uid", user_uid_pool); - roles_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:roles", roles_pool); - reshard_pool = fix_zone_pool_dup(pools, name, ".rgw.log:reshard", reshard_pool); - otp_pool = fix_zone_pool_dup(pools, name, ".rgw.otp", otp_pool); - oidc_pool = fix_zone_pool_dup(pools, name, ".rgw.meta:oidc", oidc_pool); - notif_pool = fix_zone_pool_dup(pools, name ,".rgw.log:notif", notif_pool); - - for(auto& iter : placement_pools) { - iter.second.index_pool = fix_zone_pool_dup(pools, name, "." + default_bucket_index_pool_suffix, - iter.second.index_pool); - for (auto& pi : iter.second.storage_classes.get_all()) { - if (pi.second.data_pool) { - rgw_pool& pool = pi.second.data_pool.get(); - pool = fix_zone_pool_dup(pools, name, "." + default_storage_pool_suffix, - pool); - } - } - iter.second.data_extra_pool= fix_zone_pool_dup(pools, name, "." + default_storage_extra_pool_suffix, - iter.second.data_extra_pool); - } - - return 0; -} - -int RGWZoneParams::create(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) -{ - /* check for old pools config */ - rgw_raw_obj obj(domain_root, avail_pools); - auto sysobj = sysobj_svc->get_obj(obj); - int r = sysobj.rop().stat(y, dpp); - if (r < 0) { - ldpp_dout(dpp, 10) << "couldn't find old data placement pools config, setting up new ones for the zone" << dendl; - /* a new system, let's set new placement info */ - RGWZonePlacementInfo default_placement; - default_placement.index_pool = name + "." + default_bucket_index_pool_suffix; - rgw_pool pool = name + "." + default_storage_pool_suffix; - default_placement.storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); - default_placement.data_extra_pool = name + "." + default_storage_extra_pool_suffix; - placement_pools["default-placement"] = default_placement; - } - - r = fix_pool_names(dpp, y); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: fix_pool_names returned r=" << r << dendl; - return r; - } - - r = RGWSystemMetaObj::create(dpp, y, exclusive); - if (r < 0) { - return r; - } - - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - r = set_as_default(dpp, y, true); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 10) << "WARNING: failed to set zone as default, r=" << r << dendl; - } - - return 0; -} - -rgw_pool RGWZoneParams::get_pool(CephContext *cct) const -{ - if (cct->_conf->rgw_zone_root_pool.empty()) { - return rgw_pool(RGW_DEFAULT_ZONE_ROOT_POOL); - } - - return rgw_pool(cct->_conf->rgw_zone_root_pool); -} - -const string RGWZoneParams::get_default_oid(bool old_format) const -{ - if (old_format) { - return cct->_conf->rgw_default_zone_info_oid; - } - - return cct->_conf->rgw_default_zone_info_oid + "." + realm_id; -} - -const string& RGWZoneParams::get_names_oid_prefix() const -{ - return zone_names_oid_prefix; -} - -const string& RGWZoneParams::get_info_oid_prefix(bool old_format) const -{ - return zone_info_oid_prefix; -} - -string RGWZoneParams::get_predefined_id(CephContext *cct) const { - return cct->_conf.get_val("rgw_zone_id"); -} - -const string& RGWZoneParams::get_predefined_name(CephContext *cct) const { - return cct->_conf->rgw_zone; -} - -int RGWZoneParams::init(const DoutPrefixProvider *dpp, - CephContext *cct, RGWSI_SysObj *sysobj_svc, - optional_yield y, bool setup_obj, bool old_format) -{ - if (name.empty()) { - name = cct->_conf->rgw_zone; - } - - return RGWSystemMetaObj::init(dpp, cct, sysobj_svc, y, setup_obj, old_format); -} - -int RGWZoneParams::read_default_id(const DoutPrefixProvider *dpp, string& default_id, optional_yield y, - bool old_format) -{ - if (realm_id.empty()) { - /* try using default realm */ - RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); - //no default realm exist - if (ret < 0) { - return read_id(dpp, default_zone_name, default_id, y); - } - realm_id = realm.get_id(); - } - - return RGWSystemMetaObj::read_default_id(dpp, default_id, y, old_format); -} - - -int RGWZoneParams::set_as_default(const DoutPrefixProvider *dpp, optional_yield y, bool exclusive) -{ - if (realm_id.empty()) { - /* try using default realm */ - RGWRealm realm; - int ret = realm.init(dpp, cct, sysobj_svc, y); - if (ret < 0) { - ldpp_dout(dpp, 10) << "could not read realm id: " << cpp_strerror(-ret) << dendl; - return -EINVAL; - } - realm_id = realm.get_id(); - } - - return RGWSystemMetaObj::set_as_default(dpp, y, exclusive); -} - -const string& RGWZoneParams::get_compression_type(const rgw_placement_rule& placement_rule) const -{ - static const std::string NONE{"none"}; - auto p = placement_pools.find(placement_rule.name); - if (p == placement_pools.end()) { - return NONE; - } - const auto& type = p->second.get_compression_type(placement_rule.get_storage_class()); - return !type.empty() ? type : NONE; -} - -void RGWPeriodMap::encode(bufferlist& bl) const { - ENCODE_START(2, 1, bl); - encode(id, bl); - encode(zonegroups, bl); - encode(master_zonegroup, bl); - encode(short_zone_ids, bl); - ENCODE_FINISH(bl); -} - -void RGWPeriodMap::decode(bufferlist::const_iterator& bl) { - DECODE_START(2, bl); - decode(id, bl); - decode(zonegroups, bl); - decode(master_zonegroup, bl); - if (struct_v >= 2) { - decode(short_zone_ids, bl); - } - DECODE_FINISH(bl); - - zonegroups_by_api.clear(); - for (map::iterator iter = zonegroups.begin(); - iter != zonegroups.end(); ++iter) { - RGWZoneGroup& zonegroup = iter->second; - zonegroups_by_api[zonegroup.api_name] = zonegroup; - if (zonegroup.is_master_zonegroup()) { - master_zonegroup = zonegroup.get_id(); - } - } -} - -// run an MD5 hash on the zone_id and return the first 32 bits -static uint32_t gen_short_zone_id(const std::string zone_id) -{ - unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; - MD5 hash; - // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes - hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); - hash.Update((const unsigned char *)zone_id.c_str(), zone_id.size()); - hash.Final(md5); - - uint32_t short_id; - memcpy((char *)&short_id, md5, sizeof(short_id)); - return std::max(short_id, 1u); -} - -int RGWPeriodMap::update(const RGWZoneGroup& zonegroup, CephContext *cct) -{ - if (zonegroup.is_master_zonegroup() && (!master_zonegroup.empty() && zonegroup.get_id() != master_zonegroup)) { - ldout(cct,0) << "Error updating periodmap, multiple master zonegroups configured "<< dendl; - ldout(cct,0) << "master zonegroup: " << master_zonegroup << " and " << zonegroup.get_id() <::iterator iter = zonegroups.find(zonegroup.get_id()); - if (iter != zonegroups.end()) { - RGWZoneGroup& old_zonegroup = iter->second; - if (!old_zonegroup.api_name.empty()) { - zonegroups_by_api.erase(old_zonegroup.api_name); - } - } - zonegroups[zonegroup.get_id()] = zonegroup; - - if (!zonegroup.api_name.empty()) { - zonegroups_by_api[zonegroup.api_name] = zonegroup; - } - - if (zonegroup.is_master_zonegroup()) { - master_zonegroup = zonegroup.get_id(); - } else if (master_zonegroup == zonegroup.get_id()) { - master_zonegroup = ""; - } - - for (auto& i : zonegroup.zones) { - auto& zone = i.second; - if (short_zone_ids.find(zone.id) != short_zone_ids.end()) { - continue; - } - // calculate the zone's short id - uint32_t short_id = gen_short_zone_id(zone.id); - - // search for an existing zone with the same short id - for (auto& s : short_zone_ids) { - if (s.second == short_id) { - ldout(cct, 0) << "New zone '" << zone.name << "' (" << zone.id - << ") generates the same short_zone_id " << short_id - << " as existing zone id " << s.first << dendl; - return -EEXIST; - } - } - - short_zone_ids[zone.id] = short_id; - } - - return 0; -} - -uint32_t RGWPeriodMap::get_zone_short_id(const string& zone_id) const -{ - auto i = short_zone_ids.find(zone_id); - if (i == short_zone_ids.end()) { - return 0; - } - return i->second; -} - -bool RGWPeriodMap::find_zone_by_id(const rgw_zone_id& zone_id, - RGWZoneGroup *zonegroup, - RGWZone *zone) const -{ - for (auto& iter : zonegroups) { - auto& zg = iter.second; - - auto ziter = zg.zones.find(zone_id); - if (ziter != zg.zones.end()) { - *zonegroup = zg; - *zone = ziter->second; - return true; - } - } - - return false; -} - -bool RGWPeriodMap::find_zone_by_name(const string& zone_name, - RGWZoneGroup *zonegroup, - RGWZone *zone) const -{ - for (auto& iter : zonegroups) { - auto& zg = iter.second; - for (auto& ziter : zg.zones) { - auto& z = ziter.second; - - if (z.name == zone_name) { - *zonegroup = zg; - *zone = z; - return true; - } - } - } - - return false; -} - -namespace rgw { - -int read_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view realm_id, - std::string_view realm_name, - RGWRealm& info, - std::unique_ptr* writer) -{ - if (!realm_id.empty()) { - return cfgstore->read_realm_by_id(dpp, y, realm_id, info, writer); - } - if (!realm_name.empty()) { - return cfgstore->read_realm_by_name(dpp, y, realm_name, info, writer); - } - return cfgstore->read_default_realm(dpp, y, info, writer); -} - -int create_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWRealm& info, - std::unique_ptr* writer_out) -{ - if (info.name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a realm name" << dendl; - return -EINVAL; - } - if (info.id.empty()) { - info.id = gen_random_uuid(); - } - - // if the realm already has a current_period, just make sure it exists - std::optional period; - if (!info.current_period.empty()) { - period.emplace(); - int r = cfgstore->read_period(dpp, y, info.current_period, - std::nullopt, *period); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to read realm's current_period=" - << info.current_period << " with " << cpp_strerror(r) << dendl; - return r; - } - } - - // create the realm - std::unique_ptr writer; - int r = cfgstore->create_realm(dpp, y, exclusive, info, &writer); - if (r < 0) { - return r; - } - - if (!period) { - // initialize and exclusive-create the initial period - period.emplace(); - period->id = gen_random_uuid(); - period->period_map.id = period->id; - period->epoch = FIRST_EPOCH; - period->realm_id = info.id; - period->realm_name = info.name; - - r = cfgstore->create_period(dpp, y, true, *period); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to create the initial period id=" - << period->id << " for realm " << info.name - << " with " << cpp_strerror(r) << dendl; - return r; - } - } - - // update the realm's current_period - r = realm_set_current_period(dpp, y, cfgstore, *writer, info, *period); - if (r < 0) { - return r; - } - - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - r = set_default_realm(dpp, y, cfgstore, info, true); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set realm as default: " - << cpp_strerror(r) << dendl; - } - - if (writer_out) { - *writer_out = std::move(writer); - } - return 0; -} - -int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWRealm& info, - bool exclusive) -{ - return cfgstore->write_default_realm_id(dpp, y, exclusive, info.id); -} - -int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - sal::RealmWriter& writer, RGWRealm& realm, - const RGWPeriod& period) -{ - // update realm epoch to match the period's - if (realm.epoch > period.realm_epoch) { - ldpp_dout(dpp, -1) << __func__ << " with old realm epoch " - << period.realm_epoch << ", current epoch=" << realm.epoch << dendl; - return -EINVAL; - } - if (realm.epoch == period.realm_epoch && realm.current_period != period.id) { - ldpp_dout(dpp, -1) << __func__ << " with same realm epoch " - << period.realm_epoch << ", but different period id " - << period.id << " != " << realm.current_period << dendl; - return -EINVAL; - } - - realm.epoch = period.realm_epoch; - realm.current_period = period.id; - - // update the realm object - int r = writer.write(dpp, y, realm); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to overwrite realm " - << realm.name << " with " << cpp_strerror(r) << dendl; - return r; - } - - // reflect the zonegroup and period config - (void) reflect_period(dpp, y, cfgstore, period); - return 0; -} - -int reflect_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWPeriod& info) -{ - // overwrite the local period config and zonegroup objects - constexpr bool exclusive = false; - - int r = cfgstore->write_period_config(dpp, y, exclusive, info.realm_id, - info.period_config); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to store period config for realm id=" - << info.realm_id << " with " << cpp_strerror(r) << dendl; - return r; - } - - for (auto& [zonegroup_id, zonegroup] : info.period_map.zonegroups) { - r = cfgstore->create_zonegroup(dpp, y, exclusive, zonegroup, nullptr); - if (r < 0) { - ldpp_dout(dpp, -1) << __func__ << " failed to store zonegroup id=" - << zonegroup_id << " with " << cpp_strerror(r) << dendl; - return r; - } - if (zonegroup.is_master) { - // set master as default if no default exists - constexpr bool exclusive = true; - r = set_default_zonegroup(dpp, y, cfgstore, zonegroup, exclusive); - if (r == 0) { - ldpp_dout(dpp, 1) << "Set the period's master zonegroup " - << zonegroup.name << " as the default" << dendl; - } - } - } - return 0; -} - -std::string get_staging_period_id(std::string_view realm_id) -{ - return string_cat_reserve(realm_id, ":staging"); -} - -void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info) -{ - ldpp_dout(dpp, 20) << __func__ << " realm id=" << info.realm_id - << " period id=" << info.id << dendl; - - info.predecessor_uuid = std::move(info.id); - info.id = get_staging_period_id(info.realm_id); - info.period_map.reset(); - info.realm_epoch++; -} - -int update_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, RGWPeriod& info) -{ - // clear zone short ids of removed zones. period_map.update() will add the - // remaining zones back - info.period_map.short_zone_ids.clear(); - - // list all zonegroups in the realm - rgw::sal::ListResult listing; - std::array zonegroup_names; // list in pages of 1000 - do { - int ret = cfgstore->list_zonegroup_names(dpp, y, listing.next, - zonegroup_names, listing); - if (ret < 0) { - std::cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl; - return -ret; - } - for (const auto& name : listing.entries) { - RGWZoneGroup zg; - ret = cfgstore->read_zonegroup_by_name(dpp, y, name, zg, nullptr); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to read zonegroup " - << name << ": " << cpp_strerror(-ret) << dendl; - continue; - } - - if (zg.realm_id != info.realm_id) { - ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() - << " with realm id " << zg.realm_id - << ", not on our realm " << info.realm_id << dendl; - continue; - } - - if (zg.master_zone.empty()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; - return -EINVAL; - } - - if (zg.zones.find(zg.master_zone) == zg.zones.end()) { - ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() - << " has a non existent master zone "<< dendl; - return -EINVAL; - } - - if (zg.is_master_zonegroup()) { - info.master_zonegroup = zg.get_id(); - info.master_zone = zg.master_zone; - } - - ret = info.period_map.update(zg, dpp->get_cct()); - if (ret < 0) { - return ret; - } - } // foreach name in listing.entries - } while (!listing.next.empty()); - - // read the realm's current period config - int ret = cfgstore->read_period_config(dpp, y, info.realm_id, - info.period_config); - if (ret < 0 && ret != -ENOENT) { - ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " - << cpp_strerror(ret) << dendl; - return ret; - } - - return 0; -} - -int commit_period(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, sal::Store* store, - RGWRealm& realm, sal::RealmWriter& realm_writer, - const RGWPeriod& current_period, - RGWPeriod& info, std::ostream& error_stream, - bool force_if_stale) -{ - auto zone_svc = static_cast(store)->svc()->zone; // XXX - - ldpp_dout(dpp, 20) << __func__ << " realm " << realm.id - << " period " << current_period.id << dendl; - // gateway must be in the master zone to commit - if (info.master_zone != zone_svc->get_zone_params().id) { - error_stream << "Cannot commit period on zone " - << zone_svc->get_zone_params().id << ", it must be sent to " - "the period's master zone " << info.master_zone << '.' << std::endl; - return -EINVAL; - } - // period predecessor must match current period - if (info.predecessor_uuid != current_period.id) { - error_stream << "Period predecessor " << info.predecessor_uuid - << " does not match current period " << current_period.id - << ". Use 'period pull' to get the latest period from the master, " - "reapply your changes, and try again." << std::endl; - return -EINVAL; - } - // realm epoch must be 1 greater than current period - if (info.realm_epoch != current_period.realm_epoch + 1) { - error_stream << "Period's realm epoch " << info.realm_epoch - << " does not come directly after current realm epoch " - << current_period.realm_epoch << ". Use 'realm pull' to get the " - "latest realm and period from the master zone, reapply your changes, " - "and try again." << std::endl; - return -EINVAL; - } - // did the master zone change? - if (info.master_zone != current_period.master_zone) { - // store the current metadata sync status in the period - int r = info.update_sync_status(dpp, store, current_period, - error_stream, force_if_stale); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update metadata sync status: " - << cpp_strerror(-r) << dendl; - return r; - } - // create an object with a new period id - info.period_map.id = info.id = gen_random_uuid(); - info.epoch = FIRST_EPOCH; - - constexpr bool exclusive = true; - r = cfgstore->create_period(dpp, y, exclusive, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; - return r; - } - // set as current period - r = realm_set_current_period(dpp, y, cfgstore, realm_writer, realm, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update realm's current period: " - << cpp_strerror(-r) << dendl; - return r; - } - ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period " - << info.id << dendl; - (void) cfgstore->realm_notify_new_period(dpp, y, info); - return 0; - } - // period must be based on current epoch - if (info.epoch != current_period.epoch) { - error_stream << "Period epoch " << info.epoch << " does not match " - "predecessor epoch " << current_period.epoch << ". Use " - "'period pull' to get the latest epoch from the master zone, " - "reapply your changes, and try again." << std::endl; - return -EINVAL; - } - // set period as next epoch - info.id = current_period.id; - info.epoch = current_period.epoch + 1; - info.predecessor_uuid = current_period.predecessor_uuid; - info.realm_epoch = current_period.realm_epoch; - // write the period - constexpr bool exclusive = true; - int r = cfgstore->create_period(dpp, y, exclusive, info); - if (r == -EEXIST) { - // already have this epoch (or a more recent one) - return 0; - } - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(r) << dendl; - return r; - } - r = reflect_period(dpp, y, cfgstore, info); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(r) << dendl; - return r; - } - ldpp_dout(dpp, 4) << "Committed new epoch " << info.epoch - << " for period " << info.id << dendl; - (void) cfgstore->realm_notify_new_period(dpp, y, info); - return 0; -} - - -int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view zonegroup_id, - std::string_view zonegroup_name, - RGWZoneGroup& info, - std::unique_ptr* writer) -{ - if (!zonegroup_id.empty()) { - return cfgstore->read_zonegroup_by_id(dpp, y, zonegroup_id, info, writer); - } - if (!zonegroup_name.empty()) { - return cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, info, writer); - } - - std::string realm_id; - int r = cfgstore->read_default_realm_id(dpp, y, realm_id); - if (r == -ENOENT) { - return cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, - info, writer); - } - if (r < 0) { - return r; - } - return cfgstore->read_default_zonegroup(dpp, y, realm_id, info, writer); -} - -int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWZoneGroup& info) -{ - if (info.name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a zonegroup name" << dendl; - return -EINVAL; - } - if (info.id.empty()) { - info.id = gen_random_uuid(); - } - - // insert the default placement target if it doesn't exist - constexpr std::string_view default_placement_name = "default-placement"; - - RGWZoneGroupPlacementTarget placement_target; - placement_target.name = default_placement_name; - - info.placement_targets.emplace(default_placement_name, placement_target); - if (info.default_placement.name.empty()) { - info.default_placement.name = default_placement_name; - } - - int r = cfgstore->create_zonegroup(dpp, y, exclusive, info, nullptr); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create zonegroup with " - << cpp_strerror(r) << dendl; - return r; - } - - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - r = set_default_zonegroup(dpp, y, cfgstore, info, true); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set zonegroup as default: " - << cpp_strerror(r) << dendl; - } - - return 0; -} - -int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneGroup& info, - bool exclusive) -{ - return cfgstore->write_default_zonegroup_id( - dpp, y, exclusive, info.realm_id, info.id); -} - -int add_zone_to_group(const DoutPrefixProvider* dpp, RGWZoneGroup& zonegroup, - const RGWZoneParams& zone_params, - const bool *pis_master, const bool *pread_only, - const std::list& endpoints, - const std::string *ptier_type, - const bool *psync_from_all, - const std::list& sync_from, - const std::list& sync_from_rm, - const std::string *predirect_zone, - std::optional bucket_index_max_shards, - const rgw::zone_features::set& enable_features, - const rgw::zone_features::set& disable_features) -{ - const std::string& zone_id = zone_params.id; - const std::string& zone_name = zone_params.name; - - if (zone_id.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a zone id" << dendl; - return -EINVAL; - } - if (zone_name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl; - return -EINVAL; - } - - // check for duplicate zone name on insert - if (!zonegroup.zones.count(zone_id)) { - for (const auto& [id, zone] : zonegroup.zones) { - if (zone.name == zone_name) { - ldpp_dout(dpp, 0) << "ERROR: found existing zone name " << zone_name - << " (" << id << ") in zonegroup " << zonegroup.name << dendl; - return -EEXIST; - } - } - } - - if (pis_master) { - rgw_zone_id& master_zone = zonegroup.master_zone; - if (*pis_master) { - if (!master_zone.empty() && master_zone != zone_id) { - ldpp_dout(dpp, 0) << "NOTICE: overriding master zone: " - << master_zone << dendl; - } - master_zone = zone_id; - } else if (master_zone == zone_id) { - master_zone.clear(); - } - } - - // make sure the zone's placement targets are named in the zonegroup - for (const auto& [name, placement] : zone_params.placement_pools) { - auto target = RGWZoneGroupPlacementTarget{.name = name}; - zonegroup.placement_targets.emplace(name, std::move(target)); - } - - RGWZone& zone = zonegroup.zones[zone_params.id]; - zone.id = zone_params.id; - zone.name = zone_params.name; - if (!endpoints.empty()) { - zone.endpoints = endpoints; - } - if (pread_only) { - zone.read_only = *pread_only; - } - if (ptier_type) { - zone.tier_type = *ptier_type; - } - if (psync_from_all) { - zone.sync_from_all = *psync_from_all; - } - if (predirect_zone) { - zone.redirect_zone = *predirect_zone; - } - if (bucket_index_max_shards) { - zone.bucket_index_max_shards = *bucket_index_max_shards; - } - - // add/remove sync_from - for (auto add : sync_from) { - zone.sync_from.insert(add); - } - - for (const auto& rm : sync_from_rm) { - auto i = zone.sync_from.find(rm); - if (i == zone.sync_from.end()) { - ldpp_dout(dpp, 1) << "WARNING: zone \"" << rm - << "\" was not in sync_from" << dendl; - continue; - } - zone.sync_from.erase(i); - } - - // add/remove supported features - zone.supported_features.insert(enable_features.begin(), - enable_features.end()); - - for (const auto& feature : disable_features) { - if (zonegroup.enabled_features.contains(feature)) { - ldpp_dout(dpp, -1) << "ERROR: Cannot disable zone feature \"" << feature - << "\" until it's been disabled in zonegroup " << zonegroup.name << dendl; - return -EINVAL; - } - auto i = zone.supported_features.find(feature); - if (i == zone.supported_features.end()) { - ldpp_dout(dpp, 1) << "WARNING: zone feature \"" << feature - << "\" was not enabled in zone " << zone.name << dendl; - continue; - } - zone.supported_features.erase(i); - } - - const bool log_data = zonegroup.zones.size() > 1; - for (auto& [id, zone] : zonegroup.zones) { - zone.log_data = log_data; - } - - return 0; -} - -int remove_zone_from_group(const DoutPrefixProvider* dpp, - RGWZoneGroup& zonegroup, - const rgw_zone_id& zone_id) -{ - auto z = zonegroup.zones.find(zone_id); - if (z == zonegroup.zones.end()) { - return -ENOENT; - } - zonegroup.zones.erase(z); - - if (zonegroup.master_zone == zone_id) { - // choose a new master zone - auto m = zonegroup.zones.begin(); - if (m != zonegroup.zones.end()) { - zonegroup.master_zone = m->first; - ldpp_dout(dpp, 0) << "NOTICE: promoted " << m->second.name - << " as new master_zone of zonegroup " << zonegroup.name << dendl; - } else { - zonegroup.master_zone.clear(); - ldpp_dout(dpp, 0) << "NOTICE: cleared master_zone of zonegroup " - << zonegroup.name << dendl; - } - } - - const bool log_data = zonegroup.zones.size() > 1; - for (auto& [id, zone] : zonegroup.zones) { - zone.log_data = log_data; - } - - return 0; -} - -// try to remove the given zone id from every zonegroup in the cluster -static int remove_zone_from_groups(const DoutPrefixProvider* dpp, - optional_yield y, - sal::ConfigStore* cfgstore, - const rgw_zone_id& zone_id) -{ - std::array zonegroup_names; - sal::ListResult listing; - do { - int r = cfgstore->list_zonegroup_names(dpp, y, listing.next, - zonegroup_names, listing); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to list zonegroups with " - << cpp_strerror(r) << dendl; - return r; - } - - for (const auto& name : listing.entries) { - RGWZoneGroup zonegroup; - std::unique_ptr writer; - r = cfgstore->read_zonegroup_by_name(dpp, y, name, zonegroup, &writer); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to load zonegroup " << name - << " with " << cpp_strerror(r) << dendl; - continue; - } - - r = remove_zone_from_group(dpp, zonegroup, zone_id); - if (r < 0) { - continue; - } - - // write the updated zonegroup - r = writer->write(dpp, y, zonegroup); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: failed to write zonegroup " << name - << " with " << cpp_strerror(r) << dendl; - continue; - } - ldpp_dout(dpp, 0) << "Removed zone from zonegroup " << name << dendl; - } - } while (!listing.next.empty()); - - return 0; -} - - -int read_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, - std::string_view zone_id, - std::string_view zone_name, - RGWZoneParams& info, - std::unique_ptr* writer) -{ - if (!zone_id.empty()) { - return cfgstore->read_zone_by_id(dpp, y, zone_id, info, writer); - } - if (!zone_name.empty()) { - return cfgstore->read_zone_by_name(dpp, y, zone_name, info, writer); - } - - std::string realm_id; - int r = cfgstore->read_default_realm_id(dpp, y, realm_id); - if (r == -ENOENT) { - return cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, writer); - } - if (r < 0) { - return r; - } - return cfgstore->read_default_zone(dpp, y, realm_id, info, writer); -} - -static int get_zones_pool_set(const DoutPrefixProvider *dpp, optional_yield y, - rgw::sal::ConfigStore* cfgstore, - std::string_view my_zone_id, - std::set& pools) -{ - std::array zone_names; - sal::ListResult listing; - do { - int r = cfgstore->list_zone_names(dpp, y, listing.next, - zone_names, listing); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to list zones with " << cpp_strerror(r) << dendl; - return r; - } - - for (const auto& name : listing.entries) { - RGWZoneParams info; - r = cfgstore->read_zone_by_name(dpp, y, name, info, nullptr); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to load zone " << name - << " with " << cpp_strerror(r) << dendl; - return r; - } - if (info.get_id() != my_zone_id) { - add_zone_pools(info, pools); - } - } - } while (!listing.next.empty()); - - return 0; -} - -int init_zone_pool_names(const DoutPrefixProvider *dpp, optional_yield y, - const std::set& pools, RGWZoneParams& info) -{ - info.domain_root = fix_zone_pool_dup(pools, info.name, ".rgw.meta:root", info.domain_root); - info.control_pool = fix_zone_pool_dup(pools, info.name, ".rgw.control", info.control_pool); - info.gc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:gc", info.gc_pool); - info.lc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:lc", info.lc_pool); - info.log_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log", info.log_pool); - info.intent_log_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:intent", info.intent_log_pool); - info.usage_log_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:usage", info.usage_log_pool); - info.user_keys_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.keys", info.user_keys_pool); - info.user_email_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.email", info.user_email_pool); - info.user_swift_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.swift", info.user_swift_pool); - info.user_uid_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:users.uid", info.user_uid_pool); - info.roles_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:roles", info.roles_pool); - info.reshard_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:reshard", info.reshard_pool); - info.otp_pool = fix_zone_pool_dup(pools, info.name, ".rgw.otp", info.otp_pool); - info.oidc_pool = fix_zone_pool_dup(pools, info.name, ".rgw.meta:oidc", info.oidc_pool); - info.notif_pool = fix_zone_pool_dup(pools, info.name, ".rgw.log:notif", info.notif_pool); - - for (auto& [pname, placement] : info.placement_pools) { - placement.index_pool = fix_zone_pool_dup(pools, info.name, "." + default_bucket_index_pool_suffix, placement.index_pool); - placement.data_extra_pool= fix_zone_pool_dup(pools, info.name, "." + default_storage_extra_pool_suffix, placement.data_extra_pool); - for (auto& [sname, sc] : placement.storage_classes.get_all()) { - if (sc.data_pool) { - sc.data_pool = fix_zone_pool_dup(pools, info.name, "." + default_storage_pool_suffix, *sc.data_pool); - } - } - } - - return 0; -} - -int create_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, bool exclusive, - RGWZoneParams& info, std::unique_ptr* writer) -{ - if (info.name.empty()) { - ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl; - return -EINVAL; - } - if (info.id.empty()) { - info.id = gen_random_uuid(); - } - - // add default placement with empty pool name - rgw_pool pool; - auto& placement = info.placement_pools["default-placement"]; - placement.storage_classes.set_storage_class( - RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); - - // build a set of all pool names used by other zones - std::set pools; - int r = get_zones_pool_set(dpp, y, cfgstore, info.id, pools); - if (r < 0) { - return r; - } - - // initialize pool names with the zone name prefix - r = init_zone_pool_names(dpp, y, pools, info); - if (r < 0) { - return r; - } - - r = cfgstore->create_zone(dpp, y, exclusive, info, nullptr); - if (r < 0) { - ldpp_dout(dpp, 0) << "failed to create zone with " - << cpp_strerror(r) << dendl; - return r; - } - - // try to set as default. may race with another create, so pass exclusive=true - // so we don't override an existing default - r = set_default_zone(dpp, y, cfgstore, info, true); - if (r < 0 && r != -EEXIST) { - ldpp_dout(dpp, 0) << "WARNING: failed to set zone as default: " - << cpp_strerror(r) << dendl; - } - - return 0; - -} - -int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneParams& info, - bool exclusive) -{ - return cfgstore->write_default_zone_id( - dpp, y, exclusive, info.realm_id, info.id); -} - -int delete_zone(const DoutPrefixProvider* dpp, optional_yield y, - sal::ConfigStore* cfgstore, const RGWZoneParams& info, - sal::ZoneWriter& writer) -{ - // remove this zone from any zonegroups that contain it - int r = remove_zone_from_groups(dpp, y, cfgstore, info.id); - if (r < 0) { - return r; - } - - return writer.remove(dpp, y); -} - -} // namespace rgw - -static inline int conf_to_uint64(const JSONFormattable& config, const string& key, uint64_t *pval) -{ - string sval; - if (config.find(key, &sval)) { - string err; - uint64_t val = strict_strtoll(sval.c_str(), 10, &err); - if (!err.empty()) { - return -EINVAL; - } - *pval = val; - } - return 0; -} - -int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) -{ - int r = -1; - - if (config.exists("retain_head_object")) { - string s = config["retain_head_object"]; - if (s == "true") { - retain_head_object = true; - } else { - retain_head_object = false; - } - } - - if (tier_type == "cloud-s3") { - r = t.s3.update_params(config); - } - - return r; -} - -void RGWZoneGroupPlacementTier::dump(Formatter *f) const -{ - encode_json("tier_type", tier_type, f); - encode_json("storage_class", storage_class, f); - encode_json("retain_head_object", retain_head_object, f); - - if (tier_type == "cloud-s3") { - encode_json("s3", t.s3, f); - } -} - -void RGWZoneGroupPlacementTier::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("tier_type", tier_type, obj); - JSONDecoder::decode_json("storage_class", storage_class, obj); - JSONDecoder::decode_json("retain_head_object", retain_head_object, obj); - - if (tier_type == "cloud-s3") { - JSONDecoder::decode_json("s3", t.s3, obj); - } -} - -int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config) -{ - if (config.exists("retain_head_object")) { - retain_head_object = false; - } - - if (tier_type == "cloud-s3") { - t.s3.clear_params(config); - } - - return 0; -} - -int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config) -{ - int r = -1; - - if (config.exists("endpoint")) { - endpoint = config["endpoint"]; - } - if (config.exists("target_path")) { - target_path = config["target_path"]; - } - if (config.exists("region")) { - region = config["region"]; - } - if (config.exists("host_style")) { - string s; - s = config["host_style"]; - if (s != "virtual") { - host_style = PathStyle; - } else { - host_style = VirtualStyle; - } - } - if (config.exists("target_storage_class")) { - target_storage_class = config["target_storage_class"]; - } - if (config.exists("access_key")) { - key.id = config["access_key"]; - } - if (config.exists("secret")) { - key.key = config["secret"]; - } - if (config.exists("multipart_sync_threshold")) { - r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold); - if (r < 0) { - multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - } - - if (config.exists("multipart_min_part_size")) { - r = conf_to_uint64(config, "multipart_min_part_size", &multipart_min_part_size); - if (r < 0) { - multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - } - - if (config.exists("acls")) { - const JSONFormattable& cc = config["acls"]; - if (cc.is_array()) { - for (auto& c : cc.array()) { - RGWTierACLMapping m; - m.init(c); - if (!m.source_id.empty()) { - acl_mappings[m.source_id] = m; - } - } - } else { - RGWTierACLMapping m; - m.init(cc); - if (!m.source_id.empty()) { - acl_mappings[m.source_id] = m; - } - } - } - return 0; -} - -void RGWZoneGroupPlacementTierS3::dump(Formatter *f) const -{ - encode_json("endpoint", endpoint, f); - encode_json("access_key", key.id, f); - encode_json("secret", key.key, f); - encode_json("region", region, f); - string s = (host_style == PathStyle ? "path" : "virtual"); - encode_json("host_style", s, f); - encode_json("target_storage_class", target_storage_class, f); - encode_json("target_path", target_path, f); - encode_json("acl_mappings", acl_mappings, f); - encode_json("multipart_sync_threshold", multipart_sync_threshold, f); - encode_json("multipart_min_part_size", multipart_min_part_size, f); -} - -void RGWZoneGroupPlacementTierS3::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("endpoint", endpoint, obj); - JSONDecoder::decode_json("access_key", key.id, obj); - JSONDecoder::decode_json("secret", key.key, obj); - JSONDecoder::decode_json("region", region, obj); - string s; - JSONDecoder::decode_json("host_style", s, obj); - if (s != "virtual") { - host_style = PathStyle; - } else { - host_style = VirtualStyle; - } - JSONDecoder::decode_json("target_storage_class", target_storage_class, obj); - JSONDecoder::decode_json("target_path", target_path, obj); - JSONDecoder::decode_json("acl_mappings", acl_mappings, obj); - JSONDecoder::decode_json("multipart_sync_threshold", multipart_sync_threshold, obj); - JSONDecoder::decode_json("multipart_min_part_size", multipart_min_part_size, obj); -} - -int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config) -{ - if (config.exists("endpoint")) { - endpoint.clear(); - } - if (config.exists("target_path")) { - target_path.clear(); - } - if (config.exists("region")) { - region.clear(); - } - if (config.exists("host_style")) { - /* default */ - host_style = PathStyle; - } - if (config.exists("target_storage_class")) { - target_storage_class.clear(); - } - if (config.exists("access_key")) { - key.id.clear(); - } - if (config.exists("secret")) { - key.key.clear(); - } - if (config.exists("multipart_sync_threshold")) { - multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - if (config.exists("multipart_min_part_size")) { - multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; - } - if (config.exists("acls")) { - const JSONFormattable& cc = config["acls"]; - if (cc.is_array()) { - for (auto& c : cc.array()) { - RGWTierACLMapping m; - m.init(c); - acl_mappings.erase(m.source_id); - } - } else { - RGWTierACLMapping m; - m.init(cc); - acl_mappings.erase(m.source_id); - } - } - return 0; -} - -void RGWZoneGroupPlacementTarget::dump(Formatter *f) const -{ - encode_json("name", name, f); - encode_json("tags", tags, f); - encode_json("storage_classes", storage_classes, f); - if (!tier_targets.empty()) { - encode_json("tier_targets", tier_targets, f); - } -} - -void RGWZoneGroupPlacementTarget::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("name", name, obj); - JSONDecoder::decode_json("tags", tags, obj); - JSONDecoder::decode_json("storage_classes", storage_classes, obj); - if (storage_classes.empty()) { - storage_classes.insert(RGW_STORAGE_CLASS_STANDARD); - } - if (!tier_targets.empty()) { - JSONDecoder::decode_json("tier_targets", tier_targets, obj); - } -} - -void RGWZoneGroup::dump(Formatter *f) const -{ - RGWSystemMetaObj::dump(f); - encode_json("api_name", api_name, f); - encode_json("is_master", is_master, f); - encode_json("endpoints", endpoints, f); - encode_json("hostnames", hostnames, f); - encode_json("hostnames_s3website", hostnames_s3website, f); - encode_json("master_zone", master_zone, f); - encode_json_map("zones", zones, f); /* more friendly representation */ - encode_json_map("placement_targets", placement_targets, f); /* more friendly representation */ - encode_json("default_placement", default_placement, f); - encode_json("realm_id", realm_id, f); - encode_json("sync_policy", sync_policy, f); - encode_json("enabled_features", enabled_features, f); -} - -static void decode_zones(map& zones, JSONObj *o) -{ - RGWZone z; - z.decode_json(o); - zones[z.id] = z; -} - -static void decode_placement_targets(map& targets, JSONObj *o) -{ - RGWZoneGroupPlacementTarget t; - t.decode_json(o); - targets[t.name] = t; -} - -static void decode_zonegroups(map& zonegroups, JSONObj *o) -{ - RGWZoneGroup zg; - zg.decode_json(o); - zonegroups[zg.get_id()] = zg; -} - -void RGWZoneGroup::decode_json(JSONObj *obj) -{ - RGWSystemMetaObj::decode_json(obj); - if (id.empty()) { - derr << "old format " << dendl; - JSONDecoder::decode_json("name", name, obj); - id = name; - } - JSONDecoder::decode_json("api_name", api_name, obj); - JSONDecoder::decode_json("is_master", is_master, obj); - JSONDecoder::decode_json("endpoints", endpoints, obj); - JSONDecoder::decode_json("hostnames", hostnames, obj); - JSONDecoder::decode_json("hostnames_s3website", hostnames_s3website, obj); - JSONDecoder::decode_json("master_zone", master_zone, obj); - JSONDecoder::decode_json("zones", zones, decode_zones, obj); - JSONDecoder::decode_json("placement_targets", placement_targets, decode_placement_targets, obj); - string pr; - JSONDecoder::decode_json("default_placement", pr, obj); - default_placement.from_str(pr); - JSONDecoder::decode_json("realm_id", realm_id, obj); - JSONDecoder::decode_json("sync_policy", sync_policy, obj); - JSONDecoder::decode_json("enabled_features", enabled_features, obj); -} - -void rgw_meta_sync_info::generate_test_instances(list& o) -{ - auto info = new rgw_meta_sync_info; - info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps; - info->period = "periodid"; - info->realm_epoch = 5; - o.push_back(info); - o.push_back(new rgw_meta_sync_info); -} - -void rgw_meta_sync_marker::generate_test_instances(list& o) -{ - auto marker = new rgw_meta_sync_marker; - marker->state = rgw_meta_sync_marker::IncrementalSync; - marker->marker = "01234"; - marker->realm_epoch = 5; - o.push_back(marker); - o.push_back(new rgw_meta_sync_marker); -} - -void rgw_meta_sync_status::generate_test_instances(list& o) -{ - o.push_back(new rgw_meta_sync_status); -} - -void RGWZoneParams::generate_test_instances(list &o) -{ - o.push_back(new RGWZoneParams); - o.push_back(new RGWZoneParams); -} - -void RGWPeriodLatestEpochInfo::generate_test_instances(list &o) -{ - RGWPeriodLatestEpochInfo *z = new RGWPeriodLatestEpochInfo; - o.push_back(z); - o.push_back(new RGWPeriodLatestEpochInfo); -} - -void RGWRealm::generate_test_instances(list &o) -{ - RGWRealm *z = new RGWRealm; - o.push_back(z); - o.push_back(new RGWRealm); -} - -void RGWRealm::dump(Formatter *f) const -{ - RGWSystemMetaObj::dump(f); - encode_json("current_period", current_period, f); - encode_json("epoch", epoch, f); -} - - -void RGWRealm::decode_json(JSONObj *obj) -{ - RGWSystemMetaObj::decode_json(obj); - JSONDecoder::decode_json("current_period", current_period, obj); - JSONDecoder::decode_json("epoch", epoch, obj); -} - -void RGWZoneGroup::generate_test_instances(list& o) -{ - RGWZoneGroup *r = new RGWZoneGroup; - o.push_back(r); - o.push_back(new RGWZoneGroup); -} - -void RGWZone::generate_test_instances(list &o) -{ - RGWZone *z = new RGWZone; - o.push_back(z); - o.push_back(new RGWZone); -} - -void RGWPeriod::generate_test_instances(list &o) -{ - RGWPeriod *z = new RGWPeriod; - o.push_back(z); - o.push_back(new RGWPeriod); -} - -void RGWPeriodLatestEpochInfo::dump(Formatter *f) const { - encode_json("latest_epoch", epoch, f); -} - -void RGWPeriodLatestEpochInfo::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("latest_epoch", epoch, obj); -} - -void RGWPeriod::dump(Formatter *f) const -{ - encode_json("id", id, f); - encode_json("epoch", epoch , f); - encode_json("predecessor_uuid", predecessor_uuid, f); - encode_json("sync_status", sync_status, f); - encode_json("period_map", period_map, f); - encode_json("master_zonegroup", master_zonegroup, f); - encode_json("master_zone", master_zone, f); - encode_json("period_config", period_config, f); - encode_json("realm_id", realm_id, f); - encode_json("realm_name", realm_name, f); - encode_json("realm_epoch", realm_epoch, f); -} - -void RGWPeriod::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("id", id, obj); - JSONDecoder::decode_json("epoch", epoch, obj); - JSONDecoder::decode_json("predecessor_uuid", predecessor_uuid, obj); - JSONDecoder::decode_json("sync_status", sync_status, obj); - JSONDecoder::decode_json("period_map", period_map, obj); - JSONDecoder::decode_json("master_zonegroup", master_zonegroup, obj); - JSONDecoder::decode_json("master_zone", master_zone, obj); - JSONDecoder::decode_json("period_config", period_config, obj); - JSONDecoder::decode_json("realm_id", realm_id, obj); - JSONDecoder::decode_json("realm_name", realm_name, obj); - JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); -} - -void RGWNameToId::dump(Formatter *f) const { - encode_json("obj_id", obj_id, f); -} - -void RGWNameToId::decode_json(JSONObj *obj) { - JSONDecoder::decode_json("obj_id", obj_id, obj); -} - -void RGWSystemMetaObj::dump(Formatter *f) const -{ - encode_json("id", id , f); - encode_json("name", name , f); -} - -void RGWSystemMetaObj::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("id", id, obj); - JSONDecoder::decode_json("name", name, obj); -} - -void RGWZoneStorageClass::dump(Formatter *f) const -{ - if (data_pool) { - encode_json("data_pool", data_pool.get(), f); - } - if (compression_type) { - encode_json("compression_type", compression_type.get(), f); - } -} - -void RGWZoneStorageClass::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("data_pool", data_pool, obj); - JSONDecoder::decode_json("compression_type", compression_type, obj); -} - -void RGWZoneStorageClasses::dump(Formatter *f) const -{ - for (auto& i : m) { - encode_json(i.first.c_str(), i.second, f); - } -} - -void RGWZoneStorageClasses::decode_json(JSONObj *obj) -{ - JSONFormattable f; - decode_json_obj(f, obj); - - for (auto& field : f.object()) { - JSONObj *field_obj = obj->find_obj(field.first); - assert(field_obj); - - decode_json_obj(m[field.first], field_obj); - } - standard_class = &m[RGW_STORAGE_CLASS_STANDARD]; -} - -void RGWZonePlacementInfo::dump(Formatter *f) const -{ - encode_json("index_pool", index_pool, f); - encode_json("storage_classes", storage_classes, f); - encode_json("data_extra_pool", data_extra_pool, f); - encode_json("index_type", (uint32_t)index_type, f); - - /* no real need for backward compatibility of compression_type and data_pool in here, - * rather not clutter the output */ -} - -void RGWZonePlacementInfo::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("index_pool", index_pool, obj); - JSONDecoder::decode_json("storage_classes", storage_classes, obj); - JSONDecoder::decode_json("data_extra_pool", data_extra_pool, obj); - uint32_t it; - JSONDecoder::decode_json("index_type", it, obj); - index_type = (rgw::BucketIndexType)it; - - /* backward compatibility, these are now defined in storage_classes */ - string standard_compression_type; - string *pcompression = nullptr; - if (JSONDecoder::decode_json("compression", standard_compression_type, obj)) { - pcompression = &standard_compression_type; - } - rgw_pool standard_data_pool; - rgw_pool *ppool = nullptr; - if (JSONDecoder::decode_json("data_pool", standard_data_pool, obj)) { - ppool = &standard_data_pool; - } - if (ppool || pcompression) { - storage_classes.set_storage_class(RGW_STORAGE_CLASS_STANDARD, ppool, pcompression); - } -} - -void RGWZoneParams::decode_json(JSONObj *obj) -{ - RGWSystemMetaObj::decode_json(obj); - JSONDecoder::decode_json("domain_root", domain_root, obj); - JSONDecoder::decode_json("control_pool", control_pool, obj); - JSONDecoder::decode_json("gc_pool", gc_pool, obj); - JSONDecoder::decode_json("lc_pool", lc_pool, obj); - JSONDecoder::decode_json("log_pool", log_pool, obj); - JSONDecoder::decode_json("intent_log_pool", intent_log_pool, obj); - JSONDecoder::decode_json("roles_pool", roles_pool, obj); - JSONDecoder::decode_json("reshard_pool", reshard_pool, obj); - JSONDecoder::decode_json("usage_log_pool", usage_log_pool, obj); - JSONDecoder::decode_json("user_keys_pool", user_keys_pool, obj); - JSONDecoder::decode_json("user_email_pool", user_email_pool, obj); - JSONDecoder::decode_json("user_swift_pool", user_swift_pool, obj); - JSONDecoder::decode_json("user_uid_pool", user_uid_pool, obj); - JSONDecoder::decode_json("otp_pool", otp_pool, obj); - JSONDecoder::decode_json("system_key", system_key, obj); - JSONDecoder::decode_json("placement_pools", placement_pools, obj); - JSONDecoder::decode_json("tier_config", tier_config, obj); - JSONDecoder::decode_json("realm_id", realm_id, obj); - JSONDecoder::decode_json("notif_pool", notif_pool, obj); - -} - -void RGWZone::dump(Formatter *f) const -{ - encode_json("id", id, f); - encode_json("name", name, f); - encode_json("endpoints", endpoints, f); - encode_json("log_meta", log_meta, f); - encode_json("log_data", log_data, f); - encode_json("bucket_index_max_shards", bucket_index_max_shards, f); - encode_json("read_only", read_only, f); - encode_json("tier_type", tier_type, f); - encode_json("sync_from_all", sync_from_all, f); - encode_json("sync_from", sync_from, f); - encode_json("redirect_zone", redirect_zone, f); - encode_json("supported_features", supported_features, f); -} - -void RGWZone::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("id", id, obj); - JSONDecoder::decode_json("name", name, obj); - if (id.empty()) { - id = name; - } - JSONDecoder::decode_json("endpoints", endpoints, obj); - JSONDecoder::decode_json("log_meta", log_meta, obj); - JSONDecoder::decode_json("log_data", log_data, obj); - JSONDecoder::decode_json("bucket_index_max_shards", bucket_index_max_shards, obj); - JSONDecoder::decode_json("read_only", read_only, obj); - JSONDecoder::decode_json("tier_type", tier_type, obj); - JSONDecoder::decode_json("sync_from_all", sync_from_all, true, obj); - JSONDecoder::decode_json("sync_from", sync_from, obj); - JSONDecoder::decode_json("redirect_zone", redirect_zone, obj); - JSONDecoder::decode_json("supported_features", supported_features, obj); -} - -void RGWTierACLMapping::dump(Formatter *f) const -{ - string s; - switch (type) { - case ACL_TYPE_EMAIL_USER: - s = "email"; - break; - case ACL_TYPE_GROUP: - s = "uri"; - break; - default: - s = "id"; - break; - } - encode_json("type", s, f); - encode_json("source_id", source_id, f); - encode_json("dest_id", dest_id, f); -} - -void RGWTierACLMapping::decode_json(JSONObj *obj) -{ - string s; - JSONDecoder::decode_json("type", s, obj); - if (s == "email") { - type = ACL_TYPE_EMAIL_USER; - } else if (s == "uri") { - type = ACL_TYPE_GROUP; - } else { - type = ACL_TYPE_CANON_USER; - } - - JSONDecoder::decode_json("source_id", source_id, obj); - JSONDecoder::decode_json("dest_id", dest_id, obj); -} - -void RGWPeriodMap::dump(Formatter *f) const -{ - encode_json("id", id, f); - encode_json_map("zonegroups", zonegroups, f); - encode_json("short_zone_ids", short_zone_ids, f); -} - -void RGWPeriodMap::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("id", id, obj); - JSONDecoder::decode_json("zonegroups", zonegroups, decode_zonegroups, obj); - /* backward compatability with region */ - if (zonegroups.empty()) { - JSONDecoder::decode_json("regions", zonegroups, obj); - } - /* backward compatability with region */ - if (master_zonegroup.empty()) { - JSONDecoder::decode_json("master_region", master_zonegroup, obj); - } - JSONDecoder::decode_json("short_zone_ids", short_zone_ids, obj); -} - -void RGWPeriodConfig::dump(Formatter *f) const -{ - encode_json("bucket_quota", quota.bucket_quota, f); - encode_json("user_quota", quota.user_quota, f); - encode_json("user_ratelimit", user_ratelimit, f); - encode_json("bucket_ratelimit", bucket_ratelimit, f); - encode_json("anonymous_ratelimit", anon_ratelimit, f); -} - -void RGWPeriodConfig::decode_json(JSONObj *obj) -{ - JSONDecoder::decode_json("bucket_quota", quota.bucket_quota, obj); - JSONDecoder::decode_json("user_quota", quota.user_quota, obj); - JSONDecoder::decode_json("user_ratelimit", user_ratelimit, obj); - JSONDecoder::decode_json("bucket_ratelimit", bucket_ratelimit, obj); - JSONDecoder::decode_json("anonymous_ratelimit", anon_ratelimit, obj); -} +} // namespace rgw diff --git a/src/rgw/services/svc_bi.h b/src/rgw/services/svc_bi.h index 0458901f4ba3..690825d530e7 100644 --- a/src/rgw/services/svc_bi.h +++ b/src/rgw/services/svc_bi.h @@ -17,7 +17,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWBucketInfo; struct RGWBucketEnt; diff --git a/src/rgw/services/svc_bi_rados.cc b/src/rgw/services/svc_bi_rados.cc index 9d043d9dbd83..8e7cc5ae260d 100644 --- a/src/rgw/services/svc_bi_rados.cc +++ b/src/rgw/services/svc_bi_rados.cc @@ -5,9 +5,9 @@ #include "svc_bilog_rados.h" #include "svc_zone.h" -#include "rgw/rgw_bucket.h" -#include "rgw/rgw_zone.h" -#include "rgw/rgw_datalog.h" +#include "rgw_bucket.h" +#include "rgw_zone.h" +#include "rgw_datalog.h" #include "cls/rgw/cls_rgw_client.h" diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h index 8b7e588122ef..b1fc97d459f8 100644 --- a/src/rgw/services/svc_bi_rados.h +++ b/src/rgw/services/svc_bi_rados.h @@ -16,9 +16,9 @@ #pragma once -#include "rgw/rgw_datalog.h" -#include "rgw/rgw_service.h" -#include "rgw/rgw_tools.h" +#include "rgw_datalog.h" +#include "rgw_service.h" +#include "rgw_tools.h" #include "svc_bi.h" #include "svc_rados.h" diff --git a/src/rgw/services/svc_bilog_rados.h b/src/rgw/services/svc_bilog_rados.h index a7a3b342cf4d..e9d5dbb5c0e2 100644 --- a/src/rgw/services/svc_bilog_rados.h +++ b/src/rgw/services/svc_bilog_rados.h @@ -17,7 +17,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" diff --git a/src/rgw/services/svc_bucket.h b/src/rgw/services/svc_bucket.h index f1212c9d86f0..4a526e4f2480 100644 --- a/src/rgw/services/svc_bucket.h +++ b/src/rgw/services/svc_bucket.h @@ -17,7 +17,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_bucket_types.h" diff --git a/src/rgw/services/svc_bucket_sobj.cc b/src/rgw/services/svc_bucket_sobj.cc index 99a435215e44..01db1c36eb0c 100644 --- a/src/rgw/services/svc_bucket_sobj.cc +++ b/src/rgw/services/svc_bucket_sobj.cc @@ -11,9 +11,9 @@ #include "svc_meta_be_sobj.h" #include "svc_sync_modules.h" -#include "rgw/rgw_bucket.h" -#include "rgw/rgw_tools.h" -#include "rgw/rgw_zone.h" +#include "rgw_bucket.h" +#include "rgw_tools.h" +#include "rgw_zone.h" #define dout_subsys ceph_subsys_rgw @@ -641,4 +641,4 @@ int RGWSI_Bucket_SObj::read_buckets_stats(RGWSI_Bucket_X_Ctx& ctx, } return m.size(); -} \ No newline at end of file +} diff --git a/src/rgw/services/svc_bucket_sobj.h b/src/rgw/services/svc_bucket_sobj.h index 643291536a51..93e2063a7096 100644 --- a/src/rgw/services/svc_bucket_sobj.h +++ b/src/rgw/services/svc_bucket_sobj.h @@ -17,7 +17,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_meta_be.h" #include "svc_bucket_types.h" diff --git a/src/rgw/services/svc_bucket_sync.h b/src/rgw/services/svc_bucket_sync.h index d90856b7afe8..7975e062bb6f 100644 --- a/src/rgw/services/svc_bucket_sync.h +++ b/src/rgw/services/svc_bucket_sync.h @@ -17,7 +17,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_bucket_types.h" diff --git a/src/rgw/services/svc_bucket_sync_sobj.cc b/src/rgw/services/svc_bucket_sync_sobj.cc index 2f3cfb82517d..4a29d2982af2 100644 --- a/src/rgw/services/svc_bucket_sync_sobj.cc +++ b/src/rgw/services/svc_bucket_sync_sobj.cc @@ -3,9 +3,9 @@ #include "svc_sys_obj_cache.h" #include "svc_bucket_sobj.h" -#include "rgw/rgw_bucket_sync.h" -#include "rgw/rgw_zone.h" -#include "rgw/rgw_sync_policy.h" +#include "rgw_bucket_sync.h" +#include "rgw_zone.h" +#include "rgw_sync_policy.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_bucket_sync_sobj.h b/src/rgw/services/svc_bucket_sync_sobj.h index 951affb2d4e1..779df7b996ca 100644 --- a/src/rgw/services/svc_bucket_sync_sobj.h +++ b/src/rgw/services/svc_bucket_sync_sobj.h @@ -17,7 +17,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_meta_be.h" #include "svc_bucket_sync.h" diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc index 73b9328ebc22..342146bfefa7 100644 --- a/src/rgw/services/svc_cls.cc +++ b/src/rgw/services/svc_cls.cc @@ -6,7 +6,7 @@ #include "svc_rados.h" #include "svc_zone.h" -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" #include "cls/otp/cls_otp_client.h" #include "cls/log/cls_log_client.h" diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h index ca17d0f8f2e6..d1d1d659be88 100644 --- a/src/rgw/services/svc_cls.h +++ b/src/rgw/services/svc_cls.h @@ -19,7 +19,7 @@ #include "cls/otp/cls_otp_types.h" #include "cls/log/cls_log_types.h" -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" diff --git a/src/rgw/services/svc_config_key.h b/src/rgw/services/svc_config_key.h index 1740336d1919..1c068b795f06 100644 --- a/src/rgw/services/svc_config_key.h +++ b/src/rgw/services/svc_config_key.h @@ -18,7 +18,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWSI_ConfigKey : public RGWServiceInstance { diff --git a/src/rgw/services/svc_config_key_rados.h b/src/rgw/services/svc_config_key_rados.h index e867a01f57aa..b3b995ac76de 100644 --- a/src/rgw/services/svc_config_key_rados.h +++ b/src/rgw/services/svc_config_key_rados.h @@ -20,7 +20,7 @@ #include -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_config_key.h" diff --git a/src/rgw/services/svc_finisher.h b/src/rgw/services/svc_finisher.h index 2f3ae52236aa..911b48f2b255 100644 --- a/src/rgw/services/svc_finisher.h +++ b/src/rgw/services/svc_finisher.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" class Context; class Finisher; diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc index b9622a9bf747..e8bba4556a62 100644 --- a/src/rgw/services/svc_mdlog.cc +++ b/src/rgw/services/svc_mdlog.cc @@ -6,11 +6,11 @@ #include "svc_zone.h" #include "svc_sys_obj.h" -#include "rgw/rgw_tools.h" -#include "rgw/rgw_mdlog.h" -#include "rgw/rgw_coroutine.h" -#include "rgw/rgw_cr_rados.h" -#include "rgw/rgw_zone.h" +#include "rgw_tools.h" +#include "rgw_mdlog.h" +#include "rgw_coroutine.h" +#include "rgw_cr_rados.h" +#include "rgw_zone.h" #include "common/errno.h" diff --git a/src/rgw/services/svc_mdlog.h b/src/rgw/services/svc_mdlog.h index 70c8628a56d4..703d6f6059ec 100644 --- a/src/rgw/services/svc_mdlog.h +++ b/src/rgw/services/svc_mdlog.h @@ -16,9 +16,9 @@ #pragma once -#include "rgw/rgw_service.h" -#include "rgw/rgw_period_history.h" -#include "rgw/rgw_period_puller.h" +#include "rgw_service.h" +#include "rgw_period_history.h" +#include "rgw_period_puller.h" #include "svc_meta_be.h" diff --git a/src/rgw/services/svc_meta.cc b/src/rgw/services/svc_meta.cc index 6efc3764f70b..735c39f85e89 100644 --- a/src/rgw/services/svc_meta.cc +++ b/src/rgw/services/svc_meta.cc @@ -4,7 +4,7 @@ #include "svc_meta.h" -#include "rgw/rgw_metadata.h" +#include "rgw_metadata.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_meta.h b/src/rgw/services/svc_meta.h index 8cfc8de0aaaa..b398e27fd26a 100644 --- a/src/rgw/services/svc_meta.h +++ b/src/rgw/services/svc_meta.h @@ -18,7 +18,7 @@ #include "svc_meta_be.h" -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWMetadataLog; diff --git a/src/rgw/services/svc_meta_be.cc b/src/rgw/services/svc_meta_be.cc index 9d030e314811..2cb0365c8446 100644 --- a/src/rgw/services/svc_meta_be.cc +++ b/src/rgw/services/svc_meta_be.cc @@ -4,7 +4,7 @@ #include "svc_meta_be.h" -#include "rgw/rgw_mdlog.h" +#include "rgw_mdlog.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_meta_be.h b/src/rgw/services/svc_meta_be.h index b2b5784f6ba6..97267a4e7e3e 100644 --- a/src/rgw/services/svc_meta_be.h +++ b/src/rgw/services/svc_meta_be.h @@ -19,8 +19,8 @@ #include "svc_meta_be_params.h" -#include "rgw/rgw_service.h" -#include "rgw/rgw_mdlog_types.h" +#include "rgw_service.h" +#include "rgw_mdlog_types.h" class RGWMetadataLogData; diff --git a/src/rgw/services/svc_meta_be_otp.cc b/src/rgw/services/svc_meta_be_otp.cc index ec3c6b2050be..3cabeb9d0a99 100644 --- a/src/rgw/services/svc_meta_be_otp.cc +++ b/src/rgw/services/svc_meta_be_otp.cc @@ -3,9 +3,9 @@ #include "svc_meta_be_otp.h" -#include "rgw/rgw_tools.h" -#include "rgw/rgw_metadata.h" -#include "rgw/rgw_mdlog.h" +#include "rgw_tools.h" +#include "rgw_metadata.h" +#include "rgw_mdlog.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_meta_be_otp.h b/src/rgw/services/svc_meta_be_otp.h index d58def473580..7bd9cf652ff8 100644 --- a/src/rgw/services/svc_meta_be_otp.h +++ b/src/rgw/services/svc_meta_be_otp.h @@ -18,7 +18,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_cls.h" #include "svc_meta_be.h" diff --git a/src/rgw/services/svc_meta_be_sobj.cc b/src/rgw/services/svc_meta_be_sobj.cc index 5b3ce35f78c3..c0ff402fc58e 100644 --- a/src/rgw/services/svc_meta_be_sobj.cc +++ b/src/rgw/services/svc_meta_be_sobj.cc @@ -5,9 +5,9 @@ #include "svc_meta_be_params.h" #include "svc_mdlog.h" -#include "rgw/rgw_tools.h" -#include "rgw/rgw_metadata.h" -#include "rgw/rgw_mdlog.h" +#include "rgw_tools.h" +#include "rgw_metadata.h" +#include "rgw_mdlog.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_meta_be_sobj.h b/src/rgw/services/svc_meta_be_sobj.h index a7b703f703c1..304afc8bf2ad 100644 --- a/src/rgw/services/svc_meta_be_sobj.h +++ b/src/rgw/services/svc_meta_be_sobj.h @@ -18,7 +18,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_meta_be.h" #include "svc_sys_obj.h" diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc index 093e7d76534b..b0deefcfaf14 100644 --- a/src/rgw/services/svc_notify.cc +++ b/src/rgw/services/svc_notify.cc @@ -5,13 +5,13 @@ #include "include/Context.h" #include "common/errno.h" -#include "rgw/rgw_cache.h" +#include "rgw_cache.h" #include "svc_notify.h" #include "svc_finisher.h" #include "svc_zone.h" #include "svc_rados.h" -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_notify.h b/src/rgw/services/svc_notify.h index 8531b2e23be9..38d2ab50964c 100644 --- a/src/rgw/services/svc_notify.h +++ b/src/rgw/services/svc_notify.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" diff --git a/src/rgw/services/svc_otp.cc b/src/rgw/services/svc_otp.cc index bd08fb94c269..81d8d57112ea 100644 --- a/src/rgw/services/svc_otp.cc +++ b/src/rgw/services/svc_otp.cc @@ -6,7 +6,7 @@ #include "svc_meta.h" #include "svc_meta_be_sobj.h" -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_otp.h b/src/rgw/services/svc_otp.h index f95df4073194..e639c2c923e9 100644 --- a/src/rgw/services/svc_otp.h +++ b/src/rgw/services/svc_otp.h @@ -19,7 +19,7 @@ #include "cls/otp/cls_otp_types.h" -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_otp_types.h" #include "svc_meta_be_otp.h" diff --git a/src/rgw/services/svc_quota.cc b/src/rgw/services/svc_quota.cc index 0487a6ec95c2..3108a1173c82 100644 --- a/src/rgw/services/svc_quota.cc +++ b/src/rgw/services/svc_quota.cc @@ -4,7 +4,7 @@ #include "svc_quota.h" #include "svc_zone.h" -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" const RGWQuotaInfo& RGWSI_Quota::get_bucket_quota() const { diff --git a/src/rgw/services/svc_quota.h b/src/rgw/services/svc_quota.h index 34653ff3f372..81aa0e1bd4c4 100644 --- a/src/rgw/services/svc_quota.h +++ b/src/rgw/services/svc_quota.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWSI_Quota : public RGWServiceInstance diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc index 388f0b05cfc5..7d1239cec2c6 100644 --- a/src/rgw/services/svc_rados.cc +++ b/src/rgw/services/svc_rados.cc @@ -6,8 +6,8 @@ #include "include/rados/librados.hpp" #include "common/errno.h" #include "osd/osd_types.h" -#include "rgw/rgw_tools.h" -#include "rgw/rgw_cr_rados.h" +#include "rgw_tools.h" +#include "rgw_cr_rados.h" #include "auth/AuthRegistry.h" diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h index 5e1b7cd605b0..ede029aa897b 100644 --- a/src/rgw/services/svc_rados.h +++ b/src/rgw/services/svc_rados.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "include/rados/librados.hpp" #include "common/async/yield_context.h" diff --git a/src/rgw/services/svc_role_rados.h b/src/rgw/services/svc_role_rados.h index 6acfc2f349cb..d4d3530c278c 100644 --- a/src/rgw/services/svc_role_rados.h +++ b/src/rgw/services/svc_role_rados.h @@ -15,8 +15,8 @@ #pragma once -#include "rgw/rgw_service.h" -#include "rgw/rgw_role.h" +#include "rgw_service.h" +#include "rgw_role.h" #include "svc_meta_be.h" class RGWSI_Role_RADOS: public RGWServiceInstance @@ -47,4 +47,4 @@ private: static const std::string role_name_oid_prefix = "role_names."; static const std::string role_oid_prefix = "roles."; -static const std::string role_path_oid_prefix = "role_paths."; \ No newline at end of file +static const std::string role_path_oid_prefix = "role_paths."; diff --git a/src/rgw/services/svc_sync_modules.cc b/src/rgw/services/svc_sync_modules.cc index ce9c1e8ba599..ba9e7d172583 100644 --- a/src/rgw/services/svc_sync_modules.cc +++ b/src/rgw/services/svc_sync_modules.cc @@ -4,8 +4,8 @@ #include "svc_sync_modules.h" #include "svc_zone.h" -#include "rgw/rgw_sync_module.h" -#include "rgw/rgw_zone.h" +#include "rgw_sync_module.h" +#include "rgw_zone.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_sync_modules.h b/src/rgw/services/svc_sync_modules.h index 0640ced1d57b..ea78f58178dd 100644 --- a/src/rgw/services/svc_sync_modules.h +++ b/src/rgw/services/svc_sync_modules.h @@ -3,8 +3,8 @@ #pragma once -#include "rgw/rgw_service.h" -#include "rgw/rgw_sync_module.h" +#include "rgw_service.h" +#include "rgw_sync_module.h" class RGWSI_Zone; diff --git a/src/rgw/services/svc_sys_obj.cc b/src/rgw/services/svc_sys_obj.cc index 0bd3e877bdd5..d352fb9f4dff 100644 --- a/src/rgw/services/svc_sys_obj.cc +++ b/src/rgw/services/svc_sys_obj.cc @@ -6,7 +6,7 @@ #include "svc_rados.h" #include "svc_zone.h" -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h index aaf0d347e6d5..f3e217dbde94 100644 --- a/src/rgw/services/svc_sys_obj.h +++ b/src/rgw/services/svc_sys_obj.h @@ -5,7 +5,7 @@ #include "common/static_ptr.h" -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" #include "svc_sys_obj_types.h" diff --git a/src/rgw/services/svc_sys_obj_cache.cc b/src/rgw/services/svc_sys_obj_cache.cc index ec39ed9669c0..f704e7b1d745 100644 --- a/src/rgw/services/svc_sys_obj_cache.cc +++ b/src/rgw/services/svc_sys_obj_cache.cc @@ -8,8 +8,8 @@ #include "svc_zone.h" #include "svc_notify.h" -#include "rgw/rgw_zone.h" -#include "rgw/rgw_tools.h" +#include "rgw_zone.h" +#include "rgw_tools.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_sys_obj_cache.h b/src/rgw/services/svc_sys_obj_cache.h index 79119a721de2..e1fa9f997b91 100644 --- a/src/rgw/services/svc_sys_obj_cache.h +++ b/src/rgw/services/svc_sys_obj_cache.h @@ -4,8 +4,8 @@ #pragma once #include "common/RWLock.h" -#include "rgw/rgw_service.h" -#include "rgw/rgw_cache.h" +#include "rgw_service.h" +#include "rgw_cache.h" #include "svc_sys_obj_core.h" diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc index 3b1ea66ffc00..c57fbb4eed48 100644 --- a/src/rgw/services/svc_sys_obj_core.cc +++ b/src/rgw/services/svc_sys_obj_core.cc @@ -5,7 +5,7 @@ #include "svc_rados.h" #include "svc_zone.h" -#include "rgw/rgw_tools.h" +#include "rgw_tools.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/rgw/services/svc_sys_obj_core.h b/src/rgw/services/svc_sys_obj_core.h index 46de412f6247..56b1b6d82e28 100644 --- a/src/rgw/services/svc_sys_obj_core.h +++ b/src/rgw/services/svc_sys_obj_core.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" #include "svc_sys_obj.h" diff --git a/src/rgw/services/svc_sys_obj_core_types.h b/src/rgw/services/svc_sys_obj_core_types.h index 77536dcac158..74f489d914e5 100644 --- a/src/rgw/services/svc_sys_obj_core_types.h +++ b/src/rgw/services/svc_sys_obj_core_types.h @@ -4,7 +4,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" #include "svc_sys_obj_types.h" diff --git a/src/rgw/services/svc_sys_obj_types.h b/src/rgw/services/svc_sys_obj_types.h index 5a54bef2940e..b5bc2d40d247 100644 --- a/src/rgw/services/svc_sys_obj_types.h +++ b/src/rgw/services/svc_sys_obj_types.h @@ -5,7 +5,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" struct RGWSI_SysObj_Obj_GetObjState { diff --git a/src/rgw/services/svc_tier_rados.h b/src/rgw/services/svc_tier_rados.h index d94abd9224a2..a2036b933473 100644 --- a/src/rgw/services/svc_tier_rados.h +++ b/src/rgw/services/svc_tier_rados.h @@ -18,7 +18,7 @@ #include -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_rados.h" diff --git a/src/rgw/services/svc_user.h b/src/rgw/services/svc_user.h index ef867e699430..1cb459d31cb2 100644 --- a/src/rgw/services/svc_user.h +++ b/src/rgw/services/svc_user.h @@ -19,7 +19,7 @@ #include "svc_meta_be.h" -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWUserBuckets; class RGWGetUserStats_CB; diff --git a/src/rgw/services/svc_user_rados.cc b/src/rgw/services/svc_user_rados.cc index 1866ff682f78..3226e435e51b 100644 --- a/src/rgw/services/svc_user_rados.cc +++ b/src/rgw/services/svc_user_rados.cc @@ -12,11 +12,11 @@ #include "svc_meta_be_sobj.h" #include "svc_sync_modules.h" -#include "rgw/rgw_user.h" -#include "rgw/rgw_bucket.h" -#include "rgw/rgw_tools.h" -#include "rgw/rgw_zone.h" -#include "rgw/rgw_rados.h" +#include "rgw_user.h" +#include "rgw_bucket.h" +#include "rgw_tools.h" +#include "rgw_zone.h" +#include "rgw_rados.h" #include "cls/user/cls_user_client.h" diff --git a/src/rgw/services/svc_user_rados.h b/src/rgw/services/svc_user_rados.h index d822528c5d97..ff1fe41989fb 100644 --- a/src/rgw/services/svc_user_rados.h +++ b/src/rgw/services/svc_user_rados.h @@ -16,7 +16,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" #include "svc_meta_be.h" #include "svc_user.h" diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index e11b3b6eff1d..0476bc8f7081 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -6,9 +6,9 @@ #include "svc_sys_obj.h" #include "svc_sync_modules.h" -#include "rgw/rgw_zone.h" -#include "rgw/rgw_rest_conn.h" -#include "rgw/rgw_bucket_sync.h" +#include "rgw_zone.h" +#include "rgw_rest_conn.h" +#include "rgw_bucket_sync.h" #include "common/errno.h" #include "include/random.h" diff --git a/src/rgw/services/svc_zone.h b/src/rgw/services/svc_zone.h index a03cdc48b0fa..00d4c0f4992e 100644 --- a/src/rgw/services/svc_zone.h +++ b/src/rgw/services/svc_zone.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWSI_RADOS; diff --git a/src/rgw/services/svc_zone_utils.cc b/src/rgw/services/svc_zone_utils.cc index 0641f8335046..712bb97c9ba3 100644 --- a/src/rgw/services/svc_zone_utils.cc +++ b/src/rgw/services/svc_zone_utils.cc @@ -5,7 +5,7 @@ #include "svc_rados.h" #include "svc_zone.h" -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" using namespace std; diff --git a/src/rgw/services/svc_zone_utils.h b/src/rgw/services/svc_zone_utils.h index c534e7ea346e..43e3fee8d93d 100644 --- a/src/rgw/services/svc_zone_utils.h +++ b/src/rgw/services/svc_zone_utils.h @@ -3,7 +3,7 @@ #pragma once -#include "rgw/rgw_service.h" +#include "rgw_service.h" class RGWSI_RADOS; diff --git a/src/rgw/store/dbstore/CMakeLists.txt b/src/rgw/store/dbstore/CMakeLists.txt index af5868872bf4..0d34d32970b4 100644 --- a/src/rgw/store/dbstore/CMakeLists.txt +++ b/src/rgw/store/dbstore/CMakeLists.txt @@ -25,9 +25,11 @@ set(dbstore_mgr_srcs ) add_library(dbstore_lib ${dbstore_srcs}) -target_include_directories(dbstore_lib PUBLIC "${CMAKE_SOURCE_DIR}/src/fmt/include") -target_include_directories(dbstore_lib PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw") -target_include_directories(dbstore_lib PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") +target_include_directories(dbstore_lib + PUBLIC "${CMAKE_SOURCE_DIR}/src/fmt/include" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw" + PUBLIC "${CMAKE_SOURCE_DIR}/src/rgw/store/rados" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") set(link_targets spawn) if(WITH_JAEGER) list(APPEND link_targets jaeger_base) diff --git a/src/rgw/store/dbstore/common/dbstore.h b/src/rgw/store/dbstore/common/dbstore.h index 80517c44eb82..606066c2be65 100644 --- a/src/rgw/store/dbstore/common/dbstore.h +++ b/src/rgw/store/dbstore/common/dbstore.h @@ -17,14 +17,14 @@ #define FMT_HEADER_ONLY 1 #include "fmt/format.h" #include -#include "rgw/rgw_sal_store.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_bucket.h" +#include "rgw_sal_store.h" +#include "rgw_common.h" +#include "rgw_bucket.h" #include "global/global_context.h" #include "global/global_init.h" #include "common/ceph_context.h" -#include "rgw/rgw_obj_manifest.h" -#include "rgw/rgw_multi.h" +#include "rgw_obj_manifest.h" +#include "rgw_multi.h" namespace rgw { namespace store { diff --git a/src/rgw/cls_fifo_legacy.cc b/src/rgw/store/rados/cls_fifo_legacy.cc similarity index 100% rename from src/rgw/cls_fifo_legacy.cc rename to src/rgw/store/rados/cls_fifo_legacy.cc diff --git a/src/rgw/cls_fifo_legacy.h b/src/rgw/store/rados/cls_fifo_legacy.h similarity index 100% rename from src/rgw/cls_fifo_legacy.h rename to src/rgw/store/rados/cls_fifo_legacy.h diff --git a/src/rgw/store/rados/rgw_bucket.cc b/src/rgw/store/rados/rgw_bucket.cc new file mode 100644 index 000000000000..ba8141105082 --- /dev/null +++ b/src/rgw/store/rados/rgw_bucket.cc @@ -0,0 +1,2971 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_acl_s3.h" +#include "rgw_tag_s3.h" + +#include "rgw_bucket.h" +#include "rgw_op.h" +#include "rgw_bucket_sync.h" + +#include "services/svc_zone.h" +#include "services/svc_bucket.h" +#include "services/svc_user.h" + +#include "rgw_reshard.h" + +// stolen from src/cls/version/cls_version.cc +#define VERSION_ATTR "ceph.objclass.version" + +#include "cls/user/cls_user_types.h" + +#include "rgw_sal_rados.h" + +#define dout_subsys ceph_subsys_rgw + +// seconds for timeout during RGWBucket::check_object_index +constexpr uint64_t BUCKET_TAG_QUICK_TIMEOUT = 30; + +using namespace std; + +// default number of entries to list with each bucket listing call +// (use marker to bridge between calls) +static constexpr size_t listing_max_entries = 1000; + +/* + * The tenant_name is always returned on purpose. May be empty, of course. + */ +static void parse_bucket(const string& bucket, + string *tenant_name, + string *bucket_name, + string *bucket_instance = nullptr /* optional */) +{ + /* + * expected format: [tenant/]bucket:bucket_instance + */ + int pos = bucket.find('/'); + if (pos >= 0) { + *tenant_name = bucket.substr(0, pos); + } else { + tenant_name->clear(); + } + string bn = bucket.substr(pos + 1); + pos = bn.find (':'); + if (pos < 0) { + *bucket_name = std::move(bn); + return; + } + *bucket_name = bn.substr(0, pos); + if (bucket_instance) { + *bucket_instance = bn.substr(pos + 1); + } + + /* + * deal with the possible tenant:bucket:bucket_instance case + */ + if (tenant_name->empty()) { + pos = bucket_instance->find(':'); + if (pos >= 0) { + *tenant_name = *bucket_name; + *bucket_name = bucket_instance->substr(0, pos); + *bucket_instance = bucket_instance->substr(pos + 1); + } + } +} + +static void dump_mulipart_index_results(list& objs_to_unlink, + Formatter *f) +{ + for (const auto& o : objs_to_unlink) { + f->dump_string("object", o.name); + } +} + +void check_bad_user_bucket_mapping(rgw::sal::Store* store, rgw::sal::User* user, + bool fix, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + rgw::sal::BucketList user_buckets; + string marker; + + CephContext *cct = store->ctx(); + + size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; + + do { + int ret = user->list_buckets(dpp, marker, string(), max_entries, false, user_buckets, y); + if (ret < 0) { + ldout(store->ctx(), 0) << "failed to read user buckets: " + << cpp_strerror(-ret) << dendl; + return; + } + + map>& buckets = user_buckets.get_buckets(); + for (auto i = buckets.begin(); + i != buckets.end(); + ++i) { + marker = i->first; + + auto& bucket = i->second; + + std::unique_ptr actual_bucket; + int r = store->get_bucket(dpp, user, user->get_tenant(), bucket->get_name(), &actual_bucket, null_yield); + if (r < 0) { + ldout(store->ctx(), 0) << "could not get bucket info for bucket=" << bucket << dendl; + continue; + } + + if (actual_bucket->get_name().compare(bucket->get_name()) != 0 || + actual_bucket->get_tenant().compare(bucket->get_tenant()) != 0 || + actual_bucket->get_marker().compare(bucket->get_marker()) != 0 || + actual_bucket->get_bucket_id().compare(bucket->get_bucket_id()) != 0) { + cout << "bucket info mismatch: expected " << actual_bucket << " got " << bucket << std::endl; + if (fix) { + cout << "fixing" << std::endl; + r = actual_bucket->chown(dpp, user, nullptr, null_yield); + if (r < 0) { + cerr << "failed to fix bucket: " << cpp_strerror(-r) << std::endl; + } + } + } + } + } while (user_buckets.is_truncated()); +} + +// returns true if entry is in the empty namespace. note: function +// type conforms to type RGWBucketListNameFilter +bool rgw_bucket_object_check_filter(const std::string& oid) +{ + const static std::string empty_ns; + rgw_obj_key key; // thrown away but needed for parsing + return rgw_obj_key::oid_to_key_in_ns(oid, &key, empty_ns); +} + +int rgw_remove_object(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw::sal::Bucket* bucket, rgw_obj_key& key) +{ + if (key.instance.empty()) { + key.instance = "null"; + } + + std::unique_ptr object = bucket->get_object(key); + + return object->delete_object(dpp, null_yield); +} + +static void set_err_msg(std::string *sink, std::string msg) +{ + if (sink && !msg.empty()) + *sink = msg; +} + +int RGWBucket::init(rgw::sal::Store* _store, RGWBucketAdminOpState& op_state, + optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg) +{ + if (!_store) { + set_err_msg(err_msg, "no storage!"); + return -EINVAL; + } + + store = _store; + + std::string bucket_name = op_state.get_bucket_name(); + + if (bucket_name.empty() && op_state.get_user_id().empty()) + return -EINVAL; + + user = store->get_user(op_state.get_user_id()); + std::string tenant = user->get_tenant(); + + // split possible tenant/name + auto pos = bucket_name.find('/'); + if (pos != string::npos) { + tenant = bucket_name.substr(0, pos); + bucket_name = bucket_name.substr(pos + 1); + } + + int r = store->get_bucket(dpp, user.get(), tenant, bucket_name, &bucket, y); + if (r < 0) { + set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket_name); + return r; + } + + op_state.set_bucket(bucket->clone()); + + if (!rgw::sal::User::empty(user.get())) { + r = user->load_user(dpp, y); + if (r < 0) { + set_err_msg(err_msg, "failed to fetch user info"); + return r; + } + } + + op_state.display_name = user->get_display_name(); + + clear_failure(); + return 0; +} + +bool rgw_find_bucket_by_id(const DoutPrefixProvider *dpp, CephContext *cct, rgw::sal::Store* store, + const string& marker, const string& bucket_id, rgw_bucket* bucket_out) +{ + void *handle = NULL; + bool truncated = false; + string s; + + int ret = store->meta_list_keys_init(dpp, "bucket.instance", marker, &handle); + if (ret < 0) { + cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; + store->meta_list_keys_complete(handle); + return -ret; + } + do { + list keys; + ret = store->meta_list_keys_next(dpp, handle, 1000, keys, &truncated); + if (ret < 0) { + cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; + store->meta_list_keys_complete(handle); + return -ret; + } + for (list::iterator iter = keys.begin(); iter != keys.end(); ++iter) { + s = *iter; + ret = rgw_bucket_parse_bucket_key(cct, s, bucket_out, nullptr); + if (ret < 0) { + continue; + } + if (bucket_id == bucket_out->bucket_id) { + store->meta_list_keys_complete(handle); + return true; + } + } + } while (truncated); + store->meta_list_keys_complete(handle); + return false; +} + +int RGWBucket::chown(RGWBucketAdminOpState& op_state, const string& marker, + optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg) +{ + int ret = bucket->chown(dpp, user.get(), user.get(), y, &marker); + if (ret < 0) { + set_err_msg(err_msg, "Failed to change object ownership: " + cpp_strerror(-ret)); + } + + return ret; +} + +int RGWBucket::set_quota(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, std::string *err_msg) +{ + bucket = op_state.get_bucket()->clone(); + + bucket->get_info().quota = op_state.quota; + int r = bucket->put_info(dpp, false, real_time()); + if (r < 0) { + set_err_msg(err_msg, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r)); + return r; + } + return r; +} + +int RGWBucket::remove_object(const DoutPrefixProvider *dpp, RGWBucketAdminOpState& op_state, std::string *err_msg) +{ + std::string object_name = op_state.get_object_name(); + + rgw_obj_key key(object_name); + + bucket = op_state.get_bucket()->clone(); + + int ret = rgw_remove_object(dpp, store, bucket.get(), key); + if (ret < 0) { + set_err_msg(err_msg, "unable to remove object" + cpp_strerror(-ret)); + return ret; + } + + return 0; +} + +static void dump_bucket_index(const vector& objs, Formatter *f) +{ + for (auto iter = objs.begin(); iter != objs.end(); ++iter) { + f->dump_string("object", iter->key.name); + } +} + +static void dump_bucket_usage(map& stats, Formatter *formatter) +{ + map::iterator iter; + + formatter->open_object_section("usage"); + for (iter = stats.begin(); iter != stats.end(); ++iter) { + RGWStorageStats& s = iter->second; + formatter->open_object_section(to_string(iter->first)); + s.dump(formatter); + formatter->close_section(); + } + formatter->close_section(); +} + +static void dump_index_check(map existing_stats, + map calculated_stats, + Formatter *formatter) +{ + formatter->open_object_section("check_result"); + formatter->open_object_section("existing_header"); + dump_bucket_usage(existing_stats, formatter); + formatter->close_section(); + formatter->open_object_section("calculated_header"); + dump_bucket_usage(calculated_stats, formatter); + formatter->close_section(); + formatter->close_section(); +} + +int RGWBucket::check_bad_index_multipart(RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + const DoutPrefixProvider *dpp, + std::string *err_msg) +{ + const bool fix_index = op_state.will_fix_index(); + + bucket = op_state.get_bucket()->clone(); + + rgw::sal::Bucket::ListParams params; + params.list_versions = true; + params.ns = RGW_OBJ_NS_MULTIPART; + + std::map meta_objs; + std::map all_objs; + bool is_truncated; + do { + rgw::sal::Bucket::ListResults results; + int r = bucket->list(dpp, params, listing_max_entries, results, null_yield); + if (r < 0) { + set_err_msg(err_msg, "failed to list objects in bucket=" + bucket->get_name() + + " err=" + cpp_strerror(-r)); + + return r; + } + is_truncated = results.is_truncated; + + for (const auto& o : results.objs) { + rgw_obj_index_key key = o.key; + rgw_obj obj(bucket->get_key(), key); + std::string oid = obj.get_oid(); + + int pos = oid.find_last_of('.'); + if (pos < 0) { + /* obj has no suffix */ + all_objs[key] = oid; + } else { + /* obj has suffix */ + std::string name = oid.substr(0, pos); + std::string suffix = oid.substr(pos + 1); + + if (suffix.compare("meta") == 0) { + meta_objs[name] = true; + } else { + all_objs[key] = name; + } + } + } + } while (is_truncated); + + std::list objs_to_unlink; + Formatter *f = flusher.get_formatter(); + + f->open_array_section("invalid_multipart_entries"); + + for (const auto& o : all_objs) { + const std::string& name = o.second; + if (meta_objs.find(name) == meta_objs.end()) { + objs_to_unlink.push_back(o.first); + } + + if (objs_to_unlink.size() > listing_max_entries) { + if (fix_index) { + // note: under rados this removes directly from rados index objects + int r = bucket->remove_objs_from_index(dpp, objs_to_unlink); + if (r < 0) { + set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " + + cpp_strerror(-r)); + return r; + } + } + + dump_mulipart_index_results(objs_to_unlink, f); + flusher.flush(); + objs_to_unlink.clear(); + } + } + + if (fix_index) { + // note: under rados this removes directly from rados index objects + int r = bucket->remove_objs_from_index(dpp, objs_to_unlink); + if (r < 0) { + set_err_msg(err_msg, "ERROR: remove_obj_from_index() returned error: " + + cpp_strerror(-r)); + + return r; + } + } + + dump_mulipart_index_results(objs_to_unlink, f); + f->close_section(); + flusher.flush(); + + return 0; +} + +int RGWBucket::check_object_index(const DoutPrefixProvider *dpp, + RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y, + std::string *err_msg) +{ + + bool fix_index = op_state.will_fix_index(); + + if (!fix_index) { + set_err_msg(err_msg, "check-objects flag requires fix index enabled"); + return -EINVAL; + } + + // use a quicker/shorter tag timeout during this process + bucket->set_tag_timeout(dpp, BUCKET_TAG_QUICK_TIMEOUT); + + rgw::sal::Bucket::ListResults results; + results.is_truncated = true; + + Formatter *formatter = flusher.get_formatter(); + formatter->open_object_section("objects"); + + while (results.is_truncated) { + rgw::sal::Bucket::ListParams params; + params.marker = results.next_marker; + params.force_check_filter = rgw_bucket_object_check_filter; + + int r = bucket->list(dpp, params, listing_max_entries, results, y); + + if (r == -ENOENT) { + break; + } else if (r < 0) { + set_err_msg(err_msg, "ERROR: failed operation r=" + cpp_strerror(-r)); + } + + dump_bucket_index(results.objs, formatter); + flusher.flush(); + } + + formatter->close_section(); + + // restore normal tag timeout for bucket + bucket->set_tag_timeout(dpp, 0); + + return 0; +} + + +int RGWBucket::check_index(const DoutPrefixProvider *dpp, + RGWBucketAdminOpState& op_state, + map& existing_stats, + map& calculated_stats, + std::string *err_msg) +{ + bool fix_index = op_state.will_fix_index(); + + int r = bucket->check_index(dpp, existing_stats, calculated_stats); + if (r < 0) { + set_err_msg(err_msg, "failed to check index error=" + cpp_strerror(-r)); + return r; + } + + if (fix_index) { + r = bucket->rebuild_index(dpp); + if (r < 0) { + set_err_msg(err_msg, "failed to rebuild index err=" + cpp_strerror(-r)); + return r; + } + } + + return 0; +} + +int RGWBucket::sync(RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, std::string *err_msg) +{ + if (!store->is_meta_master()) { + set_err_msg(err_msg, "ERROR: failed to update bucket sync: only allowed on meta master zone"); + return -EINVAL; + } + bool sync = op_state.will_sync_bucket(); + if (sync) { + bucket->get_info().flags &= ~BUCKET_DATASYNC_DISABLED; + } else { + bucket->get_info().flags |= BUCKET_DATASYNC_DISABLED; + } + + // when writing this metadata, RGWSI_BucketIndex_RADOS::handle_overwrite() + // will write the corresponding datalog and bilog entries + int r = bucket->put_info(dpp, false, real_time()); + if (r < 0) { + set_err_msg(err_msg, "ERROR: failed writing bucket instance info:" + cpp_strerror(-r)); + return r; + } + + return 0; +} + + +int RGWBucket::policy_bl_to_stream(bufferlist& bl, ostream& o) +{ + RGWAccessControlPolicy_S3 policy(g_ceph_context); + int ret = decode_bl(bl, policy); + if (ret < 0) { + ldout(store->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl; + } + policy.to_xml(o); + return 0; +} + +int rgw_object_get_attr(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, rgw::sal::Object* obj, + const char* attr_name, bufferlist& out_bl, optional_yield y) +{ + std::unique_ptr rop = obj->get_read_op(); + + return rop->get_attr(dpp, attr_name, out_bl, y); +} + +int RGWBucket::get_policy(RGWBucketAdminOpState& op_state, RGWAccessControlPolicy& policy, optional_yield y, const DoutPrefixProvider *dpp) +{ + int ret; + std::string object_name = op_state.get_object_name(); + + bucket = op_state.get_bucket()->clone(); + + if (!object_name.empty()) { + bufferlist bl; + std::unique_ptr obj = bucket->get_object(rgw_obj_key(object_name)); + + ret = rgw_object_get_attr(dpp, store, obj.get(), RGW_ATTR_ACL, bl, y); + if (ret < 0){ + return ret; + } + + ret = decode_bl(bl, policy); + if (ret < 0) { + ldout(store->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl; + } + return ret; + } + + map::iterator aiter = bucket->get_attrs().find(RGW_ATTR_ACL); + if (aiter == bucket->get_attrs().end()) { + return -ENOENT; + } + + ret = decode_bl(aiter->second, policy); + if (ret < 0) { + ldout(store->ctx(),0) << "failed to decode RGWAccessControlPolicy" << dendl; + } + + return ret; +} + + +int RGWBucketAdminOp::get_policy(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, + RGWAccessControlPolicy& policy, const DoutPrefixProvider *dpp) +{ + RGWBucket bucket; + + int ret = bucket.init(store, op_state, null_yield, dpp); + if (ret < 0) + return ret; + + ret = bucket.get_policy(op_state, policy, null_yield, dpp); + if (ret < 0) + return ret; + + return 0; +} + +/* Wrappers to facilitate RESTful interface */ + + +int RGWBucketAdminOp::get_policy(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, const DoutPrefixProvider *dpp) +{ + RGWAccessControlPolicy policy(store->ctx()); + + int ret = get_policy(store, op_state, policy, dpp); + if (ret < 0) + return ret; + + Formatter *formatter = flusher.get_formatter(); + + flusher.start(0); + + formatter->open_object_section("policy"); + policy.dump(formatter); + formatter->close_section(); + + flusher.flush(); + + return 0; +} + +int RGWBucketAdminOp::dump_s3_policy(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, + ostream& os, const DoutPrefixProvider *dpp) +{ + RGWAccessControlPolicy_S3 policy(store->ctx()); + + int ret = get_policy(store, op_state, policy, dpp); + if (ret < 0) + return ret; + + policy.to_xml(os); + + return 0; +} + +int RGWBucketAdminOp::unlink(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp) +{ + RGWBucket bucket; + + int ret = bucket.init(store, op_state, null_yield, dpp); + if (ret < 0) + return ret; + + return static_cast(store)->ctl()->bucket->unlink_bucket(op_state.get_user_id(), op_state.get_bucket()->get_info().bucket, null_yield, dpp, true); +} + +int RGWBucketAdminOp::link(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, string *err) +{ + if (!op_state.is_user_op()) { + set_err_msg(err, "empty user id"); + return -EINVAL; + } + + RGWBucket bucket; + int ret = bucket.init(store, op_state, null_yield, dpp, err); + if (ret < 0) + return ret; + + string bucket_id = op_state.get_bucket_id(); + std::string display_name = op_state.get_user_display_name(); + std::unique_ptr loc_bucket; + std::unique_ptr old_bucket; + + loc_bucket = op_state.get_bucket()->clone(); + + if (!bucket_id.empty() && bucket_id != loc_bucket->get_bucket_id()) { + set_err_msg(err, + "specified bucket id does not match " + loc_bucket->get_bucket_id()); + return -EINVAL; + } + + old_bucket = loc_bucket->clone(); + + loc_bucket->get_key().tenant = op_state.get_user_id().tenant; + + if (!op_state.new_bucket_name.empty()) { + auto pos = op_state.new_bucket_name.find('/'); + if (pos != string::npos) { + loc_bucket->get_key().tenant = op_state.new_bucket_name.substr(0, pos); + loc_bucket->get_key().name = op_state.new_bucket_name.substr(pos + 1); + } else { + loc_bucket->get_key().name = op_state.new_bucket_name; + } + } + + RGWObjVersionTracker objv_tracker; + RGWObjVersionTracker old_version = loc_bucket->get_info().objv_tracker; + + map::iterator aiter = loc_bucket->get_attrs().find(RGW_ATTR_ACL); + if (aiter == loc_bucket->get_attrs().end()) { + // should never happen; only pre-argonaut buckets lacked this. + ldpp_dout(dpp, 0) << "WARNING: can't bucket link because no acl on bucket=" << old_bucket << dendl; + set_err_msg(err, + "While crossing the Anavros you have displeased the goddess Hera." + " You must sacrifice your ancient bucket " + loc_bucket->get_bucket_id()); + return -EINVAL; + } + bufferlist& aclbl = aiter->second; + RGWAccessControlPolicy policy; + ACLOwner owner; + try { + auto iter = aclbl.cbegin(); + decode(policy, iter); + owner = policy.get_owner(); + } catch (buffer::error& e) { + set_err_msg(err, "couldn't decode policy"); + return -EIO; + } + + int r = static_cast(store)->ctl()->bucket->unlink_bucket(owner.get_id(), old_bucket->get_info().bucket, null_yield, dpp, false); + if (r < 0) { + set_err_msg(err, "could not unlink policy from user " + owner.get_id().to_str()); + return r; + } + + // now update the user for the bucket... + if (display_name.empty()) { + ldpp_dout(dpp, 0) << "WARNING: user " << op_state.get_user_id() << " has no display name set" << dendl; + } + + RGWAccessControlPolicy policy_instance; + policy_instance.create_default(op_state.get_user_id(), display_name); + owner = policy_instance.get_owner(); + + aclbl.clear(); + policy_instance.encode(aclbl); + + bool exclusive = false; + loc_bucket->get_info().owner = op_state.get_user_id(); + if (*loc_bucket != *old_bucket) { + loc_bucket->get_info().bucket = loc_bucket->get_key(); + loc_bucket->get_info().objv_tracker.version_for_read()->ver = 0; + exclusive = true; + } + + r = loc_bucket->put_info(dpp, exclusive, ceph::real_time()); + if (r < 0) { + set_err_msg(err, "ERROR: failed writing bucket instance info: " + cpp_strerror(-r)); + return r; + } + + /* link to user */ + RGWBucketEntryPoint ep; + ep.bucket = loc_bucket->get_info().bucket; + ep.owner = op_state.get_user_id(); + ep.creation_time = loc_bucket->get_info().creation_time; + ep.linked = true; + rgw::sal::Attrs ep_attrs; + rgw_ep_info ep_data{ep, ep_attrs}; + + r = static_cast(store)->ctl()->bucket->link_bucket(op_state.get_user_id(), loc_bucket->get_info().bucket, loc_bucket->get_info().creation_time, null_yield, dpp, true, &ep_data); + if (r < 0) { + set_err_msg(err, "failed to relink bucket"); + return r; + } + + if (*loc_bucket != *old_bucket) { + // like RGWRados::delete_bucket -- excepting no bucket_index work. + r = static_cast(store)->ctl()->bucket->remove_bucket_entrypoint_info( + old_bucket->get_key(), null_yield, dpp, + RGWBucketCtl::Bucket::RemoveParams() + .set_objv_tracker(&ep_data.ep_objv)); + if (r < 0) { + set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name()); + return r; + } + r = static_cast(store)->ctl()->bucket->remove_bucket_instance_info( + old_bucket->get_key(), old_bucket->get_info(), + null_yield, dpp, + RGWBucketCtl::BucketInstance::RemoveParams() + .set_objv_tracker(&ep_data.ep_objv)); + if (r < 0) { + set_err_msg(err, "failed to unlink old bucket " + old_bucket->get_tenant() + "/" + old_bucket->get_name()); + return r; + } + } + + return 0; +} + +int RGWBucketAdminOp::chown(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const string& marker, const DoutPrefixProvider *dpp, string *err) +{ + RGWBucket bucket; + + int ret = bucket.init(store, op_state, null_yield, dpp, err); + if (ret < 0) + return ret; + + return bucket.chown(op_state, marker, null_yield, dpp, err); + +} + +int RGWBucketAdminOp::check_index(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y, const DoutPrefixProvider *dpp) +{ + int ret; + map existing_stats; + map calculated_stats; + + + RGWBucket bucket; + + ret = bucket.init(store, op_state, null_yield, dpp); + if (ret < 0) + return ret; + + Formatter *formatter = flusher.get_formatter(); + flusher.start(0); + + ret = bucket.check_bad_index_multipart(op_state, flusher, dpp); + if (ret < 0) + return ret; + + ret = bucket.check_object_index(dpp, op_state, flusher, y); + if (ret < 0) + return ret; + + ret = bucket.check_index(dpp, op_state, existing_stats, calculated_stats); + if (ret < 0) + return ret; + + dump_index_check(existing_stats, calculated_stats, formatter); + flusher.flush(); + + return 0; +} + +int RGWBucketAdminOp::remove_bucket(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, + optional_yield y, const DoutPrefixProvider *dpp, + bool bypass_gc, bool keep_index_consistent) +{ + std::unique_ptr bucket; + std::unique_ptr user = store->get_user(op_state.get_user_id()); + + int ret = store->get_bucket(dpp, user.get(), user->get_tenant(), op_state.get_bucket_name(), + &bucket, y); + if (ret < 0) + return ret; + + if (bypass_gc) + ret = bucket->remove_bucket_bypass_gc(op_state.get_max_aio(), keep_index_consistent, y, dpp); + else + ret = bucket->remove_bucket(dpp, op_state.will_delete_children(), + false, nullptr, y); + + return ret; +} + +int RGWBucketAdminOp::remove_object(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp) +{ + RGWBucket bucket; + + int ret = bucket.init(store, op_state, null_yield, dpp); + if (ret < 0) + return ret; + + return bucket.remove_object(dpp, op_state); +} + +int RGWBucketAdminOp::sync_bucket(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp, string *err_msg) +{ + RGWBucket bucket; + int ret = bucket.init(store, op_state, null_yield, dpp, err_msg); + if (ret < 0) + { + return ret; + } + return bucket.sync(op_state, dpp, err_msg); +} + +static int bucket_stats(rgw::sal::Store* store, + const std::string& tenant_name, + const std::string& bucket_name, + Formatter *formatter, + const DoutPrefixProvider *dpp) +{ + std::unique_ptr bucket; + map stats; + + real_time mtime; + int ret = store->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield); + if (ret < 0) { + return ret; + } + + const auto& index = bucket->get_info().get_current_index(); + if (is_layout_indexless(index)) { + cerr << "error, indexless buckets do not maintain stats; bucket=" << + bucket->get_name() << std::endl; + return -EINVAL; + } + + std::string bucket_ver, master_ver; + std::string max_marker; + ret = bucket->read_stats(dpp, index, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, &max_marker); + if (ret < 0) { + cerr << "error getting bucket stats bucket=" << bucket->get_name() << " ret=" << ret << std::endl; + return ret; + } + + utime_t ut(mtime); + utime_t ctime_ut(bucket->get_creation_time()); + + formatter->open_object_section("stats"); + formatter->dump_string("bucket", bucket->get_name()); + formatter->dump_int("num_shards", + bucket->get_info().layout.current_index.layout.normal.num_shards); + formatter->dump_string("tenant", bucket->get_tenant()); + formatter->dump_string("zonegroup", bucket->get_info().zonegroup); + formatter->dump_string("placement_rule", bucket->get_info().placement_rule.to_str()); + ::encode_json("explicit_placement", bucket->get_key().explicit_placement, formatter); + formatter->dump_string("id", bucket->get_bucket_id()); + formatter->dump_string("marker", bucket->get_marker()); + formatter->dump_stream("index_type") << bucket->get_info().layout.current_index.layout.type; + ::encode_json("owner", bucket->get_info().owner, formatter); + formatter->dump_string("ver", bucket_ver); + formatter->dump_string("master_ver", master_ver); + ut.gmtime(formatter->dump_stream("mtime")); + ctime_ut.gmtime(formatter->dump_stream("creation_time")); + formatter->dump_string("max_marker", max_marker); + dump_bucket_usage(stats, formatter); + encode_json("bucket_quota", bucket->get_info().quota, formatter); + + // bucket tags + auto iter = bucket->get_attrs().find(RGW_ATTR_TAGS); + if (iter != bucket->get_attrs().end()) { + RGWObjTagSet_S3 tagset; + bufferlist::const_iterator piter{&iter->second}; + try { + tagset.decode(piter); + tagset.dump(formatter); + } catch (buffer::error& err) { + cerr << "ERROR: caught buffer:error, couldn't decode TagSet" << std::endl; + } + } + + // TODO: bucket CORS + // TODO: bucket LC + formatter->close_section(); + + return 0; +} + +int RGWBucketAdminOp::limit_check(rgw::sal::Store* store, + RGWBucketAdminOpState& op_state, + const std::list& user_ids, + RGWFormatterFlusher& flusher, optional_yield y, + const DoutPrefixProvider *dpp, + bool warnings_only) +{ + int ret = 0; + const size_t max_entries = + store->ctx()->_conf->rgw_list_buckets_max_chunk; + + const size_t safe_max_objs_per_shard = + store->ctx()->_conf->rgw_safe_max_objects_per_shard; + + uint16_t shard_warn_pct = + store->ctx()->_conf->rgw_shard_warning_threshold; + if (shard_warn_pct > 100) + shard_warn_pct = 90; + + Formatter *formatter = flusher.get_formatter(); + flusher.start(0); + + formatter->open_array_section("users"); + + for (const auto& user_id : user_ids) { + + formatter->open_object_section("user"); + formatter->dump_string("user_id", user_id); + formatter->open_array_section("buckets"); + + string marker; + rgw::sal::BucketList buckets; + do { + std::unique_ptr user = store->get_user(rgw_user(user_id)); + + ret = user->list_buckets(dpp, marker, string(), max_entries, false, buckets, y); + + if (ret < 0) + return ret; + + map>& m_buckets = buckets.get_buckets(); + + for (const auto& iter : m_buckets) { + auto& bucket = iter.second; + uint64_t num_objects = 0; + + marker = bucket->get_name(); /* Casey's location for marker update, + * as we may now not reach the end of + * the loop body */ + + ret = bucket->load_bucket(dpp, null_yield); + if (ret < 0) + continue; + + const auto& index = bucket->get_info().get_current_index(); + if (is_layout_indexless(index)) { + continue; // indexless buckets don't have stats + } + + /* need stats for num_entries */ + string bucket_ver, master_ver; + std::map stats; + ret = bucket->read_stats(dpp, index, RGW_NO_SHARD, &bucket_ver, &master_ver, stats, nullptr); + + if (ret < 0) + continue; + + for (const auto& s : stats) { + num_objects += s.second.num_objects; + } + + const uint32_t num_shards = rgw::num_shards(index.layout.normal); + uint64_t objs_per_shard = + (num_shards) ? num_objects/num_shards : num_objects; + { + bool warn; + stringstream ss; + uint64_t fill_pct = objs_per_shard * 100 / safe_max_objs_per_shard; + if (fill_pct > 100) { + ss << "OVER " << fill_pct << "%"; + warn = true; + } else if (fill_pct >= shard_warn_pct) { + ss << "WARN " << fill_pct << "%"; + warn = true; + } else { + ss << "OK"; + warn = false; + } + + if (warn || !warnings_only) { + formatter->open_object_section("bucket"); + formatter->dump_string("bucket", bucket->get_name()); + formatter->dump_string("tenant", bucket->get_tenant()); + formatter->dump_int("num_objects", num_objects); + formatter->dump_int("num_shards", num_shards); + formatter->dump_int("objects_per_shard", objs_per_shard); + formatter->dump_string("fill_status", ss.str()); + formatter->close_section(); + } + } + } + formatter->flush(cout); + } while (buckets.is_truncated()); /* foreach: bucket */ + + formatter->close_section(); + formatter->close_section(); + formatter->flush(cout); + + } /* foreach: user_id */ + + formatter->close_section(); + formatter->flush(cout); + + return ret; +} /* RGWBucketAdminOp::limit_check */ + +int RGWBucketAdminOp::info(rgw::sal::Store* store, + RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + RGWBucket bucket; + int ret = 0; + const std::string& bucket_name = op_state.get_bucket_name(); + if (!bucket_name.empty()) { + ret = bucket.init(store, op_state, null_yield, dpp); + if (-ENOENT == ret) + return -ERR_NO_SUCH_BUCKET; + else if (ret < 0) + return ret; + } + + Formatter *formatter = flusher.get_formatter(); + flusher.start(0); + + CephContext *cct = store->ctx(); + + const size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk; + + const bool show_stats = op_state.will_fetch_stats(); + const rgw_user& user_id = op_state.get_user_id(); + if (op_state.is_user_op()) { + formatter->open_array_section("buckets"); + + rgw::sal::BucketList buckets; + std::unique_ptr user = store->get_user(op_state.get_user_id()); + std::string marker; + const std::string empty_end_marker; + constexpr bool no_need_stats = false; // set need_stats to false + + do { + ret = user->list_buckets(dpp, marker, empty_end_marker, max_entries, + no_need_stats, buckets, y); + if (ret < 0) { + return ret; + } + + const std::string* marker_cursor = nullptr; + map>& m = buckets.get_buckets(); + + for (const auto& i : m) { + const std::string& obj_name = i.first; + if (!bucket_name.empty() && bucket_name != obj_name) { + continue; + } + + if (show_stats) { + bucket_stats(store, user_id.tenant, obj_name, formatter, dpp); + } else { + formatter->dump_string("bucket", obj_name); + } + + marker_cursor = &obj_name; + } // for loop + if (marker_cursor) { + marker = *marker_cursor; + } + + flusher.flush(); + } while (buckets.is_truncated()); + + formatter->close_section(); + } else if (!bucket_name.empty()) { + ret = bucket_stats(store, user_id.tenant, bucket_name, formatter, dpp); + if (ret < 0) { + return ret; + } + } else { + void *handle = nullptr; + bool truncated = true; + + formatter->open_array_section("buckets"); + ret = store->meta_list_keys_init(dpp, "bucket", string(), &handle); + while (ret == 0 && truncated) { + std::list buckets; + constexpr int max_keys = 1000; + ret = store->meta_list_keys_next(dpp, handle, max_keys, buckets, + &truncated); + for (auto& bucket_name : buckets) { + if (show_stats) { + bucket_stats(store, user_id.tenant, bucket_name, formatter, dpp); + } else { + formatter->dump_string("bucket", bucket_name); + } + } + } + store->meta_list_keys_complete(handle); + + formatter->close_section(); + } + + flusher.flush(); + + return 0; +} + +int RGWBucketAdminOp::set_quota(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, const DoutPrefixProvider *dpp) +{ + RGWBucket bucket; + + int ret = bucket.init(store, op_state, null_yield, dpp); + if (ret < 0) + return ret; + return bucket.set_quota(op_state, dpp); +} + +inline auto split_tenant(const std::string& bucket_name){ + auto p = bucket_name.find('/'); + if(p != std::string::npos) { + return std::make_pair(bucket_name.substr(0,p), bucket_name.substr(p+1)); + } + return std::make_pair(std::string(), bucket_name); +} + +using bucket_instance_ls = std::vector; +void get_stale_instances(rgw::sal::Store* store, const std::string& bucket_name, + const vector& lst, + bucket_instance_ls& stale_instances, + const DoutPrefixProvider *dpp) +{ + + bucket_instance_ls other_instances; +// first iterate over the entries, and pick up the done buckets; these +// are guaranteed to be stale + for (const auto& bucket_instance : lst){ + RGWBucketInfo binfo; + std::unique_ptr bucket; + rgw_bucket rbucket; + rgw_bucket_parse_bucket_key(store->ctx(), bucket_instance, &rbucket, nullptr); + int r = store->get_bucket(dpp, nullptr, rbucket, &bucket, null_yield); + if (r < 0){ + // this can only happen if someone deletes us right when we're processing + ldpp_dout(dpp, -1) << "Bucket instance is invalid: " << bucket_instance + << cpp_strerror(-r) << dendl; + continue; + } + binfo = bucket->get_info(); + if (binfo.reshard_status == cls_rgw_reshard_status::DONE) + stale_instances.emplace_back(std::move(binfo)); + else { + other_instances.emplace_back(std::move(binfo)); + } + } + + // Read the cur bucket info, if the bucket doesn't exist we can simply return + // all the instances + auto [tenant, bname] = split_tenant(bucket_name); + RGWBucketInfo cur_bucket_info; + std::unique_ptr cur_bucket; + int r = store->get_bucket(dpp, nullptr, tenant, bname, &cur_bucket, null_yield); + if (r < 0) { + if (r == -ENOENT) { + // bucket doesn't exist, everything is stale then + stale_instances.insert(std::end(stale_instances), + std::make_move_iterator(other_instances.begin()), + std::make_move_iterator(other_instances.end())); + } else { + // all bets are off if we can't read the bucket, just return the sureshot stale instances + ldpp_dout(dpp, -1) << "error: reading bucket info for bucket: " + << bname << cpp_strerror(-r) << dendl; + } + return; + } + + // Don't process further in this round if bucket is resharding + cur_bucket_info = cur_bucket->get_info(); + if (cur_bucket_info.reshard_status == cls_rgw_reshard_status::IN_PROGRESS) + return; + + other_instances.erase(std::remove_if(other_instances.begin(), other_instances.end(), + [&cur_bucket_info](const RGWBucketInfo& b){ + return (b.bucket.bucket_id == cur_bucket_info.bucket.bucket_id || + b.bucket.bucket_id == cur_bucket_info.new_bucket_instance_id); + }), + other_instances.end()); + + // check if there are still instances left + if (other_instances.empty()) { + return; + } + + // Now we have a bucket with instances where the reshard status is none, this + // usually happens when the reshard process couldn't complete, lockdown the + // bucket and walk through these instances to make sure no one else interferes + // with these + { + RGWBucketReshardLock reshard_lock(static_cast(store), cur_bucket->get_info(), true); + r = reshard_lock.lock(dpp); + if (r < 0) { + // most likely bucket is under reshard, return the sureshot stale instances + ldpp_dout(dpp, 5) << __func__ + << "failed to take reshard lock; reshard underway likey" << dendl; + return; + } + auto sg = make_scope_guard([&reshard_lock](){ reshard_lock.unlock();} ); + // this should be fast enough that we may not need to renew locks and check + // exit status?, should we read the values of the instances again? + stale_instances.insert(std::end(stale_instances), + std::make_move_iterator(other_instances.begin()), + std::make_move_iterator(other_instances.end())); + } + + return; +} + +static int process_stale_instances(rgw::sal::Store* store, RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + const DoutPrefixProvider *dpp, + std::function process_f) +{ + std::string marker; + void *handle; + Formatter *formatter = flusher.get_formatter(); + static constexpr auto default_max_keys = 1000; + + int ret = store->meta_list_keys_init(dpp, "bucket.instance", marker, &handle); + if (ret < 0) { + cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; + return ret; + } + + bool truncated; + + formatter->open_array_section("keys"); + auto g = make_scope_guard([&store, &handle, &formatter]() { + store->meta_list_keys_complete(handle); + formatter->close_section(); // keys + formatter->flush(cout); + }); + + do { + list keys; + + ret = store->meta_list_keys_next(dpp, handle, default_max_keys, keys, &truncated); + if (ret < 0 && ret != -ENOENT) { + cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; + return ret; + } if (ret != -ENOENT) { + // partition the list of buckets by buckets as the listing is un sorted, + // since it would minimize the reads to bucket_info + std::unordered_map> bucket_instance_map; + for (auto &key: keys) { + auto pos = key.find(':'); + if(pos != std::string::npos) + bucket_instance_map[key.substr(0,pos)].emplace_back(std::move(key)); + } + for (const auto& kv: bucket_instance_map) { + bucket_instance_ls stale_lst; + get_stale_instances(store, kv.first, kv.second, stale_lst, dpp); + process_f(stale_lst, formatter, store); + } + } + } while (truncated); + + return 0; +} + +int RGWBucketAdminOp::list_stale_instances(rgw::sal::Store* store, + RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + const DoutPrefixProvider *dpp) +{ + auto process_f = [](const bucket_instance_ls& lst, + Formatter *formatter, + rgw::sal::Store*){ + for (const auto& binfo: lst) + formatter->dump_string("key", binfo.bucket.get_key()); + }; + return process_stale_instances(store, op_state, flusher, dpp, process_f); +} + + +int RGWBucketAdminOp::clear_stale_instances(rgw::sal::Store* store, + RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + const DoutPrefixProvider *dpp) +{ + auto process_f = [dpp](const bucket_instance_ls& lst, + Formatter *formatter, + rgw::sal::Store* store){ + for (const auto &binfo: lst) { + std::unique_ptr bucket; + store->get_bucket(nullptr, binfo, &bucket); + int ret = bucket->purge_instance(dpp); + if (ret == 0){ + auto md_key = "bucket.instance:" + binfo.bucket.get_key(); + ret = store->meta_remove(dpp, md_key, null_yield); + } + formatter->open_object_section("delete_status"); + formatter->dump_string("bucket_instance", binfo.bucket.get_key()); + formatter->dump_int("status", -ret); + formatter->close_section(); + } + }; + + return process_stale_instances(store, op_state, flusher, dpp, process_f); +} + +static int fix_single_bucket_lc(rgw::sal::Store* store, + const std::string& tenant_name, + const std::string& bucket_name, + const DoutPrefixProvider *dpp) +{ + std::unique_ptr bucket; + int ret = store->get_bucket(dpp, nullptr, tenant_name, bucket_name, &bucket, null_yield); + if (ret < 0) { + // TODO: Should we handle the case where the bucket could've been removed between + // listing and fetching? + return ret; + } + + return rgw::lc::fix_lc_shard_entry(dpp, store, store->get_rgwlc()->get_lc(), bucket.get()); +} + +static void format_lc_status(Formatter* formatter, + const std::string& tenant_name, + const std::string& bucket_name, + int status) +{ + formatter->open_object_section("bucket_entry"); + std::string entry = tenant_name.empty() ? bucket_name : tenant_name + "/" + bucket_name; + formatter->dump_string("bucket", entry); + formatter->dump_int("status", status); + formatter->close_section(); // bucket_entry +} + +static void process_single_lc_entry(rgw::sal::Store* store, + Formatter *formatter, + const std::string& tenant_name, + const std::string& bucket_name, + const DoutPrefixProvider *dpp) +{ + int ret = fix_single_bucket_lc(store, tenant_name, bucket_name, dpp); + format_lc_status(formatter, tenant_name, bucket_name, -ret); +} + +int RGWBucketAdminOp::fix_lc_shards(rgw::sal::Store* store, + RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + const DoutPrefixProvider *dpp) +{ + std::string marker; + void *handle; + Formatter *formatter = flusher.get_formatter(); + static constexpr auto default_max_keys = 1000; + + bool truncated; + if (const std::string& bucket_name = op_state.get_bucket_name(); + ! bucket_name.empty()) { + const rgw_user user_id = op_state.get_user_id(); + process_single_lc_entry(store, formatter, user_id.tenant, bucket_name, dpp); + formatter->flush(cout); + } else { + int ret = store->meta_list_keys_init(dpp, "bucket", marker, &handle); + if (ret < 0) { + std::cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl; + return ret; + } + + { + formatter->open_array_section("lc_fix_status"); + auto sg = make_scope_guard([&store, &handle, &formatter](){ + store->meta_list_keys_complete(handle); + formatter->close_section(); // lc_fix_status + formatter->flush(cout); + }); + do { + list keys; + ret = store->meta_list_keys_next(dpp, handle, default_max_keys, keys, &truncated); + if (ret < 0 && ret != -ENOENT) { + std::cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl; + return ret; + } if (ret != -ENOENT) { + for (const auto &key:keys) { + auto [tenant_name, bucket_name] = split_tenant(key); + process_single_lc_entry(store, formatter, tenant_name, bucket_name, dpp); + } + } + formatter->flush(cout); // regularly flush every 1k entries + } while (truncated); + } + + } + return 0; + +} + +static bool has_object_expired(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + rgw::sal::Bucket* bucket, + const rgw_obj_key& key, utime_t& delete_at) +{ + std::unique_ptr obj = bucket->get_object(key); + bufferlist delete_at_bl; + + int ret = rgw_object_get_attr(dpp, store, obj.get(), RGW_ATTR_DELETE_AT, delete_at_bl, null_yield); + if (ret < 0) { + return false; // no delete at attr, proceed + } + + ret = decode_bl(delete_at_bl, delete_at); + if (ret < 0) { + return false; // failed to parse + } + + if (delete_at <= ceph_clock_now() && !delete_at.is_zero()) { + return true; + } + + return false; +} + +static int fix_bucket_obj_expiry(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + rgw::sal::Bucket* bucket, + RGWFormatterFlusher& flusher, bool dry_run) +{ + if (bucket->get_key().bucket_id == bucket->get_key().marker) { + ldpp_dout(dpp, -1) << "Not a resharded bucket skipping" << dendl; + return 0; // not a resharded bucket, move along + } + + Formatter *formatter = flusher.get_formatter(); + formatter->open_array_section("expired_deletion_status"); + auto sg = make_scope_guard([&formatter] { + formatter->close_section(); + formatter->flush(std::cout); + }); + + rgw::sal::Bucket::ListParams params; + rgw::sal::Bucket::ListResults results; + + params.list_versions = bucket->versioned(); + params.allow_unordered = true; + + do { + int ret = bucket->list(dpp, params, listing_max_entries, results, null_yield); + if (ret < 0) { + ldpp_dout(dpp, -1) << "ERROR failed to list objects in the bucket" << dendl; + return ret; + } + for (const auto& obj : results.objs) { + rgw_obj_key key(obj.key); + utime_t delete_at; + if (has_object_expired(dpp, store, bucket, key, delete_at)) { + formatter->open_object_section("object_status"); + formatter->dump_string("object", key.name); + formatter->dump_stream("delete_at") << delete_at; + + if (!dry_run) { + ret = rgw_remove_object(dpp, store, bucket, key); + formatter->dump_int("status", ret); + } + + formatter->close_section(); // object_status + } + } + formatter->flush(cout); // regularly flush every 1k entries + } while (results.is_truncated); + + return 0; +} + +int RGWBucketAdminOp::fix_obj_expiry(rgw::sal::Store* store, + RGWBucketAdminOpState& op_state, + RGWFormatterFlusher& flusher, + const DoutPrefixProvider *dpp, bool dry_run) +{ + RGWBucket admin_bucket; + int ret = admin_bucket.init(store, op_state, null_yield, dpp); + if (ret < 0) { + ldpp_dout(dpp, -1) << "failed to initialize bucket" << dendl; + return ret; + } + std::unique_ptr bucket; + ret = store->get_bucket(nullptr, admin_bucket.get_bucket_info(), &bucket); + if (ret < 0) { + return ret; + } + + return fix_bucket_obj_expiry(dpp, store, bucket.get(), flusher, dry_run); +} + +void RGWBucketCompleteInfo::dump(Formatter *f) const { + encode_json("bucket_info", info, f); + encode_json("attrs", attrs, f); +} + +void RGWBucketCompleteInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("bucket_info", info, obj); + JSONDecoder::decode_json("attrs", attrs, obj); +} + +class RGWBucketMetadataHandler : public RGWBucketMetadataHandlerBase { +public: + struct Svc { + RGWSI_Bucket *bucket{nullptr}; + } svc; + + struct Ctl { + RGWBucketCtl *bucket{nullptr}; + } ctl; + + RGWBucketMetadataHandler() {} + + void init(RGWSI_Bucket *bucket_svc, + RGWBucketCtl *bucket_ctl) override { + base_init(bucket_svc->ctx(), + bucket_svc->get_ep_be_handler().get()); + svc.bucket = bucket_svc; + ctl.bucket = bucket_ctl; + } + + string get_type() override { return "bucket"; } + + RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) override { + RGWBucketEntryPoint be; + + try { + decode_json_obj(be, jo); + } catch (JSONDecoder::err& e) { + return nullptr; + } + + return new RGWBucketEntryMetadataObject(be, objv, mtime); + } + + int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { + RGWObjVersionTracker ot; + RGWBucketEntryPoint be; + + real_time mtime; + map attrs; + + RGWSI_Bucket_EP_Ctx ctx(op->ctx()); + + int ret = svc.bucket->read_bucket_entrypoint_info(ctx, entry, &be, &ot, &mtime, &attrs, y, dpp); + if (ret < 0) + return ret; + + RGWBucketEntryMetadataObject *mdo = new RGWBucketEntryMetadataObject(be, ot.read_version, mtime, std::move(attrs)); + + *obj = mdo; + + return 0; + } + + int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + RGWMDLogSyncType type, bool from_remote_zone) override; + + int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) override { + RGWBucketEntryPoint be; + + real_time orig_mtime; + + RGWSI_Bucket_EP_Ctx ctx(op->ctx()); + + int ret = svc.bucket->read_bucket_entrypoint_info(ctx, entry, &be, &objv_tracker, &orig_mtime, nullptr, y, dpp); + if (ret < 0) + return ret; + + /* + * We're unlinking the bucket but we don't want to update the entrypoint here - we're removing + * it immediately and don't want to invalidate our cached objv_version or the bucket obj removal + * will incorrectly fail. + */ + ret = ctl.bucket->unlink_bucket(be.owner, be.bucket, y, dpp, false); + if (ret < 0) { + ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl; + } + + ret = svc.bucket->remove_bucket_entrypoint_info(ctx, entry, &objv_tracker, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl; + } + /* idempotent */ + return 0; + } + + int call(std::function f) { + return call(nullopt, f); + } + + int call(std::optional bectx_params, + std::function f) { + return be_handler->call(bectx_params, [&](RGWSI_MetaBackend_Handler::Op *op) { + RGWSI_Bucket_EP_Ctx ctx(op->ctx()); + return f(ctx); + }); + } +}; + +class RGWMetadataHandlerPut_Bucket : public RGWMetadataHandlerPut_SObj +{ + RGWBucketMetadataHandler *bhandler; + RGWBucketEntryMetadataObject *obj; +public: + RGWMetadataHandlerPut_Bucket(RGWBucketMetadataHandler *_handler, + RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, + RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type, from_remote_zone), + bhandler(_handler) { + obj = static_cast(_obj); + } + ~RGWMetadataHandlerPut_Bucket() {} + + void encode_obj(bufferlist *bl) override { + obj->get_ep().encode(*bl); + } + + int put_checked(const DoutPrefixProvider *dpp) override; + int put_post(const DoutPrefixProvider *dpp) override; +}; + +int RGWBucketMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + RGWMDLogSyncType type, bool from_remote_zone) +{ + RGWMetadataHandlerPut_Bucket put_op(this, op, entry, obj, objv_tracker, y, type, from_remote_zone); + return do_put_operate(&put_op, dpp); +} + +int RGWMetadataHandlerPut_Bucket::put_checked(const DoutPrefixProvider *dpp) +{ + RGWBucketEntryMetadataObject *orig_obj = static_cast(old_obj); + + if (orig_obj) { + obj->set_pattrs(&orig_obj->get_attrs()); + } + + auto& be = obj->get_ep(); + auto mtime = obj->get_mtime(); + auto pattrs = obj->get_pattrs(); + + RGWSI_Bucket_EP_Ctx ctx(op->ctx()); + + return bhandler->svc.bucket->store_bucket_entrypoint_info(ctx, entry, + be, + false, + mtime, + pattrs, + &objv_tracker, + y, + dpp); +} + +int RGWMetadataHandlerPut_Bucket::put_post(const DoutPrefixProvider *dpp) +{ + auto& be = obj->get_ep(); + + int ret; + + /* link bucket */ + if (be.linked) { + ret = bhandler->ctl.bucket->link_bucket(be.owner, be.bucket, be.creation_time, y, dpp, false); + } else { + ret = bhandler->ctl.bucket->unlink_bucket(be.owner, be.bucket, y, dpp, false); + } + + return ret; +} + +static void get_md5_digest(const RGWBucketEntryPoint *be, string& md5_digest) { + + char md5[CEPH_CRYPTO_MD5_DIGESTSIZE * 2 + 1]; + unsigned char m[CEPH_CRYPTO_MD5_DIGESTSIZE]; + bufferlist bl; + + Formatter *f = new JSONFormatter(false); + be->dump(f); + f->flush(bl); + + MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + hash.Update((const unsigned char *)bl.c_str(), bl.length()); + hash.Final(m); + + buf_to_hex(m, CEPH_CRYPTO_MD5_DIGESTSIZE, md5); + + delete f; + + md5_digest = md5; +} + +#define ARCHIVE_META_ATTR RGW_ATTR_PREFIX "zone.archive.info" + +struct archive_meta_info { + rgw_bucket orig_bucket; + + bool from_attrs(CephContext *cct, map& attrs) { + auto iter = attrs.find(ARCHIVE_META_ATTR); + if (iter == attrs.end()) { + return false; + } + + auto bliter = iter->second.cbegin(); + try { + decode(bliter); + } catch (buffer::error& err) { + ldout(cct, 0) << "ERROR: failed to decode archive meta info" << dendl; + return false; + } + + return true; + } + + void store_in_attrs(map& attrs) const { + encode(attrs[ARCHIVE_META_ATTR]); + } + + void encode(bufferlist& bl) const { + ENCODE_START(1, 1, bl); + encode(orig_bucket, bl); + ENCODE_FINISH(bl); + } + + void decode(bufferlist::const_iterator& bl) { + DECODE_START(1, bl); + decode(orig_bucket, bl); + DECODE_FINISH(bl); + } +}; +WRITE_CLASS_ENCODER(archive_meta_info) + +class RGWArchiveBucketMetadataHandler : public RGWBucketMetadataHandler { +public: + RGWArchiveBucketMetadataHandler() {} + + int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) override { + auto cct = svc.bucket->ctx(); + + RGWSI_Bucket_EP_Ctx ctx(op->ctx()); + + ldpp_dout(dpp, 5) << "SKIP: bucket removal is not allowed on archive zone: bucket:" << entry << " ... proceeding to rename" << dendl; + + string tenant_name, bucket_name; + parse_bucket(entry, &tenant_name, &bucket_name); + rgw_bucket entry_bucket; + entry_bucket.tenant = tenant_name; + entry_bucket.name = bucket_name; + + real_time mtime; + + /* read original entrypoint */ + + RGWBucketEntryPoint be; + map attrs; + int ret = svc.bucket->read_bucket_entrypoint_info(ctx, entry, &be, &objv_tracker, &mtime, &attrs, y, dpp); + if (ret < 0) { + return ret; + } + + string bi_meta_name = RGWSI_Bucket::get_bi_meta_key(be.bucket); + + /* read original bucket instance info */ + + map attrs_m; + ceph::real_time orig_mtime; + RGWBucketInfo old_bi; + + ret = ctl.bucket->read_bucket_instance_info(be.bucket, &old_bi, y, dpp, RGWBucketCtl::BucketInstance::GetParams() + .set_mtime(&orig_mtime) + .set_attrs(&attrs_m)); + if (ret < 0) { + return ret; + } + + archive_meta_info ami; + + if (!ami.from_attrs(svc.bucket->ctx(), attrs_m)) { + ami.orig_bucket = old_bi.bucket; + ami.store_in_attrs(attrs_m); + } + + /* generate a new bucket instance. We could have avoided this if we could just point a new + * bucket entry point to the old bucket instance, however, due to limitation in the way + * we index buckets under the user, bucket entrypoint and bucket instance of the same + * bucket need to have the same name, so we need to copy the old bucket instance into + * to a new entry with the new name + */ + + string new_bucket_name; + + RGWBucketInfo new_bi = old_bi; + RGWBucketEntryPoint new_be = be; + + string md5_digest; + + get_md5_digest(&new_be, md5_digest); + new_bucket_name = ami.orig_bucket.name + "-deleted-" + md5_digest; + + new_bi.bucket.name = new_bucket_name; + new_bi.objv_tracker.clear(); + + new_be.bucket.name = new_bucket_name; + + ret = ctl.bucket->store_bucket_instance_info(be.bucket, new_bi, y, dpp, RGWBucketCtl::BucketInstance::PutParams() + .set_exclusive(false) + .set_mtime(orig_mtime) + .set_attrs(&attrs_m) + .set_orig_info(&old_bi)); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket instance info for bucket=" << new_bi.bucket << " ret=" << ret << dendl; + return ret; + } + + /* store a new entrypoint */ + + RGWObjVersionTracker ot; + ot.generate_new_write_ver(cct); + + ret = svc.bucket->store_bucket_entrypoint_info(ctx, RGWSI_Bucket::get_entrypoint_meta_key(new_be.bucket), + new_be, true, mtime, &attrs, nullptr, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be.bucket << " ret=" << ret << dendl; + return ret; + } + + /* link new bucket */ + + ret = ctl.bucket->link_bucket(new_be.owner, new_be.bucket, new_be.creation_time, y, dpp, false); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to link new bucket for bucket=" << new_be.bucket << " ret=" << ret << dendl; + return ret; + } + + /* clean up old stuff */ + + ret = ctl.bucket->unlink_bucket(be.owner, entry_bucket, y, dpp, false); + if (ret < 0) { + ldpp_dout(dpp, -1) << "could not unlink bucket=" << entry << " owner=" << be.owner << dendl; + } + + // if (ret == -ECANCELED) it means that there was a race here, and someone + // wrote to the bucket entrypoint just before we removed it. The question is + // whether it was a newly created bucket entrypoint ... in which case we + // should ignore the error and move forward, or whether it is a higher version + // of the same bucket instance ... in which we should retry + ret = svc.bucket->remove_bucket_entrypoint_info(ctx, + RGWSI_Bucket::get_entrypoint_meta_key(be.bucket), + &objv_tracker, + y, + dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to put new bucket entrypoint for bucket=" << new_be.bucket << " ret=" << ret << dendl; + return ret; + } + + ret = ctl.bucket->remove_bucket_instance_info(be.bucket, old_bi, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, -1) << "could not delete bucket=" << entry << dendl; + } + + + /* idempotent */ + + return 0; + } + + int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp, + RGWMDLogSyncType type, bool from_remote_zone) override { + if (entry.find("-deleted-") != string::npos) { + RGWObjVersionTracker ot; + RGWMetadataObject *robj; + int ret = do_get(op, entry, &robj, y, dpp); + if (ret != -ENOENT) { + if (ret < 0) { + return ret; + } + ot.read_version = robj->get_version(); + delete robj; + + ret = do_remove(op, entry, ot, y, dpp); + if (ret < 0) { + return ret; + } + } + } + + return RGWBucketMetadataHandler::do_put(op, entry, obj, + objv_tracker, y, dpp, type, from_remote_zone); + } + +}; + +class RGWBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandlerBase { + int read_bucket_instance_entry(RGWSI_Bucket_BI_Ctx& ctx, + const string& entry, + RGWBucketCompleteInfo *bi, + ceph::real_time *pmtime, + optional_yield y, + const DoutPrefixProvider *dpp) { + return svc.bucket->read_bucket_instance_info(ctx, + entry, + &bi->info, + pmtime, &bi->attrs, + y, + dpp); + } + +public: + struct Svc { + RGWSI_Zone *zone{nullptr}; + RGWSI_Bucket *bucket{nullptr}; + RGWSI_BucketIndex *bi{nullptr}; + } svc; + + rgw::sal::Store* store; + + RGWBucketInstanceMetadataHandler(rgw::sal::Store* store) + : store(store) {} + + void init(RGWSI_Zone *zone_svc, + RGWSI_Bucket *bucket_svc, + RGWSI_BucketIndex *bi_svc) override { + base_init(bucket_svc->ctx(), + bucket_svc->get_bi_be_handler().get()); + svc.zone = zone_svc; + svc.bucket = bucket_svc; + svc.bi = bi_svc; + } + + string get_type() override { return "bucket.instance"; } + + RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) override { + RGWBucketCompleteInfo bci; + + try { + decode_json_obj(bci, jo); + } catch (JSONDecoder::err& e) { + return nullptr; + } + + return new RGWBucketInstanceMetadataObject(bci, objv, mtime); + } + + int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { + RGWBucketCompleteInfo bci; + real_time mtime; + + RGWSI_Bucket_BI_Ctx ctx(op->ctx()); + + int ret = svc.bucket->read_bucket_instance_info(ctx, entry, &bci.info, &mtime, &bci.attrs, y, dpp); + if (ret < 0) + return ret; + + RGWBucketInstanceMetadataObject *mdo = new RGWBucketInstanceMetadataObject(bci, bci.info.objv_tracker.read_version, mtime); + + *obj = mdo; + + return 0; + } + + int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp, + RGWMDLogSyncType sync_type, bool from_remote_zone) override; + + int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) override { + RGWBucketCompleteInfo bci; + + RGWSI_Bucket_BI_Ctx ctx(op->ctx()); + + int ret = read_bucket_instance_entry(ctx, entry, &bci, nullptr, y, dpp); + if (ret < 0 && ret != -ENOENT) + return ret; + + return svc.bucket->remove_bucket_instance_info(ctx, entry, bci.info, &bci.info.objv_tracker, y, dpp); + } + + int call(std::function f) { + return call(nullopt, f); + } + + int call(std::optional bectx_params, + std::function f) { + return be_handler->call(bectx_params, [&](RGWSI_MetaBackend_Handler::Op *op) { + RGWSI_Bucket_BI_Ctx ctx(op->ctx()); + return f(ctx); + }); + } +}; + +class RGWMetadataHandlerPut_BucketInstance : public RGWMetadataHandlerPut_SObj +{ + CephContext *cct; + RGWBucketInstanceMetadataHandler *bihandler; + RGWBucketInstanceMetadataObject *obj; +public: + RGWMetadataHandlerPut_BucketInstance(CephContext *_cct, + RGWBucketInstanceMetadataHandler *_handler, + RGWSI_MetaBackend_Handler::Op *_op, string& entry, + RGWMetadataObject *_obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, + RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, _op, entry, obj, objv_tracker, y, type, from_remote_zone), + cct(_cct), bihandler(_handler) { + obj = static_cast(_obj); + + auto& bci = obj->get_bci(); + obj->set_pattrs(&bci.attrs); + } + + void encode_obj(bufferlist *bl) override { + obj->get_bucket_info().encode(*bl); + } + + int put_check(const DoutPrefixProvider *dpp) override; + int put_checked(const DoutPrefixProvider *dpp) override; + int put_post(const DoutPrefixProvider *dpp) override; +}; + +int RGWBucketInstanceMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, + string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp, + RGWMDLogSyncType type, bool from_remote_zone) +{ + RGWMetadataHandlerPut_BucketInstance put_op(svc.bucket->ctx(), this, op, entry, obj, + objv_tracker, y, type, from_remote_zone); + return do_put_operate(&put_op, dpp); +} + +void init_default_bucket_layout(CephContext *cct, rgw::BucketLayout& layout, + const RGWZone& zone, + std::optional shards, + std::optional type) { + layout.current_index.gen = 0; + layout.current_index.layout.normal.hash_type = rgw::BucketHashType::Mod; + + layout.current_index.layout.type = + type.value_or(rgw::BucketIndexType::Normal); + + if (shards) { + layout.current_index.layout.normal.num_shards = *shards; + } else if (cct->_conf->rgw_override_bucket_index_max_shards > 0) { + layout.current_index.layout.normal.num_shards = + cct->_conf->rgw_override_bucket_index_max_shards; + } else { + layout.current_index.layout.normal.num_shards = + zone.bucket_index_max_shards; + } + + if (layout.current_index.layout.type == rgw::BucketIndexType::Normal) { + layout.logs.push_back(log_layout_from_index(0, layout.current_index)); + } +} + +int RGWMetadataHandlerPut_BucketInstance::put_check(const DoutPrefixProvider *dpp) +{ + int ret; + + RGWBucketCompleteInfo& bci = obj->get_bci(); + + RGWBucketInstanceMetadataObject *orig_obj = static_cast(old_obj); + + RGWBucketCompleteInfo *old_bci = (orig_obj ? &orig_obj->get_bci() : nullptr); + + const bool exists = (!!orig_obj); + + if (from_remote_zone) { + // don't sync bucket layout changes + if (!exists) { + // replace peer's layout with default-constructed, then apply our defaults + bci.info.layout = rgw::BucketLayout{}; + init_default_bucket_layout(cct, bci.info.layout, + bihandler->svc.zone->get_zone(), + std::nullopt, std::nullopt); + } else { + bci.info.layout = old_bci->info.layout; + } + } + + if (!exists || old_bci->info.bucket.bucket_id != bci.info.bucket.bucket_id) { + /* a new bucket, we need to select a new bucket placement for it */ + string tenant_name; + string bucket_name; + string bucket_instance; + parse_bucket(entry, &tenant_name, &bucket_name, &bucket_instance); + + RGWZonePlacementInfo rule_info; + bci.info.bucket.name = bucket_name; + bci.info.bucket.bucket_id = bucket_instance; + bci.info.bucket.tenant = tenant_name; + // if the sync module never writes data, don't require the zone to specify all placement targets + if (bihandler->svc.zone->sync_module_supports_writes()) { + ret = bihandler->svc.zone->select_bucket_location_by_rule(dpp, bci.info.placement_rule, &rule_info, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: select_bucket_placement() returned " << ret << dendl; + return ret; + } + } + bci.info.layout.current_index.layout.type = rule_info.index_type; + } else { + /* existing bucket, keep its placement */ + bci.info.bucket.explicit_placement = old_bci->info.bucket.explicit_placement; + bci.info.placement_rule = old_bci->info.placement_rule; + } + + /* record the read version (if any), store the new version */ + bci.info.objv_tracker.read_version = objv_tracker.read_version; + bci.info.objv_tracker.write_version = objv_tracker.write_version; + + return 0; +} + +int RGWMetadataHandlerPut_BucketInstance::put_checked(const DoutPrefixProvider *dpp) +{ + RGWBucketInstanceMetadataObject *orig_obj = static_cast(old_obj); + + RGWBucketInfo *orig_info = (orig_obj ? &orig_obj->get_bucket_info() : nullptr); + + auto& info = obj->get_bucket_info(); + auto mtime = obj->get_mtime(); + auto pattrs = obj->get_pattrs(); + + RGWSI_Bucket_BI_Ctx ctx(op->ctx()); + + return bihandler->svc.bucket->store_bucket_instance_info(ctx, + entry, + info, + orig_info, + false, + mtime, + pattrs, + y, + dpp); +} + +int RGWMetadataHandlerPut_BucketInstance::put_post(const DoutPrefixProvider *dpp) +{ + RGWBucketCompleteInfo& bci = obj->get_bci(); + + objv_tracker = bci.info.objv_tracker; + + int ret = bihandler->svc.bi->init_index(dpp, bci.info, bci.info.layout.current_index); + if (ret < 0) { + return ret; + } + + /* update lifecyle policy */ + { + std::unique_ptr bucket; + ret = bihandler->store->get_bucket(nullptr, bci.info, &bucket); + if (ret < 0) { + ldpp_dout(dpp, 0) << __func__ << " failed to get_bucket(...) for " + << bci.info.bucket.name + << dendl; + return ret; + } + + auto lc = bihandler->store->get_rgwlc(); + + auto lc_it = bci.attrs.find(RGW_ATTR_LC); + if (lc_it != bci.attrs.end()) { + ldpp_dout(dpp, 20) << "set lc config for " << bci.info.bucket.name << dendl; + ret = lc->set_bucket_config(bucket.get(), bci.attrs, nullptr); + if (ret < 0) { + ldpp_dout(dpp, 0) << __func__ << " failed to set lc config for " + << bci.info.bucket.name + << dendl; + return ret; + } + + } else { + ldpp_dout(dpp, 20) << "remove lc config for " << bci.info.bucket.name << dendl; + ret = lc->remove_bucket_config(bucket.get(), bci.attrs, false /* cannot merge attrs */); + if (ret < 0) { + ldpp_dout(dpp, 0) << __func__ << " failed to remove lc config for " + << bci.info.bucket.name + << dendl; + return ret; + } + } + } /* update lc */ + + return STATUS_APPLIED; +} + +class RGWArchiveBucketInstanceMetadataHandler : public RGWBucketInstanceMetadataHandler { +public: + RGWArchiveBucketInstanceMetadataHandler(rgw::sal::Store* store) + : RGWBucketInstanceMetadataHandler(store) {} + + // N.B. replication of lifecycle policy relies on logic in RGWBucketInstanceMetadataHandler::do_put(...), override with caution + + int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, optional_yield y, const DoutPrefixProvider *dpp) override { + ldpp_dout(dpp, 0) << "SKIP: bucket instance removal is not allowed on archive zone: bucket.instance:" << entry << dendl; + return 0; + } +}; + +RGWBucketCtl::RGWBucketCtl(RGWSI_Zone *zone_svc, + RGWSI_Bucket *bucket_svc, + RGWSI_Bucket_Sync *bucket_sync_svc, + RGWSI_BucketIndex *bi_svc, + RGWSI_User* user_svc) + : cct(zone_svc->ctx()) +{ + svc.zone = zone_svc; + svc.bucket = bucket_svc; + svc.bucket_sync = bucket_sync_svc; + svc.bi = bi_svc; + svc.user = user_svc; +} + +void RGWBucketCtl::init(RGWUserCtl *user_ctl, + RGWBucketMetadataHandler *_bm_handler, + RGWBucketInstanceMetadataHandler *_bmi_handler, + RGWDataChangesLog *datalog, + const DoutPrefixProvider *dpp) +{ + ctl.user = user_ctl; + + bm_handler = _bm_handler; + bmi_handler = _bmi_handler; + + bucket_be_handler = bm_handler->get_be_handler(); + bi_be_handler = bmi_handler->get_be_handler(); + + datalog->set_bucket_filter( + [this](const rgw_bucket& bucket, optional_yield y, const DoutPrefixProvider *dpp) { + return bucket_exports_data(bucket, y, dpp); + }); +} + +int RGWBucketCtl::call(std::function f) { + return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ep_ctx) { + return bmi_handler->call([&](RGWSI_Bucket_BI_Ctx& bi_ctx) { + RGWSI_Bucket_X_Ctx ctx{ep_ctx, bi_ctx}; + return f(ctx); + }); + }); +} + +int RGWBucketCtl::read_bucket_entrypoint_info(const rgw_bucket& bucket, + RGWBucketEntryPoint *info, + optional_yield y, const DoutPrefixProvider *dpp, + const Bucket::GetParams& params) +{ + return bm_handler->call(params.bectx_params, [&](RGWSI_Bucket_EP_Ctx& ctx) { + return svc.bucket->read_bucket_entrypoint_info(ctx, + RGWSI_Bucket::get_entrypoint_meta_key(bucket), + info, + params.objv_tracker, + params.mtime, + params.attrs, + y, + dpp, + params.cache_info, + params.refresh_version); + }); +} + +int RGWBucketCtl::store_bucket_entrypoint_info(const rgw_bucket& bucket, + RGWBucketEntryPoint& info, + optional_yield y, + const DoutPrefixProvider *dpp, + const Bucket::PutParams& params) +{ + return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { + return svc.bucket->store_bucket_entrypoint_info(ctx, + RGWSI_Bucket::get_entrypoint_meta_key(bucket), + info, + params.exclusive, + params.mtime, + params.attrs, + params.objv_tracker, + y, + dpp); + }); +} + +int RGWBucketCtl::remove_bucket_entrypoint_info(const rgw_bucket& bucket, + optional_yield y, + const DoutPrefixProvider *dpp, + const Bucket::RemoveParams& params) +{ + return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { + return svc.bucket->remove_bucket_entrypoint_info(ctx, + RGWSI_Bucket::get_entrypoint_meta_key(bucket), + params.objv_tracker, + y, + dpp); + }); +} + +int RGWBucketCtl::read_bucket_instance_info(const rgw_bucket& bucket, + RGWBucketInfo *info, + optional_yield y, + const DoutPrefixProvider *dpp, + const BucketInstance::GetParams& params) +{ + int ret = bmi_handler->call(params.bectx_params, [&](RGWSI_Bucket_BI_Ctx& ctx) { + return svc.bucket->read_bucket_instance_info(ctx, + RGWSI_Bucket::get_bi_meta_key(bucket), + info, + params.mtime, + params.attrs, + y, + dpp, + params.cache_info, + params.refresh_version); + }); + + if (ret < 0) { + return ret; + } + + if (params.objv_tracker) { + *params.objv_tracker = info->objv_tracker; + } + + return 0; +} + +int RGWBucketCtl::read_bucket_info(const rgw_bucket& bucket, + RGWBucketInfo *info, + optional_yield y, + const DoutPrefixProvider *dpp, + const BucketInstance::GetParams& params, + RGWObjVersionTracker *ep_objv_tracker) +{ + const rgw_bucket *b = &bucket; + + std::optional ep; + + if (b->bucket_id.empty()) { + ep.emplace(); + + int r = read_bucket_entrypoint_info(*b, &(*ep), y, dpp, RGWBucketCtl::Bucket::GetParams() + .set_bectx_params(params.bectx_params) + .set_objv_tracker(ep_objv_tracker)); + if (r < 0) { + return r; + } + + b = &ep->bucket; + } + + int ret = bmi_handler->call(params.bectx_params, [&](RGWSI_Bucket_BI_Ctx& ctx) { + return svc.bucket->read_bucket_instance_info(ctx, + RGWSI_Bucket::get_bi_meta_key(*b), + info, + params.mtime, + params.attrs, + y, dpp, + params.cache_info, + params.refresh_version); + }); + + if (ret < 0) { + return ret; + } + + if (params.objv_tracker) { + *params.objv_tracker = info->objv_tracker; + } + + return 0; +} + +int RGWBucketCtl::do_store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, + const rgw_bucket& bucket, + RGWBucketInfo& info, + optional_yield y, + const DoutPrefixProvider *dpp, + const BucketInstance::PutParams& params) +{ + if (params.objv_tracker) { + info.objv_tracker = *params.objv_tracker; + } + + return svc.bucket->store_bucket_instance_info(ctx, + RGWSI_Bucket::get_bi_meta_key(bucket), + info, + params.orig_info, + params.exclusive, + params.mtime, + params.attrs, + y, + dpp); +} + +int RGWBucketCtl::store_bucket_instance_info(const rgw_bucket& bucket, + RGWBucketInfo& info, + optional_yield y, + const DoutPrefixProvider *dpp, + const BucketInstance::PutParams& params) +{ + return bmi_handler->call([&](RGWSI_Bucket_BI_Ctx& ctx) { + return do_store_bucket_instance_info(ctx, bucket, info, y, dpp, params); + }); +} + +int RGWBucketCtl::remove_bucket_instance_info(const rgw_bucket& bucket, + RGWBucketInfo& info, + optional_yield y, + const DoutPrefixProvider *dpp, + const BucketInstance::RemoveParams& params) +{ + if (params.objv_tracker) { + info.objv_tracker = *params.objv_tracker; + } + + return bmi_handler->call([&](RGWSI_Bucket_BI_Ctx& ctx) { + return svc.bucket->remove_bucket_instance_info(ctx, + RGWSI_Bucket::get_bi_meta_key(bucket), + info, + &info.objv_tracker, + y, + dpp); + }); +} + +int RGWBucketCtl::do_store_linked_bucket_info(RGWSI_Bucket_X_Ctx& ctx, + RGWBucketInfo& info, + RGWBucketInfo *orig_info, + bool exclusive, real_time mtime, + obj_version *pep_objv, + map *pattrs, + bool create_entry_point, + optional_yield y, const DoutPrefixProvider *dpp) +{ + bool create_head = !info.has_instance_obj || create_entry_point; + + int ret = svc.bucket->store_bucket_instance_info(ctx.bi, + RGWSI_Bucket::get_bi_meta_key(info.bucket), + info, + orig_info, + exclusive, + mtime, pattrs, + y, dpp); + if (ret < 0) { + return ret; + } + + if (!create_head) + return 0; /* done! */ + + RGWBucketEntryPoint entry_point; + entry_point.bucket = info.bucket; + entry_point.owner = info.owner; + entry_point.creation_time = info.creation_time; + entry_point.linked = true; + RGWObjVersionTracker ot; + if (pep_objv && !pep_objv->tag.empty()) { + ot.write_version = *pep_objv; + } else { + ot.generate_new_write_ver(cct); + if (pep_objv) { + *pep_objv = ot.write_version; + } + } + ret = svc.bucket->store_bucket_entrypoint_info(ctx.ep, + RGWSI_Bucket::get_entrypoint_meta_key(info.bucket), + entry_point, + exclusive, + mtime, + pattrs, + &ot, + y, + dpp); + if (ret < 0) + return ret; + + return 0; +} +int RGWBucketCtl::convert_old_bucket_info(RGWSI_Bucket_X_Ctx& ctx, + const rgw_bucket& bucket, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + RGWBucketEntryPoint entry_point; + real_time ep_mtime; + RGWObjVersionTracker ot; + map attrs; + RGWBucketInfo info; + auto cct = svc.bucket->ctx(); + + ldpp_dout(dpp, 10) << "RGWRados::convert_old_bucket_info(): bucket=" << bucket << dendl; + + int ret = svc.bucket->read_bucket_entrypoint_info(ctx.ep, + RGWSI_Bucket::get_entrypoint_meta_key(bucket), + &entry_point, &ot, &ep_mtime, &attrs, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: get_bucket_entrypoint_info() returned " << ret << " bucket=" << bucket << dendl; + return ret; + } + + if (!entry_point.has_bucket_info) { + /* already converted! */ + return 0; + } + + info = entry_point.old_bucket_info; + + ot.generate_new_write_ver(cct); + + ret = do_store_linked_bucket_info(ctx, info, nullptr, false, ep_mtime, &ot.write_version, &attrs, true, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to put_linked_bucket_info(): " << ret << dendl; + return ret; + } + + return 0; +} + +int RGWBucketCtl::set_bucket_instance_attrs(RGWBucketInfo& bucket_info, + map& attrs, + RGWObjVersionTracker *objv_tracker, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + return call([&](RGWSI_Bucket_X_Ctx& ctx) { + rgw_bucket& bucket = bucket_info.bucket; + + if (!bucket_info.has_instance_obj) { + /* an old bucket object, need to convert it */ + int ret = convert_old_bucket_info(ctx, bucket, y, dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed converting old bucket info: " << ret << dendl; + return ret; + } + } + + return do_store_bucket_instance_info(ctx.bi, + bucket, + bucket_info, + y, + dpp, + BucketInstance::PutParams().set_attrs(&attrs) + .set_objv_tracker(objv_tracker) + .set_orig_info(&bucket_info)); + }); +} + + +int RGWBucketCtl::link_bucket(const rgw_user& user_id, + const rgw_bucket& bucket, + ceph::real_time creation_time, + optional_yield y, + const DoutPrefixProvider *dpp, + bool update_entrypoint, + rgw_ep_info *pinfo) +{ + return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { + return do_link_bucket(ctx, user_id, bucket, creation_time, + update_entrypoint, pinfo, y, dpp); + }); +} + +int RGWBucketCtl::do_link_bucket(RGWSI_Bucket_EP_Ctx& ctx, + const rgw_user& user_id, + const rgw_bucket& bucket, + ceph::real_time creation_time, + bool update_entrypoint, + rgw_ep_info *pinfo, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + int ret; + + RGWBucketEntryPoint ep; + RGWObjVersionTracker ot; + RGWObjVersionTracker& rot = (pinfo) ? pinfo->ep_objv : ot; + map attrs, *pattrs = nullptr; + string meta_key; + + if (update_entrypoint) { + meta_key = RGWSI_Bucket::get_entrypoint_meta_key(bucket); + if (pinfo) { + ep = pinfo->ep; + pattrs = &pinfo->attrs; + } else { + ret = svc.bucket->read_bucket_entrypoint_info(ctx, + meta_key, + &ep, &rot, + nullptr, &attrs, + y, dpp); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: store->get_bucket_entrypoint_info() returned: " + << cpp_strerror(-ret) << dendl; + } + pattrs = &attrs; + } + } + + ret = svc.user->add_bucket(dpp, user_id, bucket, creation_time, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: error adding bucket to user directory:" + << " user=" << user_id + << " bucket=" << bucket + << " err=" << cpp_strerror(-ret) + << dendl; + goto done_err; + } + + if (!update_entrypoint) + return 0; + + ep.linked = true; + ep.owner = user_id; + ep.bucket = bucket; + ret = svc.bucket->store_bucket_entrypoint_info( + ctx, meta_key, ep, false, real_time(), pattrs, &rot, y, dpp); + if (ret < 0) + goto done_err; + + return 0; + +done_err: + int r = do_unlink_bucket(ctx, user_id, bucket, true, y, dpp); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed unlinking bucket on error cleanup: " + << cpp_strerror(-r) << dendl; + } + return ret; +} + +int RGWBucketCtl::unlink_bucket(const rgw_user& user_id, const rgw_bucket& bucket, optional_yield y, const DoutPrefixProvider *dpp, bool update_entrypoint) +{ + return bm_handler->call([&](RGWSI_Bucket_EP_Ctx& ctx) { + return do_unlink_bucket(ctx, user_id, bucket, update_entrypoint, y, dpp); + }); +} + +int RGWBucketCtl::do_unlink_bucket(RGWSI_Bucket_EP_Ctx& ctx, + const rgw_user& user_id, + const rgw_bucket& bucket, + bool update_entrypoint, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + int ret = svc.user->remove_bucket(dpp, user_id, bucket, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: error removing bucket from directory: " + << cpp_strerror(-ret)<< dendl; + } + + if (!update_entrypoint) + return 0; + + RGWBucketEntryPoint ep; + RGWObjVersionTracker ot; + map attrs; + string meta_key = RGWSI_Bucket::get_entrypoint_meta_key(bucket); + ret = svc.bucket->read_bucket_entrypoint_info(ctx, meta_key, &ep, &ot, nullptr, &attrs, y, dpp); + if (ret == -ENOENT) + return 0; + if (ret < 0) + return ret; + + if (!ep.linked) + return 0; + + if (ep.owner != user_id) { + ldpp_dout(dpp, 0) << "bucket entry point user mismatch, can't unlink bucket: " << ep.owner << " != " << user_id << dendl; + return -EINVAL; + } + + ep.linked = false; + return svc.bucket->store_bucket_entrypoint_info(ctx, meta_key, ep, false, real_time(), &attrs, &ot, y, dpp); +} + +// TODO: remove RGWRados dependency for bucket listing +int RGWBucketCtl::chown(rgw::sal::Store* store, rgw::sal::Bucket* bucket, + const rgw_user& user_id, const std::string& display_name, + const std::string& marker, optional_yield y, const DoutPrefixProvider *dpp) +{ + map common_prefixes; + + rgw::sal::Bucket::ListParams params; + rgw::sal::Bucket::ListResults results; + + params.list_versions = true; + params.allow_unordered = true; + params.marker = marker; + + int count = 0; + int max_entries = 1000; + + //Loop through objects and update object acls to point to bucket owner + + do { + RGWObjectCtx obj_ctx(store); + results.objs.clear(); + int ret = bucket->list(dpp, params, max_entries, results, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: list objects failed: " << cpp_strerror(-ret) << dendl; + return ret; + } + + params.marker = results.next_marker; + count += results.objs.size(); + + for (const auto& obj : results.objs) { + std::unique_ptr r_obj = bucket->get_object(obj.key); + + ret = r_obj->get_obj_attrs(y, dpp); + if (ret < 0){ + ldpp_dout(dpp, 0) << "ERROR: failed to read object " << obj.key.name << cpp_strerror(-ret) << dendl; + continue; + } + const auto& aiter = r_obj->get_attrs().find(RGW_ATTR_ACL); + if (aiter == r_obj->get_attrs().end()) { + ldpp_dout(dpp, 0) << "ERROR: no acls found for object " << obj.key.name << " .Continuing with next object." << dendl; + continue; + } else { + bufferlist& bl = aiter->second; + RGWAccessControlPolicy policy(store->ctx()); + ACLOwner owner; + try { + decode(policy, bl); + owner = policy.get_owner(); + } catch (buffer::error& err) { + ldpp_dout(dpp, 0) << "ERROR: decode policy failed" << err.what() + << dendl; + return -EIO; + } + + //Get the ACL from the policy + RGWAccessControlList& acl = policy.get_acl(); + + //Remove grant that is set to old owner + acl.remove_canon_user_grant(owner.get_id()); + + //Create a grant and add grant + ACLGrant grant; + grant.set_canon(user_id, display_name, RGW_PERM_FULL_CONTROL); + acl.add_grant(&grant); + + //Update the ACL owner to the new user + owner.set_id(user_id); + owner.set_name(display_name); + policy.set_owner(owner); + + bl.clear(); + encode(policy, bl); + + r_obj->set_atomic(); + map attrs; + attrs[RGW_ATTR_ACL] = bl; + ret = r_obj->set_obj_attrs(dpp, &attrs, nullptr, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: modify attr failed " << cpp_strerror(-ret) << dendl; + return ret; + } + } + } + cerr << count << " objects processed in " << bucket + << ". Next marker " << params.marker.name << std::endl; + } while(results.is_truncated); + return 0; +} + +int RGWBucketCtl::read_bucket_stats(const rgw_bucket& bucket, + RGWBucketEnt *result, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + return call([&](RGWSI_Bucket_X_Ctx& ctx) { + return svc.bucket->read_bucket_stats(ctx, bucket, result, y, dpp); + }); +} + +int RGWBucketCtl::read_buckets_stats(map& m, + optional_yield y, const DoutPrefixProvider *dpp) +{ + return call([&](RGWSI_Bucket_X_Ctx& ctx) { + return svc.bucket->read_buckets_stats(ctx, m, y, dpp); + }); +} + +int RGWBucketCtl::sync_user_stats(const DoutPrefixProvider *dpp, + const rgw_user& user_id, + const RGWBucketInfo& bucket_info, + optional_yield y, + RGWBucketEnt* pent) +{ + RGWBucketEnt ent; + if (!pent) { + pent = &ent; + } + int r = svc.bi->read_stats(dpp, bucket_info, pent, null_yield); + if (r < 0) { + ldpp_dout(dpp, 20) << __func__ << "(): failed to read bucket stats (r=" << r << ")" << dendl; + return r; + } + + return svc.user->flush_bucket_stats(dpp, user_id, *pent, y); +} + +int RGWBucketCtl::get_sync_policy_handler(std::optional zone, + std::optional bucket, + RGWBucketSyncPolicyHandlerRef *phandler, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + int r = call([&](RGWSI_Bucket_X_Ctx& ctx) { + return svc.bucket_sync->get_policy_handler(ctx, zone, bucket, phandler, y, dpp); + }); + if (r < 0) { + ldpp_dout(dpp, 20) << __func__ << "(): failed to get policy handler for bucket=" << bucket << " (r=" << r << ")" << dendl; + return r; + } + return 0; +} + +int RGWBucketCtl::bucket_exports_data(const rgw_bucket& bucket, + optional_yield y, + const DoutPrefixProvider *dpp) +{ + + RGWBucketSyncPolicyHandlerRef handler; + + int r = get_sync_policy_handler(std::nullopt, bucket, &handler, y, dpp); + if (r < 0) { + return r; + } + + return handler->bucket_exports_data(); +} + +int RGWBucketCtl::bucket_imports_data(const rgw_bucket& bucket, + optional_yield y, const DoutPrefixProvider *dpp) +{ + + RGWBucketSyncPolicyHandlerRef handler; + + int r = get_sync_policy_handler(std::nullopt, bucket, &handler, y, dpp); + if (r < 0) { + return r; + } + + return handler->bucket_imports_data(); +} + +RGWBucketMetadataHandlerBase* RGWBucketMetaHandlerAllocator::alloc() +{ + return new RGWBucketMetadataHandler(); +} + +RGWBucketInstanceMetadataHandlerBase* RGWBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Store* store) +{ + return new RGWBucketInstanceMetadataHandler(store); +} + +RGWBucketMetadataHandlerBase* RGWArchiveBucketMetaHandlerAllocator::alloc() +{ + return new RGWArchiveBucketMetadataHandler(); +} + +RGWBucketInstanceMetadataHandlerBase* RGWArchiveBucketInstanceMetaHandlerAllocator::alloc(rgw::sal::Store* store) +{ + return new RGWArchiveBucketInstanceMetadataHandler(store); +} + + +void RGWBucketEntryPoint::generate_test_instances(list& o) +{ + RGWBucketEntryPoint *bp = new RGWBucketEntryPoint(); + init_bucket(&bp->bucket, "tenant", "bucket", "pool", ".index.pool", "marker", "10"); + bp->owner = "owner"; + bp->creation_time = ceph::real_clock::from_ceph_timespec({ceph_le32(2), ceph_le32(3)}); + + o.push_back(bp); + o.push_back(new RGWBucketEntryPoint); +} + +void RGWBucketEntryPoint::dump(Formatter *f) const +{ + encode_json("bucket", bucket, f); + encode_json("owner", owner, f); + utime_t ut(creation_time); + encode_json("creation_time", ut, f); + encode_json("linked", linked, f); + encode_json("has_bucket_info", has_bucket_info, f); + if (has_bucket_info) { + encode_json("old_bucket_info", old_bucket_info, f); + } +} + +void RGWBucketEntryPoint::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("bucket", bucket, obj); + JSONDecoder::decode_json("owner", owner, obj); + utime_t ut; + JSONDecoder::decode_json("creation_time", ut, obj); + creation_time = ut.to_real_time(); + JSONDecoder::decode_json("linked", linked, obj); + JSONDecoder::decode_json("has_bucket_info", has_bucket_info, obj); + if (has_bucket_info) { + JSONDecoder::decode_json("old_bucket_info", old_bucket_info, obj); + } +} + diff --git a/src/rgw/rgw_bucket.h b/src/rgw/store/rados/rgw_bucket.h similarity index 100% rename from src/rgw/rgw_bucket.h rename to src/rgw/store/rados/rgw_bucket.h diff --git a/src/rgw/rgw_bucket_sync.cc b/src/rgw/store/rados/rgw_bucket_sync.cc similarity index 100% rename from src/rgw/rgw_bucket_sync.cc rename to src/rgw/store/rados/rgw_bucket_sync.cc diff --git a/src/rgw/rgw_bucket_sync.h b/src/rgw/store/rados/rgw_bucket_sync.h similarity index 100% rename from src/rgw/rgw_bucket_sync.h rename to src/rgw/store/rados/rgw_bucket_sync.h diff --git a/src/rgw/rgw_cr_rados.cc b/src/rgw/store/rados/rgw_cr_rados.cc similarity index 100% rename from src/rgw/rgw_cr_rados.cc rename to src/rgw/store/rados/rgw_cr_rados.cc diff --git a/src/rgw/rgw_cr_rados.h b/src/rgw/store/rados/rgw_cr_rados.h similarity index 100% rename from src/rgw/rgw_cr_rados.h rename to src/rgw/store/rados/rgw_cr_rados.h diff --git a/src/rgw/rgw_cr_tools.cc b/src/rgw/store/rados/rgw_cr_tools.cc similarity index 100% rename from src/rgw/rgw_cr_tools.cc rename to src/rgw/store/rados/rgw_cr_tools.cc diff --git a/src/rgw/rgw_cr_tools.h b/src/rgw/store/rados/rgw_cr_tools.h similarity index 100% rename from src/rgw/rgw_cr_tools.h rename to src/rgw/store/rados/rgw_cr_tools.h diff --git a/src/rgw/rgw_d3n_datacache.cc b/src/rgw/store/rados/rgw_d3n_datacache.cc similarity index 100% rename from src/rgw/rgw_d3n_datacache.cc rename to src/rgw/store/rados/rgw_d3n_datacache.cc diff --git a/src/rgw/rgw_d3n_datacache.h b/src/rgw/store/rados/rgw_d3n_datacache.h similarity index 100% rename from src/rgw/rgw_d3n_datacache.h rename to src/rgw/store/rados/rgw_d3n_datacache.h diff --git a/src/rgw/rgw_data_sync.cc b/src/rgw/store/rados/rgw_data_sync.cc similarity index 100% rename from src/rgw/rgw_data_sync.cc rename to src/rgw/store/rados/rgw_data_sync.cc diff --git a/src/rgw/rgw_data_sync.h b/src/rgw/store/rados/rgw_data_sync.h similarity index 100% rename from src/rgw/rgw_data_sync.h rename to src/rgw/store/rados/rgw_data_sync.h diff --git a/src/rgw/rgw_datalog.cc b/src/rgw/store/rados/rgw_datalog.cc similarity index 100% rename from src/rgw/rgw_datalog.cc rename to src/rgw/store/rados/rgw_datalog.cc diff --git a/src/rgw/rgw_datalog.h b/src/rgw/store/rados/rgw_datalog.h similarity index 100% rename from src/rgw/rgw_datalog.h rename to src/rgw/store/rados/rgw_datalog.h diff --git a/src/rgw/rgw_datalog_notify.cc b/src/rgw/store/rados/rgw_datalog_notify.cc similarity index 100% rename from src/rgw/rgw_datalog_notify.cc rename to src/rgw/store/rados/rgw_datalog_notify.cc diff --git a/src/rgw/rgw_datalog_notify.h b/src/rgw/store/rados/rgw_datalog_notify.h similarity index 96% rename from src/rgw/rgw_datalog_notify.h rename to src/rgw/store/rados/rgw_datalog_notify.h index d32dc79b171a..4cd1b3c110f5 100644 --- a/src/rgw/rgw_datalog_notify.h +++ b/src/rgw/store/rados/rgw_datalog_notify.h @@ -6,7 +6,7 @@ #include #include -#include "rgw/rgw_datalog.h" +#include "rgw_datalog.h" namespace bc = boost::container; diff --git a/src/rgw/rgw_etag_verifier.cc b/src/rgw/store/rados/rgw_etag_verifier.cc similarity index 100% rename from src/rgw/rgw_etag_verifier.cc rename to src/rgw/store/rados/rgw_etag_verifier.cc diff --git a/src/rgw/rgw_etag_verifier.h b/src/rgw/store/rados/rgw_etag_verifier.h similarity index 100% rename from src/rgw/rgw_etag_verifier.h rename to src/rgw/store/rados/rgw_etag_verifier.h diff --git a/src/rgw/rgw_gc.cc b/src/rgw/store/rados/rgw_gc.cc similarity index 100% rename from src/rgw/rgw_gc.cc rename to src/rgw/store/rados/rgw_gc.cc diff --git a/src/rgw/rgw_gc.h b/src/rgw/store/rados/rgw_gc.h similarity index 100% rename from src/rgw/rgw_gc.h rename to src/rgw/store/rados/rgw_gc.h diff --git a/src/rgw/rgw_gc_log.cc b/src/rgw/store/rados/rgw_gc_log.cc similarity index 100% rename from src/rgw/rgw_gc_log.cc rename to src/rgw/store/rados/rgw_gc_log.cc diff --git a/src/rgw/rgw_lc_tier.cc b/src/rgw/store/rados/rgw_lc_tier.cc similarity index 100% rename from src/rgw/rgw_lc_tier.cc rename to src/rgw/store/rados/rgw_lc_tier.cc diff --git a/src/rgw/rgw_lc_tier.h b/src/rgw/store/rados/rgw_lc_tier.h similarity index 100% rename from src/rgw/rgw_lc_tier.h rename to src/rgw/store/rados/rgw_lc_tier.h diff --git a/src/rgw/rgw_log_backing.cc b/src/rgw/store/rados/rgw_log_backing.cc similarity index 100% rename from src/rgw/rgw_log_backing.cc rename to src/rgw/store/rados/rgw_log_backing.cc diff --git a/src/rgw/rgw_log_backing.h b/src/rgw/store/rados/rgw_log_backing.h similarity index 100% rename from src/rgw/rgw_log_backing.h rename to src/rgw/store/rados/rgw_log_backing.h diff --git a/src/rgw/store/rados/rgw_metadata.cc b/src/rgw/store/rados/rgw_metadata.cc new file mode 100644 index 000000000000..e3e49316eac5 --- /dev/null +++ b/src/rgw/store/rados/rgw_metadata.cc @@ -0,0 +1,233 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_metadata.h" + +#include "rgw_zone.h" +#include "rgw_mdlog.h" + +#include "services/svc_zone.h" +#include "services/svc_cls.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +const std::string RGWMetadataLogHistory::oid = "meta.history"; + +struct obj_version; + +void rgw_shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id) +{ + uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); + char buf[16]; + if (shard_id) { + *shard_id = val % max_shards; + } + snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); + name = prefix + buf; +} + +void rgw_shard_name(const string& prefix, unsigned max_shards, const string& section, const string& key, string& name) +{ + uint32_t val = ceph_str_hash_linux(key.c_str(), key.size()); + val ^= ceph_str_hash_linux(section.c_str(), section.size()); + char buf[16]; + snprintf(buf, sizeof(buf), "%u", (unsigned)(val % max_shards)); + name = prefix + buf; +} + +void rgw_shard_name(const string& prefix, unsigned shard_id, string& name) +{ + char buf[16]; + snprintf(buf, sizeof(buf), "%u", shard_id); + name = prefix + buf; +} + +int RGWMetadataLog::add_entry(const DoutPrefixProvider *dpp, const string& hash_key, const string& section, const string& key, bufferlist& bl) { + if (!svc.zone->need_to_log_metadata()) + return 0; + + string oid; + int shard_id; + + rgw_shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, &shard_id); + mark_modified(shard_id); + real_time now = real_clock::now(); + return svc.cls->timelog.add(dpp, oid, now, section, key, bl, null_yield); +} + +int RGWMetadataLog::get_shard_id(const string& hash_key, int *shard_id) +{ + string oid; + + rgw_shard_name(prefix, cct->_conf->rgw_md_log_max_shards, hash_key, oid, shard_id); + return 0; +} + +int RGWMetadataLog::store_entries_in_shard(const DoutPrefixProvider *dpp, list& entries, int shard_id, librados::AioCompletion *completion) +{ + string oid; + + mark_modified(shard_id); + rgw_shard_name(prefix, shard_id, oid); + return svc.cls->timelog.add(dpp, oid, entries, completion, false, null_yield); +} + +void RGWMetadataLog::init_list_entries(int shard_id, const real_time& from_time, const real_time& end_time, + const string& marker, void **handle) +{ + LogListCtx *ctx = new LogListCtx(); + + ctx->cur_shard = shard_id; + ctx->from_time = from_time; + ctx->end_time = end_time; + ctx->marker = marker; + + get_shard_oid(ctx->cur_shard, ctx->cur_oid); + + *handle = (void *)ctx; +} + +void RGWMetadataLog::complete_list_entries(void *handle) { + LogListCtx *ctx = static_cast(handle); + delete ctx; +} + +int RGWMetadataLog::list_entries(const DoutPrefixProvider *dpp, void *handle, + int max_entries, + list& entries, + string *last_marker, + bool *truncated) { + LogListCtx *ctx = static_cast(handle); + + if (!max_entries) { + *truncated = false; + return 0; + } + + std::string next_marker; + int ret = svc.cls->timelog.list(dpp, ctx->cur_oid, ctx->from_time, ctx->end_time, + max_entries, entries, ctx->marker, + &next_marker, truncated, null_yield); + if ((ret < 0) && (ret != -ENOENT)) + return ret; + + ctx->marker = std::move(next_marker); + if (last_marker) { + *last_marker = ctx->marker; + } + + if (ret == -ENOENT) + *truncated = false; + + return 0; +} + +int RGWMetadataLog::get_info(const DoutPrefixProvider *dpp, int shard_id, RGWMetadataLogInfo *info) +{ + string oid; + get_shard_oid(shard_id, oid); + + cls_log_header header; + + int ret = svc.cls->timelog.info(dpp, oid, &header, null_yield); + if ((ret < 0) && (ret != -ENOENT)) + return ret; + + info->marker = header.max_marker; + info->last_update = header.max_time.to_real_time(); + + return 0; +} + +static void _mdlog_info_completion(librados::completion_t cb, void *arg) +{ + auto infoc = static_cast(arg); + infoc->finish(cb); + infoc->put(); // drop the ref from get_info_async() +} + +RGWMetadataLogInfoCompletion::RGWMetadataLogInfoCompletion(info_callback_t cb) + : completion(librados::Rados::aio_create_completion((void *)this, + _mdlog_info_completion)), + callback(cb) +{ +} + +RGWMetadataLogInfoCompletion::~RGWMetadataLogInfoCompletion() +{ + completion->release(); +} + +int RGWMetadataLog::get_info_async(const DoutPrefixProvider *dpp, int shard_id, RGWMetadataLogInfoCompletion *completion) +{ + string oid; + get_shard_oid(shard_id, oid); + + completion->get(); // hold a ref until the completion fires + + return svc.cls->timelog.info_async(dpp, completion->get_io_obj(), oid, + &completion->get_header(), + completion->get_completion()); +} + +int RGWMetadataLog::trim(const DoutPrefixProvider *dpp, int shard_id, const real_time& from_time, const real_time& end_time, + const string& start_marker, const string& end_marker) +{ + string oid; + get_shard_oid(shard_id, oid); + + return svc.cls->timelog.trim(dpp, oid, from_time, end_time, start_marker, + end_marker, nullptr, null_yield); +} + +int RGWMetadataLog::lock_exclusive(const DoutPrefixProvider *dpp, int shard_id, timespan duration, string& zone_id, string& owner_id) { + string oid; + get_shard_oid(shard_id, oid); + + return svc.cls->lock.lock_exclusive(dpp, svc.zone->get_zone_params().log_pool, oid, duration, zone_id, owner_id); +} + +int RGWMetadataLog::unlock(const DoutPrefixProvider *dpp, int shard_id, string& zone_id, string& owner_id) { + string oid; + get_shard_oid(shard_id, oid); + + return svc.cls->lock.unlock(dpp, svc.zone->get_zone_params().log_pool, oid, zone_id, owner_id); +} + +void RGWMetadataLog::mark_modified(int shard_id) +{ + lock.get_read(); + if (modified_shards.find(shard_id) != modified_shards.end()) { + lock.unlock(); + return; + } + lock.unlock(); + + std::unique_lock wl{lock}; + modified_shards.insert(shard_id); +} + +void RGWMetadataLog::read_clear_modified(set &modified) +{ + std::unique_lock wl{lock}; + modified.swap(modified_shards); + modified_shards.clear(); +} + +void RGWMetadataLogInfo::dump(Formatter *f) const +{ + encode_json("marker", marker, f); + utime_t ut(last_update); + encode_json("last_update", ut, f); +} + +void RGWMetadataLogInfo::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("marker", marker, obj); + utime_t ut; + JSONDecoder::decode_json("last_update", ut, obj); + last_update = ut.to_real_time(); +} + diff --git a/src/rgw/rgw_metadata.h b/src/rgw/store/rados/rgw_metadata.h similarity index 99% rename from src/rgw/rgw_metadata.h rename to src/rgw/store/rados/rgw_metadata.h index a3bb634c4740..72283702e7e1 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/store/rados/rgw_metadata.h @@ -60,7 +60,7 @@ protected: public: RGWMetadataHandler() {} - virtual ~RGWMetadataHandler() {} + virtual ~RGWMetadataHandler(); virtual std::string get_type() = 0; void base_init(CephContext *_cct) { diff --git a/src/rgw/rgw_notify.cc b/src/rgw/store/rados/rgw_notify.cc similarity index 100% rename from src/rgw/rgw_notify.cc rename to src/rgw/store/rados/rgw_notify.cc diff --git a/src/rgw/rgw_notify.h b/src/rgw/store/rados/rgw_notify.h similarity index 100% rename from src/rgw/rgw_notify.h rename to src/rgw/store/rados/rgw_notify.h diff --git a/src/rgw/store/rados/rgw_obj_manifest.cc b/src/rgw/store/rados/rgw_obj_manifest.cc new file mode 100644 index 000000000000..3838f5cf3289 --- /dev/null +++ b/src/rgw/store/rados/rgw_obj_manifest.cc @@ -0,0 +1,404 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_obj_manifest.h" + +#include "services/svc_zone.h" +#include "rgw_rados.h" +#include "rgw_bucket.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +int RGWObjManifest::generator::create_next(uint64_t ofs) +{ + if (ofs < last_ofs) /* only going forward */ + return -EINVAL; + + uint64_t max_head_size = manifest->get_max_head_size(); + + if (ofs < max_head_size) { + manifest->set_head_size(ofs); + } + + if (ofs >= max_head_size) { + manifest->set_head_size(max_head_size); + cur_stripe = (ofs - max_head_size) / rule.stripe_max_size; + cur_stripe_size = rule.stripe_max_size; + + if (cur_part_id == 0 && max_head_size > 0) { + cur_stripe++; + } + } + + last_ofs = ofs; + manifest->set_obj_size(ofs); + + manifest->get_implicit_location(cur_part_id, cur_stripe, ofs, NULL, &cur_obj); + + return 0; +} + +int RGWObjManifest::append(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, + const RGWZoneParams& zone_params) +{ + if (explicit_objs || m.explicit_objs) { + return append_explicit(dpp, m, zonegroup, zone_params); + } + + if (rules.empty()) { + *this = m; + return 0; + } + + string override_prefix; + + if (prefix.empty()) { + prefix = m.prefix; + } + + if (prefix != m.prefix) { + override_prefix = m.prefix; + } + + map::iterator miter = m.rules.begin(); + if (miter == m.rules.end()) { + return append_explicit(dpp, m, zonegroup, zone_params); + } + + for (; miter != m.rules.end(); ++miter) { + map::reverse_iterator last_rule = rules.rbegin(); + + RGWObjManifestRule& rule = last_rule->second; + + if (rule.part_size == 0) { + rule.part_size = obj_size - rule.start_ofs; + } + + RGWObjManifestRule& next_rule = miter->second; + if (!next_rule.part_size) { + next_rule.part_size = m.obj_size - next_rule.start_ofs; + } + + string rule_prefix = prefix; + if (!rule.override_prefix.empty()) { + rule_prefix = rule.override_prefix; + } + + string next_rule_prefix = m.prefix; + if (!next_rule.override_prefix.empty()) { + next_rule_prefix = next_rule.override_prefix; + } + + if (rule.part_size != next_rule.part_size || + rule.stripe_max_size != next_rule.stripe_max_size || + rule_prefix != next_rule_prefix) { + if (next_rule_prefix != prefix) { + append_rules(m, miter, &next_rule_prefix); + } else { + append_rules(m, miter, NULL); + } + break; + } + + uint64_t expected_part_num = rule.start_part_num + 1; + if (rule.part_size > 0) { + expected_part_num = rule.start_part_num + (obj_size + next_rule.start_ofs - rule.start_ofs) / rule.part_size; + } + + if (expected_part_num != next_rule.start_part_num) { + append_rules(m, miter, NULL); + break; + } + } + + set_obj_size(obj_size + m.obj_size); + + return 0; +} + +void RGWObjManifest::append_rules(RGWObjManifest& m, map::iterator& miter, + string *override_prefix) +{ + for (; miter != m.rules.end(); ++miter) { + RGWObjManifestRule rule = miter->second; + rule.start_ofs += obj_size; + if (override_prefix) + rule.override_prefix = *override_prefix; + rules[rule.start_ofs] = rule; + } +} + +void RGWObjManifest::convert_to_explicit(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) +{ + if (explicit_objs) { + return; + } + obj_iterator iter = obj_begin(dpp); + + while (iter != obj_end(dpp)) { + RGWObjManifestPart& part = objs[iter.get_stripe_ofs()]; + const rgw_obj_select& os = iter.get_location(); + const rgw_raw_obj& raw_loc = os.get_raw_obj(zonegroup, zone_params); + part.loc_ofs = 0; + + uint64_t ofs = iter.get_stripe_ofs(); + + if (ofs == 0) { + part.loc = obj; + } else { + RGWSI_Tier_RADOS::raw_obj_to_obj(tail_placement.bucket, raw_loc, &part.loc); + } + ++iter; + uint64_t next_ofs = iter.get_stripe_ofs(); + + part.size = next_ofs - ofs; + } + + explicit_objs = true; + rules.clear(); + prefix.clear(); +} + +int RGWObjManifest::append_explicit(const DoutPrefixProvider *dpp, RGWObjManifest& m, const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) +{ + if (!explicit_objs) { + convert_to_explicit(dpp, zonegroup, zone_params); + } + if (!m.explicit_objs) { + m.convert_to_explicit(dpp, zonegroup, zone_params); + } + map::iterator iter; + uint64_t base = obj_size; + for (iter = m.objs.begin(); iter != m.objs.end(); ++iter) { + RGWObjManifestPart& part = iter->second; + objs[base + iter->first] = part; + } + obj_size += m.obj_size; + + return 0; +} + +bool RGWObjManifest::get_rule(uint64_t ofs, RGWObjManifestRule *rule) +{ + if (rules.empty()) { + return false; + } + + map::iterator iter = rules.upper_bound(ofs); + if (iter != rules.begin()) { + --iter; + } + + *rule = iter->second; + + return true; +} + +int RGWObjManifest::generator::create_begin(CephContext *cct, RGWObjManifest *_m, + const rgw_placement_rule& head_placement_rule, + const rgw_placement_rule *tail_placement_rule, + const rgw_bucket& _b, const rgw_obj& _obj) +{ + manifest = _m; + + if (!tail_placement_rule) { + manifest->set_tail_placement(head_placement_rule, _b); + } else { + rgw_placement_rule new_tail_rule = *tail_placement_rule; + new_tail_rule.inherit_from(head_placement_rule); + manifest->set_tail_placement(new_tail_rule, _b); + } + + manifest->set_head(head_placement_rule, _obj, 0); + last_ofs = 0; + + if (manifest->get_prefix().empty()) { + char buf[33]; + gen_rand_alphanumeric(cct, buf, sizeof(buf) - 1); + + string oid_prefix = "."; + oid_prefix.append(buf); + oid_prefix.append("_"); + + manifest->set_prefix(oid_prefix); + } + + bool found = manifest->get_rule(0, &rule); + if (!found) { + derr << "ERROR: manifest->get_rule() could not find rule" << dendl; + return -EIO; + } + + uint64_t head_size = manifest->get_head_size(); + + if (head_size > 0) { + cur_stripe_size = head_size; + } else { + cur_stripe_size = rule.stripe_max_size; + } + + cur_part_id = rule.start_part_num; + + manifest->get_implicit_location(cur_part_id, cur_stripe, 0, NULL, &cur_obj); + + // Normal object which not generated through copy operation + manifest->set_tail_instance(_obj.key.instance); + + return 0; +} + +void RGWObjManifestPart::generate_test_instances(std::list& o) +{ + o.push_back(new RGWObjManifestPart); + + RGWObjManifestPart *p = new RGWObjManifestPart; + rgw_bucket b; + init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12"); + + p->loc = rgw_obj(b, "object"); + p->loc_ofs = 512 * 1024; + p->size = 128 * 1024; + o.push_back(p); +} + +void RGWObjManifest::generate_test_instances(std::list& o) +{ + RGWObjManifest *m = new RGWObjManifest; + map objs; + uint64_t total_size = 0; + for (int i = 0; i<10; i++) { + RGWObjManifestPart p; + rgw_bucket b; + init_bucket(&b, "tenant", "bucket", ".pool", ".index_pool", "marker_", "12"); + p.loc = rgw_obj(b, "object"); + p.loc_ofs = 0; + p.size = 512 * 1024; + total_size += p.size; + objs[total_size] = p; + } + m->set_explicit(total_size, objs); + o.push_back(m); + o.push_back(new RGWObjManifest); +} + +void RGWObjManifestPart::dump(Formatter *f) const +{ + f->open_object_section("loc"); + loc.dump(f); + f->close_section(); + f->dump_unsigned("loc_ofs", loc_ofs); + f->dump_unsigned("size", size); +} + +void RGWObjManifest::obj_iterator::dump(Formatter *f) const +{ + f->dump_unsigned("part_ofs", part_ofs); + f->dump_unsigned("stripe_ofs", stripe_ofs); + f->dump_unsigned("ofs", ofs); + f->dump_unsigned("stripe_size", stripe_size); + f->dump_int("cur_part_id", cur_part_id); + f->dump_int("cur_stripe", cur_stripe); + f->dump_string("cur_override_prefix", cur_override_prefix); + f->dump_object("location", location); +} + +void RGWObjManifest::dump(Formatter *f) const +{ + map::const_iterator iter = objs.begin(); + f->open_array_section("objs"); + for (; iter != objs.end(); ++iter) { + f->dump_unsigned("ofs", iter->first); + f->open_object_section("part"); + iter->second.dump(f); + f->close_section(); + } + f->close_section(); + f->dump_unsigned("obj_size", obj_size); + ::encode_json("explicit_objs", explicit_objs, f); + ::encode_json("head_size", head_size, f); + ::encode_json("max_head_size", max_head_size, f); + ::encode_json("prefix", prefix, f); + ::encode_json("rules", rules, f); + ::encode_json("tail_instance", tail_instance, f); + ::encode_json("tail_placement", tail_placement, f); + + // nullptr being passed into iterators since there + // is no cct and we aren't doing anything with these + // iterators that would write do the log + f->dump_object("begin_iter", obj_begin(nullptr)); + f->dump_object("end_iter", obj_end(nullptr)); +} + +void RGWObjManifestRule::dump(Formatter *f) const +{ + encode_json("start_part_num", start_part_num, f); + encode_json("start_ofs", start_ofs, f); + encode_json("part_size", part_size, f); + encode_json("stripe_max_size", stripe_max_size, f); + encode_json("override_prefix", override_prefix, f); +} + +void rgw_obj_select::dump(Formatter *f) const +{ + f->dump_string("placement_rule", placement_rule.to_str()); + f->dump_object("obj", obj); + f->dump_object("raw_obj", raw_obj); + f->dump_bool("is_raw", is_raw); +} + +void RGWObjTier::dump(Formatter *f) const +{ + encode_json("name", name, f); + encode_json("tier_placement", tier_placement, f); + encode_json("is_multipart_upload", is_multipart_upload, f); +} + +// returns true on success, false on failure +static bool rgw_get_obj_data_pool(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params, + const rgw_placement_rule& head_placement_rule, + const rgw_obj& obj, rgw_pool *pool) +{ + if (!zone_params.get_head_data_pool(head_placement_rule, obj, pool)) { + RGWZonePlacementInfo placement; + if (!zone_params.get_placement(zonegroup.default_placement.name, &placement)) { + return false; + } + + if (!obj.in_extra_data) { + *pool = placement.get_data_pool(zonegroup.default_placement.storage_class); + } else { + *pool = placement.get_data_extra_pool(); + } + } + + return true; +} + +static bool rgw_obj_to_raw(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params, + const rgw_placement_rule& head_placement_rule, + const rgw_obj& obj, rgw_raw_obj *raw_obj) +{ + get_obj_bucket_and_oid_loc(obj, raw_obj->oid, raw_obj->loc); + + return rgw_get_obj_data_pool(zonegroup, zone_params, head_placement_rule, obj, &raw_obj->pool); +} + +rgw_raw_obj rgw_obj_select::get_raw_obj(const RGWZoneGroup& zonegroup, const RGWZoneParams& zone_params) const +{ + if (!is_raw) { + rgw_raw_obj r; + rgw_obj_to_raw(zonegroup, zone_params, placement_rule, obj, &r); + return r; + } + return raw_obj; +} + +// returns true on success, false on failure +bool RGWRados::get_obj_data_pool(const rgw_placement_rule& placement_rule, const rgw_obj& obj, rgw_pool *pool) +{ + return rgw_get_obj_data_pool(svc.zone->get_zonegroup(), svc.zone->get_zone_params(), placement_rule, obj, pool); +} + diff --git a/src/rgw/rgw_obj_manifest.h b/src/rgw/store/rados/rgw_obj_manifest.h similarity index 100% rename from src/rgw/rgw_obj_manifest.h rename to src/rgw/store/rados/rgw_obj_manifest.h diff --git a/src/rgw/rgw_object_expirer_core.cc b/src/rgw/store/rados/rgw_object_expirer_core.cc similarity index 100% rename from src/rgw/rgw_object_expirer_core.cc rename to src/rgw/store/rados/rgw_object_expirer_core.cc diff --git a/src/rgw/rgw_object_expirer_core.h b/src/rgw/store/rados/rgw_object_expirer_core.h similarity index 100% rename from src/rgw/rgw_object_expirer_core.h rename to src/rgw/store/rados/rgw_object_expirer_core.h diff --git a/src/rgw/rgw_otp.cc b/src/rgw/store/rados/rgw_otp.cc similarity index 100% rename from src/rgw/rgw_otp.cc rename to src/rgw/store/rados/rgw_otp.cc diff --git a/src/rgw/rgw_otp.h b/src/rgw/store/rados/rgw_otp.h similarity index 100% rename from src/rgw/rgw_otp.h rename to src/rgw/store/rados/rgw_otp.h diff --git a/src/rgw/store/rados/rgw_period.cc b/src/rgw/store/rados/rgw_period.cc new file mode 100644 index 000000000000..df4316c1a9b4 --- /dev/null +++ b/src/rgw/store/rados/rgw_period.cc @@ -0,0 +1,324 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_sync.h" + +#include "services/svc_zone.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; +using namespace rgw_zone_defaults; + +int RGWPeriod::get_zonegroup(RGWZoneGroup& zonegroup, + const string& zonegroup_id) const +{ + map::const_iterator iter; + if (!zonegroup_id.empty()) { + iter = period_map.zonegroups.find(zonegroup_id); + } else { + iter = period_map.zonegroups.find("default"); + } + if (iter != period_map.zonegroups.end()) { + zonegroup = iter->second; + return 0; + } + + return -ENOENT; +} + +int RGWPeriod::get_latest_epoch(const DoutPrefixProvider *dpp, epoch_t& latest_epoch, optional_yield y) +{ + RGWPeriodLatestEpochInfo info; + + int ret = read_latest_epoch(dpp, info, y); + if (ret < 0) { + return ret; + } + + latest_epoch = info.epoch; + + return 0; +} + +int RGWPeriod::delete_obj(const DoutPrefixProvider *dpp, optional_yield y) +{ + rgw_pool pool(get_pool(cct)); + + // delete the object for each period epoch + for (epoch_t e = 1; e <= epoch; e++) { + RGWPeriod p{get_id(), e}; + rgw_raw_obj oid{pool, p.get_period_oid()}; + auto sysobj = sysobj_svc->get_obj(oid); + int ret = sysobj.wop().remove(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid + << ": " << cpp_strerror(-ret) << dendl; + } + } + + // delete the .latest_epoch object + rgw_raw_obj oid{pool, get_period_oid_prefix() + get_latest_epoch_oid()}; + auto sysobj = sysobj_svc->get_obj(oid); + int ret = sysobj.wop().remove(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to delete period object " << oid + << ": " << cpp_strerror(-ret) << dendl; + } + return ret; +} + +int RGWPeriod::add_zonegroup(const DoutPrefixProvider *dpp, const RGWZoneGroup& zonegroup, optional_yield y) +{ + if (zonegroup.realm_id != realm_id) { + return 0; + } + int ret = period_map.update(zonegroup, cct); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: updating period map: " << cpp_strerror(-ret) << dendl; + return ret; + } + + return store_info(dpp, false, y); +} + +int RGWPeriod::update(const DoutPrefixProvider *dpp, optional_yield y) +{ + auto zone_svc = sysobj_svc->get_zone_svc(); + ldpp_dout(dpp, 20) << __func__ << " realm " << realm_id << " period " << get_id() << dendl; + list zonegroups; + int ret = zone_svc->list_zonegroups(dpp, zonegroups); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to list zonegroups: " << cpp_strerror(-ret) << dendl; + return ret; + } + + // clear zone short ids of removed zones. period_map.update() will add the + // remaining zones back + period_map.short_zone_ids.clear(); + + for (auto& iter : zonegroups) { + RGWZoneGroup zg(string(), iter); + ret = zg.init(dpp, cct, sysobj_svc, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: zg.init() failed: " << cpp_strerror(-ret) << dendl; + continue; + } + + if (zg.realm_id != realm_id) { + ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() << " zone realm id " << zg.realm_id << ", not on our realm " << realm_id << dendl; + continue; + } + + if (zg.master_zone.empty()) { + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; + return -EINVAL; + } + + if (zg.zones.find(zg.master_zone) == zg.zones.end()) { + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() + << " has a non existent master zone "<< dendl; + return -EINVAL; + } + + if (zg.is_master_zonegroup()) { + master_zonegroup = zg.get_id(); + master_zone = zg.master_zone; + } + + int ret = period_map.update(zg, cct); + if (ret < 0) { + return ret; + } + } + + ret = period_config.read(dpp, sysobj_svc, realm_id, y); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " + << cpp_strerror(ret) << dendl; + return ret; + } + return 0; +} + +void RGWPeriod::fork() +{ + ldout(cct, 20) << __func__ << " realm " << realm_id << " period " << id << dendl; + predecessor_uuid = id; + id = get_staging_id(realm_id); + period_map.reset(); + realm_epoch++; +} + +static int read_sync_status(const DoutPrefixProvider *dpp, rgw::sal::Store* store, rgw_meta_sync_status *sync_status) +{ + rgw::sal::RadosStore* rados_store = static_cast(store); + // initialize a sync status manager to read the status + RGWMetaSyncStatusManager mgr(rados_store, rados_store->svc()->rados->get_async_processor()); + int r = mgr.init(dpp); + if (r < 0) { + return r; + } + r = mgr.read_sync_status(dpp, sync_status); + mgr.stop(); + return r; +} + +int RGWPeriod::update_sync_status(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, /* for now */ + const RGWPeriod ¤t_period, + std::ostream& error_stream, + bool force_if_stale) +{ + rgw_meta_sync_status status; + int r = read_sync_status(dpp, store, &status); + if (r < 0) { + ldpp_dout(dpp, 0) << "period failed to read sync status: " + << cpp_strerror(-r) << dendl; + return r; + } + + std::vector markers; + + const auto current_epoch = current_period.get_realm_epoch(); + if (current_epoch != status.sync_info.realm_epoch) { + // no sync status markers for the current period + ceph_assert(current_epoch > status.sync_info.realm_epoch); + const int behind = current_epoch - status.sync_info.realm_epoch; + if (!force_if_stale && current_epoch > 1) { + error_stream << "ERROR: This zone is " << behind << " period(s) behind " + "the current master zone in metadata sync. If this zone is promoted " + "to master, any metadata changes during that time are likely to " + "be lost.\n" + "Waiting for this zone to catch up on metadata sync (see " + "'radosgw-admin sync status') is recommended.\n" + "To promote this zone to master anyway, add the flag " + "--yes-i-really-mean-it." << std::endl; + return -EINVAL; + } + // empty sync status markers - other zones will skip this period during + // incremental metadata sync + markers.resize(status.sync_info.num_shards); + } else { + markers.reserve(status.sync_info.num_shards); + for (auto& i : status.sync_markers) { + auto& marker = i.second; + // filter out markers from other periods + if (marker.realm_epoch != current_epoch) { + marker.marker.clear(); + } + markers.emplace_back(std::move(marker.marker)); + } + } + + std::swap(sync_status, markers); + return 0; +} + +int RGWPeriod::commit(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWRealm& realm, const RGWPeriod& current_period, + std::ostream& error_stream, optional_yield y, + bool force_if_stale) +{ + auto zone_svc = sysobj_svc->get_zone_svc(); + ldpp_dout(dpp, 20) << __func__ << " realm " << realm.get_id() << " period " << current_period.get_id() << dendl; + // gateway must be in the master zone to commit + if (master_zone != zone_svc->get_zone_params().get_id()) { + error_stream << "Cannot commit period on zone " + << zone_svc->get_zone_params().get_id() << ", it must be sent to " + "the period's master zone " << master_zone << '.' << std::endl; + return -EINVAL; + } + // period predecessor must match current period + if (predecessor_uuid != current_period.get_id()) { + error_stream << "Period predecessor " << predecessor_uuid + << " does not match current period " << current_period.get_id() + << ". Use 'period pull' to get the latest period from the master, " + "reapply your changes, and try again." << std::endl; + return -EINVAL; + } + // realm epoch must be 1 greater than current period + if (realm_epoch != current_period.get_realm_epoch() + 1) { + error_stream << "Period's realm epoch " << realm_epoch + << " does not come directly after current realm epoch " + << current_period.get_realm_epoch() << ". Use 'realm pull' to get the " + "latest realm and period from the master zone, reapply your changes, " + "and try again." << std::endl; + return -EINVAL; + } + // did the master zone change? + if (master_zone != current_period.get_master_zone()) { + // store the current metadata sync status in the period + int r = update_sync_status(dpp, store, current_period, error_stream, force_if_stale); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update metadata sync status: " + << cpp_strerror(-r) << dendl; + return r; + } + // create an object with a new period id + r = create(dpp, y, true); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; + return r; + } + // set as current period + r = realm.set_current_period(dpp, *this, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update realm's current period: " + << cpp_strerror(-r) << dendl; + return r; + } + ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period " + << id << dendl; + realm.notify_new_period(dpp, *this, y); + return 0; + } + // period must be based on current epoch + if (epoch != current_period.get_epoch()) { + error_stream << "Period epoch " << epoch << " does not match " + "predecessor epoch " << current_period.get_epoch() + << ". Use 'period pull' to get the latest epoch from the master zone, " + "reapply your changes, and try again." << std::endl; + return -EINVAL; + } + // set period as next epoch + set_id(current_period.get_id()); + set_epoch(current_period.get_epoch() + 1); + set_predecessor(current_period.get_predecessor()); + realm_epoch = current_period.get_realm_epoch(); + // write the period to rados + int r = store_info(dpp, false, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(-r) << dendl; + return r; + } + // set as latest epoch + r = update_latest_epoch(dpp, epoch, y); + if (r == -EEXIST) { + // already have this epoch (or a more recent one) + return 0; + } + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to set latest epoch: " << cpp_strerror(-r) << dendl; + return r; + } + r = reflect(dpp, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(-r) << dendl; + return r; + } + ldpp_dout(dpp, 4) << "Committed new epoch " << epoch + << " for period " << id << dendl; + realm.notify_new_period(dpp, *this, y); + return 0; +} + +void RGWPeriod::generate_test_instances(list &o) +{ + RGWPeriod *z = new RGWPeriod; + o.push_back(z); + o.push_back(new RGWPeriod); +} + + diff --git a/src/rgw/rgw_rest_pubsub.cc b/src/rgw/store/rados/rgw_rest_pubsub.cc similarity index 100% rename from src/rgw/rgw_rest_pubsub.cc rename to src/rgw/store/rados/rgw_rest_pubsub.cc diff --git a/src/rgw/rgw_rest_pubsub.h b/src/rgw/store/rados/rgw_rest_pubsub.h similarity index 100% rename from src/rgw/rgw_rest_pubsub.h rename to src/rgw/store/rados/rgw_rest_pubsub.h diff --git a/src/rgw/rgw_rest_pubsub_common.cc b/src/rgw/store/rados/rgw_rest_pubsub_common.cc similarity index 100% rename from src/rgw/rgw_rest_pubsub_common.cc rename to src/rgw/store/rados/rgw_rest_pubsub_common.cc diff --git a/src/rgw/rgw_rest_pubsub_common.h b/src/rgw/store/rados/rgw_rest_pubsub_common.h similarity index 100% rename from src/rgw/rgw_rest_pubsub_common.h rename to src/rgw/store/rados/rgw_rest_pubsub_common.h diff --git a/src/rgw/rgw_rest_realm.cc b/src/rgw/store/rados/rgw_rest_realm.cc similarity index 100% rename from src/rgw/rgw_rest_realm.cc rename to src/rgw/store/rados/rgw_rest_realm.cc diff --git a/src/rgw/rgw_rest_realm.h b/src/rgw/store/rados/rgw_rest_realm.h similarity index 100% rename from src/rgw/rgw_rest_realm.h rename to src/rgw/store/rados/rgw_rest_realm.h diff --git a/src/rgw/rgw_rest_user.cc b/src/rgw/store/rados/rgw_rest_user.cc similarity index 100% rename from src/rgw/rgw_rest_user.cc rename to src/rgw/store/rados/rgw_rest_user.cc diff --git a/src/rgw/rgw_rest_user.h b/src/rgw/store/rados/rgw_rest_user.h similarity index 100% rename from src/rgw/rgw_rest_user.h rename to src/rgw/store/rados/rgw_rest_user.h diff --git a/src/rgw/rgw_sal_rados.cc b/src/rgw/store/rados/rgw_sal_rados.cc similarity index 100% rename from src/rgw/rgw_sal_rados.cc rename to src/rgw/store/rados/rgw_sal_rados.cc diff --git a/src/rgw/rgw_sal_rados.h b/src/rgw/store/rados/rgw_sal_rados.h similarity index 100% rename from src/rgw/rgw_sal_rados.h rename to src/rgw/store/rados/rgw_sal_rados.h diff --git a/src/rgw/rgw_service.cc b/src/rgw/store/rados/rgw_service.cc similarity index 100% rename from src/rgw/rgw_service.cc rename to src/rgw/store/rados/rgw_service.cc diff --git a/src/rgw/rgw_service.h b/src/rgw/store/rados/rgw_service.h similarity index 99% rename from src/rgw/rgw_service.h rename to src/rgw/store/rados/rgw_service.h index 83475140d168..6a34443d3976 100644 --- a/src/rgw/rgw_service.h +++ b/src/rgw/store/rados/rgw_service.h @@ -11,7 +11,7 @@ #include "common/async/yield_context.h" -#include "rgw/rgw_common.h" +#include "rgw_common.h" struct RGWServices_Def; diff --git a/src/rgw/store/rados/rgw_sync.cc b/src/rgw/store/rados/rgw_sync.cc new file mode 100644 index 000000000000..065d20985c4d --- /dev/null +++ b/src/rgw/store/rados/rgw_sync.cc @@ -0,0 +1,2567 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_sync.h" +#include "rgw_rest_conn.h" +#include "rgw_cr_rados.h" +#include "rgw_cr_rest.h" + +#include "services/svc_zone.h" +#include "services/svc_mdlog.h" +#include "services/svc_cls.h" + +#include + +#define dout_subsys ceph_subsys_rgw + +#undef dout_prefix +#define dout_prefix (*_dout << "meta sync: ") + +using namespace std; + +static string mdlog_sync_status_oid = "mdlog.sync-status"; +static string mdlog_sync_status_shard_prefix = "mdlog.sync-status.shard"; +static string mdlog_sync_full_sync_index_prefix = "meta.full-sync.index"; + +RGWContinuousLeaseCR::~RGWContinuousLeaseCR() {} + +RGWSyncErrorLogger::RGWSyncErrorLogger(rgw::sal::RadosStore* _store, const string &oid_prefix, int _num_shards) : store(_store), num_shards(_num_shards) { + for (int i = 0; i < num_shards; i++) { + oids.push_back(get_shard_oid(oid_prefix, i)); + } +} +string RGWSyncErrorLogger::get_shard_oid(const string& oid_prefix, int shard_id) { + char buf[oid_prefix.size() + 16]; + snprintf(buf, sizeof(buf), "%s.%d", oid_prefix.c_str(), shard_id); + return string(buf); +} + +RGWCoroutine *RGWSyncErrorLogger::log_error_cr(const DoutPrefixProvider *dpp, const string& source_zone, const string& section, const string& name, uint32_t error_code, const string& message) { + cls_log_entry entry; + + rgw_sync_error_info info(source_zone, error_code, message); + bufferlist bl; + encode(info, bl); + store->svc()->cls->timelog.prepare_entry(entry, real_clock::now(), section, name, bl); + + uint32_t shard_id = ++counter % num_shards; + + + return new RGWRadosTimelogAddCR(dpp, store, oids[shard_id], entry); +} + +void RGWSyncBackoff::update_wait_time() +{ + if (cur_wait == 0) { + cur_wait = 1; + } else { + cur_wait = (cur_wait << 1); + } + if (cur_wait >= max_secs) { + cur_wait = max_secs; + } +} + +void RGWSyncBackoff::backoff_sleep() +{ + update_wait_time(); + sleep(cur_wait); +} + +void RGWSyncBackoff::backoff(RGWCoroutine *op) +{ + update_wait_time(); + op->wait(utime_t(cur_wait, 0)); +} + +int RGWBackoffControlCR::operate(const DoutPrefixProvider *dpp) { + reenter(this) { + // retry the operation until it succeeds + while (true) { + yield { + std::lock_guard l{lock}; + cr = alloc_cr(); + cr->get(); + call(cr); + } + { + std::lock_guard l{lock}; + cr->put(); + cr = NULL; + } + if (retcode >= 0) { + break; + } + if (retcode != -EBUSY && retcode != -EAGAIN) { + ldout(cct, 0) << "ERROR: RGWBackoffControlCR called coroutine returned " << retcode << dendl; + if (exit_on_error) { + return set_cr_error(retcode); + } + } + if (reset_backoff) { + backoff.reset(); + } + yield backoff.backoff(this); + } + + // run an optional finisher + yield call(alloc_finisher_cr()); + if (retcode < 0) { + ldout(cct, 0) << "ERROR: call to finisher_cr() failed: retcode=" << retcode << dendl; + return set_cr_error(retcode); + } + return set_cr_done(); + } + return 0; +} + +void rgw_mdlog_info::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("num_objects", num_shards, obj); + JSONDecoder::decode_json("period", period, obj); + JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); +} + +void rgw_mdlog_entry::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("id", id, obj); + JSONDecoder::decode_json("section", section, obj); + JSONDecoder::decode_json("name", name, obj); + utime_t ut; + JSONDecoder::decode_json("timestamp", ut, obj); + timestamp = ut.to_real_time(); + JSONDecoder::decode_json("data", log_data, obj); +} + +void rgw_mdlog_shard_data::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("marker", marker, obj); + JSONDecoder::decode_json("truncated", truncated, obj); + JSONDecoder::decode_json("entries", entries, obj); +}; + +int RGWShardCollectCR::operate(const DoutPrefixProvider *dpp) { + reenter(this) { + while (spawn_next()) { + current_running++; + + if (current_running >= max_concurrent) { + int child_ret; + yield wait_for_child(); + if (collect_next(&child_ret)) { + current_running--; + child_ret = handle_result(child_ret); + if (child_ret < 0) { + status = child_ret; + } + } + } + } + while (current_running > 0) { + int child_ret; + yield wait_for_child(); + if (collect_next(&child_ret)) { + current_running--; + child_ret = handle_result(child_ret); + if (child_ret < 0) { + status = child_ret; + } + } + } + if (status < 0) { + return set_cr_error(status); + } + return set_cr_done(); + } + return 0; +} + +class RGWReadRemoteMDLogInfoCR : public RGWShardCollectCR { + RGWMetaSyncEnv *sync_env; + + const std::string& period; + int num_shards; + map *mdlog_info; + + int shard_id; +#define READ_MDLOG_MAX_CONCURRENT 10 + + int handle_result(int r) override { + if (r == -ENOENT) { // ENOENT is not a fatal error + return 0; + } + if (r < 0) { + ldout(cct, 4) << "failed to fetch mdlog status: " << cpp_strerror(r) << dendl; + } + return r; + } +public: + RGWReadRemoteMDLogInfoCR(RGWMetaSyncEnv *_sync_env, + const std::string& period, int _num_shards, + map *_mdlog_info) : RGWShardCollectCR(_sync_env->cct, READ_MDLOG_MAX_CONCURRENT), + sync_env(_sync_env), + period(period), num_shards(_num_shards), + mdlog_info(_mdlog_info), shard_id(0) {} + bool spawn_next() override; +}; + +class RGWListRemoteMDLogCR : public RGWShardCollectCR { + RGWMetaSyncEnv *sync_env; + + const std::string& period; + map shards; + int max_entries_per_shard; + map *result; + + map::iterator iter; +#define READ_MDLOG_MAX_CONCURRENT 10 + + int handle_result(int r) override { + if (r == -ENOENT) { // ENOENT is not a fatal error + return 0; + } + if (r < 0) { + ldout(cct, 4) << "failed to list remote mdlog shard: " << cpp_strerror(r) << dendl; + } + return r; + } +public: + RGWListRemoteMDLogCR(RGWMetaSyncEnv *_sync_env, + const std::string& period, map& _shards, + int _max_entries_per_shard, + map *_result) : RGWShardCollectCR(_sync_env->cct, READ_MDLOG_MAX_CONCURRENT), + sync_env(_sync_env), period(period), + max_entries_per_shard(_max_entries_per_shard), + result(_result) { + shards.swap(_shards); + iter = shards.begin(); + } + bool spawn_next() override; +}; + +int RGWRemoteMetaLog::read_log_info(const DoutPrefixProvider *dpp, rgw_mdlog_info *log_info) +{ + rgw_http_param_pair pairs[] = { { "type", "metadata" }, + { NULL, NULL } }; + + int ret = conn->get_json_resource(dpp, "/admin/log", pairs, null_yield, *log_info); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog info" << dendl; + return ret; + } + + ldpp_dout(dpp, 20) << "remote mdlog, num_shards=" << log_info->num_shards << dendl; + + return 0; +} + +int RGWRemoteMetaLog::read_master_log_shards_info(const DoutPrefixProvider *dpp, const string &master_period, map *shards_info) +{ + if (store->svc()->zone->is_meta_master()) { + return 0; + } + + rgw_mdlog_info log_info; + int ret = read_log_info(dpp, &log_info); + if (ret < 0) { + return ret; + } + + return run(dpp, new RGWReadRemoteMDLogInfoCR(&sync_env, master_period, log_info.num_shards, shards_info)); +} + +int RGWRemoteMetaLog::read_master_log_shards_next(const DoutPrefixProvider *dpp, const string& period, map shard_markers, map *result) +{ + if (store->svc()->zone->is_meta_master()) { + return 0; + } + + return run(dpp, new RGWListRemoteMDLogCR(&sync_env, period, shard_markers, 1, result)); +} + +int RGWRemoteMetaLog::init() +{ + conn = store->svc()->zone->get_master_conn(); + + int ret = http_manager.start(); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl; + return ret; + } + + error_logger = new RGWSyncErrorLogger(store, RGW_SYNC_ERROR_LOG_SHARD_PREFIX, ERROR_LOGGER_SHARDS); + + init_sync_env(&sync_env); + + tn = sync_env.sync_tracer->add_node(sync_env.sync_tracer->root_node, "meta"); + + return 0; +} + +#define CLONE_MAX_ENTRIES 100 + +int RGWMetaSyncStatusManager::init(const DoutPrefixProvider *dpp) +{ + if (store->svc()->zone->is_meta_master()) { + return 0; + } + + if (!store->svc()->zone->get_master_conn()) { + ldpp_dout(dpp, -1) << "no REST connection to master zone" << dendl; + return -EIO; + } + + int r = rgw_init_ioctx(dpp, store->getRados()->get_rados_handle(), store->svc()->zone->get_zone_params().log_pool, ioctx, true); + if (r < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed to open log pool (" << store->svc()->zone->get_zone_params().log_pool << " ret=" << r << dendl; + return r; + } + + r = master_log.init(); + if (r < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed to init remote log, r=" << r << dendl; + return r; + } + + RGWMetaSyncEnv& sync_env = master_log.get_sync_env(); + + rgw_meta_sync_status sync_status; + r = read_sync_status(dpp, &sync_status); + if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, -1) << "ERROR: failed to read sync status, r=" << r << dendl; + return r; + } + + int num_shards = sync_status.sync_info.num_shards; + + for (int i = 0; i < num_shards; i++) { + shard_objs[i] = rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env.shard_obj_name(i)); + } + + std::unique_lock wl{ts_to_shard_lock}; + for (int i = 0; i < num_shards; i++) { + clone_markers.push_back(string()); + utime_shard ut; + ut.shard_id = i; + ts_to_shard[ut] = i; + } + + return 0; +} + +void RGWMetaSyncEnv::init(const DoutPrefixProvider *_dpp, CephContext *_cct, rgw::sal::RadosStore* _store, RGWRESTConn *_conn, + RGWAsyncRadosProcessor *_async_rados, RGWHTTPManager *_http_manager, + RGWSyncErrorLogger *_error_logger, RGWSyncTraceManager *_sync_tracer) { + dpp = _dpp; + cct = _cct; + store = _store; + conn = _conn; + async_rados = _async_rados; + http_manager = _http_manager; + error_logger = _error_logger; + sync_tracer = _sync_tracer; +} + +string RGWMetaSyncEnv::status_oid() +{ + return mdlog_sync_status_oid; +} + +string RGWMetaSyncEnv::shard_obj_name(int shard_id) +{ + char buf[mdlog_sync_status_shard_prefix.size() + 16]; + snprintf(buf, sizeof(buf), "%s.%d", mdlog_sync_status_shard_prefix.c_str(), shard_id); + + return string(buf); +} + +class RGWAsyncReadMDLogEntries : public RGWAsyncRadosRequest { + const DoutPrefixProvider *dpp; + rgw::sal::RadosStore* store; + RGWMetadataLog *mdlog; + int shard_id; + int max_entries; + +protected: + int _send_request(const DoutPrefixProvider *dpp) override { + real_time from_time; + real_time end_time; + + void *handle; + + mdlog->init_list_entries(shard_id, from_time, end_time, marker, &handle); + + int ret = mdlog->list_entries(dpp, handle, max_entries, entries, &marker, &truncated); + + mdlog->complete_list_entries(handle); + + return ret; + } +public: + string marker; + list entries; + bool truncated; + + RGWAsyncReadMDLogEntries(const DoutPrefixProvider *dpp, RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store, + RGWMetadataLog* mdlog, int _shard_id, + std::string _marker, int _max_entries) + : RGWAsyncRadosRequest(caller, cn), dpp(dpp), store(_store), mdlog(mdlog), + shard_id(_shard_id), max_entries(_max_entries), marker(std::move(_marker)) {} +}; + +class RGWReadMDLogEntriesCR : public RGWSimpleCoroutine { + RGWMetaSyncEnv *sync_env; + RGWMetadataLog *const mdlog; + int shard_id; + string marker; + string *pmarker; + int max_entries; + list *entries; + bool *truncated; + + RGWAsyncReadMDLogEntries *req{nullptr}; + +public: + RGWReadMDLogEntriesCR(RGWMetaSyncEnv *_sync_env, RGWMetadataLog* mdlog, + int _shard_id, string*_marker, int _max_entries, + list *_entries, bool *_truncated) + : RGWSimpleCoroutine(_sync_env->cct), sync_env(_sync_env), mdlog(mdlog), + shard_id(_shard_id), pmarker(_marker), max_entries(_max_entries), + entries(_entries), truncated(_truncated) {} + + ~RGWReadMDLogEntriesCR() override { + if (req) { + req->finish(); + } + } + + int send_request(const DoutPrefixProvider *dpp) override { + marker = *pmarker; + req = new RGWAsyncReadMDLogEntries(dpp, this, stack->create_completion_notifier(), + sync_env->store, mdlog, shard_id, marker, + max_entries); + sync_env->async_rados->queue(req); + return 0; + } + + int request_complete() override { + *pmarker = std::move(req->marker); + *entries = std::move(req->entries); + *truncated = req->truncated; + return req->get_ret_status(); + } +}; + + +class RGWReadRemoteMDLogShardInfoCR : public RGWCoroutine { + RGWMetaSyncEnv *env; + RGWRESTReadResource *http_op; + + const std::string& period; + int shard_id; + RGWMetadataLogInfo *shard_info; + +public: + RGWReadRemoteMDLogShardInfoCR(RGWMetaSyncEnv *env, const std::string& period, + int _shard_id, RGWMetadataLogInfo *_shard_info) + : RGWCoroutine(env->store->ctx()), env(env), http_op(NULL), + period(period), shard_id(_shard_id), shard_info(_shard_info) {} + + int operate(const DoutPrefixProvider *dpp) override { + auto store = env->store; + RGWRESTConn *conn = store->svc()->zone->get_master_conn(); + reenter(this) { + yield { + char buf[16]; + snprintf(buf, sizeof(buf), "%d", shard_id); + rgw_http_param_pair pairs[] = { { "type" , "metadata" }, + { "id", buf }, + { "period", period.c_str() }, + { "info" , NULL }, + { NULL, NULL } }; + + string p = "/admin/log/"; + + http_op = new RGWRESTReadResource(conn, p, pairs, NULL, + env->http_manager); + + init_new_io(http_op); + + int ret = http_op->aio_read(dpp); + if (ret < 0) { + ldpp_dout(env->dpp, 0) << "ERROR: failed to read from " << p << dendl; + log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; + http_op->put(); + return set_cr_error(ret); + } + + return io_block(0); + } + yield { + int ret = http_op->wait(shard_info, null_yield); + http_op->put(); + if (ret < 0) { + return set_cr_error(ret); + } + return set_cr_done(); + } + } + return 0; + } +}; + +RGWCoroutine* create_read_remote_mdlog_shard_info_cr(RGWMetaSyncEnv *env, + const std::string& period, + int shard_id, + RGWMetadataLogInfo* info) +{ + return new RGWReadRemoteMDLogShardInfoCR(env, period, shard_id, info); +} + +class RGWListRemoteMDLogShardCR : public RGWSimpleCoroutine { + RGWMetaSyncEnv *sync_env; + RGWRESTReadResource *http_op; + + const std::string& period; + int shard_id; + string marker; + uint32_t max_entries; + rgw_mdlog_shard_data *result; + +public: + RGWListRemoteMDLogShardCR(RGWMetaSyncEnv *env, const std::string& period, + int _shard_id, const string& _marker, uint32_t _max_entries, + rgw_mdlog_shard_data *_result) + : RGWSimpleCoroutine(env->store->ctx()), sync_env(env), http_op(NULL), + period(period), shard_id(_shard_id), marker(_marker), max_entries(_max_entries), result(_result) {} + + int send_request(const DoutPrefixProvider *dpp) override { + RGWRESTConn *conn = sync_env->conn; + + char buf[32]; + snprintf(buf, sizeof(buf), "%d", shard_id); + + char max_entries_buf[32]; + snprintf(max_entries_buf, sizeof(max_entries_buf), "%d", (int)max_entries); + + const char *marker_key = (marker.empty() ? "" : "marker"); + + rgw_http_param_pair pairs[] = { { "type", "metadata" }, + { "id", buf }, + { "period", period.c_str() }, + { "max-entries", max_entries_buf }, + { marker_key, marker.c_str() }, + { NULL, NULL } }; + + string p = "/admin/log/"; + + http_op = new RGWRESTReadResource(conn, p, pairs, NULL, sync_env->http_manager); + init_new_io(http_op); + + int ret = http_op->aio_read(dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to read from " << p << dendl; + log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; + http_op->put(); + return ret; + } + + return 0; + } + + int request_complete() override { + int ret = http_op->wait(result, null_yield); + http_op->put(); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to list remote mdlog shard, ret=" << ret << dendl; + return ret; + } + return 0; + } +}; + +RGWCoroutine* create_list_remote_mdlog_shard_cr(RGWMetaSyncEnv *env, + const std::string& period, + int shard_id, + const std::string& marker, + uint32_t max_entries, + rgw_mdlog_shard_data *result) +{ + return new RGWListRemoteMDLogShardCR(env, period, shard_id, marker, + max_entries, result); +} + +bool RGWReadRemoteMDLogInfoCR::spawn_next() { + if (shard_id >= num_shards) { + return false; + } + spawn(new RGWReadRemoteMDLogShardInfoCR(sync_env, period, shard_id, &(*mdlog_info)[shard_id]), false); + shard_id++; + return true; +} + +bool RGWListRemoteMDLogCR::spawn_next() { + if (iter == shards.end()) { + return false; + } + + spawn(new RGWListRemoteMDLogShardCR(sync_env, period, iter->first, iter->second, max_entries_per_shard, &(*result)[iter->first]), false); + ++iter; + return true; +} + +class RGWInitSyncStatusCoroutine : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + + rgw_meta_sync_info status; + vector shards_info; + boost::intrusive_ptr lease_cr; + boost::intrusive_ptr lease_stack; +public: + RGWInitSyncStatusCoroutine(RGWMetaSyncEnv *_sync_env, + const rgw_meta_sync_info &status) + : RGWCoroutine(_sync_env->store->ctx()), sync_env(_sync_env), + status(status), shards_info(status.num_shards), + lease_cr(nullptr), lease_stack(nullptr) {} + + ~RGWInitSyncStatusCoroutine() override { + if (lease_cr) { + lease_cr->abort(); + } + } + + int operate(const DoutPrefixProvider *dpp) override { + int ret; + reenter(this) { + yield { + set_status("acquiring sync lock"); + uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; + string lock_name = "sync_lock"; + rgw::sal::RadosStore* store = sync_env->store; + lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, store, + rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), + lock_name, lock_duration, this)); + lease_stack.reset(spawn(lease_cr.get(), false)); + } + while (!lease_cr->is_locked()) { + if (lease_cr->is_done()) { + ldpp_dout(dpp, 5) << "failed to take lease" << dendl; + set_status("lease lock failed, early abort"); + return set_cr_error(lease_cr->get_ret_status()); + } + set_sleeping(true); + yield; + } + yield { + set_status("writing sync status"); + rgw::sal::RadosStore* store = sync_env->store; + call(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, store->svc()->sysobj, + rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), + status)); + } + + if (retcode < 0) { + set_status("failed to write sync status"); + ldpp_dout(dpp, 0) << "ERROR: failed to write sync status, retcode=" << retcode << dendl; + yield lease_cr->go_down(); + return set_cr_error(retcode); + } + /* fetch current position in logs */ + set_status("fetching remote log position"); + yield { + for (int i = 0; i < (int)status.num_shards; i++) { + spawn(new RGWReadRemoteMDLogShardInfoCR(sync_env, status.period, i, + &shards_info[i]), false); + } + } + + drain_all_but_stack(lease_stack.get()); /* the lease cr still needs to run */ + + yield { + set_status("updating sync status"); + for (int i = 0; i < (int)status.num_shards; i++) { + rgw_meta_sync_marker marker; + RGWMetadataLogInfo& info = shards_info[i]; + marker.next_step_marker = info.marker; + marker.timestamp = info.last_update; + rgw::sal::RadosStore* store = sync_env->store; + spawn(new RGWSimpleRadosWriteCR(dpp, + sync_env->async_rados, + store->svc()->sysobj, + rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->shard_obj_name(i)), + marker), true); + } + } + yield { + set_status("changing sync state: build full sync maps"); + status.state = rgw_meta_sync_info::StateBuildingFullSyncMaps; + rgw::sal::RadosStore* store = sync_env->store; + call(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, store->svc()->sysobj, + rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), + status)); + } + set_status("drop lock lease"); + yield lease_cr->go_down(); + while (collect(&ret, NULL)) { + if (ret < 0) { + return set_cr_error(ret); + } + yield; + } + drain_all(); + return set_cr_done(); + } + return 0; + } +}; + +class RGWReadSyncStatusMarkersCR : public RGWShardCollectCR { + static constexpr int MAX_CONCURRENT_SHARDS = 16; + + RGWMetaSyncEnv *env; + const int num_shards; + int shard_id{0}; + map& markers; + + int handle_result(int r) override { + if (r == -ENOENT) { // ENOENT is not a fatal error + return 0; + } + if (r < 0) { + ldout(cct, 4) << "failed to read metadata sync markers: " + << cpp_strerror(r) << dendl; + } + return r; + } + public: + RGWReadSyncStatusMarkersCR(RGWMetaSyncEnv *env, int num_shards, + map& markers) + : RGWShardCollectCR(env->cct, MAX_CONCURRENT_SHARDS), + env(env), num_shards(num_shards), markers(markers) + {} + bool spawn_next() override; +}; + +bool RGWReadSyncStatusMarkersCR::spawn_next() +{ + if (shard_id >= num_shards) { + return false; + } + using CR = RGWSimpleRadosReadCR; + rgw_raw_obj obj{env->store->svc()->zone->get_zone_params().log_pool, + env->shard_obj_name(shard_id)}; + spawn(new CR(env->dpp, env->async_rados, env->store->svc()->sysobj, obj, &markers[shard_id]), false); + shard_id++; + return true; +} + +class RGWReadSyncStatusCoroutine : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + rgw_meta_sync_status *sync_status; + +public: + RGWReadSyncStatusCoroutine(RGWMetaSyncEnv *_sync_env, + rgw_meta_sync_status *_status) + : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), sync_status(_status) + {} + int operate(const DoutPrefixProvider *dpp) override; +}; + +int RGWReadSyncStatusCoroutine::operate(const DoutPrefixProvider *dpp) +{ + reenter(this) { + // read sync info + using ReadInfoCR = RGWSimpleRadosReadCR; + yield { + bool empty_on_enoent = false; // fail on ENOENT + rgw_raw_obj obj{sync_env->store->svc()->zone->get_zone_params().log_pool, + sync_env->status_oid()}; + call(new ReadInfoCR(dpp, sync_env->async_rados, sync_env->store->svc()->sysobj, obj, + &sync_status->sync_info, empty_on_enoent)); + } + if (retcode < 0) { + ldpp_dout(dpp, 4) << "failed to read sync status info with " + << cpp_strerror(retcode) << dendl; + return set_cr_error(retcode); + } + // read shard markers + using ReadMarkersCR = RGWReadSyncStatusMarkersCR; + yield call(new ReadMarkersCR(sync_env, sync_status->sync_info.num_shards, + sync_status->sync_markers)); + if (retcode < 0) { + ldpp_dout(dpp, 4) << "failed to read sync status markers with " + << cpp_strerror(retcode) << dendl; + return set_cr_error(retcode); + } + return set_cr_done(); + } + return 0; +} + +class RGWFetchAllMetaCR : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + + int num_shards; + + + int ret_status; + + list sections; + list::iterator sections_iter; + + struct meta_list_result { + list keys; + string marker; + uint64_t count{0}; + bool truncated{false}; + + void decode_json(JSONObj *obj) { + JSONDecoder::decode_json("keys", keys, obj); + JSONDecoder::decode_json("marker", marker, obj); + JSONDecoder::decode_json("count", count, obj); + JSONDecoder::decode_json("truncated", truncated, obj); + } + } result; + list::iterator iter; + + std::unique_ptr entries_index; + + boost::intrusive_ptr lease_cr; + boost::intrusive_ptr lease_stack; + bool lost_lock; + bool failed; + + string marker; + + map& markers; + + RGWSyncTraceNodeRef tn; + +public: + RGWFetchAllMetaCR(RGWMetaSyncEnv *_sync_env, int _num_shards, + map& _markers, + RGWSyncTraceNodeRef& _tn_parent) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), + num_shards(_num_shards), + ret_status(0), lease_cr(nullptr), lease_stack(nullptr), + lost_lock(false), failed(false), markers(_markers) { + tn = sync_env->sync_tracer->add_node(_tn_parent, "fetch_all_meta"); + } + + ~RGWFetchAllMetaCR() override { + } + + void append_section_from_set(set& all_sections, const string& name) { + set::iterator iter = all_sections.find(name); + if (iter != all_sections.end()) { + sections.emplace_back(std::move(*iter)); + all_sections.erase(iter); + } + } + /* + * meta sync should go in the following order: user, bucket.instance, bucket + * then whatever other sections exist (if any) + */ + void rearrange_sections() { + set all_sections; + std::move(sections.begin(), sections.end(), + std::inserter(all_sections, all_sections.end())); + sections.clear(); + + append_section_from_set(all_sections, "user"); + append_section_from_set(all_sections, "bucket.instance"); + append_section_from_set(all_sections, "bucket"); + append_section_from_set(all_sections, "roles"); + + std::move(all_sections.begin(), all_sections.end(), + std::back_inserter(sections)); + } + + int operate(const DoutPrefixProvider *dpp) override { + RGWRESTConn *conn = sync_env->conn; + + reenter(this) { + yield { + set_status(string("acquiring lock (") + sync_env->status_oid() + ")"); + uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; + string lock_name = "sync_lock"; + lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, + sync_env->store, + rgw_raw_obj(sync_env->store->svc()->zone->get_zone_params().log_pool, sync_env->status_oid()), + lock_name, lock_duration, this)); + lease_stack.reset(spawn(lease_cr.get(), false)); + } + while (!lease_cr->is_locked()) { + if (lease_cr->is_done()) { + ldpp_dout(dpp, 5) << "failed to take lease" << dendl; + set_status("lease lock failed, early abort"); + return set_cr_error(lease_cr->get_ret_status()); + } + set_sleeping(true); + yield; + } + entries_index.reset(new RGWShardedOmapCRManager(sync_env->async_rados, sync_env->store, this, num_shards, + sync_env->store->svc()->zone->get_zone_params().log_pool, + mdlog_sync_full_sync_index_prefix)); + yield { + call(new RGWReadRESTResourceCR >(cct, conn, sync_env->http_manager, + "/admin/metadata", NULL, §ions)); + } + if (get_ret_status() < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to fetch metadata sections" << dendl; + yield entries_index->finish(); + yield lease_cr->go_down(); + drain_all(); + return set_cr_error(get_ret_status()); + } + rearrange_sections(); + sections_iter = sections.begin(); + for (; sections_iter != sections.end(); ++sections_iter) { + do { + yield { +#define META_FULL_SYNC_CHUNK_SIZE "1000" + string entrypoint = string("/admin/metadata/") + *sections_iter; + rgw_http_param_pair pairs[] = { { "max-entries", META_FULL_SYNC_CHUNK_SIZE }, + { "marker", result.marker.c_str() }, + { NULL, NULL } }; + result.keys.clear(); + call(new RGWReadRESTResourceCR(cct, conn, sync_env->http_manager, + entrypoint, pairs, &result)); + } + ret_status = get_ret_status(); + if (ret_status == -ENOENT) { + set_retcode(0); /* reset coroutine status so that we don't return it */ + ret_status = 0; + } + if (ret_status < 0) { + tn->log(0, SSTR("ERROR: failed to fetch metadata section: " << *sections_iter)); + yield entries_index->finish(); + yield lease_cr->go_down(); + drain_all(); + return set_cr_error(ret_status); + } + iter = result.keys.begin(); + for (; iter != result.keys.end(); ++iter) { + if (!lease_cr->is_locked()) { + lost_lock = true; + tn->log(1, "lease is lost, abort"); + break; + } + yield; // allow entries_index consumer to make progress + + tn->log(20, SSTR("list metadata: section=" << *sections_iter << " key=" << *iter)); + string s = *sections_iter + ":" + *iter; + int shard_id; + rgw::sal::RadosStore* store = sync_env->store; + int ret = store->ctl()->meta.mgr->get_shard_id(*sections_iter, *iter, &shard_id); + if (ret < 0) { + tn->log(0, SSTR("ERROR: could not determine shard id for " << *sections_iter << ":" << *iter)); + ret_status = ret; + break; + } + if (!entries_index->append(s, shard_id)) { + break; + } + } + } while (result.truncated); + } + yield { + if (!entries_index->finish()) { + failed = true; + } + } + if (!failed) { + for (map::iterator iter = markers.begin(); iter != markers.end(); ++iter) { + int shard_id = (int)iter->first; + rgw_meta_sync_marker& marker = iter->second; + marker.total_entries = entries_index->get_total_entries(shard_id); + spawn(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, sync_env->store->svc()->sysobj, + rgw_raw_obj(sync_env->store->svc()->zone->get_zone_params().log_pool, sync_env->shard_obj_name(shard_id)), + marker), true); + } + } + + drain_all_but_stack(lease_stack.get()); /* the lease cr still needs to run */ + + yield lease_cr->go_down(); + + int ret; + while (collect(&ret, NULL)) { + if (ret < 0) { + return set_cr_error(ret); + } + yield; + } + drain_all(); + if (failed) { + yield return set_cr_error(-EIO); + } + if (lost_lock) { + yield return set_cr_error(-EBUSY); + } + + if (ret_status < 0) { + yield return set_cr_error(ret_status); + } + + yield return set_cr_done(); + } + return 0; + } +}; + +static string full_sync_index_shard_oid(int shard_id) +{ + char buf[mdlog_sync_full_sync_index_prefix.size() + 16]; + snprintf(buf, sizeof(buf), "%s.%d", mdlog_sync_full_sync_index_prefix.c_str(), shard_id); + return string(buf); +} + +class RGWReadRemoteMetadataCR : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + + RGWRESTReadResource *http_op; + + string section; + string key; + + bufferlist *pbl; + + RGWSyncTraceNodeRef tn; + +public: + RGWReadRemoteMetadataCR(RGWMetaSyncEnv *_sync_env, + const string& _section, const string& _key, bufferlist *_pbl, + const RGWSyncTraceNodeRef& _tn_parent) : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), + http_op(NULL), + section(_section), + key(_key), + pbl(_pbl) { + tn = sync_env->sync_tracer->add_node(_tn_parent, "read_remote_meta", + section + ":" + key); + } + + int operate(const DoutPrefixProvider *dpp) override { + RGWRESTConn *conn = sync_env->conn; + reenter(this) { + yield { + string key_encode; + url_encode(key, key_encode); + rgw_http_param_pair pairs[] = { { "key" , key.c_str()}, + { NULL, NULL } }; + + string p = string("/admin/metadata/") + section + "/" + key_encode; + + http_op = new RGWRESTReadResource(conn, p, pairs, NULL, sync_env->http_manager); + + init_new_io(http_op); + + int ret = http_op->aio_read(dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog data" << dendl; + log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; + http_op->put(); + return set_cr_error(ret); + } + + return io_block(0); + } + yield { + int ret = http_op->wait(pbl, null_yield); + http_op->put(); + if (ret < 0) { + return set_cr_error(ret); + } + return set_cr_done(); + } + } + return 0; + } +}; + +class RGWAsyncMetaStoreEntry : public RGWAsyncRadosRequest { + rgw::sal::RadosStore* store; + string raw_key; + bufferlist bl; + const DoutPrefixProvider *dpp; +protected: + int _send_request(const DoutPrefixProvider *dpp) override { + int ret = store->ctl()->meta.mgr->put(raw_key, bl, null_yield, dpp, RGWMDLogSyncType::APPLY_ALWAYS, true); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: can't store key: " << raw_key << " ret=" << ret << dendl; + return ret; + } + return 0; + } +public: + RGWAsyncMetaStoreEntry(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store, + const string& _raw_key, + bufferlist& _bl, + const DoutPrefixProvider *dpp) : RGWAsyncRadosRequest(caller, cn), store(_store), + raw_key(_raw_key), bl(_bl), dpp(dpp) {} +}; + + +class RGWMetaStoreEntryCR : public RGWSimpleCoroutine { + RGWMetaSyncEnv *sync_env; + string raw_key; + bufferlist bl; + + RGWAsyncMetaStoreEntry *req; + +public: + RGWMetaStoreEntryCR(RGWMetaSyncEnv *_sync_env, + const string& _raw_key, + bufferlist& _bl) : RGWSimpleCoroutine(_sync_env->cct), sync_env(_sync_env), + raw_key(_raw_key), bl(_bl), req(NULL) { + } + + ~RGWMetaStoreEntryCR() override { + if (req) { + req->finish(); + } + } + + int send_request(const DoutPrefixProvider *dpp) override { + req = new RGWAsyncMetaStoreEntry(this, stack->create_completion_notifier(), + sync_env->store, raw_key, bl, dpp); + sync_env->async_rados->queue(req); + return 0; + } + + int request_complete() override { + return req->get_ret_status(); + } +}; + +class RGWAsyncMetaRemoveEntry : public RGWAsyncRadosRequest { + rgw::sal::RadosStore* store; + string raw_key; + const DoutPrefixProvider *dpp; +protected: + int _send_request(const DoutPrefixProvider *dpp) override { + int ret = store->ctl()->meta.mgr->remove(raw_key, null_yield, dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: can't remove key: " << raw_key << " ret=" << ret << dendl; + return ret; + } + return 0; + } +public: + RGWAsyncMetaRemoveEntry(RGWCoroutine *caller, RGWAioCompletionNotifier *cn, rgw::sal::RadosStore* _store, + const string& _raw_key, const DoutPrefixProvider *dpp) : RGWAsyncRadosRequest(caller, cn), store(_store), + raw_key(_raw_key), dpp(dpp) {} +}; + + +class RGWMetaRemoveEntryCR : public RGWSimpleCoroutine { + RGWMetaSyncEnv *sync_env; + string raw_key; + + RGWAsyncMetaRemoveEntry *req; + +public: + RGWMetaRemoveEntryCR(RGWMetaSyncEnv *_sync_env, + const string& _raw_key) : RGWSimpleCoroutine(_sync_env->cct), sync_env(_sync_env), + raw_key(_raw_key), req(NULL) { + } + + ~RGWMetaRemoveEntryCR() override { + if (req) { + req->finish(); + } + } + + int send_request(const DoutPrefixProvider *dpp) override { + req = new RGWAsyncMetaRemoveEntry(this, stack->create_completion_notifier(), + sync_env->store, raw_key, dpp); + sync_env->async_rados->queue(req); + return 0; + } + + int request_complete() override { + int r = req->get_ret_status(); + if (r == -ENOENT) { + r = 0; + } + return r; + } +}; + +#define META_SYNC_UPDATE_MARKER_WINDOW 10 + + +int RGWLastCallerWinsCR::operate(const DoutPrefixProvider *dpp) { + RGWCoroutine *call_cr; + reenter(this) { + while (cr) { + call_cr = cr; + cr = nullptr; + yield call(call_cr); + /* cr might have been modified at this point */ + } + return set_cr_done(); + } + return 0; +} + +class RGWMetaSyncShardMarkerTrack : public RGWSyncShardMarkerTrack { + RGWMetaSyncEnv *sync_env; + + string marker_oid; + rgw_meta_sync_marker sync_marker; + + RGWSyncTraceNodeRef tn; + +public: + RGWMetaSyncShardMarkerTrack(RGWMetaSyncEnv *_sync_env, + const string& _marker_oid, + const rgw_meta_sync_marker& _marker, + RGWSyncTraceNodeRef& _tn) : RGWSyncShardMarkerTrack(META_SYNC_UPDATE_MARKER_WINDOW), + sync_env(_sync_env), + marker_oid(_marker_oid), + sync_marker(_marker), + tn(_tn){} + + RGWCoroutine *store_marker(const string& new_marker, uint64_t index_pos, const real_time& timestamp) override { + sync_marker.marker = new_marker; + if (index_pos > 0) { + sync_marker.pos = index_pos; + } + + if (!real_clock::is_zero(timestamp)) { + sync_marker.timestamp = timestamp; + } + + ldpp_dout(sync_env->dpp, 20) << __func__ << "(): updating marker marker_oid=" << marker_oid << " marker=" << new_marker << " realm_epoch=" << sync_marker.realm_epoch << dendl; + tn->log(20, SSTR("new marker=" << new_marker)); + rgw::sal::RadosStore* store = sync_env->store; + return new RGWSimpleRadosWriteCR(sync_env->dpp, sync_env->async_rados, + store->svc()->sysobj, + rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, marker_oid), + sync_marker); + } + + RGWOrderCallCR *allocate_order_control_cr() override { + return new RGWLastCallerWinsCR(sync_env->cct); + } +}; + +RGWMetaSyncSingleEntryCR::RGWMetaSyncSingleEntryCR(RGWMetaSyncEnv *_sync_env, + const string& _raw_key, const string& _entry_marker, + const RGWMDLogStatus& _op_status, + RGWMetaSyncShardMarkerTrack *_marker_tracker, const RGWSyncTraceNodeRef& _tn_parent) : RGWCoroutine(_sync_env->cct), + sync_env(_sync_env), + raw_key(_raw_key), entry_marker(_entry_marker), + op_status(_op_status), + pos(0), sync_status(0), + marker_tracker(_marker_tracker), tries(0) { + error_injection = (sync_env->cct->_conf->rgw_sync_meta_inject_err_probability > 0); + tn = sync_env->sync_tracer->add_node(_tn_parent, "entry", raw_key); +} + +int RGWMetaSyncSingleEntryCR::operate(const DoutPrefixProvider *dpp) { + reenter(this) { +#define NUM_TRANSIENT_ERROR_RETRIES 10 + + if (error_injection && + rand() % 10000 < cct->_conf->rgw_sync_meta_inject_err_probability * 10000.0) { + return set_cr_error(-EIO); + } + + if (op_status != MDLOG_STATUS_COMPLETE) { + tn->log(20, "skipping pending operation"); + yield call(marker_tracker->finish(entry_marker)); + if (retcode < 0) { + return set_cr_error(retcode); + } + return set_cr_done(); + } + tn->set_flag(RGW_SNS_FLAG_ACTIVE); + for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) { + yield { + pos = raw_key.find(':'); + section = raw_key.substr(0, pos); + key = raw_key.substr(pos + 1); + tn->log(10, SSTR("fetching remote metadata entry" << (tries == 0 ? "" : " (retry)"))); + call(new RGWReadRemoteMetadataCR(sync_env, section, key, &md_bl, tn)); + } + + sync_status = retcode; + + if (sync_status == -ENOENT) { + break; + } + + if (sync_status < 0) { + if (tries < NUM_TRANSIENT_ERROR_RETRIES - 1) { + ldpp_dout(dpp, 20) << *this << ": failed to fetch remote metadata entry: " << section << ":" << key << ", will retry" << dendl; + continue; + } + + tn->log(10, SSTR("failed to read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status)); + log_error() << "failed to read remote metadata entry: section=" << section << " key=" << key << " status=" << sync_status << std::endl; + yield call(sync_env->error_logger->log_error_cr(dpp, sync_env->conn->get_remote_id(), section, key, -sync_status, + string("failed to read remote metadata entry: ") + cpp_strerror(-sync_status))); + return set_cr_error(sync_status); + } + + break; + } + + retcode = 0; + for (tries = 0; tries < NUM_TRANSIENT_ERROR_RETRIES; tries++) { + if (sync_status != -ENOENT) { + tn->log(10, SSTR("storing local metadata entry: " << section << ":" << key)); + yield call(new RGWMetaStoreEntryCR(sync_env, raw_key, md_bl)); + } else { + tn->log(10, SSTR("removing local metadata entry:" << section << ":" << key)); + yield call(new RGWMetaRemoveEntryCR(sync_env, raw_key)); + if (retcode == -ENOENT) { + retcode = 0; + break; + } + } + if ((retcode < 0) && (tries < NUM_TRANSIENT_ERROR_RETRIES - 1)) { + ldpp_dout(dpp, 20) << *this << ": failed to store metadata entry: " << section << ":" << key << ", got retcode=" << retcode << ", will retry" << dendl; + continue; + } + break; + } + + sync_status = retcode; + + if (sync_status == 0 && marker_tracker) { + /* update marker */ + yield call(marker_tracker->finish(entry_marker)); + sync_status = retcode; + } + if (sync_status < 0) { + tn->log(10, SSTR("failed, status=" << sync_status)); + return set_cr_error(sync_status); + } + tn->log(10, "success"); + return set_cr_done(); + } + return 0; +} + +class RGWCloneMetaLogCoroutine : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + RGWMetadataLog *mdlog; + + const std::string& period; + int shard_id; + string marker; + bool truncated = false; + string *new_marker; + + int max_entries = CLONE_MAX_ENTRIES; + + RGWRESTReadResource *http_op = nullptr; + boost::intrusive_ptr completion; + + RGWMetadataLogInfo shard_info; + rgw_mdlog_shard_data data; + +public: + RGWCloneMetaLogCoroutine(RGWMetaSyncEnv *_sync_env, RGWMetadataLog* mdlog, + const std::string& period, int _id, + const string& _marker, string *_new_marker) + : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), mdlog(mdlog), + period(period), shard_id(_id), marker(_marker), new_marker(_new_marker) { + if (new_marker) { + *new_marker = marker; + } + } + ~RGWCloneMetaLogCoroutine() override { + if (http_op) { + http_op->put(); + } + if (completion) { + completion->cancel(); + } + } + + int operate(const DoutPrefixProvider *dpp) override; + + int state_init(); + int state_read_shard_status(); + int state_read_shard_status_complete(); + int state_send_rest_request(const DoutPrefixProvider *dpp); + int state_receive_rest_response(); + int state_store_mdlog_entries(); + int state_store_mdlog_entries_complete(); +}; + +class RGWMetaSyncShardCR : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + + const rgw_pool& pool; + const std::string& period; //< currently syncing period id + const epoch_t realm_epoch; //< realm_epoch of period + RGWMetadataLog* mdlog; //< log of syncing period + uint32_t shard_id; + rgw_meta_sync_marker& sync_marker; + boost::optional temp_marker; //< for pending updates + string marker; + string max_marker; + const std::string& period_marker; //< max marker stored in next period + + RGWRadosGetOmapKeysCR::ResultPtr omapkeys; + std::set entries; + std::set::iterator iter; + + string oid; + + RGWMetaSyncShardMarkerTrack *marker_tracker = nullptr; + + list log_entries; + list::iterator log_iter; + bool truncated = false; + + string mdlog_marker; + string raw_key; + rgw_mdlog_entry mdlog_entry; + + ceph::mutex inc_lock = ceph::make_mutex("RGWMetaSyncShardCR::inc_lock"); + ceph::condition_variable inc_cond; + + boost::asio::coroutine incremental_cr; + boost::asio::coroutine full_cr; + + boost::intrusive_ptr lease_cr; + boost::intrusive_ptr lease_stack; + + bool lost_lock = false; + + bool *reset_backoff; + + // hold a reference to the cr stack while it's in the map + using StackRef = boost::intrusive_ptr; + map stack_to_pos; + map pos_to_prev; + + bool can_adjust_marker = false; + bool done_with_period = false; + + int total_entries = 0; + + RGWSyncTraceNodeRef tn; +public: + RGWMetaSyncShardCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, + const std::string& period, epoch_t realm_epoch, + RGWMetadataLog* mdlog, uint32_t _shard_id, + rgw_meta_sync_marker& _marker, + const std::string& period_marker, bool *_reset_backoff, + RGWSyncTraceNodeRef& _tn) + : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), pool(_pool), + period(period), realm_epoch(realm_epoch), mdlog(mdlog), + shard_id(_shard_id), sync_marker(_marker), + period_marker(period_marker), + reset_backoff(_reset_backoff), tn(_tn) { + *reset_backoff = false; + } + + ~RGWMetaSyncShardCR() override { + delete marker_tracker; + if (lease_cr) { + lease_cr->abort(); + } + } + + void set_marker_tracker(RGWMetaSyncShardMarkerTrack *mt) { + delete marker_tracker; + marker_tracker = mt; + } + + int operate(const DoutPrefixProvider *dpp) override { + int r; + while (true) { + switch (sync_marker.state) { + case rgw_meta_sync_marker::FullSync: + r = full_sync(); + if (r < 0) { + ldpp_dout(dpp, 10) << "sync: full_sync: shard_id=" << shard_id << " r=" << r << dendl; + return set_cr_error(r); + } + return 0; + case rgw_meta_sync_marker::IncrementalSync: + r = incremental_sync(); + if (r < 0) { + ldpp_dout(dpp, 10) << "sync: incremental_sync: shard_id=" << shard_id << " r=" << r << dendl; + return set_cr_error(r); + } + return 0; + } + } + /* unreachable */ + return 0; + } + + void collect_children() + { + int child_ret; + RGWCoroutinesStack *child; + while (collect_next(&child_ret, &child)) { + auto iter = stack_to_pos.find(child); + if (iter == stack_to_pos.end()) { + /* some other stack that we don't care about */ + continue; + } + + string& pos = iter->second; + + if (child_ret < 0) { + ldpp_dout(sync_env->dpp, 0) << *this << ": child operation stack=" << child << " entry=" << pos << " returned " << child_ret << dendl; + // on any error code from RGWMetaSyncSingleEntryCR, we do not advance + // the sync status marker past this entry, and set + // can_adjust_marker=false to exit out of RGWMetaSyncShardCR. + // RGWMetaSyncShardControlCR will rerun RGWMetaSyncShardCR from the + // previous marker and retry + can_adjust_marker = false; + } + + map::iterator prev_iter = pos_to_prev.find(pos); + ceph_assert(prev_iter != pos_to_prev.end()); + + if (pos_to_prev.size() == 1) { + if (can_adjust_marker) { + sync_marker.marker = pos; + } + pos_to_prev.erase(prev_iter); + } else { + ceph_assert(pos_to_prev.size() > 1); + pos_to_prev.erase(prev_iter); + prev_iter = pos_to_prev.begin(); + if (can_adjust_marker) { + sync_marker.marker = prev_iter->second; + } + } + + ldpp_dout(sync_env->dpp, 4) << *this << ": adjusting marker pos=" << sync_marker.marker << dendl; + stack_to_pos.erase(iter); + } + } + + int full_sync() { +#define OMAP_GET_MAX_ENTRIES 100 + int max_entries = OMAP_GET_MAX_ENTRIES; + reenter(&full_cr) { + set_status("full_sync"); + tn->log(10, "start full sync"); + oid = full_sync_index_shard_oid(shard_id); + can_adjust_marker = true; + /* grab lock */ + yield { + uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; + string lock_name = "sync_lock"; + rgw::sal::RadosStore* store = sync_env->store; + lease_cr.reset(new RGWContinuousLeaseCR(sync_env->async_rados, store, + rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), + lock_name, lock_duration, this)); + lease_stack.reset(spawn(lease_cr.get(), false)); + lost_lock = false; + } + while (!lease_cr->is_locked()) { + if (lease_cr->is_done()) { + drain_all(); + tn->log(5, "failed to take lease"); + return lease_cr->get_ret_status(); + } + set_sleeping(true); + yield; + } + tn->log(10, "took lease"); + + /* lock succeeded, a retry now should avoid previous backoff status */ + *reset_backoff = true; + + /* prepare marker tracker */ + set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env, + sync_env->shard_obj_name(shard_id), + sync_marker, tn)); + + marker = sync_marker.marker; + + total_entries = sync_marker.pos; + + /* sync! */ + do { + if (!lease_cr->is_locked()) { + tn->log(1, "lease is lost, abort"); + lost_lock = true; + break; + } + omapkeys = std::make_shared(); + yield call(new RGWRadosGetOmapKeysCR(sync_env->store, rgw_raw_obj(pool, oid), + marker, max_entries, omapkeys)); + if (retcode < 0) { + ldpp_dout(sync_env->dpp, 0) << "ERROR: " << __func__ << "(): RGWRadosGetOmapKeysCR() returned ret=" << retcode << dendl; + tn->log(0, SSTR("ERROR: failed to list omap keys, status=" << retcode)); + yield lease_cr->go_down(); + drain_all(); + return retcode; + } + entries = std::move(omapkeys->entries); + tn->log(20, SSTR("retrieved " << entries.size() << " entries to sync")); + if (entries.size() > 0) { + tn->set_flag(RGW_SNS_FLAG_ACTIVE); /* actually have entries to sync */ + } + iter = entries.begin(); + for (; iter != entries.end(); ++iter) { + marker = *iter; + tn->log(20, SSTR("full sync: " << marker)); + total_entries++; + if (!marker_tracker->start(marker, total_entries, real_time())) { + tn->log(0, SSTR("ERROR: cannot start syncing " << marker << ". Duplicate entry?")); + } else { + // fetch remote and write locally + yield { + RGWCoroutinesStack *stack = spawn(new RGWMetaSyncSingleEntryCR(sync_env, marker, marker, MDLOG_STATUS_COMPLETE, marker_tracker, tn), false); + // stack_to_pos holds a reference to the stack + stack_to_pos[stack] = marker; + pos_to_prev[marker] = marker; + } + // limit spawn window + while (num_spawned() > static_cast(cct->_conf->rgw_meta_sync_spawn_window)) { + yield wait_for_child(); + collect_children(); + } + } + } + collect_children(); + } while (omapkeys->more && can_adjust_marker); + + tn->unset_flag(RGW_SNS_FLAG_ACTIVE); /* actually have entries to sync */ + + while (num_spawned() > 1) { + yield wait_for_child(); + collect_children(); + } + + if (!lost_lock) { + /* update marker to reflect we're done with full sync */ + if (can_adjust_marker) { + // apply updates to a temporary marker, or operate() will send us + // to incremental_sync() after we yield + temp_marker = sync_marker; + temp_marker->state = rgw_meta_sync_marker::IncrementalSync; + temp_marker->marker = std::move(temp_marker->next_step_marker); + temp_marker->next_step_marker.clear(); + temp_marker->realm_epoch = realm_epoch; + ldpp_dout(sync_env->dpp, 4) << *this << ": saving marker pos=" << temp_marker->marker << " realm_epoch=" << realm_epoch << dendl; + + using WriteMarkerCR = RGWSimpleRadosWriteCR; + yield call(new WriteMarkerCR(sync_env->dpp, sync_env->async_rados, sync_env->store->svc()->sysobj, + rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), + *temp_marker)); + } + + if (retcode < 0) { + ldpp_dout(sync_env->dpp, 0) << "ERROR: failed to set sync marker: retcode=" << retcode << dendl; + yield lease_cr->go_down(); + drain_all(); + return retcode; + } + // clean up full sync index + yield { + auto oid = full_sync_index_shard_oid(shard_id); + call(new RGWRadosRemoveCR(sync_env->store, {pool, oid})); + } + } + + /* + * if we reached here, it means that lost_lock is true, otherwise the state + * change in the previous block will prevent us from reaching here + */ + + yield lease_cr->go_down(); + + lease_cr.reset(); + + drain_all(); + + if (!can_adjust_marker) { + return -EAGAIN; + } + + if (lost_lock) { + return -EBUSY; + } + + tn->log(10, "full sync complete"); + + // apply the sync marker update + ceph_assert(temp_marker); + sync_marker = std::move(*temp_marker); + temp_marker = boost::none; + // must not yield after this point! + } + return 0; + } + + + int incremental_sync() { + reenter(&incremental_cr) { + set_status("incremental_sync"); + tn->log(10, "start incremental sync"); + can_adjust_marker = true; + /* grab lock */ + if (!lease_cr) { /* could have had a lease_cr lock from previous state */ + yield { + uint32_t lock_duration = cct->_conf->rgw_sync_lease_period; + string lock_name = "sync_lock"; + rgw::sal::RadosStore* store = sync_env->store; + lease_cr.reset( new RGWContinuousLeaseCR(sync_env->async_rados, store, + rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), + lock_name, lock_duration, this)); + lease_stack.reset(spawn(lease_cr.get(), false)); + lost_lock = false; + } + while (!lease_cr->is_locked()) { + if (lease_cr->is_done()) { + drain_all(); + tn->log(5, "failed to take lease"); + return lease_cr->get_ret_status(); + } + set_sleeping(true); + yield; + } + } + tn->log(10, "took lease"); + // if the period has advanced, we can't use the existing marker + if (sync_marker.realm_epoch < realm_epoch) { + ldpp_dout(sync_env->dpp, 4) << "clearing marker=" << sync_marker.marker + << " from old realm_epoch=" << sync_marker.realm_epoch + << " (now " << realm_epoch << ')' << dendl; + sync_marker.realm_epoch = realm_epoch; + sync_marker.marker.clear(); + } + mdlog_marker = sync_marker.marker; + set_marker_tracker(new RGWMetaSyncShardMarkerTrack(sync_env, + sync_env->shard_obj_name(shard_id), + sync_marker, tn)); + + /* + * mdlog_marker: the remote sync marker positiion + * sync_marker: the local sync marker position + * max_marker: the max mdlog position that we fetched + * marker: the current position we try to sync + * period_marker: the last marker before the next period begins (optional) + */ + marker = max_marker = sync_marker.marker; + /* inc sync */ + do { + if (!lease_cr->is_locked()) { + lost_lock = true; + tn->log(1, "lease is lost, abort"); + break; + } +#define INCREMENTAL_MAX_ENTRIES 100 + ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << " truncated=" << truncated << dendl; + if (!period_marker.empty() && period_marker <= mdlog_marker) { + tn->log(10, SSTR("finished syncing current period: mdlog_marker=" << mdlog_marker << " sync_marker=" << sync_marker.marker << " period_marker=" << period_marker)); + done_with_period = true; + break; + } + if (mdlog_marker <= max_marker || !truncated) { + /* we're at the tip, try to bring more entries */ + ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " syncing mdlog for shard_id=" << shard_id << dendl; + yield call(new RGWCloneMetaLogCoroutine(sync_env, mdlog, + period, shard_id, + mdlog_marker, &mdlog_marker)); + } + if (retcode < 0) { + tn->log(10, SSTR(*this << ": failed to fetch more log entries, retcode=" << retcode)); + yield lease_cr->go_down(); + drain_all(); + *reset_backoff = false; // back off and try again later + return retcode; + } + truncated = true; + *reset_backoff = true; /* if we got to this point, all systems function */ + if (mdlog_marker > max_marker) { + tn->set_flag(RGW_SNS_FLAG_ACTIVE); /* actually have entries to sync */ + tn->log(20, SSTR("mdlog_marker=" << mdlog_marker << " sync_marker=" << sync_marker.marker)); + marker = max_marker; + yield call(new RGWReadMDLogEntriesCR(sync_env, mdlog, shard_id, + &max_marker, INCREMENTAL_MAX_ENTRIES, + &log_entries, &truncated)); + if (retcode < 0) { + tn->log(10, SSTR("failed to list mdlog entries, retcode=" << retcode)); + yield lease_cr->go_down(); + drain_all(); + *reset_backoff = false; // back off and try again later + return retcode; + } + for (log_iter = log_entries.begin(); log_iter != log_entries.end() && !done_with_period; ++log_iter) { + if (!period_marker.empty() && period_marker <= log_iter->id) { + done_with_period = true; + if (period_marker < log_iter->id) { + tn->log(10, SSTR("found key=" << log_iter->id + << " past period_marker=" << period_marker)); + break; + } + ldpp_dout(sync_env->dpp, 10) << "found key at period_marker=" << period_marker << dendl; + // sync this entry, then return control to RGWMetaSyncCR + } + if (!mdlog_entry.convert_from(*log_iter)) { + tn->log(0, SSTR("ERROR: failed to convert mdlog entry, shard_id=" << shard_id << " log_entry: " << log_iter->id << ":" << log_iter->section << ":" << log_iter->name << ":" << log_iter->timestamp << " ... skipping entry")); + continue; + } + tn->log(20, SSTR("log_entry: " << log_iter->id << ":" << log_iter->section << ":" << log_iter->name << ":" << log_iter->timestamp)); + if (!marker_tracker->start(log_iter->id, 0, log_iter->timestamp.to_real_time())) { + ldpp_dout(sync_env->dpp, 0) << "ERROR: cannot start syncing " << log_iter->id << ". Duplicate entry?" << dendl; + } else { + raw_key = log_iter->section + ":" + log_iter->name; + yield { + RGWCoroutinesStack *stack = spawn(new RGWMetaSyncSingleEntryCR(sync_env, raw_key, log_iter->id, mdlog_entry.log_data.status, marker_tracker, tn), false); + ceph_assert(stack); + // stack_to_pos holds a reference to the stack + stack_to_pos[stack] = log_iter->id; + pos_to_prev[log_iter->id] = marker; + } + // limit spawn window + while (num_spawned() > static_cast(cct->_conf->rgw_meta_sync_spawn_window)) { + yield wait_for_child(); + collect_children(); + } + } + marker = log_iter->id; + } + } + collect_children(); + ldpp_dout(sync_env->dpp, 20) << __func__ << ":" << __LINE__ << ": shard_id=" << shard_id << " mdlog_marker=" << mdlog_marker << " max_marker=" << max_marker << " sync_marker.marker=" << sync_marker.marker << " period_marker=" << period_marker << dendl; + if (done_with_period) { + // return control to RGWMetaSyncCR and advance to the next period + tn->log(10, SSTR(*this << ": done with period")); + break; + } + if (mdlog_marker == max_marker && can_adjust_marker) { + tn->unset_flag(RGW_SNS_FLAG_ACTIVE); + yield wait(utime_t(cct->_conf->rgw_meta_sync_poll_interval, 0)); + } + } while (can_adjust_marker); + + tn->unset_flag(RGW_SNS_FLAG_ACTIVE); + + while (num_spawned() > 1) { + yield wait_for_child(); + collect_children(); + } + + yield lease_cr->go_down(); + + drain_all(); + + if (lost_lock) { + return -EBUSY; + } + + if (!can_adjust_marker) { + return -EAGAIN; + } + + return set_cr_done(); + } + /* TODO */ + return 0; + } +}; + +class RGWMetaSyncShardControlCR : public RGWBackoffControlCR +{ + RGWMetaSyncEnv *sync_env; + + const rgw_pool& pool; + const std::string& period; + epoch_t realm_epoch; + RGWMetadataLog* mdlog; + uint32_t shard_id; + rgw_meta_sync_marker sync_marker; + const std::string period_marker; + + RGWSyncTraceNodeRef tn; + + static constexpr bool exit_on_error = false; // retry on all errors +public: + RGWMetaSyncShardControlCR(RGWMetaSyncEnv *_sync_env, const rgw_pool& _pool, + const std::string& period, epoch_t realm_epoch, + RGWMetadataLog* mdlog, uint32_t _shard_id, + const rgw_meta_sync_marker& _marker, + std::string&& period_marker, + RGWSyncTraceNodeRef& _tn_parent) + : RGWBackoffControlCR(_sync_env->cct, exit_on_error), sync_env(_sync_env), + pool(_pool), period(period), realm_epoch(realm_epoch), mdlog(mdlog), + shard_id(_shard_id), sync_marker(_marker), + period_marker(std::move(period_marker)) { + tn = sync_env->sync_tracer->add_node(_tn_parent, "shard", + std::to_string(shard_id)); + } + + RGWCoroutine *alloc_cr() override { + return new RGWMetaSyncShardCR(sync_env, pool, period, realm_epoch, mdlog, + shard_id, sync_marker, period_marker, backoff_ptr(), tn); + } + + RGWCoroutine *alloc_finisher_cr() override { + rgw::sal::RadosStore* store = sync_env->store; + return new RGWSimpleRadosReadCR(sync_env->dpp, sync_env->async_rados, store->svc()->sysobj, + rgw_raw_obj(pool, sync_env->shard_obj_name(shard_id)), + &sync_marker); + } +}; + +class RGWMetaSyncCR : public RGWCoroutine { + RGWMetaSyncEnv *sync_env; + const rgw_pool& pool; + RGWPeriodHistory::Cursor cursor; //< sync position in period history + RGWPeriodHistory::Cursor next; //< next period in history + rgw_meta_sync_status sync_status; + RGWSyncTraceNodeRef tn; + + std::mutex mutex; //< protect access to shard_crs + + // TODO: it should be enough to hold a reference on the stack only, as calling + // RGWCoroutinesStack::wakeup() doesn't refer to the RGWCoroutine if it has + // already completed + using ControlCRRef = boost::intrusive_ptr; + using StackRef = boost::intrusive_ptr; + using RefPair = std::pair; + map shard_crs; + int ret{0}; + +public: + RGWMetaSyncCR(RGWMetaSyncEnv *_sync_env, const RGWPeriodHistory::Cursor &cursor, + const rgw_meta_sync_status& _sync_status, RGWSyncTraceNodeRef& _tn) + : RGWCoroutine(_sync_env->cct), sync_env(_sync_env), + pool(sync_env->store->svc()->zone->get_zone_params().log_pool), + cursor(cursor), sync_status(_sync_status), tn(_tn) {} + + ~RGWMetaSyncCR() { + } + + int operate(const DoutPrefixProvider *dpp) override { + reenter(this) { + // loop through one period at a time + tn->log(1, "start"); + for (;;) { + if (cursor == sync_env->store->svc()->mdlog->get_period_history()->get_current()) { + next = RGWPeriodHistory::Cursor{}; + if (cursor) { + ldpp_dout(dpp, 10) << "RGWMetaSyncCR on current period=" + << cursor.get_period().get_id() << dendl; + } else { + ldpp_dout(dpp, 10) << "RGWMetaSyncCR with no period" << dendl; + } + } else { + next = cursor; + next.next(); + ldpp_dout(dpp, 10) << "RGWMetaSyncCR on period=" + << cursor.get_period().get_id() << ", next=" + << next.get_period().get_id() << dendl; + } + + yield { + // get the mdlog for the current period (may be empty) + auto& period_id = sync_status.sync_info.period; + auto realm_epoch = sync_status.sync_info.realm_epoch; + auto mdlog = sync_env->store->svc()->mdlog->get_log(period_id); + + tn->log(1, SSTR("realm epoch=" << realm_epoch << " period id=" << period_id)); + + // prevent wakeup() from accessing shard_crs while we're spawning them + std::lock_guard lock(mutex); + + // sync this period on each shard + for (const auto& m : sync_status.sync_markers) { + uint32_t shard_id = m.first; + auto& marker = m.second; + + std::string period_marker; + if (next) { + // read the maximum marker from the next period's sync status + period_marker = next.get_period().get_sync_status()[shard_id]; + if (period_marker.empty()) { + // no metadata changes have occurred on this shard, skip it + ldpp_dout(dpp, 10) << "RGWMetaSyncCR: skipping shard " << shard_id + << " with empty period marker" << dendl; + continue; + } + } + + using ShardCR = RGWMetaSyncShardControlCR; + auto cr = new ShardCR(sync_env, pool, period_id, realm_epoch, + mdlog, shard_id, marker, + std::move(period_marker), tn); + auto stack = spawn(cr, false); + shard_crs[shard_id] = RefPair{cr, stack}; + } + } + // wait for each shard to complete + while (ret == 0 && num_spawned() > 0) { + yield wait_for_child(); + collect(&ret, nullptr); + } + drain_all(); + { + // drop shard cr refs under lock + std::lock_guard lock(mutex); + shard_crs.clear(); + } + if (ret < 0) { + return set_cr_error(ret); + } + // advance to the next period + ceph_assert(next); + cursor = next; + + // write the updated sync info + sync_status.sync_info.period = cursor.get_period().get_id(); + sync_status.sync_info.realm_epoch = cursor.get_epoch(); + yield call(new RGWSimpleRadosWriteCR(dpp, sync_env->async_rados, + sync_env->store->svc()->sysobj, + rgw_raw_obj(pool, sync_env->status_oid()), + sync_status.sync_info)); + } + } + return 0; + } + + void wakeup(int shard_id) { + std::lock_guard lock(mutex); + auto iter = shard_crs.find(shard_id); + if (iter == shard_crs.end()) { + return; + } + iter->second.first->wakeup(); + } +}; + +void RGWRemoteMetaLog::init_sync_env(RGWMetaSyncEnv *env) { + env->dpp = dpp; + env->cct = store->ctx(); + env->store = store; + env->conn = conn; + env->async_rados = async_rados; + env->http_manager = &http_manager; + env->error_logger = error_logger; + env->sync_tracer = store->getRados()->get_sync_tracer(); +} + +int RGWRemoteMetaLog::read_sync_status(const DoutPrefixProvider *dpp, rgw_meta_sync_status *sync_status) +{ + if (store->svc()->zone->is_meta_master()) { + return 0; + } + // cannot run concurrently with run_sync(), so run in a separate manager + RGWCoroutinesManager crs(store->ctx(), store->getRados()->get_cr_registry()); + RGWHTTPManager http_manager(store->ctx(), crs.get_completion_mgr()); + int ret = http_manager.start(); + if (ret < 0) { + ldpp_dout(dpp, 0) << "failed in http_manager.start() ret=" << ret << dendl; + return ret; + } + RGWMetaSyncEnv sync_env_local = sync_env; + sync_env_local.http_manager = &http_manager; + tn->log(20, "read sync status"); + ret = crs.run(dpp, new RGWReadSyncStatusCoroutine(&sync_env_local, sync_status)); + http_manager.stop(); + return ret; +} + +int RGWRemoteMetaLog::init_sync_status(const DoutPrefixProvider *dpp) +{ + if (store->svc()->zone->is_meta_master()) { + return 0; + } + + rgw_mdlog_info mdlog_info; + int r = read_log_info(dpp, &mdlog_info); + if (r < 0) { + ldpp_dout(dpp, -1) << "ERROR: fail to fetch master log info (r=" << r << ")" << dendl; + return r; + } + + rgw_meta_sync_info sync_info; + sync_info.num_shards = mdlog_info.num_shards; + auto cursor = store->svc()->mdlog->get_period_history()->get_current(); + if (cursor) { + sync_info.period = cursor.get_period().get_id(); + sync_info.realm_epoch = cursor.get_epoch(); + } + + return run(dpp, new RGWInitSyncStatusCoroutine(&sync_env, sync_info)); +} + +int RGWRemoteMetaLog::store_sync_info(const DoutPrefixProvider *dpp, const rgw_meta_sync_info& sync_info) +{ + tn->log(20, "store sync info"); + return run(dpp, new RGWSimpleRadosWriteCR(dpp, async_rados, store->svc()->sysobj, + rgw_raw_obj(store->svc()->zone->get_zone_params().log_pool, sync_env.status_oid()), + sync_info)); +} + +// return a cursor to the period at our sync position +static RGWPeriodHistory::Cursor get_period_at(const DoutPrefixProvider *dpp, + rgw::sal::RadosStore* store, + const rgw_meta_sync_info& info, + optional_yield y) +{ + if (info.period.empty()) { + // return an empty cursor with error=0 + return RGWPeriodHistory::Cursor{}; + } + + // look for an existing period in our history + auto cursor = store->svc()->mdlog->get_period_history()->lookup(info.realm_epoch); + if (cursor) { + // verify that the period ids match + auto& existing = cursor.get_period().get_id(); + if (existing != info.period) { + ldpp_dout(dpp, -1) << "ERROR: sync status period=" << info.period + << " does not match period=" << existing + << " in history at realm epoch=" << info.realm_epoch << dendl; + return RGWPeriodHistory::Cursor{-EEXIST}; + } + return cursor; + } + + // read the period from rados or pull it from the master + RGWPeriod period; + int r = store->svc()->mdlog->pull_period(dpp, info.period, period, y); + if (r < 0) { + ldpp_dout(dpp, -1) << "ERROR: failed to read period id " + << info.period << ": " << cpp_strerror(r) << dendl; + return RGWPeriodHistory::Cursor{r}; + } + // attach the period to our history + cursor = store->svc()->mdlog->get_period_history()->attach(dpp, std::move(period), y); + if (!cursor) { + r = cursor.get_error(); + ldpp_dout(dpp, -1) << "ERROR: failed to read period history back to " + << info.period << ": " << cpp_strerror(r) << dendl; + } + return cursor; +} + +int RGWRemoteMetaLog::run_sync(const DoutPrefixProvider *dpp, optional_yield y) +{ + if (store->svc()->zone->is_meta_master()) { + return 0; + } + + int r = 0; + + // get shard count and oldest log period from master + rgw_mdlog_info mdlog_info; + for (;;) { + if (going_down) { + ldpp_dout(dpp, 1) << __func__ << "(): going down" << dendl; + return 0; + } + r = read_log_info(dpp, &mdlog_info); + if (r == -EIO || r == -ENOENT) { + // keep retrying if master isn't alive or hasn't initialized the log + ldpp_dout(dpp, 10) << __func__ << "(): waiting for master.." << dendl; + backoff.backoff_sleep(); + continue; + } + backoff.reset(); + if (r < 0) { + ldpp_dout(dpp, -1) << "ERROR: fail to fetch master log info (r=" << r << ")" << dendl; + return r; + } + break; + } + + rgw_meta_sync_status sync_status; + do { + if (going_down) { + ldpp_dout(dpp, 1) << __func__ << "(): going down" << dendl; + return 0; + } + r = run(dpp, new RGWReadSyncStatusCoroutine(&sync_env, &sync_status)); + if (r < 0 && r != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: failed to fetch sync status r=" << r << dendl; + return r; + } + + if (!mdlog_info.period.empty()) { + // restart sync if the remote has a period, but: + // a) our status does not, or + // b) our sync period comes before the remote's oldest log period + if (sync_status.sync_info.period.empty() || + sync_status.sync_info.realm_epoch < mdlog_info.realm_epoch) { + sync_status.sync_info.state = rgw_meta_sync_info::StateInit; + string reason; + if (sync_status.sync_info.period.empty()) { + reason = "period is empty"; + } else { + reason = SSTR("sync_info realm epoch is behind: " << sync_status.sync_info.realm_epoch << " < " << mdlog_info.realm_epoch); + } + tn->log(1, "initialize sync (reason: " + reason + ")"); + ldpp_dout(dpp, 1) << "epoch=" << sync_status.sync_info.realm_epoch + << " in sync status comes before remote's oldest mdlog epoch=" + << mdlog_info.realm_epoch << ", restarting sync" << dendl; + } + } + + if (sync_status.sync_info.state == rgw_meta_sync_info::StateInit) { + ldpp_dout(dpp, 20) << __func__ << "(): init" << dendl; + sync_status.sync_info.num_shards = mdlog_info.num_shards; + auto cursor = store->svc()->mdlog->get_period_history()->get_current(); + if (cursor) { + // run full sync, then start incremental from the current period/epoch + sync_status.sync_info.period = cursor.get_period().get_id(); + sync_status.sync_info.realm_epoch = cursor.get_epoch(); + } + r = run(dpp, new RGWInitSyncStatusCoroutine(&sync_env, sync_status.sync_info)); + if (r == -EBUSY) { + backoff.backoff_sleep(); + continue; + } + backoff.reset(); + if (r < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to init sync status r=" << r << dendl; + return r; + } + } + } while (sync_status.sync_info.state == rgw_meta_sync_info::StateInit); + + auto num_shards = sync_status.sync_info.num_shards; + if (num_shards != mdlog_info.num_shards) { + ldpp_dout(dpp, -1) << "ERROR: can't sync, mismatch between num shards, master num_shards=" << mdlog_info.num_shards << " local num_shards=" << num_shards << dendl; + return -EINVAL; + } + + RGWPeriodHistory::Cursor cursor; + do { + r = run(dpp, new RGWReadSyncStatusCoroutine(&sync_env, &sync_status)); + if (r < 0 && r != -ENOENT) { + tn->log(0, SSTR("ERROR: failed to fetch sync status r=" << r)); + return r; + } + + switch ((rgw_meta_sync_info::SyncState)sync_status.sync_info.state) { + case rgw_meta_sync_info::StateBuildingFullSyncMaps: + tn->log(20, "building full sync maps"); + r = run(dpp, new RGWFetchAllMetaCR(&sync_env, num_shards, sync_status.sync_markers, tn)); + if (r == -EBUSY || r == -EIO) { + backoff.backoff_sleep(); + continue; + } + backoff.reset(); + if (r < 0) { + tn->log(0, SSTR("ERROR: failed to fetch all metadata keys (r=" << r << ")")); + return r; + } + + sync_status.sync_info.state = rgw_meta_sync_info::StateSync; + r = store_sync_info(dpp, sync_status.sync_info); + if (r < 0) { + tn->log(0, SSTR("ERROR: failed to update sync status (r=" << r << ")")); + return r; + } + /* fall through */ + case rgw_meta_sync_info::StateSync: + tn->log(20, "sync"); + // find our position in the period history (if any) + cursor = get_period_at(dpp, store, sync_status.sync_info, y); + r = cursor.get_error(); + if (r < 0) { + return r; + } + meta_sync_cr = new RGWMetaSyncCR(&sync_env, cursor, sync_status, tn); + r = run(dpp, meta_sync_cr); + if (r < 0) { + tn->log(0, "ERROR: failed to fetch all metadata keys"); + return r; + } + break; + default: + tn->log(0, "ERROR: bad sync state!"); + return -EIO; + } + } while (!going_down); + + return 0; +} + +void RGWRemoteMetaLog::wakeup(int shard_id) +{ + if (!meta_sync_cr) { + return; + } + meta_sync_cr->wakeup(shard_id); +} + +int RGWCloneMetaLogCoroutine::operate(const DoutPrefixProvider *dpp) +{ + reenter(this) { + do { + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": init request" << dendl; + return state_init(); + } + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": reading shard status" << dendl; + return state_read_shard_status(); + } + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": reading shard status complete" << dendl; + return state_read_shard_status_complete(); + } + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": sending rest request" << dendl; + return state_send_rest_request(dpp); + } + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": receiving rest response" << dendl; + return state_receive_rest_response(); + } + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": storing mdlog entries" << dendl; + return state_store_mdlog_entries(); + } + } while (truncated); + yield { + ldpp_dout(dpp, 20) << __func__ << ": shard_id=" << shard_id << ": storing mdlog entries complete" << dendl; + return state_store_mdlog_entries_complete(); + } + } + + return 0; +} + +int RGWCloneMetaLogCoroutine::state_init() +{ + data = rgw_mdlog_shard_data(); + + return 0; +} + +int RGWCloneMetaLogCoroutine::state_read_shard_status() +{ + const bool add_ref = false; // default constructs with refs=1 + + completion.reset(new RGWMetadataLogInfoCompletion( + [this](int ret, const cls_log_header& header) { + if (ret < 0) { + if (ret != -ENOENT) { + ldpp_dout(sync_env->dpp, 1) << "ERROR: failed to read mdlog info with " + << cpp_strerror(ret) << dendl; + } + } else { + shard_info.marker = header.max_marker; + shard_info.last_update = header.max_time.to_real_time(); + } + // wake up parent stack + io_complete(); + }), add_ref); + + int ret = mdlog->get_info_async(sync_env->dpp, shard_id, completion.get()); + if (ret < 0) { + ldpp_dout(sync_env->dpp, 0) << "ERROR: mdlog->get_info_async() returned ret=" << ret << dendl; + return set_cr_error(ret); + } + + return io_block(0); +} + +int RGWCloneMetaLogCoroutine::state_read_shard_status_complete() +{ + completion.reset(); + + ldpp_dout(sync_env->dpp, 20) << "shard_id=" << shard_id << " marker=" << shard_info.marker << " last_update=" << shard_info.last_update << dendl; + + marker = shard_info.marker; + + return 0; +} + +int RGWCloneMetaLogCoroutine::state_send_rest_request(const DoutPrefixProvider *dpp) +{ + RGWRESTConn *conn = sync_env->conn; + + char buf[32]; + snprintf(buf, sizeof(buf), "%d", shard_id); + + char max_entries_buf[32]; + snprintf(max_entries_buf, sizeof(max_entries_buf), "%d", max_entries); + + const char *marker_key = (marker.empty() ? "" : "marker"); + + rgw_http_param_pair pairs[] = { { "type", "metadata" }, + { "id", buf }, + { "period", period.c_str() }, + { "max-entries", max_entries_buf }, + { marker_key, marker.c_str() }, + { NULL, NULL } }; + + http_op = new RGWRESTReadResource(conn, "/admin/log", pairs, NULL, sync_env->http_manager); + + init_new_io(http_op); + + int ret = http_op->aio_read(dpp); + if (ret < 0) { + ldpp_dout(dpp, 0) << "ERROR: failed to fetch mdlog data" << dendl; + log_error() << "failed to send http operation: " << http_op->to_str() << " ret=" << ret << std::endl; + http_op->put(); + http_op = NULL; + return set_cr_error(ret); + } + + return io_block(0); +} + +int RGWCloneMetaLogCoroutine::state_receive_rest_response() +{ + int ret = http_op->wait(&data, null_yield); + if (ret < 0) { + error_stream << "http operation failed: " << http_op->to_str() << " status=" << http_op->get_http_status() << std::endl; + ldpp_dout(sync_env->dpp, 5) << "failed to wait for op, ret=" << ret << dendl; + http_op->put(); + http_op = NULL; + return set_cr_error(ret); + } + http_op->put(); + http_op = NULL; + + ldpp_dout(sync_env->dpp, 20) << "remote mdlog, shard_id=" << shard_id << " num of shard entries: " << data.entries.size() << dendl; + + truncated = ((int)data.entries.size() == max_entries); + + if (data.entries.empty()) { + if (new_marker) { + *new_marker = marker; + } + return set_cr_done(); + } + + if (new_marker) { + *new_marker = data.entries.back().id; + } + + return 0; +} + + +int RGWCloneMetaLogCoroutine::state_store_mdlog_entries() +{ + list dest_entries; + + vector::iterator iter; + for (iter = data.entries.begin(); iter != data.entries.end(); ++iter) { + rgw_mdlog_entry& entry = *iter; + ldpp_dout(sync_env->dpp, 20) << "entry: name=" << entry.name << dendl; + + cls_log_entry dest_entry; + dest_entry.id = entry.id; + dest_entry.section = entry.section; + dest_entry.name = entry.name; + dest_entry.timestamp = utime_t(entry.timestamp); + + encode(entry.log_data, dest_entry.data); + + dest_entries.push_back(dest_entry); + + marker = entry.id; + } + + RGWAioCompletionNotifier *cn = stack->create_completion_notifier(); + + int ret = mdlog->store_entries_in_shard(sync_env->dpp, dest_entries, shard_id, cn->completion()); + if (ret < 0) { + cn->put(); + ldpp_dout(sync_env->dpp, 10) << "failed to store md log entries shard_id=" << shard_id << " ret=" << ret << dendl; + return set_cr_error(ret); + } + return io_block(0); +} + +int RGWCloneMetaLogCoroutine::state_store_mdlog_entries_complete() +{ + return set_cr_done(); +} + +void rgw_meta_sync_info::decode_json(JSONObj *obj) +{ + string s; + JSONDecoder::decode_json("status", s, obj); + if (s == "init") { + state = StateInit; + } else if (s == "building-full-sync-maps") { + state = StateBuildingFullSyncMaps; + } else if (s == "sync") { + state = StateSync; + } + JSONDecoder::decode_json("num_shards", num_shards, obj); + JSONDecoder::decode_json("period", period, obj); + JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); +} + +void rgw_meta_sync_info::dump(Formatter *f) const +{ + string s; + switch ((SyncState)state) { + case StateInit: + s = "init"; + break; + case StateBuildingFullSyncMaps: + s = "building-full-sync-maps"; + break; + case StateSync: + s = "sync"; + break; + default: + s = "unknown"; + break; + } + encode_json("status", s, f); + encode_json("num_shards", num_shards, f); + encode_json("period", period, f); + encode_json("realm_epoch", realm_epoch, f); +} + + +void rgw_meta_sync_marker::decode_json(JSONObj *obj) +{ + int s; + JSONDecoder::decode_json("state", s, obj); + state = s; + JSONDecoder::decode_json("marker", marker, obj); + JSONDecoder::decode_json("next_step_marker", next_step_marker, obj); + JSONDecoder::decode_json("total_entries", total_entries, obj); + JSONDecoder::decode_json("pos", pos, obj); + utime_t ut; + JSONDecoder::decode_json("timestamp", ut, obj); + timestamp = ut.to_real_time(); + JSONDecoder::decode_json("realm_epoch", realm_epoch, obj); +} + +void rgw_meta_sync_marker::dump(Formatter *f) const +{ + encode_json("state", (int)state, f); + encode_json("marker", marker, f); + encode_json("next_step_marker", next_step_marker, f); + encode_json("total_entries", total_entries, f); + encode_json("pos", pos, f); + encode_json("timestamp", utime_t(timestamp), f); + encode_json("realm_epoch", realm_epoch, f); +} + +void rgw_meta_sync_status::decode_json(JSONObj *obj) +{ + JSONDecoder::decode_json("info", sync_info, obj); + JSONDecoder::decode_json("markers", sync_markers, obj); +} + +void rgw_meta_sync_status::dump(Formatter *f) const { + encode_json("info", sync_info, f); + encode_json("markers", sync_markers, f); +} + +void rgw_sync_error_info::dump(Formatter *f) const { + encode_json("source_zone", source_zone, f); + encode_json("error_code", error_code, f); + encode_json("message", message, f); +} + diff --git a/src/rgw/rgw_sync.h b/src/rgw/store/rados/rgw_sync.h similarity index 100% rename from src/rgw/rgw_sync.h rename to src/rgw/store/rados/rgw_sync.h diff --git a/src/rgw/rgw_sync_counters.cc b/src/rgw/store/rados/rgw_sync_counters.cc similarity index 100% rename from src/rgw/rgw_sync_counters.cc rename to src/rgw/store/rados/rgw_sync_counters.cc diff --git a/src/rgw/rgw_sync_counters.h b/src/rgw/store/rados/rgw_sync_counters.h similarity index 100% rename from src/rgw/rgw_sync_counters.h rename to src/rgw/store/rados/rgw_sync_counters.h diff --git a/src/rgw/rgw_sync_error_repo.cc b/src/rgw/store/rados/rgw_sync_error_repo.cc similarity index 100% rename from src/rgw/rgw_sync_error_repo.cc rename to src/rgw/store/rados/rgw_sync_error_repo.cc diff --git a/src/rgw/rgw_sync_error_repo.h b/src/rgw/store/rados/rgw_sync_error_repo.h similarity index 100% rename from src/rgw/rgw_sync_error_repo.h rename to src/rgw/store/rados/rgw_sync_error_repo.h diff --git a/src/rgw/rgw_sync_module.cc b/src/rgw/store/rados/rgw_sync_module.cc similarity index 100% rename from src/rgw/rgw_sync_module.cc rename to src/rgw/store/rados/rgw_sync_module.cc diff --git a/src/rgw/rgw_sync_module.h b/src/rgw/store/rados/rgw_sync_module.h similarity index 100% rename from src/rgw/rgw_sync_module.h rename to src/rgw/store/rados/rgw_sync_module.h diff --git a/src/rgw/rgw_sync_module_aws.cc b/src/rgw/store/rados/rgw_sync_module_aws.cc similarity index 100% rename from src/rgw/rgw_sync_module_aws.cc rename to src/rgw/store/rados/rgw_sync_module_aws.cc diff --git a/src/rgw/rgw_sync_module_aws.h b/src/rgw/store/rados/rgw_sync_module_aws.h similarity index 100% rename from src/rgw/rgw_sync_module_aws.h rename to src/rgw/store/rados/rgw_sync_module_aws.h diff --git a/src/rgw/rgw_sync_module_es.cc b/src/rgw/store/rados/rgw_sync_module_es.cc similarity index 100% rename from src/rgw/rgw_sync_module_es.cc rename to src/rgw/store/rados/rgw_sync_module_es.cc diff --git a/src/rgw/rgw_sync_module_es.h b/src/rgw/store/rados/rgw_sync_module_es.h similarity index 100% rename from src/rgw/rgw_sync_module_es.h rename to src/rgw/store/rados/rgw_sync_module_es.h diff --git a/src/rgw/rgw_sync_module_es_rest.cc b/src/rgw/store/rados/rgw_sync_module_es_rest.cc similarity index 100% rename from src/rgw/rgw_sync_module_es_rest.cc rename to src/rgw/store/rados/rgw_sync_module_es_rest.cc diff --git a/src/rgw/rgw_sync_module_es_rest.h b/src/rgw/store/rados/rgw_sync_module_es_rest.h similarity index 100% rename from src/rgw/rgw_sync_module_es_rest.h rename to src/rgw/store/rados/rgw_sync_module_es_rest.h diff --git a/src/rgw/rgw_sync_module_log.cc b/src/rgw/store/rados/rgw_sync_module_log.cc similarity index 100% rename from src/rgw/rgw_sync_module_log.cc rename to src/rgw/store/rados/rgw_sync_module_log.cc diff --git a/src/rgw/rgw_sync_module_log.h b/src/rgw/store/rados/rgw_sync_module_log.h similarity index 100% rename from src/rgw/rgw_sync_module_log.h rename to src/rgw/store/rados/rgw_sync_module_log.h diff --git a/src/rgw/rgw_sync_module_pubsub.cc b/src/rgw/store/rados/rgw_sync_module_pubsub.cc similarity index 100% rename from src/rgw/rgw_sync_module_pubsub.cc rename to src/rgw/store/rados/rgw_sync_module_pubsub.cc diff --git a/src/rgw/rgw_sync_module_pubsub.h b/src/rgw/store/rados/rgw_sync_module_pubsub.h similarity index 100% rename from src/rgw/rgw_sync_module_pubsub.h rename to src/rgw/store/rados/rgw_sync_module_pubsub.h diff --git a/src/rgw/rgw_sync_trace.cc b/src/rgw/store/rados/rgw_sync_trace.cc similarity index 100% rename from src/rgw/rgw_sync_trace.cc rename to src/rgw/store/rados/rgw_sync_trace.cc diff --git a/src/rgw/rgw_sync_trace.h b/src/rgw/store/rados/rgw_sync_trace.h similarity index 100% rename from src/rgw/rgw_sync_trace.h rename to src/rgw/store/rados/rgw_sync_trace.h diff --git a/src/rgw/store/rados/rgw_tools.cc b/src/rgw/store/rados/rgw_tools.cc new file mode 100644 index 000000000000..a990b2064c9f --- /dev/null +++ b/src/rgw/store/rados/rgw_tools.cc @@ -0,0 +1,414 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "common/errno.h" +#include "librados/librados_asio.h" + +#include "include/stringify.h" + +#include "rgw_tools.h" +#include "rgw_acl_s3.h" +#include "rgw_aio_throttle.h" +#include "rgw_compression.h" + +#define dout_subsys ceph_subsys_rgw + +#define READ_CHUNK_LEN (512 * 1024) + +using namespace std; + +int rgw_init_ioctx(const DoutPrefixProvider *dpp, + librados::Rados *rados, const rgw_pool& pool, + librados::IoCtx& ioctx, bool create, + bool mostly_omap) +{ + int r = rados->ioctx_create(pool.name.c_str(), ioctx); + if (r == -ENOENT && create) { + r = rados->pool_create(pool.name.c_str()); + if (r == -ERANGE) { + ldpp_dout(dpp, 0) + << __func__ + << " ERROR: librados::Rados::pool_create returned " << cpp_strerror(-r) + << " (this can be due to a pool or placement group misconfiguration, e.g." + << " pg_num < pgp_num or mon_max_pg_per_osd exceeded)" + << dendl; + } + if (r < 0 && r != -EEXIST) { + return r; + } + + r = rados->ioctx_create(pool.name.c_str(), ioctx); + if (r < 0) { + return r; + } + + r = ioctx.application_enable(pg_pool_t::APPLICATION_NAME_RGW, false); + if (r < 0 && r != -EOPNOTSUPP) { + return r; + } + + if (mostly_omap) { + // set pg_autoscale_bias + bufferlist inbl; + float bias = g_conf().get_val("rgw_rados_pool_autoscale_bias"); + int r = rados->mon_command( + "{\"prefix\": \"osd pool set\", \"pool\": \"" + + pool.name + "\", \"var\": \"pg_autoscale_bias\", \"val\": \"" + + stringify(bias) + "\"}", + inbl, NULL, NULL); + if (r < 0) { + ldpp_dout(dpp, 10) << __func__ << " warning: failed to set pg_autoscale_bias on " + << pool.name << dendl; + } + // set recovery_priority + int p = g_conf().get_val("rgw_rados_pool_recovery_priority"); + r = rados->mon_command( + "{\"prefix\": \"osd pool set\", \"pool\": \"" + + pool.name + "\", \"var\": \"recovery_priority\": \"" + + stringify(p) + "\"}", + inbl, NULL, NULL); + if (r < 0) { + ldpp_dout(dpp, 10) << __func__ << " warning: failed to set recovery_priority on " + << pool.name << dendl; + } + } + } else if (r < 0) { + return r; + } + if (!pool.ns.empty()) { + ioctx.set_namespace(pool.ns); + } + return 0; +} + +map* no_change_attrs() { + static map no_change; + return &no_change; +} + +int rgw_put_system_obj(const DoutPrefixProvider *dpp, RGWSI_SysObj* svc_sysobj, + const rgw_pool& pool, const string& oid, bufferlist& data, bool exclusive, + RGWObjVersionTracker *objv_tracker, real_time set_mtime, optional_yield y, map *pattrs) +{ + map no_attrs; + if (!pattrs) { + pattrs = &no_attrs; + } + + rgw_raw_obj obj(pool, oid); + + auto sysobj = svc_sysobj->get_obj(obj); + int ret; + + if (pattrs != no_change_attrs()) { + ret = sysobj.wop() + .set_objv_tracker(objv_tracker) + .set_exclusive(exclusive) + .set_mtime(set_mtime) + .set_attrs(*pattrs) + .write(dpp, data, y); + } else { + ret = sysobj.wop() + .set_objv_tracker(objv_tracker) + .set_exclusive(exclusive) + .set_mtime(set_mtime) + .write_data(dpp, data, y); + } + + return ret; +} + +int rgw_stat_system_obj(const DoutPrefixProvider *dpp, RGWSI_SysObj* svc_sysobj, + const rgw_pool& pool, const std::string& key, + RGWObjVersionTracker *objv_tracker, + real_time *pmtime, optional_yield y, + std::map *pattrs) +{ + rgw_raw_obj obj(pool, key); + auto sysobj = svc_sysobj->get_obj(obj); + return sysobj.rop() + .set_attrs(pattrs) + .set_last_mod(pmtime) + .stat(y, dpp); +} + + +int rgw_get_system_obj(RGWSI_SysObj* svc_sysobj, const rgw_pool& pool, const string& key, bufferlist& bl, + RGWObjVersionTracker *objv_tracker, real_time *pmtime, optional_yield y, + const DoutPrefixProvider *dpp, map *pattrs, + rgw_cache_entry_info *cache_info, + boost::optional refresh_version, bool raw_attrs) +{ + const rgw_raw_obj obj(pool, key); + auto sysobj = svc_sysobj->get_obj(obj); + auto rop = sysobj.rop(); + return rop.set_attrs(pattrs) + .set_last_mod(pmtime) + .set_objv_tracker(objv_tracker) + .set_raw_attrs(raw_attrs) + .set_cache_info(cache_info) + .set_refresh_version(refresh_version) + .read(dpp, &bl, y); +} + +int rgw_delete_system_obj(const DoutPrefixProvider *dpp, + RGWSI_SysObj *sysobj_svc, const rgw_pool& pool, const string& oid, + RGWObjVersionTracker *objv_tracker, optional_yield y) +{ + auto sysobj = sysobj_svc->get_obj(rgw_raw_obj{pool, oid}); + rgw_raw_obj obj(pool, oid); + return sysobj.wop() + .set_objv_tracker(objv_tracker) + .remove(dpp, y); +} + +int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid, + librados::ObjectReadOperation *op, bufferlist* pbl, + optional_yield y, int flags) +{ + // given a yield_context, call async_operate() to yield the coroutine instead + // of blocking + if (y) { + auto& context = y.get_io_context(); + auto& yield = y.get_yield_context(); + boost::system::error_code ec; + auto bl = librados::async_operate( + context, ioctx, oid, op, flags, yield[ec]); + if (pbl) { + *pbl = std::move(bl); + } + return -ec.value(); + } + // work on asio threads should be asynchronous, so warn when they block + if (is_asio_thread) { + ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; + } + return ioctx.operate(oid, op, nullptr, flags); +} + +int rgw_rados_operate(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid, + librados::ObjectWriteOperation *op, optional_yield y, + int flags) +{ + if (y) { + auto& context = y.get_io_context(); + auto& yield = y.get_yield_context(); + boost::system::error_code ec; + librados::async_operate(context, ioctx, oid, op, flags, yield[ec]); + return -ec.value(); + } + if (is_asio_thread) { + ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; + } + return ioctx.operate(oid, op, flags); +} + +int rgw_rados_notify(const DoutPrefixProvider *dpp, librados::IoCtx& ioctx, const std::string& oid, + bufferlist& bl, uint64_t timeout_ms, bufferlist* pbl, + optional_yield y) +{ + if (y) { + auto& context = y.get_io_context(); + auto& yield = y.get_yield_context(); + boost::system::error_code ec; + auto reply = librados::async_notify(context, ioctx, oid, + bl, timeout_ms, yield[ec]); + if (pbl) { + *pbl = std::move(reply); + } + return -ec.value(); + } + if (is_asio_thread) { + ldpp_dout(dpp, 20) << "WARNING: blocking librados call" << dendl; + } + return ioctx.notify2(oid, bl, timeout_ms, pbl); +} + +void rgw_filter_attrset(map& unfiltered_attrset, const string& check_prefix, + map *attrset) +{ + attrset->clear(); + map::iterator iter; + for (iter = unfiltered_attrset.lower_bound(check_prefix); + iter != unfiltered_attrset.end(); ++iter) { + if (!boost::algorithm::starts_with(iter->first, check_prefix)) + break; + (*attrset)[iter->first] = iter->second; + } +} + +RGWDataAccess::RGWDataAccess(rgw::sal::Store* _store) : store(_store) +{ +} + + +int RGWDataAccess::Bucket::finish_init() +{ + auto iter = attrs.find(RGW_ATTR_ACL); + if (iter == attrs.end()) { + return 0; + } + + bufferlist::const_iterator bliter = iter->second.begin(); + try { + policy.decode(bliter); + } catch (buffer::error& err) { + return -EIO; + } + + return 0; +} + +int RGWDataAccess::Bucket::init(const DoutPrefixProvider *dpp, optional_yield y) +{ + std::unique_ptr bucket; + int ret = sd->store->get_bucket(dpp, nullptr, tenant, name, &bucket, y); + if (ret < 0) { + return ret; + } + + bucket_info = bucket->get_info(); + mtime = bucket->get_modification_time(); + attrs = bucket->get_attrs(); + + return finish_init(); +} + +int RGWDataAccess::Bucket::init(const RGWBucketInfo& _bucket_info, + const map& _attrs) +{ + bucket_info = _bucket_info; + attrs = _attrs; + + return finish_init(); +} + +int RGWDataAccess::Bucket::get_object(const rgw_obj_key& key, + ObjectRef *obj) { + obj->reset(new Object(sd, shared_from_this(), key)); + return 0; +} + +int RGWDataAccess::Object::put(bufferlist& data, + map& attrs, + const DoutPrefixProvider *dpp, + optional_yield y) +{ + rgw::sal::Store* store = sd->store; + CephContext *cct = store->ctx(); + + string tag; + append_rand_alpha(cct, tag, tag, 32); + + RGWBucketInfo& bucket_info = bucket->bucket_info; + + rgw::BlockingAioThrottle aio(store->ctx()->_conf->rgw_put_obj_min_window_size); + + std::unique_ptr b; + store->get_bucket(NULL, bucket_info, &b); + std::unique_ptr obj = b->get_object(key); + + auto& owner = bucket->policy.get_owner(); + + string req_id = store->zone_unique_id(store->get_new_req_id()); + + std::unique_ptr processor; + processor = store->get_atomic_writer(dpp, y, std::move(obj), + owner.get_id(), + nullptr, olh_epoch, req_id); + + int ret = processor->prepare(y); + if (ret < 0) + return ret; + + rgw::sal::DataProcessor *filter = processor.get(); + + CompressorRef plugin; + boost::optional compressor; + + const auto& compression_type = store->get_compression_type(bucket_info.placement_rule); + if (compression_type != "none") { + plugin = Compressor::create(store->ctx(), compression_type); + if (!plugin) { + ldpp_dout(dpp, 1) << "Cannot load plugin for compression type " + << compression_type << dendl; + } else { + compressor.emplace(store->ctx(), plugin, filter); + filter = &*compressor; + } + } + + off_t ofs = 0; + auto obj_size = data.length(); + + RGWMD5Etag etag_calc; + + do { + size_t read_len = std::min(data.length(), (unsigned int)cct->_conf->rgw_max_chunk_size); + + bufferlist bl; + + data.splice(0, read_len, &bl); + etag_calc.update(bl); + + ret = filter->process(std::move(bl), ofs); + if (ret < 0) + return ret; + + ofs += read_len; + } while (data.length() > 0); + + ret = filter->process({}, ofs); + if (ret < 0) { + return ret; + } + bool has_etag_attr = false; + auto iter = attrs.find(RGW_ATTR_ETAG); + if (iter != attrs.end()) { + bufferlist& bl = iter->second; + etag = bl.to_str(); + has_etag_attr = true; + } + + if (!aclbl) { + RGWAccessControlPolicy_S3 policy(cct); + + policy.create_canned(bucket->policy.get_owner(), bucket->policy.get_owner(), string()); /* default private policy */ + + policy.encode(aclbl.emplace()); + } + + if (etag.empty()) { + etag_calc.finish(&etag); + } + + if (!has_etag_attr) { + bufferlist etagbl; + etagbl.append(etag); + attrs[RGW_ATTR_ETAG] = etagbl; + } + attrs[RGW_ATTR_ACL] = *aclbl; + + string *puser_data = nullptr; + if (user_data) { + puser_data = &(*user_data); + } + + return processor->complete(obj_size, etag, + &mtime, mtime, + attrs, delete_at, + nullptr, nullptr, + puser_data, + nullptr, nullptr, y); +} + +void RGWDataAccess::Object::set_policy(const RGWAccessControlPolicy& policy) +{ + policy.encode(aclbl.emplace()); +} + +void rgw_complete_aio_completion(librados::AioCompletion* c, int r) { + auto pc = c->pc; + librados::CB_AioCompleteAndSafe cb(pc); + cb(r); +} diff --git a/src/rgw/rgw_tools.h b/src/rgw/store/rados/rgw_tools.h similarity index 100% rename from src/rgw/rgw_tools.h rename to src/rgw/store/rados/rgw_tools.h diff --git a/src/rgw/rgw_trim_bilog.cc b/src/rgw/store/rados/rgw_trim_bilog.cc similarity index 100% rename from src/rgw/rgw_trim_bilog.cc rename to src/rgw/store/rados/rgw_trim_bilog.cc diff --git a/src/rgw/rgw_trim_bilog.h b/src/rgw/store/rados/rgw_trim_bilog.h similarity index 99% rename from src/rgw/rgw_trim_bilog.h rename to src/rgw/store/rados/rgw_trim_bilog.h index f4432eec9faa..5b9c4cdd7ec1 100644 --- a/src/rgw/rgw_trim_bilog.h +++ b/src/rgw/store/rados/rgw_trim_bilog.h @@ -24,7 +24,7 @@ #include "include/encoding.h" #include "common/ceph_time.h" #include "common/dout.h" -#include "rgw/rgw_common.h" +#include "rgw_common.h" class RGWCoroutine; class RGWHTTPManager; diff --git a/src/rgw/rgw_trim_datalog.cc b/src/rgw/store/rados/rgw_trim_datalog.cc similarity index 100% rename from src/rgw/rgw_trim_datalog.cc rename to src/rgw/store/rados/rgw_trim_datalog.cc diff --git a/src/rgw/rgw_trim_datalog.h b/src/rgw/store/rados/rgw_trim_datalog.h similarity index 100% rename from src/rgw/rgw_trim_datalog.h rename to src/rgw/store/rados/rgw_trim_datalog.h diff --git a/src/rgw/rgw_trim_mdlog.cc b/src/rgw/store/rados/rgw_trim_mdlog.cc similarity index 100% rename from src/rgw/rgw_trim_mdlog.cc rename to src/rgw/store/rados/rgw_trim_mdlog.cc diff --git a/src/rgw/rgw_trim_mdlog.h b/src/rgw/store/rados/rgw_trim_mdlog.h similarity index 100% rename from src/rgw/rgw_trim_mdlog.h rename to src/rgw/store/rados/rgw_trim_mdlog.h diff --git a/src/rgw/store/rados/rgw_user.cc b/src/rgw/store/rados/rgw_user.cc new file mode 100644 index 000000000000..27a0dfd8fa29 --- /dev/null +++ b/src/rgw/store/rados/rgw_user.cc @@ -0,0 +1,2768 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "common/errno.h" + +#include "rgw_user.h" + +#include "rgw_bucket.h" + +#include "services/svc_user.h" +#include "services/svc_meta.h" + +#define dout_subsys ceph_subsys_rgw + +using namespace std; + +extern void op_type_to_str(uint32_t mask, char *buf, int len); + +static string key_type_to_str(int key_type) { + switch (key_type) { + case KEY_TYPE_SWIFT: + return "swift"; + break; + + default: + return "s3"; + break; + } +} + +static bool char_is_unreserved_url(char c) +{ + if (isalnum(c)) + return true; + + switch (c) { + case '-': + case '.': + case '_': + case '~': + return true; + default: + return false; + } +} + +static bool validate_access_key(string& key) +{ + const char *p = key.c_str(); + while (*p) { + if (!char_is_unreserved_url(*p)) + return false; + p++; + } + return true; +} + +static void set_err_msg(std::string *sink, std::string msg) +{ + if (sink && !msg.empty()) + *sink = msg; +} + +/* + * Dump either the full user info or a subset to a formatter. + * + * NOTE: It is the caller's responsibility to ensure that the + * formatter is flushed at the correct time. + */ + +static void dump_subusers_info(Formatter *f, RGWUserInfo &info) +{ + map::iterator uiter; + + f->open_array_section("subusers"); + for (uiter = info.subusers.begin(); uiter != info.subusers.end(); ++uiter) { + RGWSubUser& u = uiter->second; + f->open_object_section("user"); + string s; + info.user_id.to_str(s); + f->dump_format("id", "%s:%s", s.c_str(), u.name.c_str()); + char buf[256]; + rgw_perm_to_str(u.perm_mask, buf, sizeof(buf)); + f->dump_string("permissions", buf); + f->close_section(); + } + f->close_section(); +} + +static void dump_access_keys_info(Formatter *f, RGWUserInfo &info) +{ + map::iterator kiter; + f->open_array_section("keys"); + for (kiter = info.access_keys.begin(); kiter != info.access_keys.end(); ++kiter) { + RGWAccessKey& k = kiter->second; + const char *sep = (k.subuser.empty() ? "" : ":"); + const char *subuser = (k.subuser.empty() ? "" : k.subuser.c_str()); + f->open_object_section("key"); + string s; + info.user_id.to_str(s); + f->dump_format("user", "%s%s%s", s.c_str(), sep, subuser); + f->dump_string("access_key", k.id); + f->dump_string("secret_key", k.key); + f->close_section(); + } + f->close_section(); +} + +static void dump_swift_keys_info(Formatter *f, RGWUserInfo &info) +{ + map::iterator kiter; + f->open_array_section("swift_keys"); + for (kiter = info.swift_keys.begin(); kiter != info.swift_keys.end(); ++kiter) { + RGWAccessKey& k = kiter->second; + const char *sep = (k.subuser.empty() ? "" : ":"); + const char *subuser = (k.subuser.empty() ? "" : k.subuser.c_str()); + f->open_object_section("key"); + string s; + info.user_id.to_str(s); + f->dump_format("user", "%s%s%s", s.c_str(), sep, subuser); + f->dump_string("secret_key", k.key); + f->close_section(); + } + f->close_section(); +} + +static void dump_user_info(Formatter *f, RGWUserInfo &info, + RGWStorageStats *stats = NULL) +{ + f->open_object_section("user_info"); + encode_json("tenant", info.user_id.tenant, f); + encode_json("user_id", info.user_id.id, f); + encode_json("display_name", info.display_name, f); + encode_json("email", info.user_email, f); + encode_json("suspended", (int)info.suspended, f); + encode_json("max_buckets", (int)info.max_buckets, f); + + dump_subusers_info(f, info); + dump_access_keys_info(f, info); + dump_swift_keys_info(f, info); + + encode_json("caps", info.caps, f); + + char buf[256]; + op_type_to_str(info.op_mask, buf, sizeof(buf)); + encode_json("op_mask", (const char *)buf, f); + encode_json("system", (bool)info.system, f); + encode_json("admin", (bool)info.admin, f); + encode_json("default_placement", info.default_placement.name, f); + encode_json("default_storage_class", info.default_placement.storage_class, f); + encode_json("placement_tags", info.placement_tags, f); + encode_json("bucket_quota", info.quota.bucket_quota, f); + encode_json("user_quota", info.quota.user_quota, f); + encode_json("temp_url_keys", info.temp_url_keys, f); + + string user_source_type; + switch ((RGWIdentityType)info.type) { + case TYPE_RGW: + user_source_type = "rgw"; + break; + case TYPE_KEYSTONE: + user_source_type = "keystone"; + break; + case TYPE_LDAP: + user_source_type = "ldap"; + break; + case TYPE_NONE: + user_source_type = "none"; + break; + default: + user_source_type = "none"; + break; + } + encode_json("type", user_source_type, f); + encode_json("mfa_ids", info.mfa_ids, f); + if (stats) { + encode_json("stats", *stats, f); + } + f->close_section(); +} + +static int user_add_helper(RGWUserAdminOpState& op_state, std::string *err_msg) +{ + int ret = 0; + const rgw_user& uid = op_state.get_user_id(); + std::string user_email = op_state.get_user_email(); + std::string display_name = op_state.get_display_name(); + + // fail if the user exists already + if (op_state.has_existing_user()) { + if (op_state.found_by_email) { + set_err_msg(err_msg, "email: " + user_email + + " is the email address of an existing user"); + ret = -ERR_EMAIL_EXIST; + } else if (op_state.found_by_key) { + set_err_msg(err_msg, "duplicate key provided"); + ret = -ERR_KEY_EXIST; + } else { + set_err_msg(err_msg, "user: " + uid.to_str() + " exists"); + ret = -EEXIST; + } + return ret; + } + + // fail if the user_info has already been populated + if (op_state.is_populated()) { + set_err_msg(err_msg, "cannot overwrite already populated user"); + return -EEXIST; + } + + // fail if the display name was not included + if (display_name.empty()) { + set_err_msg(err_msg, "no display name specified"); + return -EINVAL; + } + + return ret; +} + +RGWAccessKeyPool::RGWAccessKeyPool(RGWUser* usr) +{ + if (!usr) { + return; + } + + user = usr; + + store = user->get_store(); +} + +int RGWAccessKeyPool::init(RGWUserAdminOpState& op_state) +{ + if (!op_state.is_initialized()) { + keys_allowed = false; + return -EINVAL; + } + + const rgw_user& uid = op_state.get_user_id(); + if (uid.compare(RGW_USER_ANON_ID) == 0) { + keys_allowed = false; + return -EINVAL; + } + + swift_keys = op_state.get_swift_keys(); + access_keys = op_state.get_access_keys(); + + keys_allowed = true; + + return 0; +} + +RGWUserAdminOpState::RGWUserAdminOpState(rgw::sal::Store* store) +{ + user = store->get_user(rgw_user(RGW_USER_ANON_ID)); +} + +void RGWUserAdminOpState::set_user_id(const rgw_user& id) +{ + if (id.empty()) + return; + + user->get_info().user_id = id; +} + +void RGWUserAdminOpState::set_subuser(std::string& _subuser) +{ + if (_subuser.empty()) + return; + + size_t pos = _subuser.find(":"); + if (pos != string::npos) { + rgw_user tmp_id; + tmp_id.from_str(_subuser.substr(0, pos)); + if (tmp_id.tenant.empty()) { + user->get_info().user_id.id = tmp_id.id; + } else { + user->get_info().user_id = tmp_id; + } + subuser = _subuser.substr(pos+1); + } else { + subuser = _subuser; + } + + subuser_specified = true; +} + +void RGWUserAdminOpState::set_user_info(RGWUserInfo& user_info) +{ + user->get_info() = user_info; +} + +void RGWUserAdminOpState::set_user_version_tracker(RGWObjVersionTracker& objv_tracker) +{ + user->get_version_tracker() = objv_tracker; +} + +const rgw_user& RGWUserAdminOpState::get_user_id() +{ + return user->get_id(); +} + +RGWUserInfo& RGWUserAdminOpState::get_user_info() +{ + return user->get_info(); +} + +map* RGWUserAdminOpState::get_swift_keys() +{ + return &user->get_info().swift_keys; +} + +map* RGWUserAdminOpState::get_access_keys() +{ + return &user->get_info().access_keys; +} + +map* RGWUserAdminOpState::get_subusers() +{ + return &user->get_info().subusers; +} + +RGWUserCaps *RGWUserAdminOpState::get_caps_obj() +{ + return &user->get_info().caps; +} + +std::string RGWUserAdminOpState::build_default_swift_kid() +{ + if (user->get_id().empty() || subuser.empty()) + return ""; + + std::string kid; + user->get_id().to_str(kid); + kid.append(":"); + kid.append(subuser); + + return kid; +} + +std::string RGWUserAdminOpState::generate_subuser() { + if (user->get_id().empty()) + return ""; + + std::string generated_subuser; + user->get_id().to_str(generated_subuser); + std::string rand_suffix; + + int sub_buf_size = RAND_SUBUSER_LEN + 1; + char sub_buf[RAND_SUBUSER_LEN + 1]; + + gen_rand_alphanumeric_upper(g_ceph_context, sub_buf, sub_buf_size); + + rand_suffix = sub_buf; + if (rand_suffix.empty()) + return ""; + + generated_subuser.append(rand_suffix); + subuser = generated_subuser; + + return generated_subuser; +} + +/* + * Do a fairly exhaustive search for an existing key matching the parameters + * given. Also handles the case where no key type was specified and updates + * the operation state if needed. + */ + +bool RGWAccessKeyPool::check_existing_key(RGWUserAdminOpState& op_state) +{ + bool existing_key = false; + + int key_type = op_state.get_key_type(); + std::string kid = op_state.get_access_key(); + std::map::iterator kiter; + std::string swift_kid = op_state.build_default_swift_kid(); + + RGWUserInfo dup_info; + + if (kid.empty() && swift_kid.empty()) + return false; + + switch (key_type) { + case KEY_TYPE_SWIFT: + kiter = swift_keys->find(swift_kid); + + existing_key = (kiter != swift_keys->end()); + if (existing_key) + op_state.set_access_key(swift_kid); + + break; + case KEY_TYPE_S3: + kiter = access_keys->find(kid); + existing_key = (kiter != access_keys->end()); + + break; + default: + kiter = access_keys->find(kid); + + existing_key = (kiter != access_keys->end()); + if (existing_key) { + op_state.set_key_type(KEY_TYPE_S3); + break; + } + + kiter = swift_keys->find(kid); + + existing_key = (kiter != swift_keys->end()); + if (existing_key) { + op_state.set_key_type(KEY_TYPE_SWIFT); + break; + } + + // handle the case where the access key was not provided in user:key format + if (swift_kid.empty()) + return false; + + kiter = swift_keys->find(swift_kid); + + existing_key = (kiter != swift_keys->end()); + if (existing_key) { + op_state.set_access_key(swift_kid); + op_state.set_key_type(KEY_TYPE_SWIFT); + } + } + + op_state.set_existing_key(existing_key); + + return existing_key; +} + +int RGWAccessKeyPool::check_op(RGWUserAdminOpState& op_state, + std::string *err_msg) +{ + RGWUserInfo dup_info; + + if (!op_state.is_populated()) { + set_err_msg(err_msg, "user info was not populated"); + return -EINVAL; + } + + if (!keys_allowed) { + set_err_msg(err_msg, "keys not allowed for this user"); + return -EACCES; + } + + int32_t key_type = op_state.get_key_type(); + + // if a key type wasn't specified + if (key_type < 0) { + if (op_state.has_subuser()) { + key_type = KEY_TYPE_SWIFT; + } else { + key_type = KEY_TYPE_S3; + } + } + + op_state.set_key_type(key_type); + + /* see if the access key was specified */ + if (key_type == KEY_TYPE_S3 && !op_state.will_gen_access() && + op_state.get_access_key().empty()) { + set_err_msg(err_msg, "empty access key"); + return -ERR_INVALID_ACCESS_KEY; + } + + // don't check for secret key because we may be doing a removal + + if (check_existing_key(op_state)) { + op_state.set_access_key_exist(); + } + return 0; +} + +// Generate a new random key +int RGWAccessKeyPool::generate_key(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, + optional_yield y, std::string *err_msg) +{ + std::string id; + std::string key; + + std::pair key_pair; + RGWAccessKey new_key; + std::unique_ptr duplicate_check; + + int key_type = op_state.get_key_type(); + bool gen_access = op_state.will_gen_access(); + bool gen_secret = op_state.will_gen_secret(); + + if (!keys_allowed) { + set_err_msg(err_msg, "access keys not allowed for this user"); + return -EACCES; + } + + if (op_state.has_existing_key()) { + set_err_msg(err_msg, "cannot create existing key"); + return -ERR_KEY_EXIST; + } + + if (!gen_access) { + id = op_state.get_access_key(); + } + + if (!id.empty()) { + switch (key_type) { + case KEY_TYPE_SWIFT: + if (store->get_user_by_swift(dpp, id, y, &duplicate_check) >= 0) { + set_err_msg(err_msg, "existing swift key in RGW system:" + id); + return -ERR_KEY_EXIST; + } + break; + case KEY_TYPE_S3: + if (store->get_user_by_access_key(dpp, id, y, &duplicate_check) >= 0) { + set_err_msg(err_msg, "existing S3 key in RGW system:" + id); + return -ERR_KEY_EXIST; + } + } + } + + //key's subuser + if (op_state.has_subuser()) { + //create user and subuser at the same time, user's s3 key should not be set this + if (!op_state.key_type_setbycontext || (key_type == KEY_TYPE_SWIFT)) { + new_key.subuser = op_state.get_subuser(); + } + } + + //Secret key + if (!gen_secret) { + if (op_state.get_secret_key().empty()) { + set_err_msg(err_msg, "empty secret key"); + return -ERR_INVALID_SECRET_KEY; + } + + key = op_state.get_secret_key(); + } else { + char secret_key_buf[SECRET_KEY_LEN + 1]; + gen_rand_alphanumeric_plain(g_ceph_context, secret_key_buf, sizeof(secret_key_buf)); + key = secret_key_buf; + } + + // Generate the access key + if (key_type == KEY_TYPE_S3 && gen_access) { + char public_id_buf[PUBLIC_ID_LEN + 1]; + + do { + int id_buf_size = sizeof(public_id_buf); + gen_rand_alphanumeric_upper(g_ceph_context, public_id_buf, id_buf_size); + id = public_id_buf; + if (!validate_access_key(id)) + continue; + + } while (!store->get_user_by_access_key(dpp, id, y, &duplicate_check)); + } + + if (key_type == KEY_TYPE_SWIFT) { + id = op_state.build_default_swift_kid(); + if (id.empty()) { + set_err_msg(err_msg, "empty swift access key"); + return -ERR_INVALID_ACCESS_KEY; + } + + // check that the access key doesn't exist + if (store->get_user_by_swift(dpp, id, y, &duplicate_check) >= 0) { + set_err_msg(err_msg, "cannot create existing swift key"); + return -ERR_KEY_EXIST; + } + } + + // finally create the new key + new_key.id = id; + new_key.key = key; + + key_pair.first = id; + key_pair.second = new_key; + + if (key_type == KEY_TYPE_S3) { + access_keys->insert(key_pair); + } else if (key_type == KEY_TYPE_SWIFT) { + swift_keys->insert(key_pair); + } + + return 0; +} + +// modify an existing key +int RGWAccessKeyPool::modify_key(RGWUserAdminOpState& op_state, std::string *err_msg) +{ + std::string id; + std::string key = op_state.get_secret_key(); + int key_type = op_state.get_key_type(); + + RGWAccessKey modify_key; + + pair key_pair; + map::iterator kiter; + + switch (key_type) { + case KEY_TYPE_S3: + id = op_state.get_access_key(); + if (id.empty()) { + set_err_msg(err_msg, "no access key specified"); + return -ERR_INVALID_ACCESS_KEY; + } + break; + case KEY_TYPE_SWIFT: + id = op_state.build_default_swift_kid(); + if (id.empty()) { + set_err_msg(err_msg, "no subuser specified"); + return -EINVAL; + } + break; + default: + set_err_msg(err_msg, "invalid key type"); + return -ERR_INVALID_KEY_TYPE; + } + + if (!op_state.has_existing_key()) { + set_err_msg(err_msg, "key does not exist"); + return -ERR_INVALID_ACCESS_KEY; + } + + key_pair.first = id; + + if (key_type == KEY_TYPE_SWIFT) { + modify_key.id = id; + modify_key.subuser = op_state.get_subuser(); + } else if (key_type == KEY_TYPE_S3) { + kiter = access_keys->find(id); + if (kiter != access_keys->end()) { + modify_key = kiter->second; + } + } + + if (op_state.will_gen_secret()) { + char secret_key_buf[SECRET_KEY_LEN + 1]; + int key_buf_size = sizeof(secret_key_buf); + gen_rand_alphanumeric_plain(g_ceph_context, secret_key_buf, key_buf_size); + key = secret_key_buf; + } + + if (key.empty()) { + set_err_msg(err_msg, "empty secret key"); + return -ERR_INVALID_SECRET_KEY; + } + + // update the access key with the new secret key + modify_key.key = key; + + key_pair.second = modify_key; + + + if (key_type == KEY_TYPE_S3) { + (*access_keys)[id] = modify_key; + } else if (key_type == KEY_TYPE_SWIFT) { + (*swift_keys)[id] = modify_key; + } + + return 0; +} + +int RGWAccessKeyPool::execute_add(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, + std::string *err_msg, bool defer_user_update, + optional_yield y) +{ + int ret = 0; + + std::string subprocess_msg; + int key_op = GENERATE_KEY; + + // set the op + if (op_state.has_existing_key()) + key_op = MODIFY_KEY; + + switch (key_op) { + case GENERATE_KEY: + ret = generate_key(dpp, op_state, y, &subprocess_msg); + break; + case MODIFY_KEY: + ret = modify_key(op_state, &subprocess_msg); + break; + } + + if (ret < 0) { + set_err_msg(err_msg, subprocess_msg); + return ret; + } + + // store the updated info + if (!defer_user_update) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +int RGWAccessKeyPool::add(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, optional_yield y, + std::string *err_msg) +{ + return add(dpp, op_state, err_msg, false, y); +} + +int RGWAccessKeyPool::add(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, std::string *err_msg, + bool defer_user_update, optional_yield y) +{ + int ret; + std::string subprocess_msg; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); + return ret; + } + + ret = execute_add(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to add access key, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWAccessKeyPool::execute_remove(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, + std::string *err_msg, + bool defer_user_update, + optional_yield y) +{ + int ret = 0; + + int key_type = op_state.get_key_type(); + std::string id = op_state.get_access_key(); + map::iterator kiter; + map *keys_map; + + if (!op_state.has_existing_key()) { + set_err_msg(err_msg, "unable to find access key, with key type: " + + key_type_to_str(key_type)); + return -ERR_INVALID_ACCESS_KEY; + } + + if (key_type == KEY_TYPE_S3) { + keys_map = access_keys; + } else if (key_type == KEY_TYPE_SWIFT) { + keys_map = swift_keys; + } else { + keys_map = NULL; + set_err_msg(err_msg, "invalid access key"); + return -ERR_INVALID_ACCESS_KEY; + } + + kiter = keys_map->find(id); + if (kiter == keys_map->end()) { + set_err_msg(err_msg, "key not found"); + return -ERR_INVALID_ACCESS_KEY; + } + + keys_map->erase(kiter); + + if (!defer_user_update) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +int RGWAccessKeyPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, + std::string *err_msg) +{ + return remove(dpp, op_state, err_msg, false, y); +} + +int RGWAccessKeyPool::remove(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, + std::string *err_msg, bool defer_user_update, + optional_yield y) +{ + int ret; + + std::string subprocess_msg; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); + return ret; + } + + ret = execute_remove(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to remove access key, " + subprocess_msg); + return ret; + } + + return 0; +} + +// remove all keys associated with a subuser +int RGWAccessKeyPool::remove_subuser_keys(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, + std::string *err_msg, + bool defer_user_update, + optional_yield y) +{ + int ret = 0; + + if (!op_state.is_populated()) { + set_err_msg(err_msg, "user info was not populated"); + return -EINVAL; + } + + if (!op_state.has_subuser()) { + set_err_msg(err_msg, "no subuser specified"); + return -EINVAL; + } + + std::string swift_kid = op_state.build_default_swift_kid(); + if (swift_kid.empty()) { + set_err_msg(err_msg, "empty swift access key"); + return -EINVAL; + } + + map::iterator kiter; + map *keys_map; + + // a subuser can have at most one swift key + keys_map = swift_keys; + kiter = keys_map->find(swift_kid); + if (kiter != keys_map->end()) { + keys_map->erase(kiter); + } + + // a subuser may have multiple s3 key pairs + std::string subuser_str = op_state.get_subuser(); + keys_map = access_keys; + RGWUserInfo user_info = op_state.get_user_info(); + auto user_kiter = user_info.access_keys.begin(); + for (; user_kiter != user_info.access_keys.end(); ++user_kiter) { + if (user_kiter->second.subuser == subuser_str) { + kiter = keys_map->find(user_kiter->first); + if (kiter != keys_map->end()) { + keys_map->erase(kiter); + } + } + } + + if (!defer_user_update) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +RGWSubUserPool::RGWSubUserPool(RGWUser *usr) +{ + if (!usr) { + return; + } + + user = usr; + + subusers_allowed = true; + store = user->get_store(); +} + +int RGWSubUserPool::init(RGWUserAdminOpState& op_state) +{ + if (!op_state.is_initialized()) { + subusers_allowed = false; + return -EINVAL; + } + + const rgw_user& uid = op_state.get_user_id(); + if (uid.compare(RGW_USER_ANON_ID) == 0) { + subusers_allowed = false; + return -EACCES; + } + + subuser_map = op_state.get_subusers(); + if (subuser_map == NULL) { + subusers_allowed = false; + return -EINVAL; + } + + subusers_allowed = true; + + return 0; +} + +bool RGWSubUserPool::exists(std::string subuser) +{ + if (subuser.empty()) + return false; + + if (!subuser_map) + return false; + + if (subuser_map->count(subuser)) + return true; + + return false; +} + +int RGWSubUserPool::check_op(RGWUserAdminOpState& op_state, + std::string *err_msg) +{ + bool existing = false; + std::string subuser = op_state.get_subuser(); + + if (!op_state.is_populated()) { + set_err_msg(err_msg, "user info was not populated"); + return -EINVAL; + } + + if (!subusers_allowed) { + set_err_msg(err_msg, "subusers not allowed for this user"); + return -EACCES; + } + + if (subuser.empty() && !op_state.will_gen_subuser()) { + set_err_msg(err_msg, "empty subuser name"); + return -EINVAL; + } + + if (op_state.get_subuser_perm() == RGW_PERM_INVALID) { + set_err_msg(err_msg, "invalid subuser access"); + return -EINVAL; + } + + //set key type when it not set or set by context + if ((op_state.get_key_type() < 0) || op_state.key_type_setbycontext) { + op_state.set_key_type(KEY_TYPE_SWIFT); + op_state.key_type_setbycontext = true; + } + + // check if the subuser exists + if (!subuser.empty()) + existing = exists(subuser); + + op_state.set_existing_subuser(existing); + + return 0; +} + +int RGWSubUserPool::execute_add(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, + std::string *err_msg, bool defer_user_update, + optional_yield y) +{ + int ret = 0; + std::string subprocess_msg; + + RGWSubUser subuser; + std::pair subuser_pair; + std::string subuser_str = op_state.get_subuser(); + + subuser_pair.first = subuser_str; + + // assumes key should be created + if (op_state.has_key_op()) { + ret = user->keys.add(dpp, op_state, &subprocess_msg, true, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to create subuser key, " + subprocess_msg); + return ret; + } + } + + // create the subuser + subuser.name = subuser_str; + + if (op_state.has_subuser_perm()) + subuser.perm_mask = op_state.get_subuser_perm(); + + // insert the subuser into user info + subuser_pair.second = subuser; + subuser_map->insert(subuser_pair); + + // attempt to save the subuser + if (!defer_user_update) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +int RGWSubUserPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, + std::string *err_msg) +{ + return add(dpp, op_state, err_msg, false, y); +} + +int RGWSubUserPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update, optional_yield y) +{ + std::string subprocess_msg; + int ret; + int32_t key_type = op_state.get_key_type(); + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); + return ret; + } + + if (op_state.get_access_key_exist()) { + set_err_msg(err_msg, "cannot create existing key"); + return -ERR_KEY_EXIST; + } + + if (key_type == KEY_TYPE_S3 && op_state.get_access_key().empty()) { + op_state.set_gen_access(); + } + + if (op_state.get_secret_key().empty()) { + op_state.set_gen_secret(); + } + + ret = execute_add(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to create subuser, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWSubUserPool::execute_remove(const DoutPrefixProvider *dpp, + RGWUserAdminOpState& op_state, + std::string *err_msg, bool defer_user_update, + optional_yield y) +{ + int ret = 0; + std::string subprocess_msg; + + std::string subuser_str = op_state.get_subuser(); + + map::iterator siter; + siter = subuser_map->find(subuser_str); + if (siter == subuser_map->end()){ + set_err_msg(err_msg, "subuser not found: " + subuser_str); + return -ERR_NO_SUCH_SUBUSER; + } + if (!op_state.has_existing_subuser()) { + set_err_msg(err_msg, "subuser not found: " + subuser_str); + return -ERR_NO_SUCH_SUBUSER; + } + + // always purge all associate keys + user->keys.remove_subuser_keys(dpp, op_state, &subprocess_msg, true, y); + + // remove the subuser from the user info + subuser_map->erase(siter); + + // attempt to save the subuser + if (!defer_user_update) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +int RGWSubUserPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, + std::string *err_msg) +{ + return remove(dpp, op_state, err_msg, false, y); +} + +int RGWSubUserPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, + bool defer_user_update, optional_yield y) +{ + std::string subprocess_msg; + int ret; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); + return ret; + } + + ret = execute_remove(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to remove subuser, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWSubUserPool::execute_modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, bool defer_user_update, optional_yield y) +{ + int ret = 0; + std::string subprocess_msg; + std::map::iterator siter; + std::pair subuser_pair; + + std::string subuser_str = op_state.get_subuser(); + RGWSubUser subuser; + + if (!op_state.has_existing_subuser()) { + set_err_msg(err_msg, "subuser does not exist"); + return -ERR_NO_SUCH_SUBUSER; + } + + subuser_pair.first = subuser_str; + + siter = subuser_map->find(subuser_str); + subuser = siter->second; + + if (op_state.has_key_op()) { + ret = user->keys.add(dpp, op_state, &subprocess_msg, true, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to create subuser keys, " + subprocess_msg); + return ret; + } + } + + if (op_state.has_subuser_perm()) + subuser.perm_mask = op_state.get_subuser_perm(); + + subuser_pair.second = subuser; + + subuser_map->erase(siter); + subuser_map->insert(subuser_pair); + + // attempt to save the subuser + if (!defer_user_update) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +int RGWSubUserPool::modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) +{ + return RGWSubUserPool::modify(dpp, op_state, y, err_msg, false); +} + +int RGWSubUserPool::modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg, bool defer_user_update) +{ + std::string subprocess_msg; + int ret; + + RGWSubUser subuser; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse request, " + subprocess_msg); + return ret; + } + + ret = execute_modify(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to modify subuser, " + subprocess_msg); + return ret; + } + + return 0; +} + +RGWUserCapPool::RGWUserCapPool(RGWUser *usr) +{ + if (!usr) { + return; + } + user = usr; + caps_allowed = true; +} + +int RGWUserCapPool::init(RGWUserAdminOpState& op_state) +{ + if (!op_state.is_initialized()) { + caps_allowed = false; + return -EINVAL; + } + + const rgw_user& uid = op_state.get_user_id(); + if (uid.compare(RGW_USER_ANON_ID) == 0) { + caps_allowed = false; + return -EACCES; + } + + caps = op_state.get_caps_obj(); + if (!caps) { + caps_allowed = false; + return -ERR_INVALID_CAP; + } + + caps_allowed = true; + + return 0; +} + +int RGWUserCapPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, + std::string *err_msg) +{ + return add(dpp, op_state, err_msg, false, y); +} + +int RGWUserCapPool::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, + bool defer_save, optional_yield y) +{ + int ret = 0; + std::string caps_str = op_state.get_caps(); + + if (!op_state.is_populated()) { + set_err_msg(err_msg, "user info was not populated"); + return -EINVAL; + } + + if (!caps_allowed) { + set_err_msg(err_msg, "caps not allowed for this user"); + return -EACCES; + } + + if (caps_str.empty()) { + set_err_msg(err_msg, "empty user caps"); + return -ERR_INVALID_CAP; + } + + int r = caps->add_from_string(caps_str); + if (r < 0) { + set_err_msg(err_msg, "unable to add caps: " + caps_str); + return r; + } + + if (!defer_save) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +int RGWUserCapPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, + std::string *err_msg) +{ + return remove(dpp, op_state, err_msg, false, y); +} + +int RGWUserCapPool::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, + bool defer_save, optional_yield y) +{ + int ret = 0; + + std::string caps_str = op_state.get_caps(); + + if (!op_state.is_populated()) { + set_err_msg(err_msg, "user info was not populated"); + return -EINVAL; + } + + if (!caps_allowed) { + set_err_msg(err_msg, "caps not allowed for this user"); + return -EACCES; + } + + if (caps_str.empty()) { + set_err_msg(err_msg, "empty user caps"); + return -ERR_INVALID_CAP; + } + + int r = caps->remove_from_string(caps_str); + if (r < 0) { + set_err_msg(err_msg, "unable to remove caps: " + caps_str); + return r; + } + + if (!defer_save) + ret = user->update(dpp, op_state, err_msg, y); + + if (ret < 0) + return ret; + + return 0; +} + +RGWUser::RGWUser() : caps(this), keys(this), subusers(this) +{ + init_default(); +} + +int RGWUser::init(const DoutPrefixProvider *dpp, rgw::sal::Store* storage, + RGWUserAdminOpState& op_state, optional_yield y) +{ + init_default(); + int ret = init_storage(storage); + if (ret < 0) + return ret; + + ret = init(dpp, op_state, y); + if (ret < 0) + return ret; + + return 0; +} + +void RGWUser::init_default() +{ + // use anonymous user info as a placeholder + rgw_get_anon_user(old_info); + user_id = RGW_USER_ANON_ID; + + clear_populated(); +} + +int RGWUser::init_storage(rgw::sal::Store* storage) +{ + if (!storage) { + return -EINVAL; + } + + store = storage; + + clear_populated(); + + /* API wrappers */ + keys = RGWAccessKeyPool(this); + caps = RGWUserCapPool(this); + subusers = RGWSubUserPool(this); + + return 0; +} + +int RGWUser::init(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y) +{ + bool found = false; + std::string swift_user; + user_id = op_state.get_user_id(); + std::string user_email = op_state.get_user_email(); + std::string access_key = op_state.get_access_key(); + std::string subuser = op_state.get_subuser(); + + int key_type = op_state.get_key_type(); + if (key_type == KEY_TYPE_SWIFT) { + swift_user = op_state.get_access_key(); + access_key.clear(); + } + + std::unique_ptr user; + + clear_populated(); + + if (user_id.empty() && !subuser.empty()) { + size_t pos = subuser.find(':'); + if (pos != string::npos) { + user_id = subuser.substr(0, pos); + op_state.set_user_id(user_id); + } + } + + if (!user_id.empty() && (user_id.compare(RGW_USER_ANON_ID) != 0)) { + user = store->get_user(user_id); + found = (user->load_user(dpp, y) >= 0); + op_state.found_by_uid = found; + } + if (store->ctx()->_conf.get_val("rgw_user_unique_email")) { + if (!user_email.empty() && !found) { + found = (store->get_user_by_email(dpp, user_email, y, &user) >= 0); + op_state.found_by_email = found; + } + } + if (!swift_user.empty() && !found) { + found = (store->get_user_by_swift(dpp, swift_user, y, &user) >= 0); + op_state.found_by_key = found; + } + if (!access_key.empty() && !found) { + found = (store->get_user_by_access_key(dpp, access_key, y, &user) >= 0); + op_state.found_by_key = found; + } + + op_state.set_existing_user(found); + if (found) { + op_state.set_user_info(user->get_info()); + op_state.set_populated(); + op_state.objv = user->get_version_tracker(); + op_state.set_user_version_tracker(user->get_version_tracker()); + + old_info = user->get_info(); + set_populated(); + } + + if (user_id.empty()) { + user_id = user->get_id(); + } + op_state.set_initialized(); + + // this may have been called by a helper object + int ret = init_members(op_state); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUser::init_members(RGWUserAdminOpState& op_state) +{ + int ret = 0; + + ret = keys.init(op_state); + if (ret < 0) + return ret; + + ret = subusers.init(op_state); + if (ret < 0) + return ret; + + ret = caps.init(op_state); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUser::update(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, + optional_yield y) +{ + int ret; + std::string subprocess_msg; + rgw::sal::User* user = op_state.get_user(); + + if (!store) { + set_err_msg(err_msg, "couldn't initialize storage"); + return -EINVAL; + } + + RGWUserInfo *pold_info = (is_populated() ? &old_info : nullptr); + + ret = user->store_user(dpp, y, false, pold_info); + op_state.objv = user->get_version_tracker(); + op_state.set_user_version_tracker(user->get_version_tracker()); + + if (ret < 0) { + set_err_msg(err_msg, "unable to store user info"); + return ret; + } + + old_info = user->get_info(); + set_populated(); + + return 0; +} + +int RGWUser::check_op(RGWUserAdminOpState& op_state, std::string *err_msg) +{ + int ret = 0; + const rgw_user& uid = op_state.get_user_id(); + + if (uid.compare(RGW_USER_ANON_ID) == 0) { + set_err_msg(err_msg, "unable to perform operations on the anonymous user"); + return -EINVAL; + } + + if (is_populated() && user_id.compare(uid) != 0) { + set_err_msg(err_msg, "user id mismatch, operation id: " + uid.to_str() + + " does not match: " + user_id.to_str()); + + return -EINVAL; + } + + ret = rgw_validate_tenant_name(uid.tenant); + if (ret) { + set_err_msg(err_msg, + "invalid tenant only alphanumeric and _ characters are allowed"); + return ret; + } + + //set key type when it not set or set by context + if ((op_state.get_key_type() < 0) || op_state.key_type_setbycontext) { + op_state.set_key_type(KEY_TYPE_S3); + op_state.key_type_setbycontext = true; + } + + return 0; +} + +// update swift_keys with new user id +static void rename_swift_keys(const rgw_user& user, + std::map& keys) +{ + std::string user_id; + user.to_str(user_id); + + auto modify_keys = std::move(keys); + for ([[maybe_unused]] auto& [k, key] : modify_keys) { + std::string id = user_id + ":" + key.subuser; + key.id = id; + keys[id] = std::move(key); + } +} + +int RGWUser::execute_rename(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, optional_yield y) +{ + int ret; + bool populated = op_state.is_populated(); + + if (!op_state.has_existing_user() && !populated) { + set_err_msg(err_msg, "user not found"); + return -ENOENT; + } + + if (!populated) { + ret = init(dpp, op_state, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to retrieve user info"); + return ret; + } + } + + std::unique_ptr old_user = store->get_user(op_state.get_user_info().user_id); + std::unique_ptr new_user = store->get_user(op_state.get_new_uid()); + if (old_user->get_tenant() != new_user->get_tenant()) { + set_err_msg(err_msg, "users have to be under the same tenant namespace " + + old_user->get_tenant() + " != " + new_user->get_tenant()); + return -EINVAL; + } + + // create a stub user and write only the uid index and buckets object + std::unique_ptr user; + user = store->get_user(new_user->get_id()); + + const bool exclusive = !op_state.get_overwrite_new_user(); // overwrite if requested + + ret = user->store_user(dpp, y, exclusive); + if (ret == -EEXIST) { + set_err_msg(err_msg, "user name given by --new-uid already exists"); + return ret; + } + if (ret < 0) { + set_err_msg(err_msg, "unable to store new user info"); + return ret; + } + + RGWAccessControlPolicy policy_instance; + policy_instance.create_default(new_user->get_id(), old_user->get_display_name()); + + //unlink and link buckets to new user + string marker; + CephContext *cct = store->ctx(); + size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; + rgw::sal::BucketList buckets; + + do { + ret = old_user->list_buckets(dpp, marker, "", max_buckets, false, buckets, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to list user buckets"); + return ret; + } + + auto& m = buckets.get_buckets(); + + for (auto it = m.begin(); it != m.end(); ++it) { + auto& bucket = it->second; + marker = it->first; + + ret = bucket->load_bucket(dpp, y); + if (ret < 0) { + set_err_msg(err_msg, "failed to fetch bucket info for bucket=" + bucket->get_name()); + return ret; + } + + ret = bucket->set_acl(dpp, policy_instance, y); + if (ret < 0) { + set_err_msg(err_msg, "failed to set acl on bucket " + bucket->get_name()); + return ret; + } + + ret = bucket->chown(dpp, new_user.get(), old_user.get(), y); + if (ret < 0) { + set_err_msg(err_msg, "failed to run bucket chown" + cpp_strerror(-ret)); + return ret; + } + } + + } while (buckets.is_truncated()); + + // update the 'stub user' with all of the other fields and rewrite all of the + // associated index objects + RGWUserInfo& user_info = op_state.get_user_info(); + user_info.user_id = new_user->get_id(); + op_state.objv = user->get_version_tracker(); + op_state.set_user_version_tracker(user->get_version_tracker()); + + rename_swift_keys(new_user->get_id(), user_info.swift_keys); + + return update(dpp, op_state, err_msg, y); +} + +int RGWUser::execute_add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, + optional_yield y) +{ + const rgw_user& uid = op_state.get_user_id(); + std::string user_email = op_state.get_user_email(); + std::string display_name = op_state.get_display_name(); + + // set the user info + RGWUserInfo user_info; + user_id = uid; + user_info.user_id = user_id; + user_info.display_name = display_name; + user_info.type = TYPE_RGW; + + if (!user_email.empty()) + user_info.user_email = user_email; + + CephContext *cct = store->ctx(); + if (op_state.max_buckets_specified) { + user_info.max_buckets = op_state.get_max_buckets(); + } else { + user_info.max_buckets = + cct->_conf.get_val("rgw_user_max_buckets"); + } + + user_info.suspended = op_state.get_suspension_status(); + user_info.admin = op_state.admin; + user_info.system = op_state.system; + + if (op_state.op_mask_specified) + user_info.op_mask = op_state.get_op_mask(); + + if (op_state.has_bucket_quota()) { + user_info.quota.bucket_quota = op_state.get_bucket_quota(); + } else { + rgw_apply_default_bucket_quota(user_info.quota.bucket_quota, cct->_conf); + } + + if (op_state.temp_url_key_specified) { + map::iterator iter; + for (iter = op_state.temp_url_keys.begin(); + iter != op_state.temp_url_keys.end(); ++iter) { + user_info.temp_url_keys[iter->first] = iter->second; + } + } + + if (op_state.has_user_quota()) { + user_info.quota.user_quota = op_state.get_user_quota(); + } else { + rgw_apply_default_user_quota(user_info.quota.user_quota, cct->_conf); + } + + if (op_state.default_placement_specified) { + user_info.default_placement = op_state.default_placement; + } + + if (op_state.placement_tags_specified) { + user_info.placement_tags = op_state.placement_tags; + } + + // update the request + op_state.set_user_info(user_info); + op_state.set_populated(); + + // update the helper objects + int ret = init_members(op_state); + if (ret < 0) { + set_err_msg(err_msg, "unable to initialize user"); + return ret; + } + + // see if we need to add an access key + std::string subprocess_msg; + bool defer_user_update = true; + if (op_state.has_key_op()) { + ret = keys.add(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to create access key, " + subprocess_msg); + return ret; + } + } + + // see if we need to add some caps + if (op_state.has_caps_op()) { + ret = caps.add(dpp, op_state, &subprocess_msg, defer_user_update, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to add user capabilities, " + subprocess_msg); + return ret; + } + } + + ret = update(dpp, op_state, err_msg, y); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUser::add(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) +{ + std::string subprocess_msg; + int ret = user_add_helper(op_state, &subprocess_msg); + if (ret != 0) { + set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); + return ret; + } + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); + return ret; + } + + ret = execute_add(dpp, op_state, &subprocess_msg, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to create user, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWUser::rename(RGWUserAdminOpState& op_state, optional_yield y, const DoutPrefixProvider *dpp, std::string *err_msg) +{ + std::string subprocess_msg; + int ret; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); + return ret; + } + + ret = execute_rename(dpp, op_state, &subprocess_msg, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to rename user, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWUser::execute_remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, optional_yield y) +{ + int ret; + + bool purge_data = op_state.will_purge_data(); + rgw::sal::User* user = op_state.get_user(); + + if (!op_state.has_existing_user()) { + set_err_msg(err_msg, "user does not exist"); + return -ENOENT; + } + + rgw::sal::BucketList buckets; + string marker; + CephContext *cct = store->ctx(); + size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; + do { + ret = user->list_buckets(dpp, marker, string(), max_buckets, false, buckets, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to read user bucket info"); + return ret; + } + + auto& m = buckets.get_buckets(); + if (!m.empty() && !purge_data) { + set_err_msg(err_msg, "must specify purge data to remove user with buckets"); + return -EEXIST; // change to code that maps to 409: conflict + } + + for (auto it = m.begin(); it != m.end(); ++it) { + ret = it->second->remove_bucket(dpp, true, false, nullptr, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to delete user data"); + return ret; + } + + marker = it->first; + } + + } while (buckets.is_truncated()); + + ret = user->remove_user(dpp, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to remove user from RADOS"); + return ret; + } + + op_state.clear_populated(); + clear_populated(); + + return 0; +} + +int RGWUser::remove(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) +{ + std::string subprocess_msg; + int ret; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); + return ret; + } + + ret = execute_remove(dpp, op_state, &subprocess_msg, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to remove user, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWUser::execute_modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, std::string *err_msg, optional_yield y) +{ + bool populated = op_state.is_populated(); + int ret = 0; + std::string subprocess_msg; + std::string op_email = op_state.get_user_email(); + std::string display_name = op_state.get_display_name(); + + RGWUserInfo user_info; + std::unique_ptr duplicate_check; + + // ensure that the user info has been populated or is populate-able + if (!op_state.has_existing_user() && !populated) { + set_err_msg(err_msg, "user not found"); + return -ENOENT; + } + + // if the user hasn't already been populated...attempt to + if (!populated) { + ret = init(dpp, op_state, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to retrieve user info"); + return ret; + } + } + + // ensure that we can modify the user's attributes + if (user_id.compare(RGW_USER_ANON_ID) == 0) { + set_err_msg(err_msg, "unable to modify anonymous user's info"); + return -EACCES; + } + + user_info = old_info; + + std::string old_email = old_info.user_email; + if (!op_email.empty()) { + // make sure we are not adding a duplicate email + if (old_email != op_email) { + ret = store->get_user_by_email(dpp, op_email, y, &duplicate_check); + if (ret >= 0 && duplicate_check->get_id().compare(user_id) != 0) { + set_err_msg(err_msg, "cannot add duplicate email"); + return -ERR_EMAIL_EXIST; + } + } + user_info.user_email = op_email; + } else if (op_email.empty() && op_state.user_email_specified) { + ldpp_dout(dpp, 10) << "removing email index: " << user_info.user_email << dendl; + /* will be physically removed later when calling update() */ + user_info.user_email.clear(); + } + + // update the remaining user info + if (!display_name.empty()) + user_info.display_name = display_name; + + if (op_state.max_buckets_specified) + user_info.max_buckets = op_state.get_max_buckets(); + + if (op_state.admin_specified) + user_info.admin = op_state.admin; + + if (op_state.system_specified) + user_info.system = op_state.system; + + if (op_state.temp_url_key_specified) { + map::iterator iter; + for (iter = op_state.temp_url_keys.begin(); + iter != op_state.temp_url_keys.end(); ++iter) { + user_info.temp_url_keys[iter->first] = iter->second; + } + } + + if (op_state.op_mask_specified) + user_info.op_mask = op_state.get_op_mask(); + + if (op_state.has_bucket_quota()) + user_info.quota.bucket_quota = op_state.get_bucket_quota(); + + if (op_state.has_user_quota()) + user_info.quota.user_quota = op_state.get_user_quota(); + + if (op_state.has_suspension_op()) { + __u8 suspended = op_state.get_suspension_status(); + user_info.suspended = suspended; + + rgw::sal::BucketList buckets; + + if (user_id.empty()) { + set_err_msg(err_msg, "empty user id passed...aborting"); + return -EINVAL; + } + + string marker; + CephContext *cct = store->ctx(); + size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk; + std::unique_ptr user = store->get_user(user_id); + do { + ret = user->list_buckets(dpp, marker, string(), max_buckets, false, buckets, y); + if (ret < 0) { + set_err_msg(err_msg, "could not get buckets for uid: " + user_id.to_str()); + return ret; + } + + auto& m = buckets.get_buckets(); + + vector bucket_names; + for (auto iter = m.begin(); iter != m.end(); ++iter) { + auto& bucket = iter->second; + bucket_names.push_back(bucket->get_key()); + + marker = iter->first; + } + + ret = store->set_buckets_enabled(dpp, bucket_names, !suspended); + if (ret < 0) { + set_err_msg(err_msg, "failed to modify bucket"); + return ret; + } + + } while (buckets.is_truncated()); + } + + if (op_state.mfa_ids_specified) { + user_info.mfa_ids = op_state.mfa_ids; + } + + if (op_state.default_placement_specified) { + user_info.default_placement = op_state.default_placement; + } + + if (op_state.placement_tags_specified) { + user_info.placement_tags = op_state.placement_tags; + } + + op_state.set_user_info(user_info); + + // if we're supposed to modify keys, do so + if (op_state.has_key_op()) { + ret = keys.add(dpp, op_state, &subprocess_msg, true, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to create or modify keys, " + subprocess_msg); + return ret; + } + } + + ret = update(dpp, op_state, err_msg, y); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUser::modify(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, optional_yield y, std::string *err_msg) +{ + std::string subprocess_msg; + int ret; + + ret = check_op(op_state, &subprocess_msg); + if (ret < 0) { + set_err_msg(err_msg, "unable to parse parameters, " + subprocess_msg); + return ret; + } + + ret = execute_modify(dpp, op_state, &subprocess_msg, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to modify user, " + subprocess_msg); + return ret; + } + + return 0; +} + +int RGWUser::info(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, RGWUserInfo& fetched_info, + optional_yield y, std::string *err_msg) +{ + int ret = init(dpp, op_state, y); + if (ret < 0) { + set_err_msg(err_msg, "unable to fetch user info"); + return ret; + } + + fetched_info = op_state.get_user_info(); + + return 0; +} + +int RGWUser::info(RGWUserInfo& fetched_info, std::string *err_msg) +{ + if (!is_populated()) { + set_err_msg(err_msg, "no user info saved"); + return -EINVAL; + } + + fetched_info = old_info; + + return 0; +} + +int RGWUser::list(const DoutPrefixProvider *dpp, RGWUserAdminOpState& op_state, RGWFormatterFlusher& flusher) +{ + Formatter *formatter = flusher.get_formatter(); + void *handle = nullptr; + std::string metadata_key = "user"; + if (op_state.max_entries > 1000) { + op_state.max_entries = 1000; + } + + int ret = store->meta_list_keys_init(dpp, metadata_key, op_state.marker, &handle); + if (ret < 0) { + return ret; + } + + bool truncated = false; + uint64_t count = 0; + uint64_t left = 0; + flusher.start(0); + + // open the result object section + formatter->open_object_section("result"); + + // open the user id list array section + formatter->open_array_section("keys"); + do { + std::list keys; + left = op_state.max_entries - count; + ret = store->meta_list_keys_next(dpp, handle, left, keys, &truncated); + if (ret < 0 && ret != -ENOENT) { + return ret; + } if (ret != -ENOENT) { + for (std::list::iterator iter = keys.begin(); iter != keys.end(); ++iter) { + formatter->dump_string("key", *iter); + ++count; + } + } + } while (truncated && left > 0); + // close user id list section + formatter->close_section(); + + formatter->dump_bool("truncated", truncated); + formatter->dump_int("count", count); + if (truncated) { + formatter->dump_string("marker", store->meta_get_marker(handle)); + } + + // close result object section + formatter->close_section(); + + store->meta_list_keys_complete(handle); + + flusher.flush(); + return 0; +} + +int RGWUserAdminOp_User::list(const DoutPrefixProvider *dpp, rgw::sal::Store* store, RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher) +{ + RGWUser user; + + int ret = user.init_storage(store); + if (ret < 0) + return ret; + + ret = user.list(dpp, op_state, flusher); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUserAdminOp_User::info(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + std::unique_ptr ruser; + + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + ruser = store->get_user(info.user_id); + + if (op_state.sync_stats) { + ret = rgw_user_sync_all_stats(dpp, store, ruser.get(), y); + if (ret < 0) { + return ret; + } + } + + RGWStorageStats stats; + RGWStorageStats *arg_stats = NULL; + if (op_state.fetch_stats) { + int ret = ruser->read_stats(dpp, y, &stats); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + + arg_stats = &stats; + } + + if (formatter) { + flusher.start(0); + + dump_user_info(formatter, info, arg_stats); + flusher.flush(); + } + + return 0; +} + +int RGWUserAdminOp_User::create(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.add(dpp, op_state, y, NULL); + if (ret < 0) { + if (ret == -EEXIST) + ret = -ERR_USER_EXIST; + return ret; + } + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + dump_user_info(formatter, info); + flusher.flush(); + } + + return 0; +} + +int RGWUserAdminOp_User::modify(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + Formatter *formatter = flusher.get_formatter(); + + ret = user.modify(dpp, op_state, y, NULL); + if (ret < 0) { + if (ret == -ENOENT) + ret = -ERR_NO_SUCH_USER; + return ret; + } + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + dump_user_info(formatter, info); + flusher.flush(); + } + + return 0; +} + +int RGWUserAdminOp_User::remove(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + + ret = user.remove(dpp, op_state, y, NULL); + + if (ret == -ENOENT) + ret = -ERR_NO_SUCH_USER; + return ret; +} + +int RGWUserAdminOp_Subuser::create(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.subusers.add(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + dump_subusers_info(formatter, info); + flusher.flush(); + } + + return 0; +} + +int RGWUserAdminOp_Subuser::modify(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.subusers.modify(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + dump_subusers_info(formatter, info); + flusher.flush(); + } + + return 0; +} + +int RGWUserAdminOp_Subuser::remove(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + ret = user.subusers.remove(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUserAdminOp_Key::create(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.keys.add(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + int key_type = op_state.get_key_type(); + + if (key_type == KEY_TYPE_SWIFT) + dump_swift_keys_info(formatter, info); + + else if (key_type == KEY_TYPE_S3) + dump_access_keys_info(formatter, info); + + flusher.flush(); + } + + return 0; +} + +int RGWUserAdminOp_Key::remove(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, + optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + + ret = user.keys.remove(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + return 0; +} + +int RGWUserAdminOp_Caps::add(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.caps.add(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + info.caps.dump(formatter); + flusher.flush(); + } + + return 0; +} + + +int RGWUserAdminOp_Caps::remove(const DoutPrefixProvider *dpp, + rgw::sal::Store* store, + RGWUserAdminOpState& op_state, + RGWFormatterFlusher& flusher, optional_yield y) +{ + RGWUserInfo info; + RGWUser user; + int ret = user.init(dpp, store, op_state, y); + if (ret < 0) + return ret; + + if (!op_state.has_existing_user()) + return -ERR_NO_SUCH_USER; + + Formatter *formatter = flusher.get_formatter(); + + ret = user.caps.remove(dpp, op_state, y, NULL); + if (ret < 0) + return ret; + + ret = user.info(info, NULL); + if (ret < 0) + return ret; + + if (formatter) { + flusher.start(0); + + info.caps.dump(formatter); + flusher.flush(); + } + + return 0; +} + +class RGWUserMetadataHandler : public RGWMetadataHandler_GenericMetaBE { +public: + struct Svc { + RGWSI_User *user{nullptr}; + } svc; + + RGWUserMetadataHandler(RGWSI_User *user_svc) { + base_init(user_svc->ctx(), user_svc->get_be_handler()); + svc.user = user_svc; + } + + ~RGWUserMetadataHandler() {} + + string get_type() override { return "user"; } + + int do_get(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWMetadataObject **obj, optional_yield y, const DoutPrefixProvider *dpp) override { + RGWUserCompleteInfo uci; + RGWObjVersionTracker objv_tracker; + real_time mtime; + + rgw_user user = RGWSI_User::user_from_meta_key(entry); + + int ret = svc.user->read_user_info(op->ctx(), user, &uci.info, &objv_tracker, + &mtime, nullptr, &uci.attrs, + y, dpp); + if (ret < 0) { + return ret; + } + + RGWUserMetadataObject *mdo = new RGWUserMetadataObject(uci, objv_tracker.read_version, mtime); + *obj = mdo; + + return 0; + } + + RGWMetadataObject *get_meta_obj(JSONObj *jo, const obj_version& objv, const ceph::real_time& mtime) override { + RGWUserCompleteInfo uci; + + try { + decode_json_obj(uci, jo); + } catch (JSONDecoder::err& e) { + return nullptr; + } + + return new RGWUserMetadataObject(uci, objv, mtime); + } + + int do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp, + RGWMDLogSyncType type, bool from_remote_zone) override; + + int do_remove(RGWSI_MetaBackend_Handler::Op *op, string& entry, RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp) override { + RGWUserInfo info; + + rgw_user user = RGWSI_User::user_from_meta_key(entry); + + int ret = svc.user->read_user_info(op->ctx(), user, &info, nullptr, + nullptr, nullptr, nullptr, + y, dpp); + if (ret < 0) { + return ret; + } + + return svc.user->remove_user_info(op->ctx(), info, &objv_tracker, + y, dpp); + } +}; + +class RGWMetadataHandlerPut_User : public RGWMetadataHandlerPut_SObj +{ + RGWUserMetadataHandler *uhandler; + RGWUserMetadataObject *uobj; +public: + RGWMetadataHandlerPut_User(RGWUserMetadataHandler *_handler, + RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *obj, RGWObjVersionTracker& objv_tracker, + optional_yield y, + RGWMDLogSyncType type, bool from_remote_zone) : RGWMetadataHandlerPut_SObj(_handler, op, entry, obj, objv_tracker, y, type, from_remote_zone), + uhandler(_handler) { + uobj = static_cast(obj); + } + + int put_checked(const DoutPrefixProvider *dpp) override; +}; + +int RGWUserMetadataHandler::do_put(RGWSI_MetaBackend_Handler::Op *op, string& entry, + RGWMetadataObject *obj, + RGWObjVersionTracker& objv_tracker, + optional_yield y, const DoutPrefixProvider *dpp, + RGWMDLogSyncType type, bool from_remote_zone) +{ + RGWMetadataHandlerPut_User put_op(this, op, entry, obj, objv_tracker, y, type, from_remote_zone); + return do_put_operate(&put_op, dpp); +} + +int RGWMetadataHandlerPut_User::put_checked(const DoutPrefixProvider *dpp) +{ + RGWUserMetadataObject *orig_obj = static_cast(old_obj); + RGWUserCompleteInfo& uci = uobj->get_uci(); + + map *pattrs{nullptr}; + if (uci.has_attrs) { + pattrs = &uci.attrs; + } + + RGWUserInfo *pold_info = (orig_obj ? &orig_obj->get_uci().info : nullptr); + + auto mtime = obj->get_mtime(); + + int ret = uhandler->svc.user->store_user_info(op->ctx(), uci.info, pold_info, + &objv_tracker, mtime, + false, pattrs, y, dpp); + if (ret < 0) { + return ret; + } + + return STATUS_APPLIED; +} + + +RGWUserCtl::RGWUserCtl(RGWSI_Zone *zone_svc, + RGWSI_User *user_svc, + RGWUserMetadataHandler *_umhandler) : umhandler(_umhandler) { + svc.zone = zone_svc; + svc.user = user_svc; + be_handler = umhandler->get_be_handler(); +} + +template +class optional_default +{ + const std::optional& opt; + std::optional def; + const T *p; +public: + optional_default(const std::optional& _o) : opt(_o) { + if (opt) { + p = &(*opt); + } else { + def = T(); + p = &(*def); + } + } + + const T *operator->() { + return p; + } + + const T& operator*() { + return *p; + } +}; + +int RGWUserCtl::get_info_by_uid(const DoutPrefixProvider *dpp, + const rgw_user& uid, + RGWUserInfo *info, + optional_yield y, + const GetParams& params) + +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->read_user_info(op->ctx(), + uid, + info, + params.objv_tracker, + params.mtime, + params.cache_info, + params.attrs, + y, + dpp); + }); +} + +int RGWUserCtl::get_info_by_email(const DoutPrefixProvider *dpp, + const string& email, + RGWUserInfo *info, + optional_yield y, + const GetParams& params) +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->get_user_info_by_email(op->ctx(), email, + info, + params.objv_tracker, + params.mtime, + y, + dpp); + }); +} + +int RGWUserCtl::get_info_by_swift(const DoutPrefixProvider *dpp, + const string& swift_name, + RGWUserInfo *info, + optional_yield y, + const GetParams& params) +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->get_user_info_by_swift(op->ctx(), swift_name, + info, + params.objv_tracker, + params.mtime, + y, + dpp); + }); +} + +int RGWUserCtl::get_info_by_access_key(const DoutPrefixProvider *dpp, + const string& access_key, + RGWUserInfo *info, + optional_yield y, + const GetParams& params) +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->get_user_info_by_access_key(op->ctx(), access_key, + info, + params.objv_tracker, + params.mtime, + y, + dpp); + }); +} + +int RGWUserCtl::get_attrs_by_uid(const DoutPrefixProvider *dpp, + const rgw_user& user_id, + map *pattrs, + optional_yield y, + RGWObjVersionTracker *objv_tracker) +{ + RGWUserInfo user_info; + + return get_info_by_uid(dpp, user_id, &user_info, y, RGWUserCtl::GetParams() + .set_attrs(pattrs) + .set_objv_tracker(objv_tracker)); +} + +int RGWUserCtl::store_info(const DoutPrefixProvider *dpp, + const RGWUserInfo& info, optional_yield y, + const PutParams& params) +{ + string key = RGWSI_User::get_meta_key(info.user_id); + + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->store_user_info(op->ctx(), info, + params.old_info, + params.objv_tracker, + params.mtime, + params.exclusive, + params.attrs, + y, + dpp); + }); +} + +int RGWUserCtl::remove_info(const DoutPrefixProvider *dpp, + const RGWUserInfo& info, optional_yield y, + const RemoveParams& params) + +{ + string key = RGWSI_User::get_meta_key(info.user_id); + + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->remove_user_info(op->ctx(), info, + params.objv_tracker, + y, dpp); + }); +} + +int RGWUserCtl::list_buckets(const DoutPrefixProvider *dpp, + const rgw_user& user, + const string& marker, + const string& end_marker, + uint64_t max, + bool need_stats, + RGWUserBuckets *buckets, + bool *is_truncated, + optional_yield y, + uint64_t default_max) +{ + if (!max) { + max = default_max; + } + + int ret = svc.user->list_buckets(dpp, user, marker, end_marker, + max, buckets, is_truncated, y); + if (ret < 0) { + return ret; + } + if (need_stats) { + map& m = buckets->get_buckets(); + ret = ctl.bucket->read_buckets_stats(m, y, dpp); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: could not get stats for buckets" << dendl; + return ret; + } + } + return 0; +} + +int RGWUserCtl::read_stats(const DoutPrefixProvider *dpp, + const rgw_user& user, RGWStorageStats *stats, + optional_yield y, + ceph::real_time *last_stats_sync, + ceph::real_time *last_stats_update) +{ + return be_handler->call([&](RGWSI_MetaBackend_Handler::Op *op) { + return svc.user->read_stats(dpp, op->ctx(), user, stats, + last_stats_sync, last_stats_update, y); + }); +} + +RGWMetadataHandler *RGWUserMetaHandlerAllocator::alloc(RGWSI_User *user_svc) { + return new RGWUserMetadataHandler(user_svc); +} + +void rgw_user::dump(Formatter *f) const +{ + ::encode_json("user", *this, f); +} + diff --git a/src/rgw/rgw_user.h b/src/rgw/store/rados/rgw_user.h similarity index 100% rename from src/rgw/rgw_user.h rename to src/rgw/store/rados/rgw_user.h diff --git a/src/rgw/store/rados/rgw_zone.cc b/src/rgw/store/rados/rgw_zone.cc new file mode 100644 index 000000000000..4fa6a52c9124 --- /dev/null +++ b/src/rgw/store/rados/rgw_zone.cc @@ -0,0 +1,1287 @@ +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab ft=cpp + +#include "rgw_zone.h" +#include "rgw_realm_watcher.h" +#include "rgw_sal_config.h" +#include "rgw_sync.h" + +#include "services/svc_zone.h" + +#define dout_context g_ceph_context +#define dout_subsys ceph_subsys_rgw + +using namespace std; +using namespace rgw_zone_defaults; + +RGWMetaSyncStatusManager::~RGWMetaSyncStatusManager(){} + +#define FIRST_EPOCH 1 + +struct RGWAccessKey; + +/// Generate a random uuid for realm/period/zonegroup/zone ids +static std::string gen_random_uuid() +{ + uuid_d uuid; + uuid.generate_random(); + return uuid.to_string(); +} + +void RGWDefaultZoneGroupInfo::dump(Formatter *f) const { + encode_json("default_zonegroup", default_zonegroup, f); +} + +void RGWDefaultZoneGroupInfo::decode_json(JSONObj *obj) { + + JSONDecoder::decode_json("default_zonegroup", default_zonegroup, obj); + /* backward compatability with region */ + if (default_zonegroup.empty()) { + JSONDecoder::decode_json("default_region", default_zonegroup, obj); + } +} + +int RGWZoneGroup::create_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) +{ + name = default_zonegroup_name; + api_name = default_zonegroup_name; + is_master = true; + + RGWZoneGroupPlacementTarget placement_target; + placement_target.name = "default-placement"; + placement_targets[placement_target.name] = placement_target; + default_placement.name = "default-placement"; + + RGWZoneParams zone_params(default_zone_name); + + int r = zone_params.init(dpp, cct, sysobj_svc, y, false); + if (r < 0) { + ldpp_dout(dpp, 0) << "create_default: error initializing zone params: " << cpp_strerror(-r) << dendl; + return r; + } + + r = zone_params.create_default(dpp, y); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "create_default: error in create_default zone params: " << cpp_strerror(-r) << dendl; + return r; + } else if (r == -EEXIST) { + ldpp_dout(dpp, 10) << "zone_params::create_default() returned -EEXIST, we raced with another default zone_params creation" << dendl; + zone_params.clear_id(); + r = zone_params.init(dpp, cct, sysobj_svc, y); + if (r < 0) { + ldpp_dout(dpp, 0) << "create_default: error in init existing zone params: " << cpp_strerror(-r) << dendl; + return r; + } + ldpp_dout(dpp, 20) << "zone_params::create_default() " << zone_params.get_name() << " id " << zone_params.get_id() + << dendl; + } + + RGWZone& default_zone = zones[zone_params.get_id()]; + default_zone.name = zone_params.get_name(); + default_zone.id = zone_params.get_id(); + master_zone = default_zone.id; + + // enable all supported features + enabled_features.insert(rgw::zone_features::supported.begin(), + rgw::zone_features::supported.end()); + default_zone.supported_features = enabled_features; + + r = create(dpp, y); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "error storing zone group info: " << cpp_strerror(-r) << dendl; + return r; + } + + if (r == -EEXIST) { + ldpp_dout(dpp, 10) << "create_default() returned -EEXIST, we raced with another zonegroup creation" << dendl; + id.clear(); + r = init(dpp, cct, sysobj_svc, y); + if (r < 0) { + return r; + } + } + + if (old_format) { + name = id; + } + + post_process_params(dpp, y); + + return 0; +} + +int RGWZoneGroup::equals(const string& other_zonegroup) const +{ + if (is_master && other_zonegroup.empty()) + return true; + + return (id == other_zonegroup); +} + +int RGWZoneGroup::add_zone(const DoutPrefixProvider *dpp, + const RGWZoneParams& zone_params, bool *is_master, bool *read_only, + const list& endpoints, const string *ptier_type, + bool *psync_from_all, list& sync_from, list& sync_from_rm, + string *predirect_zone, std::optional bucket_index_max_shards, + RGWSyncModulesManager *sync_mgr, + const rgw::zone_features::set& enable_features, + const rgw::zone_features::set& disable_features, + optional_yield y) +{ + auto& zone_id = zone_params.get_id(); + auto& zone_name = zone_params.get_name(); + + // check for duplicate zone name on insert + if (!zones.count(zone_id)) { + for (const auto& zone : zones) { + if (zone.second.name == zone_name) { + ldpp_dout(dpp, 0) << "ERROR: found existing zone name " << zone_name + << " (" << zone.first << ") in zonegroup " << get_name() << dendl; + return -EEXIST; + } + } + } + + if (is_master) { + if (*is_master) { + if (!master_zone.empty() && master_zone != zone_id) { + ldpp_dout(dpp, 0) << "NOTICE: overriding master zone: " << master_zone << dendl; + } + master_zone = zone_id; + } else if (master_zone == zone_id) { + master_zone.clear(); + } + } + + RGWZone& zone = zones[zone_id]; + zone.name = zone_name; + zone.id = zone_id; + if (!endpoints.empty()) { + zone.endpoints = endpoints; + } + if (read_only) { + zone.read_only = *read_only; + } + if (ptier_type) { + zone.tier_type = *ptier_type; + if (!sync_mgr->get_module(*ptier_type, nullptr)) { + ldpp_dout(dpp, 0) << "ERROR: could not found sync module: " << *ptier_type + << ", valid sync modules: " + << sync_mgr->get_registered_module_names() + << dendl; + return -ENOENT; + } + } + + if (psync_from_all) { + zone.sync_from_all = *psync_from_all; + } + + if (predirect_zone) { + zone.redirect_zone = *predirect_zone; + } + + if (bucket_index_max_shards) { + zone.bucket_index_max_shards = *bucket_index_max_shards; + } + + for (auto add : sync_from) { + zone.sync_from.insert(add); + } + + for (auto rm : sync_from_rm) { + zone.sync_from.erase(rm); + } + + zone.supported_features.insert(enable_features.begin(), + enable_features.end()); + + for (const auto& feature : disable_features) { + if (enabled_features.contains(feature)) { + lderr(cct) << "ERROR: Cannot disable zone feature \"" << feature + << "\" until it's been disabled in zonegroup " << name << dendl; + return -EINVAL; + } + auto i = zone.supported_features.find(feature); + if (i == zone.supported_features.end()) { + ldout(cct, 1) << "WARNING: zone feature \"" << feature + << "\" was not enabled in zone " << zone.name << dendl; + continue; + } + zone.supported_features.erase(i); + } + + post_process_params(dpp, y); + + return update(dpp,y); +} + + +int RGWZoneGroup::rename_zone(const DoutPrefixProvider *dpp, + const RGWZoneParams& zone_params, + optional_yield y) +{ + RGWZone& zone = zones[zone_params.get_id()]; + zone.name = zone_params.get_name(); + + return update(dpp, y); +} + +void RGWZoneGroup::post_process_params(const DoutPrefixProvider *dpp, optional_yield y) +{ + bool log_data = zones.size() > 1; + + if (master_zone.empty()) { + auto iter = zones.begin(); + if (iter != zones.end()) { + master_zone = iter->first; + } + } + + for (auto& item : zones) { + RGWZone& zone = item.second; + zone.log_data = log_data; + + RGWZoneParams zone_params(zone.id, zone.name); + int ret = zone_params.init(dpp, cct, sysobj_svc, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: could not read zone params for zone id=" << zone.id << " name=" << zone.name << dendl; + continue; + } + + for (auto& pitem : zone_params.placement_pools) { + const string& placement_name = pitem.first; + if (placement_targets.find(placement_name) == placement_targets.end()) { + RGWZoneGroupPlacementTarget placement_target; + placement_target.name = placement_name; + placement_targets[placement_name] = placement_target; + } + } + } + + if (default_placement.empty() && !placement_targets.empty()) { + default_placement.init(placement_targets.begin()->first, RGW_STORAGE_CLASS_STANDARD); + } +} + +int RGWZoneGroup::remove_zone(const DoutPrefixProvider *dpp, const std::string& zone_id, optional_yield y) +{ + auto iter = zones.find(zone_id); + if (iter == zones.end()) { + ldpp_dout(dpp, 0) << "zone id " << zone_id << " is not a part of zonegroup " + << name << dendl; + return -ENOENT; + } + + zones.erase(iter); + + post_process_params(dpp, y); + + return update(dpp, y); +} + +void RGWDefaultSystemMetaObjInfo::dump(Formatter *f) const { + encode_json("default_id", default_id, f); +} + +void RGWDefaultSystemMetaObjInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("default_id", default_id, obj); +} + +int RGWSystemMetaObj::rename(const DoutPrefixProvider *dpp, const string& new_name, optional_yield y) +{ + string new_id; + int ret = read_id(dpp, new_name, new_id, y); + if (!ret) { + return -EEXIST; + } + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "Error read_id " << new_name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + string old_name = name; + name = new_name; + ret = update(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Error storing new obj info " << new_name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + ret = store_name(dpp, true, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Error storing new name " << new_name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + /* delete old name */ + rgw_pool pool(get_pool(cct)); + string oid = get_names_oid_prefix() + old_name; + rgw_raw_obj old_name_obj(pool, oid); + auto sysobj = sysobj_svc->get_obj(old_name_obj); + ret = sysobj.wop().remove(dpp, y); + if (ret < 0) { + ldpp_dout(dpp, 0) << "Error delete old obj name " << old_name << ": " << cpp_strerror(-ret) << dendl; + return ret; + } + + return ret; +} + +int RGWSystemMetaObj::read(const DoutPrefixProvider *dpp, optional_yield y) +{ + int ret = read_id(dpp, name, id, y); + if (ret < 0) { + return ret; + } + + return read_info(dpp, id, y); +} + +int RGWZoneParams::create_default(const DoutPrefixProvider *dpp, optional_yield y, bool old_format) +{ + name = default_zone_name; + + int r = create(dpp, y); + if (r < 0) { + return r; + } + + if (old_format) { + name = id; + } + + return r; +} + +const string& RGWZoneParams::get_compression_type(const rgw_placement_rule& placement_rule) const +{ + static const std::string NONE{"none"}; + auto p = placement_pools.find(placement_rule.name); + if (p == placement_pools.end()) { + return NONE; + } + const auto& type = p->second.get_compression_type(placement_rule.get_storage_class()); + return !type.empty() ? type : NONE; +} + +// run an MD5 hash on the zone_id and return the first 32 bits +static uint32_t gen_short_zone_id(const std::string zone_id) +{ + unsigned char md5[CEPH_CRYPTO_MD5_DIGESTSIZE]; + MD5 hash; + // Allow use of MD5 digest in FIPS mode for non-cryptographic purposes + hash.SetFlags(EVP_MD_CTX_FLAG_NON_FIPS_ALLOW); + hash.Update((const unsigned char *)zone_id.c_str(), zone_id.size()); + hash.Final(md5); + + uint32_t short_id; + memcpy((char *)&short_id, md5, sizeof(short_id)); + return std::max(short_id, 1u); +} + +int RGWPeriodMap::update(const RGWZoneGroup& zonegroup, CephContext *cct) +{ + if (zonegroup.is_master_zonegroup() && (!master_zonegroup.empty() && zonegroup.get_id() != master_zonegroup)) { + ldout(cct,0) << "Error updating periodmap, multiple master zonegroups configured "<< dendl; + ldout(cct,0) << "master zonegroup: " << master_zonegroup << " and " << zonegroup.get_id() <::iterator iter = zonegroups.find(zonegroup.get_id()); + if (iter != zonegroups.end()) { + RGWZoneGroup& old_zonegroup = iter->second; + if (!old_zonegroup.api_name.empty()) { + zonegroups_by_api.erase(old_zonegroup.api_name); + } + } + zonegroups[zonegroup.get_id()] = zonegroup; + + if (!zonegroup.api_name.empty()) { + zonegroups_by_api[zonegroup.api_name] = zonegroup; + } + + if (zonegroup.is_master_zonegroup()) { + master_zonegroup = zonegroup.get_id(); + } else if (master_zonegroup == zonegroup.get_id()) { + master_zonegroup = ""; + } + + for (auto& i : zonegroup.zones) { + auto& zone = i.second; + if (short_zone_ids.find(zone.id) != short_zone_ids.end()) { + continue; + } + // calculate the zone's short id + uint32_t short_id = gen_short_zone_id(zone.id); + + // search for an existing zone with the same short id + for (auto& s : short_zone_ids) { + if (s.second == short_id) { + ldout(cct, 0) << "New zone '" << zone.name << "' (" << zone.id + << ") generates the same short_zone_id " << short_id + << " as existing zone id " << s.first << dendl; + return -EEXIST; + } + } + + short_zone_ids[zone.id] = short_id; + } + + return 0; +} + +uint32_t RGWPeriodMap::get_zone_short_id(const string& zone_id) const +{ + auto i = short_zone_ids.find(zone_id); + if (i == short_zone_ids.end()) { + return 0; + } + return i->second; +} + +bool RGWPeriodMap::find_zone_by_name(const string& zone_name, + RGWZoneGroup *zonegroup, + RGWZone *zone) const +{ + for (auto& iter : zonegroups) { + auto& zg = iter.second; + for (auto& ziter : zg.zones) { + auto& z = ziter.second; + + if (z.name == zone_name) { + *zonegroup = zg; + *zone = z; + return true; + } + } + } + + return false; +} + +namespace rgw { + +int read_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view realm_id, + std::string_view realm_name, + RGWRealm& info, + std::unique_ptr* writer) +{ + if (!realm_id.empty()) { + return cfgstore->read_realm_by_id(dpp, y, realm_id, info, writer); + } + if (!realm_name.empty()) { + return cfgstore->read_realm_by_name(dpp, y, realm_name, info, writer); + } + return cfgstore->read_default_realm(dpp, y, info, writer); +} + +int create_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWRealm& info, + std::unique_ptr* writer_out) +{ + if (info.name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a realm name" << dendl; + return -EINVAL; + } + if (info.id.empty()) { + info.id = gen_random_uuid(); + } + + // if the realm already has a current_period, just make sure it exists + std::optional period; + if (!info.current_period.empty()) { + period.emplace(); + int r = cfgstore->read_period(dpp, y, info.current_period, + std::nullopt, *period); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to read realm's current_period=" + << info.current_period << " with " << cpp_strerror(r) << dendl; + return r; + } + } + + // create the realm + std::unique_ptr writer; + int r = cfgstore->create_realm(dpp, y, exclusive, info, &writer); + if (r < 0) { + return r; + } + + if (!period) { + // initialize and exclusive-create the initial period + period.emplace(); + period->id = gen_random_uuid(); + period->period_map.id = period->id; + period->epoch = FIRST_EPOCH; + period->realm_id = info.id; + period->realm_name = info.name; + + r = cfgstore->create_period(dpp, y, true, *period); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to create the initial period id=" + << period->id << " for realm " << info.name + << " with " << cpp_strerror(r) << dendl; + return r; + } + } + + // update the realm's current_period + r = realm_set_current_period(dpp, y, cfgstore, *writer, info, *period); + if (r < 0) { + return r; + } + + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + r = set_default_realm(dpp, y, cfgstore, info, true); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "WARNING: failed to set realm as default: " + << cpp_strerror(r) << dendl; + } + + if (writer_out) { + *writer_out = std::move(writer); + } + return 0; +} + +int set_default_realm(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWRealm& info, + bool exclusive) +{ + return cfgstore->write_default_realm_id(dpp, y, exclusive, info.id); +} + +int realm_set_current_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + sal::RealmWriter& writer, RGWRealm& realm, + const RGWPeriod& period) +{ + // update realm epoch to match the period's + if (realm.epoch > period.realm_epoch) { + ldpp_dout(dpp, -1) << __func__ << " with old realm epoch " + << period.realm_epoch << ", current epoch=" << realm.epoch << dendl; + return -EINVAL; + } + if (realm.epoch == period.realm_epoch && realm.current_period != period.id) { + ldpp_dout(dpp, -1) << __func__ << " with same realm epoch " + << period.realm_epoch << ", but different period id " + << period.id << " != " << realm.current_period << dendl; + return -EINVAL; + } + + realm.epoch = period.realm_epoch; + realm.current_period = period.id; + + // update the realm object + int r = writer.write(dpp, y, realm); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to overwrite realm " + << realm.name << " with " << cpp_strerror(r) << dendl; + return r; + } + + // reflect the zonegroup and period config + (void) reflect_period(dpp, y, cfgstore, period); + return 0; +} + +int reflect_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWPeriod& info) +{ + // overwrite the local period config and zonegroup objects + constexpr bool exclusive = false; + + int r = cfgstore->write_period_config(dpp, y, exclusive, info.realm_id, + info.period_config); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to store period config for realm id=" + << info.realm_id << " with " << cpp_strerror(r) << dendl; + return r; + } + + for (auto& [zonegroup_id, zonegroup] : info.period_map.zonegroups) { + r = cfgstore->create_zonegroup(dpp, y, exclusive, zonegroup, nullptr); + if (r < 0) { + ldpp_dout(dpp, -1) << __func__ << " failed to store zonegroup id=" + << zonegroup_id << " with " << cpp_strerror(r) << dendl; + return r; + } + if (zonegroup.is_master) { + // set master as default if no default exists + constexpr bool exclusive = true; + r = set_default_zonegroup(dpp, y, cfgstore, zonegroup, exclusive); + if (r == 0) { + ldpp_dout(dpp, 1) << "Set the period's master zonegroup " + << zonegroup.name << " as the default" << dendl; + } + } + } + return 0; +} + +std::string get_staging_period_id(std::string_view realm_id) +{ + return string_cat_reserve(realm_id, ":staging"); +} + +void fork_period(const DoutPrefixProvider* dpp, RGWPeriod& info) +{ + ldpp_dout(dpp, 20) << __func__ << " realm id=" << info.realm_id + << " period id=" << info.id << dendl; + + info.predecessor_uuid = std::move(info.id); + info.id = get_staging_period_id(info.realm_id); + info.period_map.reset(); + info.realm_epoch++; +} + +int update_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, RGWPeriod& info) +{ + // clear zone short ids of removed zones. period_map.update() will add the + // remaining zones back + info.period_map.short_zone_ids.clear(); + + // list all zonegroups in the realm + rgw::sal::ListResult listing; + std::array zonegroup_names; // list in pages of 1000 + do { + int ret = cfgstore->list_zonegroup_names(dpp, y, listing.next, + zonegroup_names, listing); + if (ret < 0) { + std::cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl; + return -ret; + } + for (const auto& name : listing.entries) { + RGWZoneGroup zg; + ret = cfgstore->read_zonegroup_by_name(dpp, y, name, zg, nullptr); + if (ret < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to read zonegroup " + << name << ": " << cpp_strerror(-ret) << dendl; + continue; + } + + if (zg.realm_id != info.realm_id) { + ldpp_dout(dpp, 20) << "skipping zonegroup " << zg.get_name() + << " with realm id " << zg.realm_id + << ", not on our realm " << info.realm_id << dendl; + continue; + } + + if (zg.master_zone.empty()) { + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() << " should have a master zone " << dendl; + return -EINVAL; + } + + if (zg.zones.find(zg.master_zone) == zg.zones.end()) { + ldpp_dout(dpp, 0) << "ERROR: zonegroup " << zg.get_name() + << " has a non existent master zone "<< dendl; + return -EINVAL; + } + + if (zg.is_master_zonegroup()) { + info.master_zonegroup = zg.get_id(); + info.master_zone = zg.master_zone; + } + + ret = info.period_map.update(zg, dpp->get_cct()); + if (ret < 0) { + return ret; + } + } // foreach name in listing.entries + } while (!listing.next.empty()); + + // read the realm's current period config + int ret = cfgstore->read_period_config(dpp, y, info.realm_id, + info.period_config); + if (ret < 0 && ret != -ENOENT) { + ldpp_dout(dpp, 0) << "ERROR: failed to read period config: " + << cpp_strerror(ret) << dendl; + return ret; + } + + return 0; +} + +int commit_period(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, sal::Store* store, + RGWRealm& realm, sal::RealmWriter& realm_writer, + const RGWPeriod& current_period, + RGWPeriod& info, std::ostream& error_stream, + bool force_if_stale) +{ + auto zone_svc = static_cast(store)->svc()->zone; // XXX + + ldpp_dout(dpp, 20) << __func__ << " realm " << realm.id + << " period " << current_period.id << dendl; + // gateway must be in the master zone to commit + if (info.master_zone != zone_svc->get_zone_params().id) { + error_stream << "Cannot commit period on zone " + << zone_svc->get_zone_params().id << ", it must be sent to " + "the period's master zone " << info.master_zone << '.' << std::endl; + return -EINVAL; + } + // period predecessor must match current period + if (info.predecessor_uuid != current_period.id) { + error_stream << "Period predecessor " << info.predecessor_uuid + << " does not match current period " << current_period.id + << ". Use 'period pull' to get the latest period from the master, " + "reapply your changes, and try again." << std::endl; + return -EINVAL; + } + // realm epoch must be 1 greater than current period + if (info.realm_epoch != current_period.realm_epoch + 1) { + error_stream << "Period's realm epoch " << info.realm_epoch + << " does not come directly after current realm epoch " + << current_period.realm_epoch << ". Use 'realm pull' to get the " + "latest realm and period from the master zone, reapply your changes, " + "and try again." << std::endl; + return -EINVAL; + } + // did the master zone change? + if (info.master_zone != current_period.master_zone) { + // store the current metadata sync status in the period + int r = info.update_sync_status(dpp, store, current_period, + error_stream, force_if_stale); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update metadata sync status: " + << cpp_strerror(-r) << dendl; + return r; + } + // create an object with a new period id + info.period_map.id = info.id = gen_random_uuid(); + info.epoch = FIRST_EPOCH; + + constexpr bool exclusive = true; + r = cfgstore->create_period(dpp, y, exclusive, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create new period: " << cpp_strerror(-r) << dendl; + return r; + } + // set as current period + r = realm_set_current_period(dpp, y, cfgstore, realm_writer, realm, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update realm's current period: " + << cpp_strerror(-r) << dendl; + return r; + } + ldpp_dout(dpp, 4) << "Promoted to master zone and committed new period " + << info.id << dendl; + (void) cfgstore->realm_notify_new_period(dpp, y, info); + return 0; + } + // period must be based on current epoch + if (info.epoch != current_period.epoch) { + error_stream << "Period epoch " << info.epoch << " does not match " + "predecessor epoch " << current_period.epoch << ". Use " + "'period pull' to get the latest epoch from the master zone, " + "reapply your changes, and try again." << std::endl; + return -EINVAL; + } + // set period as next epoch + info.id = current_period.id; + info.epoch = current_period.epoch + 1; + info.predecessor_uuid = current_period.predecessor_uuid; + info.realm_epoch = current_period.realm_epoch; + // write the period + constexpr bool exclusive = true; + int r = cfgstore->create_period(dpp, y, exclusive, info); + if (r == -EEXIST) { + // already have this epoch (or a more recent one) + return 0; + } + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to store period: " << cpp_strerror(r) << dendl; + return r; + } + r = reflect_period(dpp, y, cfgstore, info); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to update local objects: " << cpp_strerror(r) << dendl; + return r; + } + ldpp_dout(dpp, 4) << "Committed new epoch " << info.epoch + << " for period " << info.id << dendl; + (void) cfgstore->realm_notify_new_period(dpp, y, info); + return 0; +} + + +int read_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view zonegroup_id, + std::string_view zonegroup_name, + RGWZoneGroup& info, + std::unique_ptr* writer) +{ + if (!zonegroup_id.empty()) { + return cfgstore->read_zonegroup_by_id(dpp, y, zonegroup_id, info, writer); + } + if (!zonegroup_name.empty()) { + return cfgstore->read_zonegroup_by_name(dpp, y, zonegroup_name, info, writer); + } + + std::string realm_id; + int r = cfgstore->read_default_realm_id(dpp, y, realm_id); + if (r == -ENOENT) { + return cfgstore->read_zonegroup_by_name(dpp, y, default_zonegroup_name, + info, writer); + } + if (r < 0) { + return r; + } + return cfgstore->read_default_zonegroup(dpp, y, realm_id, info, writer); +} + +int create_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWZoneGroup& info) +{ + if (info.name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a zonegroup name" << dendl; + return -EINVAL; + } + if (info.id.empty()) { + info.id = gen_random_uuid(); + } + + // insert the default placement target if it doesn't exist + constexpr std::string_view default_placement_name = "default-placement"; + + RGWZoneGroupPlacementTarget placement_target; + placement_target.name = default_placement_name; + + info.placement_targets.emplace(default_placement_name, placement_target); + if (info.default_placement.name.empty()) { + info.default_placement.name = default_placement_name; + } + + int r = cfgstore->create_zonegroup(dpp, y, exclusive, info, nullptr); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create zonegroup with " + << cpp_strerror(r) << dendl; + return r; + } + + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + r = set_default_zonegroup(dpp, y, cfgstore, info, true); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "WARNING: failed to set zonegroup as default: " + << cpp_strerror(r) << dendl; + } + + return 0; +} + +int set_default_zonegroup(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneGroup& info, + bool exclusive) +{ + return cfgstore->write_default_zonegroup_id( + dpp, y, exclusive, info.realm_id, info.id); +} + +int remove_zone_from_group(const DoutPrefixProvider* dpp, + RGWZoneGroup& zonegroup, + const rgw_zone_id& zone_id) +{ + auto z = zonegroup.zones.find(zone_id); + if (z == zonegroup.zones.end()) { + return -ENOENT; + } + zonegroup.zones.erase(z); + + if (zonegroup.master_zone == zone_id) { + // choose a new master zone + auto m = zonegroup.zones.begin(); + if (m != zonegroup.zones.end()) { + zonegroup.master_zone = m->first; + ldpp_dout(dpp, 0) << "NOTICE: promoted " << m->second.name + << " as new master_zone of zonegroup " << zonegroup.name << dendl; + } else { + zonegroup.master_zone.clear(); + ldpp_dout(dpp, 0) << "NOTICE: cleared master_zone of zonegroup " + << zonegroup.name << dendl; + } + } + + const bool log_data = zonegroup.zones.size() > 1; + for (auto& [id, zone] : zonegroup.zones) { + zone.log_data = log_data; + } + + return 0; +} + +// try to remove the given zone id from every zonegroup in the cluster +static int remove_zone_from_groups(const DoutPrefixProvider* dpp, + optional_yield y, + sal::ConfigStore* cfgstore, + const rgw_zone_id& zone_id) +{ + std::array zonegroup_names; + sal::ListResult listing; + do { + int r = cfgstore->list_zonegroup_names(dpp, y, listing.next, + zonegroup_names, listing); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to list zonegroups with " + << cpp_strerror(r) << dendl; + return r; + } + + for (const auto& name : listing.entries) { + RGWZoneGroup zonegroup; + std::unique_ptr writer; + r = cfgstore->read_zonegroup_by_name(dpp, y, name, zonegroup, &writer); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to load zonegroup " << name + << " with " << cpp_strerror(r) << dendl; + continue; + } + + r = remove_zone_from_group(dpp, zonegroup, zone_id); + if (r < 0) { + continue; + } + + // write the updated zonegroup + r = writer->write(dpp, y, zonegroup); + if (r < 0) { + ldpp_dout(dpp, 0) << "WARNING: failed to write zonegroup " << name + << " with " << cpp_strerror(r) << dendl; + continue; + } + ldpp_dout(dpp, 0) << "Removed zone from zonegroup " << name << dendl; + } + } while (!listing.next.empty()); + + return 0; +} + + +int read_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, + std::string_view zone_id, + std::string_view zone_name, + RGWZoneParams& info, + std::unique_ptr* writer) +{ + if (!zone_id.empty()) { + return cfgstore->read_zone_by_id(dpp, y, zone_id, info, writer); + } + if (!zone_name.empty()) { + return cfgstore->read_zone_by_name(dpp, y, zone_name, info, writer); + } + + std::string realm_id; + int r = cfgstore->read_default_realm_id(dpp, y, realm_id); + if (r == -ENOENT) { + return cfgstore->read_zone_by_name(dpp, y, default_zone_name, info, writer); + } + if (r < 0) { + return r; + } + return cfgstore->read_default_zone(dpp, y, realm_id, info, writer); +} + +extern int get_zones_pool_set(const DoutPrefixProvider *dpp, optional_yield y, + rgw::sal::ConfigStore* cfgstore, + std::string_view my_zone_id, + std::set& pools); + +int create_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, bool exclusive, + RGWZoneParams& info, std::unique_ptr* writer) +{ + if (info.name.empty()) { + ldpp_dout(dpp, -1) << __func__ << " requires a zone name" << dendl; + return -EINVAL; + } + if (info.id.empty()) { + info.id = gen_random_uuid(); + } + + // add default placement with empty pool name + rgw_pool pool; + auto& placement = info.placement_pools["default-placement"]; + placement.storage_classes.set_storage_class( + RGW_STORAGE_CLASS_STANDARD, &pool, nullptr); + + // build a set of all pool names used by other zones + std::set pools; + int r = get_zones_pool_set(dpp, y, cfgstore, info.id, pools); + if (r < 0) { + return r; + } + + // initialize pool names with the zone name prefix + r = init_zone_pool_names(dpp, y, pools, info); + if (r < 0) { + return r; + } + + r = cfgstore->create_zone(dpp, y, exclusive, info, nullptr); + if (r < 0) { + ldpp_dout(dpp, 0) << "failed to create zone with " + << cpp_strerror(r) << dendl; + return r; + } + + // try to set as default. may race with another create, so pass exclusive=true + // so we don't override an existing default + r = set_default_zone(dpp, y, cfgstore, info, true); + if (r < 0 && r != -EEXIST) { + ldpp_dout(dpp, 0) << "WARNING: failed to set zone as default: " + << cpp_strerror(r) << dendl; + } + + return 0; + +} + +int set_default_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneParams& info, + bool exclusive) +{ + return cfgstore->write_default_zone_id( + dpp, y, exclusive, info.realm_id, info.id); +} + +int delete_zone(const DoutPrefixProvider* dpp, optional_yield y, + sal::ConfigStore* cfgstore, const RGWZoneParams& info, + sal::ZoneWriter& writer) +{ + // remove this zone from any zonegroups that contain it + int r = remove_zone_from_groups(dpp, y, cfgstore, info.id); + if (r < 0) { + return r; + } + + return writer.remove(dpp, y); +} + +} // namespace rgw + +static inline int conf_to_uint64(const JSONFormattable& config, const string& key, uint64_t *pval) +{ + string sval; + if (config.find(key, &sval)) { + string err; + uint64_t val = strict_strtoll(sval.c_str(), 10, &err); + if (!err.empty()) { + return -EINVAL; + } + *pval = val; + } + return 0; +} + +int RGWZoneGroupPlacementTier::update_params(const JSONFormattable& config) +{ + int r = -1; + + if (config.exists("retain_head_object")) { + string s = config["retain_head_object"]; + if (s == "true") { + retain_head_object = true; + } else { + retain_head_object = false; + } + } + + if (tier_type == "cloud-s3") { + r = t.s3.update_params(config); + } + + return r; +} + +int RGWZoneGroupPlacementTier::clear_params(const JSONFormattable& config) +{ + if (config.exists("retain_head_object")) { + retain_head_object = false; + } + + if (tier_type == "cloud-s3") { + t.s3.clear_params(config); + } + + return 0; +} + +int RGWZoneGroupPlacementTierS3::update_params(const JSONFormattable& config) +{ + int r = -1; + + if (config.exists("endpoint")) { + endpoint = config["endpoint"]; + } + if (config.exists("target_path")) { + target_path = config["target_path"]; + } + if (config.exists("region")) { + region = config["region"]; + } + if (config.exists("host_style")) { + string s; + s = config["host_style"]; + if (s != "virtual") { + host_style = PathStyle; + } else { + host_style = VirtualStyle; + } + } + if (config.exists("target_storage_class")) { + target_storage_class = config["target_storage_class"]; + } + if (config.exists("access_key")) { + key.id = config["access_key"]; + } + if (config.exists("secret")) { + key.key = config["secret"]; + } + if (config.exists("multipart_sync_threshold")) { + r = conf_to_uint64(config, "multipart_sync_threshold", &multipart_sync_threshold); + if (r < 0) { + multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + } + + if (config.exists("multipart_min_part_size")) { + r = conf_to_uint64(config, "multipart_min_part_size", &multipart_min_part_size); + if (r < 0) { + multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + } + + if (config.exists("acls")) { + const JSONFormattable& cc = config["acls"]; + if (cc.is_array()) { + for (auto& c : cc.array()) { + RGWTierACLMapping m; + m.init(c); + if (!m.source_id.empty()) { + acl_mappings[m.source_id] = m; + } + } + } else { + RGWTierACLMapping m; + m.init(cc); + if (!m.source_id.empty()) { + acl_mappings[m.source_id] = m; + } + } + } + return 0; +} + +int RGWZoneGroupPlacementTierS3::clear_params(const JSONFormattable& config) +{ + if (config.exists("endpoint")) { + endpoint.clear(); + } + if (config.exists("target_path")) { + target_path.clear(); + } + if (config.exists("region")) { + region.clear(); + } + if (config.exists("host_style")) { + /* default */ + host_style = PathStyle; + } + if (config.exists("target_storage_class")) { + target_storage_class.clear(); + } + if (config.exists("access_key")) { + key.id.clear(); + } + if (config.exists("secret")) { + key.key.clear(); + } + if (config.exists("multipart_sync_threshold")) { + multipart_sync_threshold = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + if (config.exists("multipart_min_part_size")) { + multipart_min_part_size = DEFAULT_MULTIPART_SYNC_PART_SIZE; + } + if (config.exists("acls")) { + const JSONFormattable& cc = config["acls"]; + if (cc.is_array()) { + for (auto& c : cc.array()) { + RGWTierACLMapping m; + m.init(c); + acl_mappings.erase(m.source_id); + } + } else { + RGWTierACLMapping m; + m.init(cc); + acl_mappings.erase(m.source_id); + } + } + return 0; +} + +void rgw_meta_sync_info::generate_test_instances(list& o) +{ + auto info = new rgw_meta_sync_info; + info->state = rgw_meta_sync_info::StateBuildingFullSyncMaps; + info->period = "periodid"; + info->realm_epoch = 5; + o.push_back(info); + o.push_back(new rgw_meta_sync_info); +} + +void rgw_meta_sync_marker::generate_test_instances(list& o) +{ + auto marker = new rgw_meta_sync_marker; + marker->state = rgw_meta_sync_marker::IncrementalSync; + marker->marker = "01234"; + marker->realm_epoch = 5; + o.push_back(marker); + o.push_back(new rgw_meta_sync_marker); +} + +void rgw_meta_sync_status::generate_test_instances(list& o) +{ + o.push_back(new rgw_meta_sync_status); +} + +void RGWZoneParams::generate_test_instances(list &o) +{ + o.push_back(new RGWZoneParams); + o.push_back(new RGWZoneParams); +} + +void RGWPeriodLatestEpochInfo::generate_test_instances(list &o) +{ + RGWPeriodLatestEpochInfo *z = new RGWPeriodLatestEpochInfo; + o.push_back(z); + o.push_back(new RGWPeriodLatestEpochInfo); +} + +void RGWZoneGroup::generate_test_instances(list& o) +{ + RGWZoneGroup *r = new RGWZoneGroup; + o.push_back(r); + o.push_back(new RGWZoneGroup); +} + +void RGWPeriodLatestEpochInfo::dump(Formatter *f) const { + encode_json("latest_epoch", epoch, f); +} + +void RGWPeriodLatestEpochInfo::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("latest_epoch", epoch, obj); +} + +void RGWNameToId::dump(Formatter *f) const { + encode_json("obj_id", obj_id, f); +} + +void RGWNameToId::decode_json(JSONObj *obj) { + JSONDecoder::decode_json("obj_id", obj_id, obj); +} + diff --git a/src/rgw/rgw_zone.h b/src/rgw/store/rados/rgw_zone.h similarity index 99% rename from src/rgw/rgw_zone.h rename to src/rgw/store/rados/rgw_zone.h index 5042959ded92..da1400c5432b 100644 --- a/src/rgw/rgw_zone.h +++ b/src/rgw/store/rados/rgw_zone.h @@ -396,6 +396,7 @@ struct RGWZoneParams : RGWSystemMetaObj { RGWZoneParams(const rgw_zone_id& id, const std::string& name) : RGWSystemMetaObj(id.id, name) {} RGWZoneParams(const rgw_zone_id& id, const std::string& name, const std::string& _realm_id) : RGWSystemMetaObj(id.id, name), realm_id(_realm_id) {} + virtual ~RGWZoneParams(); rgw_pool get_pool(CephContext *cct) const override; const std::string get_default_oid(bool old_format = false) const override; @@ -923,6 +924,7 @@ struct RGWZoneGroup : public RGWSystemMetaObj { const std::string& _realm_id, const std::list& _endpoints) : RGWSystemMetaObj(_name, cct , sysobj_svc), endpoints(_endpoints), is_master(_is_master), realm_id(_realm_id) {} + virtual ~RGWZoneGroup(); bool is_master_zonegroup() const { return is_master;} void update_master(const DoutPrefixProvider *dpp, bool _is_master, optional_yield y) { diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 0c16d6a9eb4a..0a38a4698600 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -319,7 +319,8 @@ add_executable(ceph_test_librgw_file_nfsns ) target_include_directories(ceph_test_librgw_file_nfsns PUBLIC "${LUA_INCLUDE_DIR}" - SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(ceph_test_librgw_file_nfsns rgw librados @@ -350,7 +351,8 @@ add_executable(ceph_test_librgw_file_marker ) target_include_directories(ceph_test_librgw_file_marker PUBLIC "${LUA_INCLUDE_DIR}" - SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(ceph_test_librgw_file_marker rgw librados @@ -364,11 +366,11 @@ install(TARGETS ceph_test_librgw_file_marker DESTINATION ${CMAKE_INSTALL_BINDIR} # ceph_test_librgw_file_xattr (attribute ops) add_executable(ceph_test_librgw_file_xattr - librgw_file_xattr.cc - ) + librgw_file_xattr.cc) target_include_directories(ceph_test_librgw_file_xattr PUBLIC "${LUA_INCLUDE_DIR}" - SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(ceph_test_librgw_file_xattr rgw librados @@ -395,6 +397,9 @@ add_executable(test_rgw_ldap ${CMAKE_SOURCE_DIR}/src/rgw/rgw_ldap.cc test_rgw_ldap.cc ) +target_include_directories(test_rgw_ldap + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(test_rgw_ldap librados ceph-common diff --git a/src/test/librgw_file_marker.cc b/src/test/librgw_file_marker.cc index 9095fd9bfcf2..085c991013ce 100644 --- a/src/test/librgw_file_marker.cc +++ b/src/test/librgw_file_marker.cc @@ -20,8 +20,8 @@ #include "include/rados/librgw.h" #include "include/rados/rgw_file.h" -#include "rgw/rgw_file.h" -#include "rgw/rgw_lib_frontend.h" // direct requests +#include "rgw_file.h" +#include "rgw_lib_frontend.h" // direct requests #include "gtest/gtest.h" #include "common/ceph_argparse.h" diff --git a/src/test/librgw_file_nfsns.cc b/src/test/librgw_file_nfsns.cc index 2ce65519c017..911401f61e6c 100644 --- a/src/test/librgw_file_nfsns.cc +++ b/src/test/librgw_file_nfsns.cc @@ -20,9 +20,9 @@ #include "include/rados/librgw.h" #include "include/rados/rgw_file.h" -#include "rgw/rgw_file.h" +#include "rgw_file.h" #include "rgw_lib.h" -#include "rgw/rgw_lib_frontend.h" // direct requests +#include "rgw_lib_frontend.h" // direct requests #include "gtest/gtest.h" #include "common/ceph_argparse.h" diff --git a/src/test/librgw_file_xattr.cc b/src/test/librgw_file_xattr.cc index 13c75becc2e0..e112c40d16ef 100644 --- a/src/test/librgw_file_xattr.cc +++ b/src/test/librgw_file_xattr.cc @@ -23,7 +23,7 @@ #include "include/rados/librgw.h" #include "include/rados/rgw_file.h" -#include "rgw/rgw_file.h" +#include "rgw_file.h" #include "gtest/gtest.h" #include "common/ceph_argparse.h" diff --git a/src/test/rgw/CMakeLists.txt b/src/test/rgw/CMakeLists.txt index 55921737665d..1883d3949bb2 100644 --- a/src/test/rgw/CMakeLists.txt +++ b/src/test/rgw/CMakeLists.txt @@ -175,52 +175,73 @@ target_link_libraries(unittest_rgw_iam_policy add_executable(unittest_rgw_string test_rgw_string.cc) add_ceph_unittest(unittest_rgw_string) +target_include_directories(unittest_rgw_string + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") # unitttest_rgw_dmclock_queue add_executable(unittest_rgw_dmclock_scheduler test_rgw_dmclock_scheduler.cc $) add_ceph_unittest(unittest_rgw_dmclock_scheduler) +target_include_directories(unittest_rgw_dmclock_scheduler + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_dmclock_scheduler rgw_schedulers global ${UNITTEST_LIBS}) if(WITH_RADOSGW_AMQP_ENDPOINT) add_executable(unittest_rgw_amqp test_rgw_amqp.cc) add_ceph_unittest(unittest_rgw_amqp) + target_include_directories(unittest_rgw_amqp + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_amqp ${rgw_libs}) endif() # unittest_rgw_xml add_executable(unittest_rgw_xml test_rgw_xml.cc) add_ceph_unittest(unittest_rgw_xml) - +target_include_directories(unittest_rgw_xml + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_xml ${rgw_libs} ${EXPAT_LIBRARIES}) # unittest_rgw_lc add_executable(unittest_rgw_lc test_rgw_lc.cc) add_ceph_unittest(unittest_rgw_lc) -target_include_directories(unittest_rgw_lc SYSTEM PRIVATE - "${CMAKE_SOURCE_DIR}/src/rgw") +target_include_directories(unittest_rgw_lc + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_lc rgw_common ${rgw_libs} ${EXPAT_LIBRARIES}) # unittest_rgw_arn add_executable(unittest_rgw_arn test_rgw_arn.cc) add_ceph_unittest(unittest_rgw_arn) - +target_include_directories(unittest_rgw_arn + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_arn ${rgw_libs}) # unittest_rgw_kms add_executable(unittest_rgw_kms test_rgw_kms.cc) add_ceph_unittest(unittest_rgw_kms) - +target_include_directories(unittest_rgw_kms + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_kms ${rgw_libs}) # unittest_rgw_url add_executable(unittest_rgw_url test_rgw_url.cc) add_ceph_unittest(unittest_rgw_url) - +target_include_directories(unittest_rgw_url + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_url ${rgw_libs}) add_executable(ceph_test_rgw_gc_log test_rgw_gc_log.cc $) +target_include_directories(ceph_test_rgw_gc_log + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(ceph_test_rgw_gc_log ${rgw_libs} radostest-cxx) install(TARGETS ceph_test_rgw_gc_log DESTINATION ${CMAKE_INSTALL_BINDIR}) @@ -229,15 +250,24 @@ add_ceph_test(test-ceph-diff-sorted.sh # unittest_cls_fifo_legacy add_executable(unittest_cls_fifo_legacy test_cls_fifo_legacy.cc) +target_include_directories(unittest_cls_fifo_legacy + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_cls_fifo_legacy radostest-cxx ${UNITTEST_LIBS} ${rgw_libs}) # unittest_log_backing add_executable(unittest_log_backing test_log_backing.cc) +target_include_directories(unittest_log_backing + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_log_backing radostest-cxx ${UNITTEST_LIBS} ${rgw_libs}) add_executable(unittest_rgw_lua test_rgw_lua.cc) add_ceph_unittest(unittest_rgw_lua) +target_include_directories(unittest_rgw_lua + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw" + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw/store/rados") target_link_libraries(unittest_rgw_lua ${rgw_libs}) diff --git a/src/test/rgw/bench_rgw_ratelimit.cc b/src/test/rgw/bench_rgw_ratelimit.cc index 6c682fcc01b4..2bf7753ad3e6 100644 --- a/src/test/rgw/bench_rgw_ratelimit.cc +++ b/src/test/rgw/bench_rgw_ratelimit.cc @@ -1,5 +1,5 @@ -#include "rgw/rgw_ratelimit.h" -#include "rgw/rgw_common.h" +#include "rgw_ratelimit.h" +#include "rgw_common.h" #include "random" #include #include diff --git a/src/test/rgw/bench_rgw_ratelimit_gc.cc b/src/test/rgw/bench_rgw_ratelimit_gc.cc index 7b07dc2b0fc8..ae422e1dada2 100644 --- a/src/test/rgw/bench_rgw_ratelimit_gc.cc +++ b/src/test/rgw/bench_rgw_ratelimit_gc.cc @@ -1,5 +1,5 @@ -#include "rgw/rgw_ratelimit.h" -#include "rgw/rgw_common.h" +#include "rgw_ratelimit.h" +#include "rgw_common.h" #include "random" #include #include diff --git a/src/test/rgw/test_cls_fifo_legacy.cc b/src/test/rgw/test_cls_fifo_legacy.cc index e4c4375c4de3..64c53ec03e43 100644 --- a/src/test/rgw/test_cls_fifo_legacy.cc +++ b/src/test/rgw/test_cls_fifo_legacy.cc @@ -25,8 +25,8 @@ #include "test/librados/test_cxx.h" #include "global/global_context.h" -#include "rgw/rgw_tools.h" -#include "rgw/cls_fifo_legacy.h" +#include "rgw_tools.h" +#include "cls_fifo_legacy.h" #include "gtest/gtest.h" diff --git a/src/test/rgw/test_http_manager.cc b/src/test/rgw/test_http_manager.cc index b58b60dedc81..f2daeddca792 100644 --- a/src/test/rgw/test_http_manager.cc +++ b/src/test/rgw/test_http_manager.cc @@ -11,8 +11,8 @@ * Foundation. See file COPYING. * */ -#include "rgw/rgw_rados.h" -#include "rgw/rgw_http_client.h" +#include "rgw_rados.h" +#include "rgw_http_client.h" #include "global/global_init.h" #include "common/ceph_argparse.h" #include diff --git a/src/test/rgw/test_log_backing.cc b/src/test/rgw/test_log_backing.cc index 6c9a1b9e3c0d..4c81b2d57d2c 100644 --- a/src/test/rgw/test_log_backing.cc +++ b/src/test/rgw/test_log_backing.cc @@ -30,8 +30,8 @@ #include "cls/log/cls_log_client.h" -#include "rgw/rgw_tools.h" -#include "rgw/cls_fifo_legacy.h" +#include "rgw_tools.h" +#include "cls_fifo_legacy.h" #include "gtest/gtest.h" diff --git a/src/test/rgw/test_rgw_amqp.cc b/src/test/rgw/test_rgw_amqp.cc index e4beb34d1845..bf8671771d9e 100644 --- a/src/test/rgw/test_rgw_amqp.cc +++ b/src/test/rgw/test_rgw_amqp.cc @@ -1,7 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "rgw/rgw_amqp.h" +#include "rgw_amqp.h" #include "common/ceph_context.h" #include "amqp_mock.h" #include diff --git a/src/test/rgw/test_rgw_arn.cc b/src/test/rgw/test_rgw_arn.cc index 334c5ecfa868..83445a2750d7 100644 --- a/src/test/rgw/test_rgw_arn.cc +++ b/src/test/rgw/test_rgw_arn.cc @@ -1,7 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "rgw/rgw_arn.h" +#include "rgw_arn.h" #include using namespace rgw; diff --git a/src/test/rgw/test_rgw_bencode.cc b/src/test/rgw/test_rgw_bencode.cc index a6ba793944bb..c149d532cef1 100644 --- a/src/test/rgw/test_rgw_bencode.cc +++ b/src/test/rgw/test_rgw_bencode.cc @@ -2,7 +2,7 @@ // vim: ts=8 sw=2 smarttab #include "gtest/gtest.h" -#include "rgw/rgw_torrent.h" +#include "rgw_torrent.h" using namespace std; diff --git a/src/test/rgw/test_rgw_bucket_sync_cache.cc b/src/test/rgw/test_rgw_bucket_sync_cache.cc index 7809817fe50b..22ec1005e70c 100644 --- a/src/test/rgw/test_rgw_bucket_sync_cache.cc +++ b/src/test/rgw/test_rgw_bucket_sync_cache.cc @@ -12,7 +12,7 @@ * Foundation. See file COPYING. */ -#include "rgw/rgw_bucket_sync_cache.h" +#include "rgw_bucket_sync_cache.h" #include using namespace rgw::bucket_sync; diff --git a/src/test/rgw/test_rgw_common.h b/src/test/rgw/test_rgw_common.h index 40a48ee320a8..664e0b22e804 100644 --- a/src/test/rgw/test_rgw_common.h +++ b/src/test/rgw/test_rgw_common.h @@ -14,9 +14,9 @@ #include #include "common/ceph_json.h" #include "common/Formatter.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_rados.h" -#include "rgw/rgw_zone.h" +#include "rgw_common.h" +#include "rgw_rados.h" +#include "rgw_zone.h" #ifndef CEPH_TEST_RGW_COMMON_H #define CEPH_TEST_RGW_COMMON_H diff --git a/src/test/rgw/test_rgw_compression.cc b/src/test/rgw/test_rgw_compression.cc index 81b213e24b32..a34653530393 100644 --- a/src/test/rgw/test_rgw_compression.cc +++ b/src/test/rgw/test_rgw_compression.cc @@ -2,7 +2,7 @@ // vim: ts=8 sw=2 smarttab #include "gtest/gtest.h" -#include "rgw/rgw_compression.h" +#include "rgw_compression.h" class ut_get_sink : public RGWGetObj_Filter { bufferlist sink; diff --git a/src/test/rgw/test_rgw_crypto.cc b/src/test/rgw/test_rgw_crypto.cc index bc6dfd17494e..92df403fd09b 100644 --- a/src/test/rgw/test_rgw_crypto.cc +++ b/src/test/rgw/test_rgw_crypto.cc @@ -14,9 +14,9 @@ #include #include "global/global_init.h" #include "common/ceph_argparse.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_rados.h" -#include "rgw/rgw_crypt.h" +#include "rgw_common.h" +#include "rgw_rados.h" +#include "rgw_crypt.h" #include #include "include/ceph_assert.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/test/rgw/test_rgw_dmclock_scheduler.cc b/src/test/rgw/test_rgw_dmclock_scheduler.cc index ca4aa02012d9..92800767c99a 100644 --- a/src/test/rgw/test_rgw_dmclock_scheduler.cc +++ b/src/test/rgw/test_rgw_dmclock_scheduler.cc @@ -14,8 +14,8 @@ //#define BOOST_ASIO_ENABLE_HANDLER_TRACKING -#include "rgw/rgw_dmclock_sync_scheduler.h" -#include "rgw/rgw_dmclock_async_scheduler.h" +#include "rgw_dmclock_sync_scheduler.h" +#include "rgw_dmclock_async_scheduler.h" #include #include diff --git a/src/test/rgw/test_rgw_gc_log.cc b/src/test/rgw/test_rgw_gc_log.cc index e45ad50b6df6..ae8c4d37204e 100644 --- a/src/test/rgw/test_rgw_gc_log.cc +++ b/src/test/rgw/test_rgw_gc_log.cc @@ -1,7 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "rgw/rgw_gc_log.h" +#include "rgw_gc_log.h" #include "test/librados/test_cxx.h" #include "gtest/gtest.h" diff --git a/src/test/rgw/test_rgw_iam_policy.cc b/src/test/rgw/test_rgw_iam_policy.cc index dac4d2cef388..ec3600e01b60 100644 --- a/src/test/rgw/test_rgw_iam_policy.cc +++ b/src/test/rgw/test_rgw_iam_policy.cc @@ -23,9 +23,9 @@ #include "common/code_environment.h" #include "common/ceph_context.h" #include "global/global_init.h" -#include "rgw/rgw_auth.h" -#include "rgw/rgw_iam_policy.h" -#include "rgw/rgw_op.h" +#include "rgw_auth.h" +#include "rgw_iam_policy.h" +#include "rgw_op.h" #include "rgw_sal_rados.h" diff --git a/src/test/rgw/test_rgw_kms.cc b/src/test/rgw/test_rgw_kms.cc index 14319d739670..49ee747f7335 100644 --- a/src/test/rgw/test_rgw_kms.cc +++ b/src/test/rgw/test_rgw_kms.cc @@ -4,9 +4,9 @@ #include #include #include "common/ceph_context.h" -#include "rgw/rgw_common.h" +#include "rgw_common.h" #define FORTEST_VIRTUAL virtual -#include "rgw/rgw_kms.cc" +#include "rgw_kms.cc" using ::testing::_; using ::testing::Action; diff --git a/src/test/rgw/test_rgw_lua.cc b/src/test/rgw/test_rgw_lua.cc index 91c01c19b7d0..68c1b115940f 100644 --- a/src/test/rgw/test_rgw_lua.cc +++ b/src/test/rgw/test_rgw_lua.cc @@ -1,12 +1,12 @@ #include #include "common/ceph_context.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_auth.h" -#include "rgw/rgw_process.h" -#include "rgw/rgw_sal_rados.h" -#include "rgw/rgw_lua_request.h" -#include "rgw/rgw_lua_background.h" -#include "rgw/rgw_lua_data_filter.h" +#include "rgw_common.h" +#include "rgw_auth.h" +#include "rgw_process.h" +#include "rgw_sal_rados.h" +#include "rgw_lua_request.h" +#include "rgw_lua_background.h" +#include "rgw_lua_data_filter.h" using namespace std; using namespace rgw; diff --git a/src/test/rgw/test_rgw_manifest.cc b/src/test/rgw/test_rgw_manifest.cc index ee92248d0088..acde46d44cc4 100644 --- a/src/test/rgw/test_rgw_manifest.cc +++ b/src/test/rgw/test_rgw_manifest.cc @@ -14,8 +14,8 @@ #include #include "global/global_init.h" #include "common/ceph_argparse.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_rados.h" +#include "rgw_common.h" +#include "rgw_rados.h" #include "test_rgw_common.h" #include diff --git a/src/test/rgw/test_rgw_obj.cc b/src/test/rgw/test_rgw_obj.cc index ba26aec355fe..53d7897ae2cf 100644 --- a/src/test/rgw/test_rgw_obj.cc +++ b/src/test/rgw/test_rgw_obj.cc @@ -14,9 +14,9 @@ #include #include "common/ceph_json.h" #include "common/Formatter.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_rados.h" -#include "rgw/services/svc_tier_rados.h" +#include "rgw_common.h" +#include "rgw_rados.h" +#include "services/svc_tier_rados.h" #include "test_rgw_common.h" #include diff --git a/src/test/rgw/test_rgw_period_history.cc b/src/test/rgw/test_rgw_period_history.cc index a3589934f447..25ea87d3a2f5 100644 --- a/src/test/rgw/test_rgw_period_history.cc +++ b/src/test/rgw/test_rgw_period_history.cc @@ -11,9 +11,9 @@ * Foundation. See file COPYING. * */ -#include "rgw/rgw_period_history.h" -#include "rgw/rgw_rados.h" -#include "rgw/rgw_zone.h" +#include "rgw_period_history.h" +#include "rgw_rados.h" +#include "rgw_zone.h" #include "global/global_init.h" #include "common/ceph_argparse.h" #include diff --git a/src/test/rgw/test_rgw_putobj.cc b/src/test/rgw/test_rgw_putobj.cc index f26a9c2d3858..35abc3036e55 100644 --- a/src/test/rgw/test_rgw_putobj.cc +++ b/src/test/rgw/test_rgw_putobj.cc @@ -12,7 +12,7 @@ * */ -#include "rgw/rgw_putobj.h" +#include "rgw_putobj.h" #include inline bufferlist string_buf(const char* buf) { diff --git a/src/test/rgw/test_rgw_ratelimit.cc b/src/test/rgw/test_rgw_ratelimit.cc index 875ec8392ae4..01be4df48a02 100644 --- a/src/test/rgw/test_rgw_ratelimit.cc +++ b/src/test/rgw/test_rgw_ratelimit.cc @@ -2,7 +2,7 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include -#include "rgw/rgw_ratelimit.h" +#include "rgw_ratelimit.h" using namespace std::chrono_literals; diff --git a/src/test/rgw/test_rgw_reshard.cc b/src/test/rgw/test_rgw_reshard.cc index acb1b597afb9..da41b967f051 100644 --- a/src/test/rgw/test_rgw_reshard.cc +++ b/src/test/rgw/test_rgw_reshard.cc @@ -12,7 +12,7 @@ * */ -#include "rgw/rgw_reshard.h" +#include "rgw_reshard.h" #include diff --git a/src/test/rgw/test_rgw_reshard_wait.cc b/src/test/rgw/test_rgw_reshard_wait.cc index 1db8bad391a6..06caae34adef 100644 --- a/src/test/rgw/test_rgw_reshard_wait.cc +++ b/src/test/rgw/test_rgw_reshard_wait.cc @@ -12,7 +12,7 @@ * */ -#include "rgw/rgw_reshard.h" +#include "rgw_reshard.h" #include #include diff --git a/src/test/rgw/test_rgw_string.cc b/src/test/rgw/test_rgw_string.cc index 5d20718ae030..90a0b00c8c0f 100644 --- a/src/test/rgw/test_rgw_string.cc +++ b/src/test/rgw/test_rgw_string.cc @@ -12,7 +12,7 @@ * */ -#include "rgw/rgw_string.h" +#include "rgw_string.h" #include const std::string abc{"abc"}; diff --git a/src/test/rgw/test_rgw_throttle.cc b/src/test/rgw/test_rgw_throttle.cc index 9f6dc6ac23eb..d67f2c6ceb65 100644 --- a/src/test/rgw/test_rgw_throttle.cc +++ b/src/test/rgw/test_rgw_throttle.cc @@ -12,7 +12,7 @@ * */ -#include "rgw/rgw_aio_throttle.h" +#include "rgw_aio_throttle.h" #include #include diff --git a/src/test/rgw/test_rgw_url.cc b/src/test/rgw/test_rgw_url.cc index b61e1d595921..92731dfad623 100644 --- a/src/test/rgw/test_rgw_url.cc +++ b/src/test/rgw/test_rgw_url.cc @@ -1,7 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "rgw/rgw_url.h" +#include "rgw_url.h" #include #include diff --git a/src/test/rgw/test_rgw_xml.cc b/src/test/rgw/test_rgw_xml.cc index 50c42c7af7ed..fa9f21157787 100644 --- a/src/test/rgw/test_rgw_xml.cc +++ b/src/test/rgw/test_rgw_xml.cc @@ -1,7 +1,7 @@ // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab -#include "rgw/rgw_xml.h" +#include "rgw_xml.h" #include #include #include diff --git a/src/test/test_cors.cc b/src/test/test_cors.cc index 2df022f80a3d..186f47a735ee 100644 --- a/src/test/test_cors.cc +++ b/src/test/test_cors.cc @@ -17,8 +17,8 @@ extern "C"{ #include "common/ceph_argparse.h" #include "common/Finisher.h" #include "global/global_init.h" -#include "rgw/rgw_cors.h" -#include "rgw/rgw_cors_s3.h" +#include "rgw_cors.h" +#include "rgw_cors_s3.h" using namespace std; diff --git a/src/test/test_rgw_admin_log.cc b/src/test/test_rgw_admin_log.cc index 73767663916c..fe072ead18a1 100644 --- a/src/test/test_rgw_admin_log.cc +++ b/src/test/test_rgw_admin_log.cc @@ -32,11 +32,11 @@ extern "C"{ #include "common/ceph_argparse.h" #include "common/Finisher.h" #include "global/global_init.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_datalog.h" -#include "rgw/rgw_mdlog.h" -#include "rgw/rgw_bucket.h" -#include "rgw/rgw_rados.h" +#include "rgw_common.h" +#include "rgw_datalog.h" +#include "rgw_mdlog.h" +#include "rgw_bucket.h" +#include "rgw_rados.h" #include "include/utime.h" #include "include/object.h" #include diff --git a/src/test/test_rgw_admin_meta.cc b/src/test/test_rgw_admin_meta.cc index 734836782e09..4699a876dfeb 100644 --- a/src/test/test_rgw_admin_meta.cc +++ b/src/test/test_rgw_admin_meta.cc @@ -31,8 +31,8 @@ extern "C"{ #include "common/ceph_argparse.h" #include "common/Finisher.h" #include "global/global_init.h" -#include "rgw/rgw_common.h" -#include "rgw/rgw_rados.h" +#include "rgw_common.h" +#include "rgw_rados.h" #include using namespace std; diff --git a/src/test/test_rgw_ldap.cc b/src/test/test_rgw_ldap.cc index 007f7db301f5..8c1c30191355 100644 --- a/src/test/test_rgw_ldap.cc +++ b/src/test/test_rgw_ldap.cc @@ -19,8 +19,8 @@ #include #include -#include "rgw/rgw_ldap.h" -#include "rgw/rgw_token.h" +#include "rgw_ldap.h" +#include "rgw_token.h" #include "gtest/gtest.h" #include "common/ceph_argparse.h" diff --git a/src/test/test_rgw_token.cc b/src/test/test_rgw_token.cc index f2defbf30515..0a1fc06a3b02 100644 --- a/src/test/test_rgw_token.cc +++ b/src/test/test_rgw_token.cc @@ -22,8 +22,8 @@ #include "common/debug.h" #include "include/ceph_assert.h" #include "gtest/gtest.h" -#include "rgw/rgw_token.h" -#include "rgw/rgw_b64.h" +#include "rgw_token.h" +#include "rgw_b64.h" #define dout_subsys ceph_subsys_rgw diff --git a/src/tools/ceph-dencoder/CMakeLists.txt b/src/tools/ceph-dencoder/CMakeLists.txt index 9441010abf54..b40da0833ee2 100644 --- a/src/tools/ceph-dencoder/CMakeLists.txt +++ b/src/tools/ceph-dencoder/CMakeLists.txt @@ -57,6 +57,8 @@ if(WITH_RADOSGW) add_denc_mod(denc-mod-rgw rgw_types.cc ${CMAKE_SOURCE_DIR}/src/rgw/rgw_dencoder.cc) + target_include_directories(denc-mod-rgw + SYSTEM PRIVATE "${CMAKE_SOURCE_DIR}/src/rgw") target_link_libraries(denc-mod-rgw rgw_a cls_rgw_client diff --git a/src/tools/ceph-dencoder/rgw_types.h b/src/tools/ceph-dencoder/rgw_types.h index 4c136970bae9..dd5c3a8cb7f1 100644 --- a/src/tools/ceph-dencoder/rgw_types.h +++ b/src/tools/ceph-dencoder/rgw_types.h @@ -1,12 +1,12 @@ #ifdef WITH_RADOSGW -#include "rgw/rgw_rados.h" +#include "rgw_rados.h" TYPE(RGWOLHInfo) TYPE(RGWObjManifestPart) TYPE(RGWObjManifest) TYPE(objexp_hint_entry) -#include "rgw/rgw_zone.h" +#include "rgw_zone.h" TYPE(RGWZoneParams) TYPE(RGWZone) TYPE(RGWZoneGroup) @@ -14,7 +14,7 @@ TYPE(RGWRealm) TYPE(RGWPeriod) TYPE(RGWPeriodLatestEpochInfo) -#include "rgw/rgw_acl.h" +#include "rgw_acl.h" TYPE(ACLPermission) TYPE(ACLGranteeType) TYPE(ACLGrant) @@ -22,12 +22,12 @@ TYPE(RGWAccessControlList) TYPE(ACLOwner) TYPE(RGWAccessControlPolicy) -#include "rgw/rgw_cache.h" +#include "rgw_cache.h" TYPE(ObjectMetaInfo) TYPE(ObjectCacheInfo) TYPE(RGWCacheNotifyInfo) -#include "rgw/rgw_lc.h" +#include "rgw_lc.h" TYPE(RGWLifecycleConfiguration) #include "cls/rgw/cls_rgw_types.h" @@ -110,7 +110,7 @@ TYPE(cls::journal::ObjectSetPosition) TYPE(cls::journal::Client) TYPE(cls::journal::Tag) -#include "rgw/rgw_common.h" +#include "rgw_common.h" TYPE(RGWAccessKey) TYPE(RGWSubUser) TYPE(RGWUserInfo) @@ -119,23 +119,23 @@ TYPE(RGWBucketInfo) TYPE(RGWBucketEnt) TYPE(rgw_obj) -#include "rgw/rgw_log.h" +#include "rgw_log.h" TYPE(rgw_log_entry) -#include "rgw/rgw_meta_sync_status.h" +#include "rgw_meta_sync_status.h" TYPE(rgw_meta_sync_info) TYPE(rgw_meta_sync_marker) TYPE(rgw_meta_sync_status) -#include "rgw/rgw_multi.h" +#include "rgw_multi.h" TYPE(RGWUploadPartInfo) -#include "rgw/rgw_data_sync.h" +#include "rgw_data_sync.h" TYPE(rgw_data_sync_info) TYPE(rgw_data_sync_marker) TYPE(rgw_data_sync_status) -#include "rgw/rgw_bucket_encryption.h" +#include "rgw_bucket_encryption.h" TYPE(RGWBucketEncryptionConfig) #endif -- 2.47.3