From 746ac03a3d8f8207b32c3953b1fd84634c0ba309 Mon Sep 17 00:00:00 2001 From: Yuval Lifshitz Date: Tue, 13 May 2025 12:12:19 +0000 Subject: [PATCH] rgw/logginag: make unique portion of log object name orderd Fixes: https://tracker.ceph.com/issues/71308 Signed-off-by: Yuval Lifshitz (cherry picked from commit 198fa75258bc82d4b9d681275ea7a61a24df8799) --- doc/radosgw/bucket_logging.rst | 4 +++- src/rgw/rgw_bucket_logging.cc | 33 ++++++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/doc/radosgw/bucket_logging.rst b/doc/radosgw/bucket_logging.rst index b0b95d8e1e681..bffc9f0374df7 100644 --- a/doc/radosgw/bucket_logging.rst +++ b/doc/radosgw/bucket_logging.rst @@ -26,6 +26,8 @@ in different objects in the log bucket. - Source and log bucket must be in the same zonegroup - Source and log buckets may belong to different accounts (with proper bucket policy set) - The log bucket may have object lock enabled with default retention period + - The 16 bit unique ID part of the log object name is an incrementing counter, or a random, + alphanumeric string if the counter is not available .. toctree:: @@ -157,7 +159,7 @@ For example: :: - fish/testid//all-log/2024/08/06/2024-08-06-10-11-18-1HMU3UMWOJKNQJ0X + fish/testid//all-log/2024/08/06/2024-08-06-10-11-18-0000000000000002 Log Records ~~~~~~~~~~~ diff --git a/src/rgw/rgw_bucket_logging.cc b/src/rgw/rgw_bucket_logging.cc index d2429aad40ab7..aa39a509248d2 100644 --- a/src/rgw/rgw_bucket_logging.cc +++ b/src/rgw/rgw_bucket_logging.cc @@ -9,6 +9,7 @@ #include "rgw_sal.h" #include "rgw_op.h" #include "rgw_auth_s3.h" +#include #define dout_subsys ceph_subsys_rgw @@ -171,6 +172,7 @@ std::string configuration::to_json_str() const { return ss.str(); } +// create a random string of N characters template std::string unique_string() { static const std::string possible_characters{"0123456789ABCDEFGHIJKLMNOPQRSTUVWXY"}; @@ -183,6 +185,27 @@ std::string unique_string() { return str; } +// create an incremental string of N characters if possible +// fallback to a random string of N characters if not +template +std::string incremental_string(const DoutPrefixProvider *dpp, std::optional old_name) { + // for the fist time we create a string of zeros + if (!old_name) { + return std::string(N, '0'); + } + const auto str_counter = old_name->substr(old_name->length() - N+1, N); + try { + auto counter = boost::lexical_cast(str_counter); + ++counter; + static const auto format = fmt::format("{{:0>{}}}", N); + return fmt::vformat(format, fmt::make_format_args(counter)); + } catch (const boost::bad_lexical_cast& e) { + ldpp_dout(dpp, 5) << "WARNING: failed to convert string '" << str_counter << + "' to counter. " <(); + } +} + constexpr size_t UniqueStringLength = 16; ceph::coarse_real_time time_from_name(const std::string& obj_name, const DoutPrefixProvider *dpp) { @@ -220,13 +243,13 @@ int new_logging_object(const configuration& conf, std::string& obj_name, const DoutPrefixProvider *dpp, optional_yield y, - bool init_obj, + std::optional old_name, RGWObjVersionTracker* objv_tracker) { const auto tt = ceph::coarse_real_time::clock::to_time_t(ceph::coarse_real_time::clock::now()); std::tm t{}; localtime_r(&tt, &t); - const auto unique = unique_string(); + const auto unique = incremental_string(dpp, old_name); switch (conf.obj_key_format) { case KeyFormat::Simple: @@ -251,7 +274,7 @@ int new_logging_object(const configuration& conf, break; } const auto& target_bucket_id = target_bucket->get_key(); - int ret = target_bucket->set_logging_object_name(obj_name, conf.target_prefix, y, dpp, init_obj, objv_tracker); + int ret = target_bucket->set_logging_object_name(obj_name, conf.target_prefix, y, dpp, (old_name == std::nullopt), objv_tracker); if (ret == -EEXIST || ret == -ECANCELED) { if (ret = target_bucket->get_logging_object_name(obj_name, conf.target_prefix, y, dpp, nullptr); ret < 0) { ldpp_dout(dpp, 1) << "ERROR: failed to get name of logging object of bucket '" << @@ -330,7 +353,7 @@ int rollover_logging_object(const configuration& conf, return -EINVAL; } const auto old_obj = obj_name; - const int ret = new_logging_object(conf, target_bucket, obj_name, dpp, y, false, objv_tracker); + const int ret = new_logging_object(conf, target_bucket, obj_name, dpp, y, old_obj, objv_tracker); if (ret == -ECANCELED) { ldpp_dout(dpp, 20) << "INFO: rollover already performed for object '" << old_obj << "' to logging bucket '" << target_bucket->get_key() << "'. ret = " << ret << dendl; @@ -455,7 +478,7 @@ int log_record(rgw::sal::Driver* driver, } } else if (ret == -ENOENT) { // try to create the temporary log object for the first time - ret = new_logging_object(conf, target_bucket, obj_name, dpp, y, true, nullptr); + ret = new_logging_object(conf, target_bucket, obj_name, dpp, y, std::nullopt, nullptr); if (ret == 0) { ldpp_dout(dpp, 20) << "INFO: first time logging for bucket '" << target_bucket_id << "' and prefix '" << conf.target_prefix << "'" << dendl; -- 2.39.5