From bbece46124ffd16a2ffd05b1b763424789413f04 Mon Sep 17 00:00:00 2001 From: Samarah Date: Thu, 26 Jun 2025 20:31:08 +0000 Subject: [PATCH] rgw: Add parameter to `get_storage` to indicate radosgw-admin call Signed-off-by: Samarah --- src/rgw/driver/d4n/rgw_sal_d4n.cc | 12 +-- src/rgw/driver/d4n/rgw_sal_d4n.h | 2 +- src/rgw/radosgw-admin/radosgw-admin.cc | 2 +- src/rgw/rgw_sal.cc | 6 +- src/rgw/rgw_sal.h | 7 +- src/rgw/rgw_ssd_driver.cc | 108 +++++++++++++------------ src/rgw/rgw_ssd_driver.h | 3 +- src/test/rgw/test_ssd_driver.cc | 2 +- 8 files changed, 73 insertions(+), 69 deletions(-) diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 9c3e0730547..de759eb6097 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -36,16 +36,16 @@ static inline Object* nextObject(Object* t) return dynamic_cast(t)->get_next(); } -D4NFilterDriver::D4NFilterDriver(Driver* _next, boost::asio::io_context& io_context) : FilterDriver(_next), - io_context(io_context), - y(null_yield) +D4NFilterDriver::D4NFilterDriver(Driver* _next, boost::asio::io_context& io_context, bool admin) : FilterDriver(_next), + io_context(io_context), + y(null_yield) { rgw::cache::Partition partition_info; partition_info.location = g_conf()->rgw_d4n_l1_datacache_persistent_path; partition_info.name = "d4n"; partition_info.type = "read-cache"; partition_info.size = g_conf()->rgw_d4n_l1_datacache_size; - cacheDriver = std::make_unique(partition_info); + cacheDriver = std::make_unique(partition_info, admin); } D4NFilterDriver::~D4NFilterDriver() = default; @@ -3104,9 +3104,9 @@ int D4NFilterMultipartUpload::complete(const DoutPrefixProvider *dpp, extern "C" { -rgw::sal::Driver* newD4NFilter(rgw::sal::Driver* next, void* io_context) +rgw::sal::Driver* newD4NFilter(rgw::sal::Driver* next, void* io_context, bool admin) { - rgw::sal::D4NFilterDriver* driver = new rgw::sal::D4NFilterDriver(next, *static_cast(io_context)); + rgw::sal::D4NFilterDriver* driver = new rgw::sal::D4NFilterDriver(next, *static_cast(io_context), admin); return driver; } diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.h b/src/rgw/driver/d4n/rgw_sal_d4n.h index d091ede2b73..65b4f31733d 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.h +++ b/src/rgw/driver/d4n/rgw_sal_d4n.h @@ -63,7 +63,7 @@ class D4NFilterDriver : public FilterDriver { optional_yield y; public: - D4NFilterDriver(Driver* _next, boost::asio::io_context& io_context); + D4NFilterDriver(Driver* _next, boost::asio::io_context& io_context, bool admin); virtual ~D4NFilterDriver(); virtual int initialize(CephContext *cct, const DoutPrefixProvider *dpp) override; diff --git a/src/rgw/radosgw-admin/radosgw-admin.cc b/src/rgw/radosgw-admin/radosgw-admin.cc index 2a79d9d421e..821ead35e4b 100644 --- a/src/rgw/radosgw-admin/radosgw-admin.cc +++ b/src/rgw/radosgw-admin/radosgw-admin.cc @@ -4641,7 +4641,7 @@ int main(int argc, const char **argv) null_yield, cfgstore.get(), need_cache && g_conf()->rgw_cache_enabled, - need_gc); + need_gc, true /* admin */); } if (!driver) { cerr << "couldn't init storage provider" << std::endl; diff --git a/src/rgw/rgw_sal.cc b/src/rgw/rgw_sal.cc index 786da6987b9..fe85704d3c8 100644 --- a/src/rgw/rgw_sal.cc +++ b/src/rgw/rgw_sal.cc @@ -65,7 +65,7 @@ extern rgw::sal::Driver* newPOSIXDriver(rgw::sal::Driver* next); #endif extern rgw::sal::Driver* newBaseFilter(rgw::sal::Driver* next); #ifdef WITH_RADOSGW_D4N -extern rgw::sal::Driver* newD4NFilter(rgw::sal::Driver* next, boost::asio::io_context& io_context); +extern rgw::sal::Driver* newD4NFilter(rgw::sal::Driver* next, boost::asio::io_context& io_context, bool admin); #endif } @@ -84,7 +84,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* bool use_cache, bool use_gc, bool background_tasks, - optional_yield y, rgw::sal::ConfigStore* cfgstore) + optional_yield y, rgw::sal::ConfigStore* cfgstore, bool admin) { rgw::sal::Driver* driver{nullptr}; @@ -215,7 +215,7 @@ rgw::sal::Driver* DriverManager::init_storage_provider(const DoutPrefixProvider* #ifdef WITH_RADOSGW_D4N else if (cfg.filter_name.compare("d4n") == 0) { rgw::sal::Driver* next = driver; - driver = newD4NFilter(next, io_context); + driver = newD4NFilter(next, io_context, admin); if (driver->initialize(cct, dpp) < 0) { delete driver; diff --git a/src/rgw/rgw_sal.h b/src/rgw/rgw_sal.h index 65b3702d7e6..7a3a55559cc 100644 --- a/src/rgw/rgw_sal.h +++ b/src/rgw/rgw_sal.h @@ -1943,7 +1943,8 @@ public: optional_yield y, rgw::sal::ConfigStore* cfgstore, bool use_cache = true, - bool use_gc = true) { + bool use_gc = true, + bool admin = false) { rgw::sal::Driver* driver = init_storage_provider(dpp, cct, cfg, io_context, site_config, use_gc_thread, @@ -1954,7 +1955,7 @@ public: run_reshard_thread, run_notification_thread, use_cache, use_gc, - background_tasks, y, cfgstore); + background_tasks, y, cfgstore, admin); return driver; } /** Get a stripped down driver by service name */ @@ -1984,7 +1985,7 @@ public: bool run_notification_thread, bool use_metadata_cache, bool use_gc, bool background_tasks, - optional_yield y, rgw::sal::ConfigStore* cfgstore); + optional_yield y, rgw::sal::ConfigStore* cfgstore, bool admin); /** Initialize a new raw Driver */ static rgw::sal::Driver* init_raw_storage_provider(const DoutPrefixProvider* dpp, CephContext* cct, diff --git a/src/rgw/rgw_ssd_driver.cc b/src/rgw/rgw_ssd_driver.cc index 773abe907f9..37f3d308bdb 100644 --- a/src/rgw/rgw_ssd_driver.cc +++ b/src/rgw/rgw_ssd_driver.cc @@ -116,59 +116,61 @@ int SSDDriver::initialize(const DoutPrefixProvider* dpp) partition_info.location += "/"; } - try { - if (efs::exists(partition_info.location)) { - if (dpp->get_cct()->_conf->rgw_d4n_l1_evict_cache_on_start) { - ldpp_dout(dpp, 5) << "initialize: evicting the persistent storage directory on start" << dendl; - - uid_t uid = dpp->get_cct()->get_set_uid(); - gid_t gid = dpp->get_cct()->get_set_gid(); - - ldpp_dout(dpp, 5) << "initialize:: uid is " << uid << " and gid is " << gid << dendl; - ldpp_dout(dpp, 5) << "initialize:: changing permissions for datacache directory." << dendl; - - if (uid) { - if (chown(partition_info.location.c_str(), uid, gid) == -1) { - ldpp_dout(dpp, 5) << "initialize: chown return error: " << strerror(errno) << dendl; - } - - if (chmod(partition_info.location.c_str(), S_IRWXU|S_IRWXG|S_IRWXO) == -1) { - ldpp_dout(dpp, 5) << "initialize: chmod return error: " << strerror(errno) << dendl; - } - } - - for (auto& p : efs::directory_iterator(partition_info.location)) { - efs::remove_all(p.path()); - } - } - } else { - ldpp_dout(dpp, 5) << "initialize:: creating the persistent storage directory on start: " << partition_info.location << dendl; - std::error_code ec; - if (!efs::create_directories(partition_info.location, ec)) { - ldpp_dout(dpp, 0) << "initialize::: ERROR initializing the cache storage directory: '" << partition_info.location << - "' : " << ec.value() << dendl; - } else { - uid_t uid = dpp->get_cct()->get_set_uid(); - gid_t gid = dpp->get_cct()->get_set_gid(); - - ldpp_dout(dpp, 5) << "initialize:: uid is " << uid << " and gid is " << gid << dendl; - ldpp_dout(dpp, 5) << "initialize:: changing permissions for datacache directory." << dendl; - - if (uid) { - if (chown(partition_info.location.c_str(), uid, gid) == -1) { - ldpp_dout(dpp, 5) << "initialize: chown return error: " << strerror(errno) << dendl; - } - - if (chmod(partition_info.location.c_str(), S_IRWXU|S_IRWXG|S_IRWXO) == -1) { - ldpp_dout(dpp, 5) << "initialize: chmod return error: " << strerror(errno) << dendl; - } - } - } - } - } catch (const efs::filesystem_error& e) { - ldpp_dout(dpp, 0) << "initialize::: ERROR initializing the cache storage directory '" << partition_info.location << - "' : " << e.what() << dendl; - //return -EINVAL; Should return error from here? + if (!admin) { // Only initialize or evict cache if radosgw-admin is not responsible for call + try { + if (efs::exists(partition_info.location)) { + if (dpp->get_cct()->_conf->rgw_d4n_l1_evict_cache_on_start) { + ldpp_dout(dpp, 5) << "initialize: evicting the persistent storage directory on start" << dendl; + + uid_t uid = dpp->get_cct()->get_set_uid(); + gid_t gid = dpp->get_cct()->get_set_gid(); + + ldpp_dout(dpp, 5) << "initialize:: uid is " << uid << " and gid is " << gid << dendl; + ldpp_dout(dpp, 5) << "initialize:: changing permissions for datacache directory." << dendl; + + if (uid) { + if (chown(partition_info.location.c_str(), uid, gid) == -1) { + ldpp_dout(dpp, 5) << "initialize: chown return error: " << strerror(errno) << dendl; + } + + if (chmod(partition_info.location.c_str(), S_IRWXU|S_IRWXG|S_IRWXO) == -1) { + ldpp_dout(dpp, 5) << "initialize: chmod return error: " << strerror(errno) << dendl; + } + } + + for (auto& p : efs::directory_iterator(partition_info.location)) { + efs::remove_all(p.path()); + } + } + } else { + ldpp_dout(dpp, 5) << "initialize:: creating the persistent storage directory on start: " << partition_info.location << dendl; + std::error_code ec; + if (!efs::create_directories(partition_info.location, ec)) { + ldpp_dout(dpp, 0) << "initialize::: ERROR initializing the cache storage directory: '" << partition_info.location << + "' : " << ec.value() << dendl; + } else { + uid_t uid = dpp->get_cct()->get_set_uid(); + gid_t gid = dpp->get_cct()->get_set_gid(); + + ldpp_dout(dpp, 5) << "initialize:: uid is " << uid << " and gid is " << gid << dendl; + ldpp_dout(dpp, 5) << "initialize:: changing permissions for datacache directory." << dendl; + + if (uid) { + if (chown(partition_info.location.c_str(), uid, gid) == -1) { + ldpp_dout(dpp, 5) << "initialize: chown return error: " << strerror(errno) << dendl; + } + + if (chmod(partition_info.location.c_str(), S_IRWXU|S_IRWXG|S_IRWXO) == -1) { + ldpp_dout(dpp, 5) << "initialize: chmod return error: " << strerror(errno) << dendl; + } + } + } + } + } catch (const efs::filesystem_error& e) { + ldpp_dout(dpp, 0) << "initialize::: ERROR initializing the cache storage directory '" << partition_info.location << + "' : " << e.what() << dendl; + //return -EINVAL; Should return error from here? + } } #if defined(HAVE_LIBAIO) && defined(__GLIBC__) diff --git a/src/rgw/rgw_ssd_driver.h b/src/rgw/rgw_ssd_driver.h index dedf539e0ff..49d85244cc4 100644 --- a/src/rgw/rgw_ssd_driver.h +++ b/src/rgw/rgw_ssd_driver.h @@ -8,7 +8,7 @@ namespace rgw { namespace cache { class SSDDriver : public CacheDriver { public: - SSDDriver(Partition& partition_info) : partition_info(partition_info) {} + SSDDriver(Partition& partition_info, bool admin) : partition_info(partition_info), admin(admin) {} virtual ~SSDDriver() {} virtual int initialize(const DoutPrefixProvider* dpp) override; @@ -39,6 +39,7 @@ private: uint64_t free_space; CephContext* cct; std::mutex cache_lock; + bool admin; struct libaio_read_handler { rgw::Aio* throttle = nullptr; diff --git a/src/test/rgw/test_ssd_driver.cc b/src/test/rgw/test_ssd_driver.cc index 1ab3009acc2..01ea4e7be31 100644 --- a/src/test/rgw/test_ssd_driver.cc +++ b/src/test/rgw/test_ssd_driver.cc @@ -87,7 +87,7 @@ class SSDDriverFixture: public ::testing::Test { protected: virtual void SetUp() { rgw::cache::Partition partition_info{.name = "d4n", .type = "read-cache", .location = "rgw_d4n_datacache", .size = 5368709120}; - cacheDriver = new rgw::cache::SSDDriver{partition_info}; + cacheDriver = new rgw::cache::SSDDriver{partition_info, false}; ASSERT_NE(cacheDriver, nullptr); -- 2.39.5