From a4524dcceb36113ebe3b13ae4ac1cbe7f405dfcc Mon Sep 17 00:00:00 2001 From: Pritha Srivastava Date: Wed, 27 Sep 2023 16:04:47 +0530 Subject: [PATCH] rgw/cache: adding d4n config options for ssd backed cache driver. Signed-off-by: Pritha Srivastava --- src/common/options/rgw.yaml.in | 54 +++++++++++++++++++++++++++++++ src/rgw/driver/d4n/rgw_sal_d4n.cc | 4 +-- src/rgw/rgw_ssd_driver.cc | 14 ++++---- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/src/common/options/rgw.yaml.in b/src/common/options/rgw.yaml.in index 52e69dae959..9f87a037ee2 100644 --- a/src/common/options/rgw.yaml.in +++ b/src/common/options/rgw.yaml.in @@ -3612,6 +3612,60 @@ options: see_also: - rgw_thread_pool_size with_legacy: true +- name: rgw_d4n_l1_datacache_persistent_path + type: str + level: advanced + desc: path for the directory for storing the local cache objects data for d4n + default: /tmp/rgw_d4n_datacache/ + services: + - rgw + with_legacy: true +- name: rgw_d4n_l1_datacache_size + type: size + level: advanced + desc: datacache maximum size on disk in bytes + default: 1_G + services: + - rgw + with_legacy: true +- name: rgw_d4n_l1_evict_cache_on_start + type: bool + level: advanced + desc: clear the content of the persistent data cache directory on start + default: true + services: + - rgw + with_legacy: true +- name: rgw_d4n_l1_fadvise + type: int + level: advanced + desc: posix_fadvise() flag for access pattern of cache files + long_desc: for example to bypass the page-cache - + POSIX_FADV_DONTNEED=4 + default: 4 + services: + - rgw + with_legacy: true +- name: rgw_d4n_libaio_aio_threads + type: int + level: advanced + desc: specifies the maximum number of worker threads that may be used by libaio + default: 20 + services: + - rgw + see_also: + - rgw_thread_pool_size + with_legacy: true +- name: rgw_d4n_libaio_aio_num + type: int + level: advanced + desc: specifies the maximum number of simultaneous I/O requests that libaio expects to enqueue + default: 64 + services: + - rgw + see_also: + - rgw_thread_pool_size + with_legacy: true - name: rgw_backend_store type: str level: advanced diff --git a/src/rgw/driver/d4n/rgw_sal_d4n.cc b/src/rgw/driver/d4n/rgw_sal_d4n.cc index 3a264eae350..1a17915bb87 100644 --- a/src/rgw/driver/d4n/rgw_sal_d4n.cc +++ b/src/rgw/driver/d4n/rgw_sal_d4n.cc @@ -41,10 +41,10 @@ static inline Object* nextObject(Object* t) D4NFilterDriver::D4NFilterDriver(Driver* _next) : FilterDriver(_next) { rgw::cache::Partition partition_info; - partition_info.location = g_conf()->rgw_d3n_l1_datacache_persistent_path; + partition_info.location = g_conf()->rgw_d4n_l1_datacache_persistent_path; partition_info.name = "d4n"; partition_info.type = "read-cache"; - partition_info.size = g_conf()->rgw_d3n_l1_datacache_size; + partition_info.size = g_conf()->rgw_d4n_l1_datacache_size; cacheDriver = new rgw::cache::SSDDriver(partition_info); objDir = new rgw::d4n::ObjectDirectory(); diff --git a/src/rgw/rgw_ssd_driver.cc b/src/rgw/rgw_ssd_driver.cc index b4862219544..c6e974d6513 100644 --- a/src/rgw/rgw_ssd_driver.cc +++ b/src/rgw/rgw_ssd_driver.cc @@ -70,7 +70,7 @@ int SSDDriver::initialize(CephContext* cct, const DoutPrefixProvider* dpp) } try { if (efs::exists(partition_info.location)) { - if (cct->_conf->rgw_d3n_l1_evict_cache_on_start) { + if (cct->_conf->rgw_d4n_l1_evict_cache_on_start) { ldpp_dout(dpp, 5) << "initialize: evicting the persistent storage directory on start" << dendl; for (auto& p : efs::directory_iterator(partition_info.location)) { efs::remove_all(p.path()); @@ -89,8 +89,8 @@ int SSDDriver::initialize(CephContext* cct, const DoutPrefixProvider* dpp) #if defined(HAVE_LIBAIO) && defined(__GLIBC__) // libaio setup struct aioinit ainit{0}; - ainit.aio_threads = cct->_conf.get_val("rgw_d3n_libaio_aio_threads"); - ainit.aio_num = cct->_conf.get_val("rgw_d3n_libaio_aio_num"); + ainit.aio_threads = cct->_conf.get_val("rgw_d4n_libaio_aio_threads"); + ainit.aio_num = cct->_conf.get_val("rgw_d4n_libaio_aio_num"); ainit.aio_idle_time = 120; aio_init(&ainit); #endif @@ -328,8 +328,8 @@ int SSDDriver::AsyncWriteRequest::prepare_libaio_write_op(const DoutPrefixProvid ldpp_dout(dpp, 0) << "ERROR: AsyncWriteRequest::prepare_libaio_write_op: open file failed, errno=" << errno << ", location='" << location.c_str() << "'" << dendl; return r; } - if (dpp->get_cct()->_conf->rgw_d3n_l1_fadvise != POSIX_FADV_NORMAL) - posix_fadvise(fd, 0, 0, dpp->get_cct()->_conf->rgw_d3n_l1_fadvise); + if (dpp->get_cct()->_conf->rgw_d4n_l1_fadvise != POSIX_FADV_NORMAL) + posix_fadvise(fd, 0, 0, dpp->get_cct()->_conf->rgw_d4n_l1_fadvise); cb->aio_fildes = fd; data = malloc(len); @@ -361,8 +361,8 @@ int SSDDriver::AsyncReadOp::init(const DoutPrefixProvider *dpp, CephContext* cct ldpp_dout(dpp, 1) << "ERROR: SSDCache: " << __func__ << "(): can't open " << file_path << " : " << " error: " << err << dendl; return -err; } - if (cct->_conf->rgw_d3n_l1_fadvise != POSIX_FADV_NORMAL) { - posix_fadvise(aio_cb->aio_fildes, 0, 0, g_conf()->rgw_d3n_l1_fadvise); + if (cct->_conf->rgw_d4n_l1_fadvise != POSIX_FADV_NORMAL) { + posix_fadvise(aio_cb->aio_fildes, 0, 0, g_conf()->rgw_d4n_l1_fadvise); } bufferptr bp(read_len); -- 2.39.5