From: Pritha Srivastava Date: Mon, 14 Aug 2023 06:39:50 +0000 (+0530) Subject: rgw/cache: this commit squashes commits related to removing X-Git-Tag: testing/wip-batrick-testing-20240411.154038~45^2~67 X-Git-Url: http://git.apps.os.sepia.ceph.com/?a=commitdiff_plain;h=af131995917665d92043c3a536644f05df3a1231;p=ceph-ci.git rgw/cache: this commit squashes commits related to removing cache_aio_read()/cache_aio_write() from cache api, ssd driver and redis driver. rgw/cache: removing cache_aio_read()/cache_aio_write() interfaces from the Cache API. And moving cache_read_op from rgw_aio.cc to ssd cache driver. rgw/cache: changes related to cache_aio_read()/cache_aio_write() in redis cache driver. Signed-off-by: Pritha Srivastava --- diff --git a/src/rgw/rgw_aio.cc b/src/rgw/rgw_aio.cc index da71ea5071a..cfbedfccecc 100644 --- a/src/rgw/rgw_aio.cc +++ b/src/rgw/rgw_aio.cc @@ -146,14 +146,4 @@ Aio::OpFunc Aio::d3n_cache_op(const DoutPrefixProvider *dpp, optional_yield y, return d3n_cache_aio_abstract(dpp, y, read_ofs, read_len, cache_location); } -Aio::OpFunc Aio::cache_read_op(const DoutPrefixProvider *dpp, optional_yield y, rgw::cache::CacheDriver* cache_driver, - off_t read_ofs, off_t read_len, const std::string& key) { - return [dpp, y, cache_driver, read_ofs, read_len, key] (Aio* aio, AioResult& r) mutable { - ceph_assert(y); - auto c = cache_driver->get_cache_aio_request_ptr(dpp); - ldpp_dout(dpp, 20) << "Cache: cache_read_op(): Read From Cache, oid=" << r.obj.oid << dendl; - c->cache_aio_read(dpp, y, key, read_ofs, read_len, aio, r); - }; -} - } // namespace rgw diff --git a/src/rgw/rgw_cache_driver.h b/src/rgw/rgw_cache_driver.h index 5d5199fdf51..9dcc1425e7c 100644 --- a/src/rgw/rgw_cache_driver.h +++ b/src/rgw/rgw_cache_driver.h @@ -19,14 +19,6 @@ struct Entry { int localWeight; }; -class CacheAioRequest { - public: - CacheAioRequest() {} - virtual ~CacheAioRequest() = default; - virtual void cache_aio_read(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, off_t ofs, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) = 0; - virtual void cache_aio_write(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& bl, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) = 0; -}; - class CacheDriver { public: CacheDriver() {} @@ -46,8 +38,6 @@ class CacheDriver { virtual std::string get_attr(const DoutPrefixProvider* dpp, const std::string& key, const std::string& attr_name) = 0; virtual int set_attr(const DoutPrefixProvider* dpp, const std::string& key, const std::string& attr_name, const std::string& attr_val) = 0; - virtual std::unique_ptr get_cache_aio_request_ptr(const DoutPrefixProvider* dpp) = 0; - /* Entry */ virtual bool key_exists(const DoutPrefixProvider* dpp, const std::string& key) = 0; virtual std::vector list_entries(const DoutPrefixProvider* dpp) = 0; diff --git a/src/rgw/rgw_redis_driver.cc b/src/rgw/rgw_redis_driver.cc index 44ceaac9363..83c1bc64e3f 100644 --- a/src/rgw/rgw_redis_driver.cc +++ b/src/rgw/rgw_redis_driver.cc @@ -640,12 +640,12 @@ int RedisDriver::set_attr(const DoutPrefixProvider* dpp, const std::string& key, return result - 1; } - +#if 0 std::unique_ptr RedisDriver::get_cache_aio_request_ptr(const DoutPrefixProvider* dpp) { return std::make_unique(this); } - +#endif rgw::AioResultList RedisDriver::get_async(const DoutPrefixProvider* dpp, optional_yield y, rgw::Aio* aio, const std::string& key, off_t ofs, uint64_t len, uint64_t cost, uint64_t id) { rgw_raw_obj r_obj; @@ -731,7 +731,7 @@ auto RedisDriver::get_async(const DoutPrefixProvider *dpp, ExecutionContext& ctx }*/ //return init.result.get(); } - +#if 0 void RedisCacheAioRequest::cache_aio_read(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, off_t ofs, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) { using namespace boost::asio; @@ -745,7 +745,7 @@ void RedisCacheAioRequest::cache_aio_read(const DoutPrefixProvider* dpp, optiona void RedisCacheAioRequest::cache_aio_write(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& bl, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) { } - +#endif int RedisDriver::put_async(const DoutPrefixProvider* dpp, const std::string& key, bufferlist& bl, uint64_t len, rgw::sal::Attrs& attrs) { return 0; diff --git a/src/rgw/rgw_redis_driver.h b/src/rgw/rgw_redis_driver.h index 832bcedf2c2..cbaae3c2820 100644 --- a/src/rgw/rgw_redis_driver.h +++ b/src/rgw/rgw_redis_driver.h @@ -12,18 +12,6 @@ namespace rgw { namespace cache { -class RedisDriver; - -class RedisCacheAioRequest: public CacheAioRequest { - public: - RedisCacheAioRequest(RedisDriver* cache_driver) : cache_driver(cache_driver) {} - virtual ~RedisCacheAioRequest() = default; - virtual void cache_aio_read(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, off_t ofs, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) override; - virtual void cache_aio_write(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& bl, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) override; - private: - RedisDriver* cache_driver; -}; - class RedisDriver : public CacheDriver { public: RedisDriver(Partition& _partition_info) : partition_info(_partition_info), @@ -61,7 +49,6 @@ class RedisDriver : public CacheDriver { virtual std::string get_attr(const DoutPrefixProvider* dpp, const std::string& key, const std::string& attr_name) override; virtual int set_attr(const DoutPrefixProvider* dpp, const std::string& key, const std::string& attr_name, const std::string& attr_val) override; - virtual std::unique_ptr get_cache_aio_request_ptr(const DoutPrefixProvider* dpp) override; virtual rgw::AioResultList get_async(const DoutPrefixProvider* dpp, optional_yield y, rgw::Aio* aio, const std::string& key, off_t ofs, uint64_t len, uint64_t cost, uint64_t id) override; virtual int put_async(const DoutPrefixProvider* dpp, const std::string& key, bufferlist& bl, uint64_t len, rgw::sal::Attrs& attrs) override; diff --git a/src/rgw/rgw_ssd_driver.cc b/src/rgw/rgw_ssd_driver.cc index 5cd2ccf0d95..317590afcb0 100644 --- a/src/rgw/rgw_ssd_driver.cc +++ b/src/rgw/rgw_ssd_driver.cc @@ -217,16 +217,61 @@ int SSDDriver::get(const DoutPrefixProvider* dpp, const std::string& key, off_t return 0; } -std::unique_ptr SSDDriver::get_cache_aio_request_ptr(const DoutPrefixProvider* dpp) +template +auto SSDDriver::AsyncReadOp::create(const Executor1& ex1, CompletionHandler&& handler) { - return std::make_unique(this); + auto p = Completion::create(ex1, std::move(handler)); + return p; +} + +template +auto SSDDriver::get_async(const DoutPrefixProvider *dpp, ExecutionContext& ctx, const std::string& key, + off_t read_ofs, off_t read_len, CompletionToken&& token) +{ + std::string location = partition_info.location + key; + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): location=" << location << dendl; + + using Op = AsyncReadOp; + using Signature = typename Op::Signature; + boost::asio::async_completion init(token); + auto p = Op::create(ctx.get_executor(), init.completion_handler); + auto& op = p->user_data; + + int ret = op.init(dpp, cct, location, read_ofs, read_len, p.get()); + if(0 == ret) { + ret = ::aio_read(op.aio_cb.get()); + } + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): ::aio_read(), ret=" << ret << dendl; + if(ret < 0) { + auto ec = boost::system::error_code{-ret, boost::system::system_category()}; + ceph::async::post(std::move(p), ec, bufferlist{}); + } else { + (void)p.release(); + } + return init.result.get(); +} + +rgw::Aio::OpFunc SSDDriver::ssd_cache_read_op(const DoutPrefixProvider *dpp, optional_yield y, rgw::cache::CacheDriver* cache_driver, + off_t read_ofs, off_t read_len, const std::string& key) { + return [this, dpp, y, read_ofs, read_len, key] (Aio* aio, AioResult& r) mutable { + ceph_assert(y); + ldpp_dout(dpp, 20) << "SSDCache: cache_read_op(): Read From Cache, oid=" << r.obj.oid << dendl; + + using namespace boost::asio; + spawn::yield_context yield = y.get_yield_context(); + async_completion init(yield); + auto ex = get_associated_executor(init.completion_handler); + + ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key=" << key << dendl; + this->get_async(dpp, y.get_io_context(), key, read_ofs, read_len, bind_executor(ex, SSDDriver::libaio_handler{aio, r})); + }; } rgw::AioResultList SSDDriver::get_async(const DoutPrefixProvider* dpp, optional_yield y, rgw::Aio* aio, const std::string& key, off_t ofs, uint64_t len, uint64_t cost, uint64_t id) { rgw_raw_obj r_obj; r_obj.oid = key; - return aio->get(r_obj, rgw::Aio::cache_read_op(dpp, y, this, ofs, len, key), cost, id); + return aio->get(r_obj, ssd_cache_read_op(dpp, y, this, ofs, len, key), cost, id); } void SSDDriver::libaio_write_completion_cb(AsyncWriteRequest* c) @@ -318,7 +363,6 @@ int SSDDriver::AsyncWriteRequest::prepare_libaio_write_op(const DoutPrefixProvid void SSDDriver::AsyncWriteRequest::libaio_write_cb(sigval sigval) { SSDDriver::AsyncWriteRequest* c = static_cast(sigval.sival_ptr); - ldpp_dout(c->dpp, 20) << "SSDCache: " << __func__ << "()" << dendl; c->priv_data->libaio_write_completion_cb(c); } @@ -364,40 +408,6 @@ void SSDDriver::AsyncReadOp::libaio_cb_aio_dispatch(sigval sigval) ceph::async::dispatch(std::move(p), ec, std::move(op.result)); } -template -auto SSDDriver::AsyncReadOp::create(const Executor1& ex1, CompletionHandler&& handler) -{ - auto p = Completion::create(ex1, std::move(handler)); - return p; -} - -template -auto SSDDriver::get_async(const DoutPrefixProvider *dpp, ExecutionContext& ctx, const std::string& key, - off_t read_ofs, off_t read_len, CompletionToken&& token) -{ - std::string location = partition_info.location + key; - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): location=" << location << dendl; - - using Op = AsyncReadOp; - using Signature = typename Op::Signature; - boost::asio::async_completion init(token); - auto p = Op::create(ctx.get_executor(), init.completion_handler); - auto& op = p->user_data; - - int ret = op.init(dpp, cct, location, read_ofs, read_len, p.get()); - if(0 == ret) { - ret = ::aio_read(op.aio_cb.get()); - } - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): ::aio_read(), ret=" << ret << dendl; - if(ret < 0) { - auto ec = boost::system::error_code{-ret, boost::system::system_category()}; - ceph::async::post(std::move(p), ec, bufferlist{}); - } else { - (void)p.release(); - } - return init.result.get(); -} - int SSDDriver::update_attrs(const DoutPrefixProvider* dpp, const std::string& key, rgw::sal::Attrs& attrs) { std::string location = partition_info.location + key; @@ -522,21 +532,4 @@ int SSDDriver::delete_attr(const DoutPrefixProvider* dpp, const std::string& key return removexattr(location.c_str(), attr_name.c_str()); } - -void SSDCacheAioRequest::cache_aio_read(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, off_t ofs, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) -{ - using namespace boost::asio; - spawn::yield_context yield = y.get_yield_context(); - async_completion init(yield); - auto ex = get_associated_executor(init.completion_handler); - - ldpp_dout(dpp, 20) << "SSDCache: " << __func__ << "(): key=" << key << dendl; - cache_driver->get_async(dpp, y.get_io_context(), key, ofs, len, bind_executor(ex, SSDDriver::libaio_handler{aio, r})); -} - -void SSDCacheAioRequest::cache_aio_write(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& bl, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) -{ - //TODO - implement cache_aio_write -} - } } // namespace rgw::cache diff --git a/src/rgw/rgw_ssd_driver.h b/src/rgw/rgw_ssd_driver.h index 81a60442869..6f0c4773c75 100644 --- a/src/rgw/rgw_ssd_driver.h +++ b/src/rgw/rgw_ssd_driver.h @@ -4,19 +4,7 @@ #include "rgw_common.h" #include "rgw_cache_driver.h" -namespace rgw { namespace cache { //cal stands for Cache Abstraction Layer - -class SSDDriver; - -class SSDCacheAioRequest: public CacheAioRequest { -public: - SSDCacheAioRequest(SSDDriver* cache_driver) : cache_driver(cache_driver) {} - virtual ~SSDCacheAioRequest() = default; - virtual void cache_aio_read(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, off_t ofs, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) override; - virtual void cache_aio_write(const DoutPrefixProvider* dpp, optional_yield y, const std::string& key, bufferlist& bl, uint64_t len, rgw::Aio* aio, rgw::AioResult& r) override; -private: - SSDDriver* cache_driver; -}; +namespace rgw { namespace cache { class SSDDriver : public CacheDriver { public: @@ -49,8 +37,6 @@ public: static std::optional get_partition_info(const DoutPrefixProvider* dpp, const std::string& name, const std::string& type); static std::vector list_partitions(const DoutPrefixProvider* dpp); - virtual std::unique_ptr get_cache_aio_request_ptr(const DoutPrefixProvider* dpp) override; - struct libaio_handler { rgw::Aio* throttle = nullptr; rgw::AioResult& r; @@ -61,9 +47,7 @@ public: throttle->put(r); } }; - template - auto get_async(const DoutPrefixProvider *dpp, ExecutionContext& ctx, const std::string& key, - off_t read_ofs, off_t read_len, CompletionToken&& token); + protected: inline static std::unordered_map partitions; std::unordered_map entries; @@ -91,6 +75,13 @@ struct libaio_aiocb_deleter { } }; +template + auto get_async(const DoutPrefixProvider *dpp, ExecutionContext& ctx, const std::string& key, + off_t read_ofs, off_t read_len, CompletionToken&& token); + +rgw::Aio::OpFunc ssd_cache_read_op(const DoutPrefixProvider *dpp, optional_yield y, rgw::cache::CacheDriver* cache_driver, + off_t read_ofs, off_t read_len, const std::string& key); + using unique_aio_cb_ptr = std::unique_ptr; struct AsyncReadOp {