From: Adam C. Emerson Date: Tue, 6 Dec 2022 00:51:25 +0000 (-0500) Subject: rgw: Remove `RGWSI_RADOS` X-Git-Tag: v19.0.0~21^2 X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=refs%2Fpull%2F50359%2Fhead;p=ceph.git rgw: Remove `RGWSI_RADOS` And remove the last places that reference it. Signed-off-by: Adam C. Emerson --- diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index d54c5b6afa95..00a9e4127e1e 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -48,7 +48,6 @@ set(librgw_common_srcs services/svc_otp.cc services/svc_quota.cc services/svc_sync_modules.cc - services/svc_rados.cc services/svc_role_rados.cc services/svc_sys_obj.cc services/svc_sys_obj_cache.cc diff --git a/src/rgw/driver/rados/rgw_cr_rados.cc b/src/rgw/driver/rados/rgw_cr_rados.cc index cacba3cf44bc..f903233cb02c 100644 --- a/src/rgw/driver/rados/rgw_cr_rados.cc +++ b/src/rgw/driver/rados/rgw_cr_rados.cc @@ -476,29 +476,7 @@ RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, } RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, - RGWSI_RADOS::Obj& obj, - RGWObjVersionTracker* objv_tracker) - : RGWSimpleCoroutine(store->ctx()), - ioctx(librados::IoCtx(obj.get_ref().pool.ioctx())), - oid(obj.get_ref().obj.oid), - objv_tracker(objv_tracker) -{ - set_description() << "remove dest=" << oid; -} - -RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, - RGWSI_RADOS::Obj&& obj, - RGWObjVersionTracker* objv_tracker) - : RGWSimpleCoroutine(store->ctx()), - ioctx(std::move(obj.get_ref().pool.ioctx())), - oid(std::move(obj.get_ref().obj.oid)), - objv_tracker(objv_tracker) -{ - set_description() << "remove dest=" << oid; -} - -RGWRadosRemoveOidCR::RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, - rgw_rados_ref&& obj, + rgw_rados_ref obj, RGWObjVersionTracker* objv_tracker) : RGWSimpleCoroutine(store->ctx()), ioctx(std::move(obj.ioctx)), diff --git a/src/rgw/driver/rados/rgw_cr_rados.h b/src/rgw/driver/rados/rgw_cr_rados.h index 0afb74d93a96..2cf99f98cc8e 100644 --- a/src/rgw/driver/rados/rgw_cr_rados.h +++ b/src/rgw/driver/rados/rgw_cr_rados.h @@ -753,15 +753,7 @@ public: RGWObjVersionTracker* objv_tracker = nullptr); RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, - RGWSI_RADOS::Obj& obj, - RGWObjVersionTracker* objv_tracker = nullptr); - - RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, - RGWSI_RADOS::Obj&& obj, - RGWObjVersionTracker* objv_tracker = nullptr); - - RGWRadosRemoveOidCR(rgw::sal::RadosStore* store, - rgw_rados_ref&& obj, + rgw_rados_ref obj, RGWObjVersionTracker* objv_tracker = nullptr); int send_request(const DoutPrefixProvider *dpp) override; diff --git a/src/rgw/driver/rados/rgw_d3n_datacache.h b/src/rgw/driver/rados/rgw_d3n_datacache.h index 196f892329f8..58ac95d093d0 100644 --- a/src/rgw/driver/rados/rgw_d3n_datacache.h +++ b/src/rgw/driver/rados/rgw_d3n_datacache.h @@ -195,10 +195,10 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const return 0; } - auto obj = d->rgwrados->svc.rados->obj(read_obj); - r = obj.open(dpp); + rgw_rados_ref ref; + r = rgw_get_rados_ref(dpp, d->rgwrados->get_rados_handle(), read_obj, &ref); if (r < 0) { - lsubdout(g_ceph_context, rgw, 4) << "failed to open rados context for " << read_obj << dendl; + ldpp_dout(dpp, 4) << "failed to open rados context for " << read_obj << dendl; return r; } @@ -208,8 +208,7 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const const uint64_t cost = len; const uint64_t id = obj_ofs; // use logical object offset for sorting replies - auto& ref = obj.get_ref(); - auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.ioctx, std::move(op), d->yield), cost, id); return d->flush(std::move(completed)); } else { ldpp_dout(dpp, 20) << "D3nDataCache::" << __func__ << "(): oid=" << read_obj.oid << ", is_head_obj=" << is_head_obj << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << ", len=" << len << dendl; @@ -221,20 +220,19 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const const uint64_t id = obj_ofs; // use logical object offset for sorting replies oid = read_obj.oid; - auto obj = d->rgwrados->svc.rados->obj(read_obj); - r = obj.open(dpp); + rgw_rados_ref ref; + r = rgw_get_rados_ref(dpp, d->rgwrados->get_rados_handle(), read_obj, &ref); if (r < 0) { - lsubdout(g_ceph_context, rgw, 0) << "D3nDataCache: Error: failed to open rados context for " << read_obj << ", r=" << r << dendl; + ldpp_dout(dpp, 4) << "failed to open rados context for " << read_obj << dendl; return r; } - auto& ref = obj.get_ref(); const bool is_compressed = (astate->attrset.find(RGW_ATTR_COMPRESSION) != astate->attrset.end()); const bool is_encrypted = (astate->attrset.find(RGW_ATTR_CRYPT_MODE) != astate->attrset.end()); if (read_ofs != 0 || astate->size != astate->accounted_size || is_compressed || is_encrypted) { d->d3n_bypass_cache_write = true; lsubdout(g_ceph_context, rgw, 5) << "D3nDataCache: " << __func__ << "(): Note - bypassing datacache: oid=" << read_obj.oid << ", read_ofs!=0 = " << read_ofs << ", size=" << astate->size << " != accounted_size=" << astate->accounted_size << ", is_compressed=" << is_compressed << ", is_encrypted=" << is_encrypted << dendl; - auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.ioctx, std::move(op), d->yield), cost, id); r = d->flush(std::move(completed)); return r; } @@ -251,7 +249,7 @@ int D3nRGWDataCache::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const } else { // Write To Cache ldpp_dout(dpp, 20) << "D3nDataCache: " << __func__ << "(): WRITE TO CACHE: oid=" << read_obj.oid << ", obj-ofs=" << obj_ofs << ", read_ofs=" << read_ofs << " len=" << len << dendl; - auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); + auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.ioctx, std::move(op), d->yield), cost, id); return d->flush(std::move(completed)); } } diff --git a/src/rgw/driver/rados/rgw_data_sync.cc b/src/rgw/driver/rados/rgw_data_sync.cc index 7b8c0b7343d9..b5effb979c9b 100644 --- a/src/rgw/driver/rados/rgw_data_sync.cc +++ b/src/rgw/driver/rados/rgw_data_sync.cc @@ -1457,7 +1457,7 @@ public: } if (complete->timestamp != ceph::real_time{}) { tn->log(10, SSTR("writing " << *complete << " to error repo for retry")); - yield call(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo, + yield call(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, rgw::error_repo::encode_key(complete->bs, complete->gen), complete->timestamp)); if (retcode < 0) { @@ -1465,7 +1465,7 @@ public: } } } else if (complete->retry) { - yield call(rgw::error_repo::remove_cr(sync_env->driver->svc()->rados, error_repo, + yield call(rgw::error_repo::remove_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, rgw::error_repo::encode_key(complete->bs, complete->gen), complete->timestamp)); if (retcode < 0) { @@ -1529,7 +1529,7 @@ public: if (retcode == -ENOENT) { // don't retry if bucket instance does not exist tn->log(10, SSTR("bucket instance or log layout does not exist on source for bucket " << source_bs.bucket)); - yield call(rgw::error_repo::remove_cr(sync_env->driver->svc()->rados, error_repo, + yield call(rgw::error_repo::remove_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, error_marker, timestamp)); return set_cr_done(); } else if (retcode < 0) { @@ -1544,7 +1544,7 @@ public: pool = sync_env->svc->zone->get_zone_params().log_pool; error_repo = datalog_oid_for_error_repo(sc, sync_env->driver, pool, source_bs); tn->log(10, SSTR("writing shard_id " << sid << " of gen " << each->gen << " to error repo for retry")); - yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo, + yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, rgw::error_repo::encode_key(bs, each->gen), timestamp), sc->lcc.adj_concurrency(cct->_conf->rgw_data_sync_spawn_window), [&](uint64_t stack_id, int ret) { @@ -1563,7 +1563,7 @@ public: }); // once everything succeeds, remove the full sync obligation from the error repo - yield call(rgw::error_repo::remove_cr(sync_env->driver->svc()->rados, error_repo, + yield call(rgw::error_repo::remove_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, error_marker, timestamp)); return set_cr_done(); } @@ -1648,7 +1648,7 @@ public: if (retcode < 0) { tn->log(10, SSTR("full sync: failed to read remote bucket info. Writing " << source_bs.shard_id << " to error repo for retry")); - yield call(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo, + yield call(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, rgw::error_repo::encode_key(source_bs, std::nullopt), timestamp)); if (retcode < 0) { @@ -1670,7 +1670,7 @@ public: timestamp = timestamp_for_bucket_shard(sync_env->driver, sync_status, source_bs); if (retcode < 0) { tn->log(10, SSTR("Write " << source_bs.shard_id << " to error repo for retry")); - yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->svc()->rados, error_repo, + yield_spawn_window(rgw::error_repo::write_cr(sync_env->driver->getRados()->get_rados_handle(), error_repo, rgw::error_repo::encode_key(source_bs, each->gen), timestamp), sc->lcc.adj_concurrency(cct->_conf->rgw_data_sync_spawn_window), std::nullopt); } else { @@ -2016,7 +2016,7 @@ public: } if (retcode < 0) { tn->log(1, SSTR("failed to parse bucket shard: " << error_marker)); - spawn(rgw::error_repo::remove_cr(sc->env->driver->svc()->rados, + spawn(rgw::error_repo::remove_cr(sc->env->driver->getRados()->get_rados_handle(), error_repo, error_marker, entry_timestamp), false); @@ -5160,8 +5160,11 @@ int RGWBucketShardIncrementalSyncCR::operate(const DoutPrefixProvider *dpp) } yield { // delete the shard status object - auto status_obj = sync_env->svc->rados->obj(marker_tracker.get_obj()); - retcode = status_obj.open(dpp); + rgw_rados_ref status_obj; + retcode = rgw_get_rados_ref(dpp, + sync_env->driver->getRados()->get_rados_handle(), + marker_tracker.get_obj(), + &status_obj); if (retcode < 0) { return set_cr_error(retcode); } @@ -6009,7 +6012,7 @@ int RGWSyncBucketCR::operate(const DoutPrefixProvider *dpp) // use the error repo and sync status timestamp from the datalog shard corresponding to source_bs error_repo = datalog_oid_for_error_repo(sc, sc->env->driver, pool, source_bs); - yield call(rgw::error_repo::write_cr(sc->env->driver->svc()->rados, error_repo, + yield call(rgw::error_repo::write_cr(sc->env->driver->getRados()->get_rados_handle(), error_repo, rgw::error_repo::encode_key(source_bs, current_gen), ceph::real_clock::zero())); if (retcode < 0) { diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.cc b/src/rgw/driver/rados/rgw_object_expirer_core.cc index 3f20ce26ba8a..269043f94df1 100644 --- a/src/rgw/driver/rados/rgw_object_expirer_core.cc +++ b/src/rgw/driver/rados/rgw_object_expirer_core.cc @@ -32,7 +32,6 @@ #include "rgw_zone.h" #include "rgw_sal_rados.h" -#include "services/svc_rados.h" #include "services/svc_zone.h" #include "services/svc_sys_obj.h" #include "services/svc_bi_rados.h" @@ -108,8 +107,11 @@ int RGWObjExpStore::objexp_hint_add(const DoutPrefixProvider *dpp, cls_timeindex_add(op, utime_t(delete_at), keyext, hebl); string shard_name = objexp_hint_get_shardname(objexp_key_shard(obj_key, cct->_conf->rgw_objexp_hints_num_shards)); - auto obj = rados_svc->obj(rgw_raw_obj(driver->svc()->zone->get_zone_params().log_pool, shard_name)); - int r = obj.open(dpp); + rgw_rados_ref obj; + int r = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(), + { driver->svc()->zone->get_zone_params().log_pool, + shard_name }, + &obj); if (r < 0) { ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl; return r; @@ -131,8 +133,10 @@ int RGWObjExpStore::objexp_hint_list(const DoutPrefixProvider *dpp, cls_timeindex_list(op, utime_t(start_time), utime_t(end_time), marker, max_entries, entries, out_marker, truncated); - auto obj = rados_svc->obj(rgw_raw_obj(driver->svc()->zone->get_zone_params().log_pool, oid)); - int r = obj.open(dpp); + rgw_rados_ref obj; + int r = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(), + { driver->svc()->zone->get_zone_params().log_pool, + oid }, &obj); if (r < 0) { ldpp_dout(dpp, 0) << "ERROR: " << __func__ << "(): failed to open obj=" << obj << " (r=" << r << ")" << dendl; return r; diff --git a/src/rgw/driver/rados/rgw_object_expirer_core.h b/src/rgw/driver/rados/rgw_object_expirer_core.h index b616570b82f9..d23457fe75ff 100644 --- a/src/rgw/driver/rados/rgw_object_expirer_core.h +++ b/src/rgw/driver/rados/rgw_object_expirer_core.h @@ -30,19 +30,16 @@ #include "rgw_sal_rados.h" -class RGWSI_RADOS; class RGWSI_Zone; class RGWBucketInfo; class cls_timeindex_entry; class RGWObjExpStore { CephContext *cct; - RGWSI_RADOS *rados_svc; rgw::sal::RadosStore* driver; public: - RGWObjExpStore(CephContext *_cct, RGWSI_RADOS *_rados_svc, rgw::sal::RadosStore* _driver) : cct(_cct), - rados_svc(_rados_svc), - driver(_driver) {} + RGWObjExpStore(CephContext *_cct, rgw::sal::RadosStore* _driver) : cct(_cct), + driver(_driver) {} int objexp_hint_add(const DoutPrefixProvider *dpp, const ceph::real_time& delete_at, @@ -101,7 +98,7 @@ protected: public: explicit RGWObjectExpirer(rgw::sal::Driver* _driver) : driver(_driver), - exp_store(_driver->ctx(), static_cast(driver)->svc()->rados, static_cast(driver)), + exp_store(_driver->ctx(), static_cast(driver)), worker(NULL) { } ~RGWObjectExpirer() { diff --git a/src/rgw/driver/rados/rgw_putobj_processor.cc b/src/rgw/driver/rados/rgw_putobj_processor.cc index 14661935a77c..67e8a3a998ef 100644 --- a/src/rgw/driver/rados/rgw_putobj_processor.cc +++ b/src/rgw/driver/rados/rgw_putobj_processor.cc @@ -126,8 +126,8 @@ void RadosWriter::add_write_hint(librados::ObjectWriteOperation& op) { int RadosWriter::set_stripe_obj(const rgw_raw_obj& raw_obj) { - stripe_obj = store->svc.rados->obj(raw_obj); - return stripe_obj.open(dpp); + return rgw_get_rados_ref(dpp, store->get_rados_handle(), raw_obj, + &stripe_obj); } int RadosWriter::process(bufferlist&& bl, uint64_t offset) @@ -145,8 +145,9 @@ int RadosWriter::process(bufferlist&& bl, uint64_t offset) op.write(offset, data); } constexpr uint64_t id = 0; // unused - auto& ref = stripe_obj.get_ref(); - auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); + auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx, + std::move(op), y), + cost, id); return process_completed(c, &written); } @@ -160,8 +161,9 @@ int RadosWriter::write_exclusive(const bufferlist& data) op.write_full(data); constexpr uint64_t id = 0; // unused - auto& ref = stripe_obj.get_ref(); - auto c = aio->get(ref.obj, Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); + auto c = aio->get(stripe_obj.obj, Aio::librados_op(stripe_obj.ioctx, + std::move(op), y), + cost, id); auto d = aio->drain(); c.splice(c.end(), d); return process_completed(c, &written); diff --git a/src/rgw/driver/rados/rgw_putobj_processor.h b/src/rgw/driver/rados/rgw_putobj_processor.h index 9a21c0c793a3..ce163e36f777 100644 --- a/src/rgw/driver/rados/rgw_putobj_processor.h +++ b/src/rgw/driver/rados/rgw_putobj_processor.h @@ -18,7 +18,6 @@ #include #include "rgw_putobj.h" -#include "services/svc_rados.h" #include "services/svc_tier_rados.h" #include "rgw_sal.h" #include "rgw_obj_manifest.h" @@ -70,7 +69,7 @@ class RadosWriter : public rgw::sal::DataProcessor { const RGWBucketInfo& bucket_info; RGWObjectCtx& obj_ctx; const rgw_obj head_obj; - RGWSI_RADOS::Obj stripe_obj; // current stripe object + rgw_rados_ref stripe_obj; // current stripe object RawObjSet written; // set of written objects for deletion const DoutPrefixProvider *dpp; optional_yield y; diff --git a/src/rgw/driver/rados/rgw_rados.cc b/src/rgw/driver/rados/rgw_rados.cc index 66d6b12338d1..592d7a828bc1 100644 --- a/src/rgw/driver/rados/rgw_rados.cc +++ b/src/rgw/driver/rados/rgw_rados.cc @@ -1020,7 +1020,7 @@ void RGWRados::finalize() { /* Before joining any sync threads, drain outstanding requests & * mark the async_processor as going_down() */ - if (svc.rados) { + if (svc.async_processor) { svc.async_processor->stop(); } @@ -1172,7 +1172,7 @@ int RGWRados::update_service_map(const DoutPrefixProvider *dpp, std::mapget_sync_module(); @@ -2394,7 +2394,9 @@ bool RGWRados::obj_to_raw(const rgw_placement_rule& placement_rule, const rgw_ob std::string RGWRados::get_cluster_fsid(const DoutPrefixProvider *dpp, optional_yield y) { - return svc.rados->cluster_fsid(); + std::string s; + rados.cluster_fsid(&s); + return s; } int RGWRados::get_obj_head_ioctx(const DoutPrefixProvider *dpp, @@ -4778,8 +4780,10 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx, ref_tag = tag + '\0'; cls_refcount_get(op, ref_tag, true); - auto obj = svc.rados->obj(miter.get_location().get_raw_obj(this)); - ret = obj.open(dpp); + rgw_rados_ref obj; + ret = rgw_get_rados_ref(dpp, driver->getRados()->get_rados_handle(), + miter.get_location().get_raw_obj(this), + &obj); if (ret < 0) { ldpp_dout(dpp, 0) << "failed to open rados context for " << obj << dendl; goto done_ret; @@ -4787,8 +4791,9 @@ int RGWRados::copy_obj(RGWObjectCtx& obj_ctx, static constexpr uint64_t cost = 1; // 1 throttle unit per request static constexpr uint64_t id = 0; // ids unused - auto& ref = obj.get_ref(); - rgw::AioResultList completed = aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); + rgw::AioResultList completed = + aio->get(obj.obj, rgw::Aio::librados_op(obj.ioctx, std::move(op), y), + cost, id); ret = rgw::check_for_errors(completed); all_results.splice(all_results.end(), completed); if (ret < 0) { @@ -4855,19 +4860,20 @@ done_ret: if (r.result < 0) { continue; // skip errors } - auto obj = svc.rados->obj(r.obj); - ret2 = obj.open(dpp); + rgw_rados_ref obj; + ret2 = rgw_get_rados_ref(dpp, get_rados_handle(), r.obj, &obj); if (ret2 < 0) { continue; } - auto& ref = obj.get_ref(); ObjectWriteOperation op; cls_refcount_put(op, ref_tag, true); static constexpr uint64_t cost = 1; // 1 throttle unit per request static constexpr uint64_t id = 0; // ids unused - rgw::AioResultList completed = aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), y), cost, id); + rgw::AioResultList completed = + aio->get(obj.obj, rgw::Aio::librados_op(obj.ioctx, std::move(op), y), + cost, id); ret2 = rgw::check_for_errors(completed); if (ret2 < 0) { ldpp_dout(dpp, 0) << "ERROR: cleanup after error failed to drop reference on obj=" << r.obj << dendl; @@ -7156,8 +7162,9 @@ int RGWRados::get_obj_iterate_cb(const DoutPrefixProvider *dpp, } } - auto obj = d->rgwrados->svc.rados->obj(read_obj); - int r = obj.open(dpp); + rgw_rados_ref obj; + int r = rgw_get_rados_ref(dpp, d->rgwrados->get_rados_handle(), read_obj, + &obj); if (r < 0) { ldpp_dout(dpp, 4) << "failed to open rados context for " << read_obj << dendl; return r; @@ -7169,8 +7176,7 @@ int RGWRados::get_obj_iterate_cb(const DoutPrefixProvider *dpp, const uint64_t cost = len; const uint64_t id = obj_ofs; // use logical object offset for sorting replies - auto& ref = obj.get_ref(); - auto completed = d->aio->get(ref.obj, rgw::Aio::librados_op(ref.pool.ioctx(), std::move(op), d->yield), cost, id); + auto completed = d->aio->get(obj.obj, rgw::Aio::librados_op(obj.ioctx, std::move(op), d->yield), cost, id); return d->flush(std::move(completed)); } diff --git a/src/rgw/driver/rados/rgw_rados.h b/src/rgw/driver/rados/rgw_rados.h index f5d9b68b259e..986dc091163a 100644 --- a/src/rgw/driver/rados/rgw_rados.h +++ b/src/rgw/driver/rados/rgw_rados.h @@ -33,7 +33,6 @@ #include "rgw_aio.h" #include "rgw_d3n_cacherequest.h" -#include "services/svc_rados.h" #include "services/svc_bi_rados.h" #include "common/Throttle.h" #include "common/ceph_mutex.h" diff --git a/src/rgw/driver/rados/rgw_service.cc b/src/rgw/driver/rados/rgw_service.cc index 12e9cd82aec5..5b78472dfe3b 100644 --- a/src/rgw/driver/rados/rgw_service.cc +++ b/src/rgw/driver/rados/rgw_service.cc @@ -17,7 +17,6 @@ #include "services/svc_meta_be_otp.h" #include "services/svc_notify.h" #include "services/svc_otp.h" -#include "services/svc_rados.h" #include "services/svc_zone.h" #include "services/svc_zone_utils.h" #include "services/svc_quota.h" @@ -52,7 +51,7 @@ int RGWServices_Def::init(CephContext *cct, bool have_cache, bool raw, bool run_sync, - librados::Rados* radoshandle, + librados::Rados* rados, optional_yield y, const DoutPrefixProvider *dpp) { @@ -70,7 +69,6 @@ int RGWServices_Def::init(CephContext *cct, meta_be_otp = std::make_unique(cct); notify = std::make_unique(cct); otp = std::make_unique(cct); - rados = std::make_unique(cct); zone = std::make_unique(cct); zone_utils = std::make_unique(cct); quota = std::make_unique(cct); @@ -90,7 +88,7 @@ int RGWServices_Def::init(CephContext *cct, async_processor->start(); finisher->init(); - bi_rados->init(zone.get(), radoshandle, bilog_rados.get(), datalog_rados.get()); + bi_rados->init(zone.get(), rados, bilog_rados.get(), datalog_rados.get()); bilog_rados->init(bi_rados.get()); bucket_sobj->init(zone.get(), sysobj.get(), sysobj_cache.get(), bi_rados.get(), meta.get(), meta_be_sobj.get(), @@ -99,28 +97,27 @@ int RGWServices_Def::init(CephContext *cct, sysobj.get(), sysobj_cache.get(), bucket_sobj.get()); - cls->init(zone.get(), radoshandle); - config_key_rados->init(radoshandle); - mdlog->init(radoshandle, zone.get(), sysobj.get(), cls.get(), + cls->init(zone.get(), rados); + config_key_rados->init(rados); + mdlog->init(rados, zone.get(), sysobj.get(), cls.get(), async_processor.get()); meta->init(sysobj.get(), mdlog.get(), meta_bes); meta_be_sobj->init(sysobj.get(), mdlog.get()); meta_be_otp->init(sysobj.get(), mdlog.get(), cls.get()); - notify->init(zone.get(), radoshandle, finisher.get()); + notify->init(zone.get(), rados, finisher.get()); otp->init(zone.get(), meta.get(), meta_be_otp.get()); - rados->init(); - zone->init(sysobj.get(), radoshandle, sync_modules.get(), bucket_sync_sobj.get()); - zone_utils->init(radoshandle, zone.get()); + zone->init(sysobj.get(), rados, sync_modules.get(), bucket_sync_sobj.get()); + zone_utils->init(rados, zone.get()); quota->init(zone.get()); sync_modules->init(zone.get()); - sysobj_core->core_init(radoshandle, zone.get()); + sysobj_core->core_init(rados, zone.get()); if (have_cache) { - sysobj_cache->init(radoshandle, zone.get(), notify.get()); - sysobj->init(radoshandle, sysobj_cache.get()); + sysobj_cache->init(rados, zone.get(), notify.get()); + sysobj->init(rados, sysobj_cache.get()); } else { - sysobj->init(radoshandle, sysobj_core.get()); + sysobj->init(rados, sysobj_core.get()); } - user_rados->init(radoshandle, zone.get(), sysobj.get(), sysobj_cache.get(), + user_rados->init(rados, zone.get(), sysobj.get(), sysobj_cache.get(), meta.get(), meta_be_sobj.get(), sync_modules.get()); role_rados->init(zone.get(), meta.get(), meta_be_sobj.get(), sysobj.get()); @@ -140,12 +137,6 @@ int RGWServices_Def::init(CephContext *cct, } } - r = rados->start(y, dpp); - if (r < 0) { - ldpp_dout(dpp, 0) << "ERROR: failed to start rados service (" << cpp_strerror(-r) << dendl; - return r; - } - if (!raw) { r = zone->start(y, dpp); if (r < 0) { @@ -155,7 +146,7 @@ int RGWServices_Def::init(CephContext *cct, r = datalog_rados->start(dpp, &zone->get_zone(), zone->get_zone_params(), - rados->get_rados_handle()); + rados); if (r < 0) { ldpp_dout(dpp, 0) << "ERROR: failed to start datalog_rados service (" << cpp_strerror(-r) << dendl; return r; @@ -306,18 +297,17 @@ void RGWServices_Def::shutdown() zone_utils->shutdown(); zone->shutdown(); async_processor->stop(); - rados->shutdown(); has_shutdown = true; } int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw, - bool run_sync, librados::Rados* radoshandle, + bool run_sync, librados::Rados* rados, optional_yield y, const DoutPrefixProvider *dpp) { cct = _cct; - int r = _svc.init(cct, have_cache, raw, run_sync, radoshandle, y, dpp); + int r = _svc.init(cct, have_cache, raw, run_sync, rados, y, dpp); if (r < 0) { return r; } @@ -340,7 +330,6 @@ int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw, meta_be_otp = _svc.meta_be_otp.get(); notify = _svc.notify.get(); otp = _svc.otp.get(); - rados = _svc.rados.get(); zone = _svc.zone.get(); zone_utils = _svc.zone_utils.get(); quota = _svc.quota.get(); diff --git a/src/rgw/driver/rados/rgw_service.h b/src/rgw/driver/rados/rgw_service.h index ad80a6c88266..7c05f043a47c 100644 --- a/src/rgw/driver/rados/rgw_service.h +++ b/src/rgw/driver/rados/rgw_service.h @@ -62,7 +62,6 @@ class RGWSI_MetaBackend_SObj; class RGWSI_MetaBackend_OTP; class RGWSI_Notify; class RGWSI_OTP; -class RGWSI_RADOS; class RGWSI_Zone; class RGWSI_ZoneUtils; class RGWSI_Quota; @@ -94,7 +93,6 @@ struct RGWServices_Def std::unique_ptr meta_be_otp; std::unique_ptr notify; std::unique_ptr otp; - std::unique_ptr rados; std::unique_ptr zone; std::unique_ptr zone_utils; std::unique_ptr quota; @@ -111,7 +109,7 @@ struct RGWServices_Def ~RGWServices_Def(); int init(CephContext *cct, bool have_cache, bool raw_storage, bool run_sync, - librados::Rados* radoshandle, optional_yield y, + librados::Rados* rados, optional_yield y, const DoutPrefixProvider *dpp); void shutdown(); }; @@ -141,7 +139,6 @@ struct RGWServices RGWSI_MetaBackend *meta_be_otp{nullptr}; RGWSI_Notify *notify{nullptr}; RGWSI_OTP *otp{nullptr}; - RGWSI_RADOS *rados{nullptr}; RGWSI_Zone *zone{nullptr}; RGWSI_ZoneUtils *zone_utils{nullptr}; RGWSI_Quota *quota{nullptr}; @@ -154,18 +151,18 @@ struct RGWServices RGWAsyncRadosProcessor* async_processor; int do_init(CephContext *cct, bool have_cache, bool raw_storage, - bool run_sync, librados::Rados* radoshandle, optional_yield y, + bool run_sync, librados::Rados* rados, optional_yield y, const DoutPrefixProvider *dpp); int init(CephContext *cct, bool have_cache, bool run_sync, - librados::Rados* radoshandle, optional_yield y, + librados::Rados* rados, optional_yield y, const DoutPrefixProvider *dpp) { - return do_init(cct, have_cache, false, run_sync, radoshandle, y, dpp); + return do_init(cct, have_cache, false, run_sync, rados, y, dpp); } - int init_raw(CephContext *cct, bool have_cache, librados::Rados* radoshandle, + int init_raw(CephContext *cct, bool have_cache, librados::Rados* rados, optional_yield y, const DoutPrefixProvider *dpp) { - return do_init(cct, have_cache, true, false, radoshandle, y, dpp); + return do_init(cct, have_cache, true, false, rados, y, dpp); } void shutdown() { _svc.shutdown(); diff --git a/src/rgw/driver/rados/rgw_sync_error_repo.cc b/src/rgw/driver/rados/rgw_sync_error_repo.cc index 44305b60b6b2..aca671039ee1 100644 --- a/src/rgw/driver/rados/rgw_sync_error_repo.cc +++ b/src/rgw/driver/rados/rgw_sync_error_repo.cc @@ -15,7 +15,6 @@ #include "rgw_sync_error_repo.h" #include "rgw_coroutine.h" #include "rgw_sal.h" -#include "services/svc_rados.h" #include "cls/cmpomap/client.h" namespace rgw::error_repo { @@ -118,16 +117,17 @@ int remove(librados::ObjectWriteOperation& op, } class RGWErrorRepoWriteCR : public RGWSimpleCoroutine { - RGWSI_RADOS::Obj obj; + librados::Rados* rados; + const rgw_raw_obj& raw_obj; std::string key; ceph::real_time timestamp; boost::intrusive_ptr cn; public: - RGWErrorRepoWriteCR(RGWSI_RADOS* rados, const rgw_raw_obj& raw_obj, + RGWErrorRepoWriteCR(librados::Rados* rados, const rgw_raw_obj& raw_obj, const std::string& key, ceph::real_time timestamp) - : RGWSimpleCoroutine(rados->ctx()), - obj(rados->obj(raw_obj)), + : RGWSimpleCoroutine(static_cast(rados->cct())), + raw_obj(raw_obj), key(key), timestamp(timestamp) {} @@ -137,13 +137,14 @@ class RGWErrorRepoWriteCR : public RGWSimpleCoroutine { if (r < 0) { return r; } - r = obj.open(dpp); + rgw_rados_ref ref; + r = rgw_get_rados_ref(dpp, rados, raw_obj, &ref); if (r < 0) { return r; } cn = stack->create_completion_notifier(); - return obj.aio_operate(cn->completion(), &op); + return ref.aio_operate(cn->completion(), &op); } int request_complete() override { @@ -151,7 +152,7 @@ class RGWErrorRepoWriteCR : public RGWSimpleCoroutine { } }; -RGWCoroutine* write_cr(RGWSI_RADOS* rados, +RGWCoroutine* write_cr(librados::Rados* rados, const rgw_raw_obj& obj, const std::string& key, ceph::real_time timestamp) @@ -161,16 +162,17 @@ RGWCoroutine* write_cr(RGWSI_RADOS* rados, class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine { - RGWSI_RADOS::Obj obj; + librados::Rados* rados; + const rgw_raw_obj& raw_obj; std::string key; ceph::real_time timestamp; boost::intrusive_ptr cn; public: - RGWErrorRepoRemoveCR(RGWSI_RADOS* rados, const rgw_raw_obj& raw_obj, + RGWErrorRepoRemoveCR(librados::Rados* rados, const rgw_raw_obj& raw_obj, const std::string& key, ceph::real_time timestamp) - : RGWSimpleCoroutine(rados->ctx()), - obj(rados->obj(raw_obj)), + : RGWSimpleCoroutine(static_cast(rados->cct())), + raw_obj(raw_obj), key(key), timestamp(timestamp) {} @@ -180,13 +182,14 @@ class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine { if (r < 0) { return r; } - r = obj.open(dpp); + rgw_rados_ref ref; + r = rgw_get_rados_ref(dpp, rados, raw_obj, &ref); if (r < 0) { return r; } cn = stack->create_completion_notifier(); - return obj.aio_operate(cn->completion(), &op); + return ref.aio_operate(cn->completion(), &op); } int request_complete() override { @@ -194,7 +197,7 @@ class RGWErrorRepoRemoveCR : public RGWSimpleCoroutine { } }; -RGWCoroutine* remove_cr(RGWSI_RADOS* rados, +RGWCoroutine* remove_cr(librados::Rados* rados, const rgw_raw_obj& obj, const std::string& key, ceph::real_time timestamp) diff --git a/src/rgw/driver/rados/rgw_sync_error_repo.h b/src/rgw/driver/rados/rgw_sync_error_repo.h index 60525d281f0f..7760c9fe198d 100644 --- a/src/rgw/driver/rados/rgw_sync_error_repo.h +++ b/src/rgw/driver/rados/rgw_sync_error_repo.h @@ -19,7 +19,6 @@ #include "include/buffer_fwd.h" #include "common/ceph_time.h" -class RGWSI_RADOS; class RGWCoroutine; struct rgw_raw_obj; struct rgw_bucket_shard; @@ -42,7 +41,7 @@ ceph::real_time decode_value(const ceph::bufferlist& bl); int write(librados::ObjectWriteOperation& op, const std::string& key, ceph::real_time timestamp); -RGWCoroutine* write_cr(RGWSI_RADOS* rados, +RGWCoroutine* write_cr(librados::Rados* rados, const rgw_raw_obj& obj, const std::string& key, ceph::real_time timestamp); @@ -51,7 +50,7 @@ RGWCoroutine* write_cr(RGWSI_RADOS* rados, int remove(librados::ObjectWriteOperation& op, const std::string& key, ceph::real_time timestamp); -RGWCoroutine* remove_cr(RGWSI_RADOS* rados, +RGWCoroutine* remove_cr(librados::Rados* rados, const rgw_raw_obj& obj, const std::string& key, ceph::real_time timestamp); diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h index 35f5daba2ed2..c6c11f8bc00b 100644 --- a/src/rgw/services/svc_bi_rados.h +++ b/src/rgw/services/svc_bi_rados.h @@ -21,7 +21,6 @@ #include "rgw_tools.h" #include "svc_bi.h" -#include "svc_rados.h" #include "svc_tier_rados.h" struct rgw_bucket_dir_header; diff --git a/src/rgw/services/svc_bilog_rados.h b/src/rgw/services/svc_bilog_rados.h index e9d5dbb5c0e2..e9c948d3fa27 100644 --- a/src/rgw/services/svc_bilog_rados.h +++ b/src/rgw/services/svc_bilog_rados.h @@ -1,4 +1,3 @@ - // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- // vim: ts=8 sw=2 smarttab ft=cpp @@ -19,11 +18,6 @@ #include "rgw_service.h" -#include "svc_rados.h" - - - - class RGWSI_BILog_RADOS : public RGWServiceInstance { public: diff --git a/src/rgw/services/svc_cls.cc b/src/rgw/services/svc_cls.cc index 101108981c32..740d9ab03072 100644 --- a/src/rgw/services/svc_cls.cc +++ b/src/rgw/services/svc_cls.cc @@ -3,7 +3,6 @@ #include "svc_cls.h" -#include "svc_rados.h" #include "svc_zone.h" #include "rgw_zone.h" diff --git a/src/rgw/services/svc_cls.h b/src/rgw/services/svc_cls.h index c5ee361fc56d..6648714dbc8f 100644 --- a/src/rgw/services/svc_cls.h +++ b/src/rgw/services/svc_cls.h @@ -21,8 +21,6 @@ #include "rgw_service.h" -#include "svc_rados.h" - #include "driver/rados/rgw_tools.h" diff --git a/src/rgw/services/svc_mdlog.cc b/src/rgw/services/svc_mdlog.cc index 603718dc96e5..46a158ceb77b 100644 --- a/src/rgw/services/svc_mdlog.cc +++ b/src/rgw/services/svc_mdlog.cc @@ -2,7 +2,6 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "svc_mdlog.h" -#include "svc_rados.h" #include "svc_zone.h" #include "svc_sys_obj.h" diff --git a/src/rgw/services/svc_notify.cc b/src/rgw/services/svc_notify.cc index 041354a376f7..5593dee9ae29 100644 --- a/src/rgw/services/svc_notify.cc +++ b/src/rgw/services/svc_notify.cc @@ -9,7 +9,6 @@ #include "svc_notify.h" #include "svc_finisher.h" #include "svc_zone.h" -#include "svc_rados.h" #include "rgw_zone.h" diff --git a/src/rgw/services/svc_notify.h b/src/rgw/services/svc_notify.h index 618853d02c65..4f7e9d17ee37 100644 --- a/src/rgw/services/svc_notify.h +++ b/src/rgw/services/svc_notify.h @@ -5,8 +5,6 @@ #include "rgw_service.h" -#include "svc_rados.h" - #include "rgw_tools.h" diff --git a/src/rgw/services/svc_rados.cc b/src/rgw/services/svc_rados.cc deleted file mode 100644 index c45853368680..000000000000 --- a/src/rgw/services/svc_rados.cc +++ /dev/null @@ -1,433 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#include "svc_rados.h" - -#include "include/rados/librados.hpp" -#include "common/errno.h" -#include "osd/osd_types.h" -#include "rgw_tools.h" -#include "rgw_cr_rados.h" - -#include "auth/AuthRegistry.h" - -#define dout_subsys ceph_subsys_rgw - -using namespace std; - -RGWSI_RADOS::RGWSI_RADOS(CephContext *cct) : RGWServiceInstance(cct) -{ -} - -RGWSI_RADOS::~RGWSI_RADOS() -{ -} - -int RGWSI_RADOS::do_start(optional_yield, const DoutPrefixProvider *dpp) -{ - int ret = rados.init_with_context(cct); - if (ret < 0) { - return ret; - } - ret = rados.connect(); - if (ret < 0) { - return ret; - } - - return 0; -} - -void RGWSI_RADOS::shutdown() -{ - rados.shutdown(); -} - -librados::Rados* RGWSI_RADOS::get_rados_handle() -{ - return &rados; -} - -std::string RGWSI_RADOS::cluster_fsid() -{ - std::string fsid; - (void) get_rados_handle()->cluster_fsid(&fsid); - return fsid; -} - -uint64_t RGWSI_RADOS::instance_id() -{ - return get_rados_handle()->get_instance_id(); -} - -int RGWSI_RADOS::open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, - const OpenParams& params) -{ - return rgw_init_ioctx(dpp, get_rados_handle(), pool, io_ctx, - params.create, - params.mostly_omap); -} - -int RGWSI_RADOS::pool_iterate(const DoutPrefixProvider *dpp, - librados::IoCtx& io_ctx, - librados::NObjectIterator& iter, - uint32_t num, vector& objs, - const rgw::AccessListFilter& filter, - bool *is_truncated) -{ - if (iter == io_ctx.nobjects_end()) - return -ENOENT; - - uint32_t i; - - for (i = 0; i < num && iter != io_ctx.nobjects_end(); ++i, ++iter) { - rgw_bucket_dir_entry e; - - string oid = iter->get_oid(); - ldpp_dout(dpp, 20) << "RGWRados::pool_iterate: got " << oid << dendl; - - // fill it in with initial values; we may correct later - if (filter && filter(oid, oid)) - continue; - - e.key = oid; - objs.push_back(e); - } - - if (is_truncated) - *is_truncated = (iter != io_ctx.nobjects_end()); - - return objs.size(); -} - -RGWSI_RADOS::Obj::Obj(Pool& pool, const string& oid) : rados_svc(pool.rados_svc) -{ - ref.pool = pool; - ref.obj = rgw_raw_obj(pool.get_pool(), oid); -} - -void RGWSI_RADOS::Obj::init(const rgw_raw_obj& obj) -{ - ref.pool = RGWSI_RADOS::Pool(rados_svc, obj.pool); - ref.obj = obj; -} - -int RGWSI_RADOS::Obj::open(const DoutPrefixProvider *dpp) -{ - int r = ref.pool.open(dpp); - if (r < 0) { - return r; - } - - ref.pool.ioctx().locator_set_key(ref.obj.loc); - - return 0; -} - -int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op, - optional_yield y, int flags) -{ - return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, y, flags); -} - -int RGWSI_RADOS::Obj::operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op, - bufferlist *pbl, optional_yield y, int flags) -{ - return rgw_rados_operate(dpp, ref.pool.ioctx(), ref.obj.oid, op, pbl, y, flags); -} - -int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op) -{ - return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op); -} - -int RGWSI_RADOS::Obj::aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op, - bufferlist *pbl) -{ - return ref.pool.ioctx().aio_operate(ref.obj.oid, c, op, pbl); -} - -int RGWSI_RADOS::Obj::watch(uint64_t *handle, librados::WatchCtx2 *ctx) -{ - return ref.pool.ioctx().watch2(ref.obj.oid, handle, ctx); -} - -int RGWSI_RADOS::Obj::aio_watch(librados::AioCompletion *c, uint64_t *handle, librados::WatchCtx2 *ctx) -{ - return ref.pool.ioctx().aio_watch(ref.obj.oid, c, handle, ctx); -} - -int RGWSI_RADOS::Obj::unwatch(uint64_t handle) -{ - return ref.pool.ioctx().unwatch2(handle); -} - -int RGWSI_RADOS::Obj::notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms, - bufferlist *pbl, optional_yield y) -{ - return rgw_rados_notify(dpp, ref.pool.ioctx(), ref.obj.oid, bl, timeout_ms, pbl, y); -} - -void RGWSI_RADOS::Obj::notify_ack(uint64_t notify_id, - uint64_t cookie, - bufferlist& bl) -{ - ref.pool.ioctx().notify_ack(ref.obj.oid, notify_id, cookie, bl); -} - -uint64_t RGWSI_RADOS::Obj::get_last_version() -{ - return ref.pool.ioctx().get_last_version(); -} - -int RGWSI_RADOS::Pool::create(const DoutPrefixProvider *dpp) -{ - librados::Rados *rad = rados_svc->get_rados_handle(); - int r = rad->pool_create(pool.name.c_str()); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: pool_create returned " << r << dendl; - return r; - } - librados::IoCtx io_ctx; - r = rad->ioctx_create(pool.name.c_str(), io_ctx); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: ioctx_create returned " << r << dendl; - return r; - } - r = io_ctx.application_enable(pg_pool_t::APPLICATION_NAME_RGW, false); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: application_enable returned " << r << dendl; - return r; - } - return 0; -} - -int RGWSI_RADOS::Pool::create(const DoutPrefixProvider *dpp, const vector& pools, vector *retcodes) -{ - vector completions; - vector rets; - - librados::Rados *rad = rados_svc->get_rados_handle(); - for (auto iter = pools.begin(); iter != pools.end(); ++iter) { - librados::PoolAsyncCompletion *c = librados::Rados::pool_async_create_completion(); - completions.push_back(c); - auto& pool = *iter; - int ret = rad->pool_create_async(pool.name.c_str(), c); - rets.push_back(ret); - } - - vector::iterator riter; - vector::iterator citer; - - bool error = false; - ceph_assert(rets.size() == completions.size()); - for (riter = rets.begin(), citer = completions.begin(); riter != rets.end(); ++riter, ++citer) { - int r = *riter; - librados::PoolAsyncCompletion *c = *citer; - if (r == 0) { - c->wait(); - r = c->get_return_value(); - if (r < 0) { - ldpp_dout(dpp, 0) << "WARNING: async pool_create returned " << r << dendl; - error = true; - } - } - c->release(); - retcodes->push_back(r); - } - if (error) { - return 0; - } - - std::vector io_ctxs; - retcodes->clear(); - for (auto pool : pools) { - io_ctxs.emplace_back(); - int ret = rad->ioctx_create(pool.name.c_str(), io_ctxs.back()); - if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: ioctx_create returned " << ret << dendl; - error = true; - } - retcodes->push_back(ret); - } - if (error) { - return 0; - } - - completions.clear(); - for (auto &io_ctx : io_ctxs) { - librados::PoolAsyncCompletion *c = - librados::Rados::pool_async_create_completion(); - completions.push_back(c); - int ret = io_ctx.application_enable_async(pg_pool_t::APPLICATION_NAME_RGW, - false, c); - ceph_assert(ret == 0); - } - - retcodes->clear(); - for (auto c : completions) { - c->wait(); - int ret = c->get_return_value(); - if (ret == -EOPNOTSUPP) { - ret = 0; - } else if (ret < 0) { - ldpp_dout(dpp, 0) << "WARNING: async application_enable returned " << ret - << dendl; - error = true; - } - c->release(); - retcodes->push_back(ret); - } - return 0; -} - -int RGWSI_RADOS::Pool::lookup() -{ - librados::Rados *rad = rados_svc->get_rados_handle(); - int ret = rad->pool_lookup(pool.name.c_str()); - if (ret < 0) { - return ret; - } - - return 0; -} - -int RGWSI_RADOS::Pool::open(const DoutPrefixProvider *dpp, const OpenParams& params) -{ - return rados_svc->open_pool_ctx(dpp, pool, state.ioctx, params); -} - -int RGWSI_RADOS::Pool::List::init(const DoutPrefixProvider *dpp, const string& marker, rgw::AccessListFilter filter) -{ - if (ctx.initialized) { - return -EINVAL; - } - - if (!pool) { - return -EINVAL; - } - - int r = pool->rados_svc->open_pool_ctx(dpp, pool->pool, ctx.ioctx); - if (r < 0) { - return r; - } - - librados::ObjectCursor oc; - if (!oc.from_str(marker)) { - ldpp_dout(dpp, 10) << "failed to parse cursor: " << marker << dendl; - return -EINVAL; - } - - try { - ctx.iter = ctx.ioctx.nobjects_begin(oc); - } catch (const std::system_error& e) { - r = -e.code().value(); - ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what() - << ", returning " << r << dendl; - return r; - } catch (const std::exception& e) { - ldpp_dout(dpp, 10) << "nobjects_begin threw " << e.what() - << ", returning -5" << dendl; - return -EIO; - } - ctx.filter = std::move(filter); - ctx.initialized = true; - - return 0; -} - -int RGWSI_RADOS::Pool::List::get_next(const DoutPrefixProvider *dpp, - int max, - std::vector *oids, - bool *is_truncated) -{ - if (!ctx.initialized) { - return -EINVAL; - } - vector objs; - int r = pool->rados_svc->pool_iterate(dpp, ctx.ioctx, ctx.iter, max, objs, ctx.filter, is_truncated); - if (r < 0) { - if(r != -ENOENT) { - ldpp_dout(dpp, 10) << "failed to list objects pool_iterate returned r=" << r << dendl; - } - return r; - } - - for (auto& o : objs) { - oids->push_back(o.key.name); - } - - return oids->size(); -} - -RGWSI_RADOS::Obj RGWSI_RADOS::Handle::obj(const rgw_raw_obj& o) -{ - return RGWSI_RADOS::Obj(rados_svc, o); -} -int RGWSI_RADOS::Handle::watch_flush() -{ - librados::Rados *rad = rados_svc->get_rados_handle(); - return rad->watch_flush(); -} - -int RGWSI_RADOS::Handle::mon_command(std::string cmd, - const bufferlist& inbl, - bufferlist *outbl, - std::string *outs) -{ - librados::Rados *rad = rados_svc->get_rados_handle(); - return rad->mon_command(cmd, inbl, outbl, outs); -} - -int RGWSI_RADOS::Pool::List::get_marker(string *marker) -{ - if (!ctx.initialized) { - return -EINVAL; - } - - *marker = ctx.iter.get_cursor().to_str(); - return 0; -} - -int RGWSI_RADOS::clog_warn(const string& msg) -{ - string cmd = - "{" - "\"prefix\": \"log\", " - "\"level\": \"warn\", " - "\"logtext\": [\"" + msg + "\"]" - "}"; - - bufferlist inbl; - auto h = handle(); - return h.mon_command(cmd, inbl, nullptr, nullptr); -} - -bool RGWSI_RADOS::check_secure_mon_conn(const DoutPrefixProvider *dpp) const -{ - AuthRegistry reg(cct); - - reg.refresh_config(); - - std::vector methods; - std::vector modes; - - reg.get_supported_methods(CEPH_ENTITY_TYPE_MON, &methods, &modes); - ldpp_dout(dpp, 20) << __func__ << "(): auth registry supported: methods=" << methods << " modes=" << modes << dendl; - - for (auto method : methods) { - if (!reg.is_secure_method(method)) { - ldpp_dout(dpp, 20) << __func__ << "(): method " << method << " is insecure" << dendl; - return false; - } - } - - for (auto mode : modes) { - if (!reg.is_secure_mode(mode)) { - ldpp_dout(dpp, 20) << __func__ << "(): mode " << mode << " is insecure" << dendl; - return false; - } - } - - return true; -} - diff --git a/src/rgw/services/svc_rados.h b/src/rgw/services/svc_rados.h deleted file mode 100644 index bed6522e2ae2..000000000000 --- a/src/rgw/services/svc_rados.h +++ /dev/null @@ -1,232 +0,0 @@ -// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- -// vim: ts=8 sw=2 smarttab ft=cpp - -#pragma once - -#include "rgw_service.h" - -#include "include/rados/librados.hpp" -#include "common/async/yield_context.h" - -#include "rgw_tools.h" - -class RGWAsyncRadosProcessor; - -class RGWSI_RADOS : public RGWServiceInstance -{ - librados::Rados rados; - - int do_start(optional_yield, const DoutPrefixProvider *dpp) override; - -public: - struct OpenParams { - bool create{true}; - bool mostly_omap{false}; - - OpenParams() {} - - OpenParams& set_create(bool _create) { - create = _create; - return *this; - } - OpenParams& set_mostly_omap(bool _mostly_omap) { - mostly_omap = _mostly_omap; - return *this; - } - }; - -private: - int open_pool_ctx(const DoutPrefixProvider *dpp, const rgw_pool& pool, librados::IoCtx& io_ctx, - const OpenParams& params = {}); - int pool_iterate(const DoutPrefixProvider *dpp, - librados::IoCtx& ioctx, - librados::NObjectIterator& iter, - uint32_t num, std::vector& objs, - const rgw::AccessListFilter& filter, - bool *is_truncated); - -public: - RGWSI_RADOS(CephContext *cct); - ~RGWSI_RADOS(); - librados::Rados* get_rados_handle(); - - void init() {} - void shutdown() override; - - std::string cluster_fsid(); - uint64_t instance_id(); - bool check_secure_mon_conn(const DoutPrefixProvider *dpp) const; - - int clog_warn(const std::string& msg); - - class Handle; - - class Pool { - friend class RGWSI_RADOS; - friend Handle; - friend class Obj; - - RGWSI_RADOS *rados_svc{nullptr}; - rgw_pool pool; - - struct State { - librados::IoCtx ioctx; - } state; - - Pool(RGWSI_RADOS *_rados_svc, - const rgw_pool& _pool) : rados_svc(_rados_svc), - pool(_pool) {} - - Pool(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {} - public: - Pool() {} - - int create(const DoutPrefixProvider *dpp); - int create(const DoutPrefixProvider *dpp, const std::vector& pools, std::vector *retcodes); - int lookup(); - int open(const DoutPrefixProvider *dpp, const OpenParams& params = {}); - - const rgw_pool& get_pool() { - return pool; - } - - librados::IoCtx& ioctx() & { - return state.ioctx; - } - - librados::IoCtx&& ioctx() && { - return std::move(state.ioctx); - } - - struct List { - Pool *pool{nullptr}; - - struct Ctx { - bool initialized{false}; - librados::IoCtx ioctx; - librados::NObjectIterator iter; - rgw::AccessListFilter filter; - } ctx; - - List() {} - List(Pool *_pool) : pool(_pool) {} - - int init(const DoutPrefixProvider *dpp, const std::string& marker, - rgw::AccessListFilter filter); - int get_next(const DoutPrefixProvider *dpp, int max, - std::vector *oids, - bool *is_truncated); - - int get_marker(std::string *marker); - }; - - List op() { - return List(this); - } - - friend List; - }; - - - struct rados_ref { - RGWSI_RADOS::Pool pool; - rgw_raw_obj obj; - }; - - class Obj { - friend class RGWSI_RADOS; - friend class Handle; - - RGWSI_RADOS *rados_svc{nullptr}; - rados_ref ref; - - void init(const rgw_raw_obj& obj); - - Obj(RGWSI_RADOS *_rados_svc, const rgw_raw_obj& _obj) - : rados_svc(_rados_svc) { - init(_obj); - } - - Obj(Pool& pool, const std::string& oid); - - public: - Obj() {} - - int open(const DoutPrefixProvider *dpp); - - int operate(const DoutPrefixProvider *dpp, librados::ObjectWriteOperation *op, optional_yield y, - int flags = 0); - int operate(const DoutPrefixProvider *dpp, librados::ObjectReadOperation *op, bufferlist *pbl, - optional_yield y, int flags = 0); - int aio_operate(librados::AioCompletion *c, librados::ObjectWriteOperation *op); - int aio_operate(librados::AioCompletion *c, librados::ObjectReadOperation *op, - bufferlist *pbl); - - int watch(uint64_t *handle, librados::WatchCtx2 *ctx); - int aio_watch(librados::AioCompletion *c, uint64_t *handle, librados::WatchCtx2 *ctx); - int unwatch(uint64_t handle); - int notify(const DoutPrefixProvider *dpp, bufferlist& bl, uint64_t timeout_ms, - bufferlist *pbl, optional_yield y); - void notify_ack(uint64_t notify_id, - uint64_t cookie, - bufferlist& bl); - - uint64_t get_last_version(); - - rados_ref& get_ref() { return ref; } - const rados_ref& get_ref() const { return ref; } - - const rgw_raw_obj& get_raw_obj() const { - return ref.obj; - } - }; - - class Handle { - friend class RGWSI_RADOS; - - RGWSI_RADOS *rados_svc{nullptr}; - - Handle(RGWSI_RADOS *_rados_svc) : rados_svc(_rados_svc) {} - public: - Obj obj(const rgw_raw_obj& o); - - Pool pool(const rgw_pool& p) { - return Pool(rados_svc, p); - } - - int watch_flush(); - - int mon_command(std::string cmd, - const bufferlist& inbl, - bufferlist *outbl, - std::string *outs); - }; - - Handle handle() { - return Handle(this); - } - - Obj obj(const rgw_raw_obj& o) { - return Obj(this, o); - } - - Obj obj(Pool& pool, const std::string& oid) { - return Obj(pool, oid); - } - - Pool pool() { - return Pool(this); - } - - Pool pool(const rgw_pool& p) { - return Pool(this, p); - } - - friend Obj; - friend Pool; - friend Pool::List; -}; - -inline std::ostream& operator<<(std::ostream& out, const RGWSI_RADOS::Obj& obj) { - return out << obj.get_raw_obj(); -} diff --git a/src/rgw/services/svc_sys_obj.cc b/src/rgw/services/svc_sys_obj.cc index 310e60514d2c..38e7acdfe0be 100644 --- a/src/rgw/services/svc_sys_obj.cc +++ b/src/rgw/services/svc_sys_obj.cc @@ -3,7 +3,6 @@ #include "svc_sys_obj.h" #include "svc_sys_obj_core.h" -#include "svc_rados.h" #include "svc_zone.h" #include "rgw_zone.h" diff --git a/src/rgw/services/svc_sys_obj.h b/src/rgw/services/svc_sys_obj.h index 3739d68d150e..0bba2fbe01dc 100644 --- a/src/rgw/services/svc_sys_obj.h +++ b/src/rgw/services/svc_sys_obj.h @@ -7,7 +7,6 @@ #include "rgw_service.h" -#include "svc_rados.h" #include "svc_sys_obj_types.h" #include "svc_sys_obj_core_types.h" diff --git a/src/rgw/services/svc_sys_obj_core.cc b/src/rgw/services/svc_sys_obj_core.cc index 68878cdcceb2..fb9c7edce238 100644 --- a/src/rgw/services/svc_sys_obj_core.cc +++ b/src/rgw/services/svc_sys_obj_core.cc @@ -2,7 +2,6 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "svc_sys_obj_core.h" -#include "svc_rados.h" #include "svc_zone.h" #include "rgw_tools.h" diff --git a/src/rgw/services/svc_sys_obj_core.h b/src/rgw/services/svc_sys_obj_core.h index 1b34d7078317..d9dfc02f01bc 100644 --- a/src/rgw/services/svc_sys_obj_core.h +++ b/src/rgw/services/svc_sys_obj_core.h @@ -5,7 +5,6 @@ #include "rgw_service.h" -#include "svc_rados.h" #include "svc_sys_obj.h" #include "svc_sys_obj_core_types.h" diff --git a/src/rgw/services/svc_sys_obj_core_types.h b/src/rgw/services/svc_sys_obj_core_types.h index 7bec012f5c29..404f0fdde686 100644 --- a/src/rgw/services/svc_sys_obj_core_types.h +++ b/src/rgw/services/svc_sys_obj_core_types.h @@ -7,7 +7,6 @@ #include "rgw_tools.h" #include "rgw_service.h" -#include "svc_rados.h" #include "svc_sys_obj_types.h" diff --git a/src/rgw/services/svc_tier_rados.h b/src/rgw/services/svc_tier_rados.h index f7424b410387..0327e4658818 100644 --- a/src/rgw/services/svc_tier_rados.h +++ b/src/rgw/services/svc_tier_rados.h @@ -20,8 +20,6 @@ #include "rgw_service.h" -#include "svc_rados.h" - extern const std::string MP_META_SUFFIX; class RGWMPObj { diff --git a/src/rgw/services/svc_zone.cc b/src/rgw/services/svc_zone.cc index a44d53727b16..70cf40eb6cb6 100644 --- a/src/rgw/services/svc_zone.cc +++ b/src/rgw/services/svc_zone.cc @@ -2,7 +2,6 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "svc_zone.h" -#include "svc_rados.h" #include "svc_sys_obj.h" #include "svc_sync_modules.h" diff --git a/src/rgw/services/svc_zone_utils.cc b/src/rgw/services/svc_zone_utils.cc index be9f861f02cc..c809974e6d5f 100644 --- a/src/rgw/services/svc_zone_utils.cc +++ b/src/rgw/services/svc_zone_utils.cc @@ -2,7 +2,6 @@ // vim: ts=8 sw=2 smarttab ft=cpp #include "svc_zone_utils.h" -#include "svc_rados.h" #include "svc_zone.h" #undef FMT_HEADER_ONLY