From ac2a69d8d0d2779ace50f0ef01ae3dda901bebd3 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Tue, 11 Jun 2019 15:07:04 -0700 Subject: [PATCH] rgw: add svc.bilog_rados Signed-off-by: Yehuda Sadeh --- src/rgw/CMakeLists.txt | 1 + src/rgw/rgw_rados.cc | 49 --------------------- src/rgw/rgw_rados.h | 3 -- src/rgw/rgw_service.cc | 5 ++- src/rgw/rgw_service.h | 3 ++ src/rgw/services/svc_bi_rados.cc | 35 ++++++++++++++- src/rgw/services/svc_bi_rados.h | 11 ++++- src/rgw/services/svc_bilog_rados.cc | 68 +++++++++++++++++++++++++++++ src/rgw/services/svc_bilog_rados.h | 46 +++++++++++++++++++ src/rgw/services/svc_bucket_sobj.cc | 36 +-------------- src/rgw/services/svc_bucket_sobj.h | 4 -- 11 files changed, 168 insertions(+), 93 deletions(-) create mode 100644 src/rgw/services/svc_bilog_rados.cc create mode 100644 src/rgw/services/svc_bilog_rados.h diff --git a/src/rgw/CMakeLists.txt b/src/rgw/CMakeLists.txt index c3134104592..88c50ca3030 100644 --- a/src/rgw/CMakeLists.txt +++ b/src/rgw/CMakeLists.txt @@ -22,6 +22,7 @@ endfunction() set(librgw_common_srcs services/svc_finisher.cc services/svc_bi_rados.cc + services/svc_bilog_rados.cc services/svc_bucket.cc services/svc_bucket_sobj.cc services/svc_cls.cc diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 2018b566a7f..68a26b3959c 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -7851,55 +7851,6 @@ int RGWRados::list_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, stri return 0; } -int RGWRados::trim_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& start_marker, string& end_marker) -{ - librados::IoCtx index_ctx; - map bucket_objs; - - BucketIndexShardsManager start_marker_mgr; - BucketIndexShardsManager end_marker_mgr; - - int r = open_bucket_index(bucket_info, index_ctx, bucket_objs, shard_id); - if (r < 0) { - return r; - } - - r = start_marker_mgr.from_string(start_marker, shard_id); - if (r < 0) { - return r; - } - - r = end_marker_mgr.from_string(end_marker, shard_id); - if (r < 0) { - return r; - } - - return CLSRGWIssueBILogTrim(index_ctx, start_marker_mgr, end_marker_mgr, bucket_objs, - cct->_conf->rgw_bucket_index_max_aio)(); -} - -int RGWRados::resync_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id) -{ - librados::IoCtx index_ctx; - map bucket_objs; - int r = open_bucket_index(bucket_info, index_ctx, bucket_objs, shard_id); - if (r < 0) - return r; - - return CLSRGWIssueResyncBucketBILog(index_ctx, bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); -} - -int RGWRados::stop_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id) -{ - librados::IoCtx index_ctx; - map bucket_objs; - int r = open_bucket_index(bucket_info, index_ctx, bucket_objs, shard_id); - if (r < 0) - return r; - - return CLSRGWIssueBucketBILogStop(index_ctx, bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); -} - int RGWRados::bi_get_instance(const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent) { diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index af0a59c20a9..d300c1aa47d 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -1369,9 +1369,6 @@ public: int cls_bucket_head(const RGWBucketInfo& bucket_info, int shard_id, vector& headers, map *bucket_instance_ids = NULL); int cls_bucket_head_async(const RGWBucketInfo& bucket_info, int shard_id, RGWGetDirHeader_CB *ctx, int *num_aio); int list_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, uint32_t max, std::list& result, bool *truncated); - int trim_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id, string& marker, string& end_marker); - int resync_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id); - int stop_bi_log_entries(RGWBucketInfo& bucket_info, int shard_id); int get_bi_log_status(RGWBucketInfo& bucket_info, int shard_id, map& max_marker); int bi_get_instance(const RGWBucketInfo& bucket_info, const rgw_obj& obj, rgw_bucket_dir_entry *dirent); diff --git a/src/rgw/rgw_service.cc b/src/rgw/rgw_service.cc index 533eb4047b7..899a6c01243 100644 --- a/src/rgw/rgw_service.cc +++ b/src/rgw/rgw_service.cc @@ -47,6 +47,7 @@ int RGWServices_Def::init(CephContext *cct, finisher = std::make_unique(cct); bucket_sobj = std::make_unique(cct); bi_rados = std::make_unique(cct); + bilog_rados = std::make_unique(cct); cls = std::make_unique(cct); mdlog = std::make_unique(cct); meta = std::make_unique(cct); @@ -70,7 +71,8 @@ int RGWServices_Def::init(CephContext *cct, vector meta_bes{meta_be_sobj.get(), meta_be_otp.get()}; finisher->init(); - bi_rados->init(zone.get(), rados.get()); + bi_rados->init(zone.get(), rados.get(), bilog_rados.get()); + bilog_rados->init(bi_rados.get()); bucket->init(zone.get(), sysobj.get(), sysobj_cache.get(), bi_rados.get(), meta.get(), meta_be_sobj.get(), sync_modules.get()); @@ -249,6 +251,7 @@ int RGWServices::do_init(CephContext *_cct, bool have_cache, bool raw) finisher = _svc.finisher.get(); bi_rados = _svc.bi_rados.get(); bi = bi_rados; + bilog_rados = _svc.bilog_rados.get(); bucket_sobj = _svc.bucket_sobj.get(); bucket = _svc.bucket.get(); cls = _svc.cls.get(); diff --git a/src/rgw/rgw_service.h b/src/rgw/rgw_service.h index c2b99a83a6f..979385894c1 100644 --- a/src/rgw/rgw_service.h +++ b/src/rgw/rgw_service.h @@ -49,6 +49,7 @@ class RGWSI_Bucket; class RGWSI_Bucket_SObj; class RGWSI_BucketIndex; class RGWSI_BucketIndex_RADOS; +class RGWSI_BILog_RADOS; class RGWSI_Cls; class RGWSI_MDLog; class RGWSI_Meta; @@ -76,6 +77,7 @@ struct RGWServices_Def std::unique_ptr finisher; std::unique_ptr bucket_sobj; std::unique_ptr bi_rados; + std::unique_ptr bilog_rados; std::unique_ptr cls; std::unique_ptr mdlog; std::unique_ptr meta; @@ -112,6 +114,7 @@ struct RGWServices RGWSI_Bucket_SObj *bucket_sobj{nullptr}; RGWSI_BucketIndex *bi{nullptr}; RGWSI_BucketIndex_RADOS *bi_rados{nullptr}; + RGWSI_BILog_RADOS *bilog_rados{nullptr}; RGWSI_Cls *cls{nullptr}; RGWSI_MDLog *mdlog{nullptr}; RGWSI_Meta *meta{nullptr}; diff --git a/src/rgw/services/svc_bi_rados.cc b/src/rgw/services/svc_bi_rados.cc index 66a52390fad..c280c846b8f 100644 --- a/src/rgw/services/svc_bi_rados.cc +++ b/src/rgw/services/svc_bi_rados.cc @@ -1,5 +1,6 @@ #include "svc_bi_rados.h" +#include "svc_bilog_rados.h" #include "svc_zone.h" #include "rgw/rgw_zone.h" @@ -15,10 +16,12 @@ RGWSI_BucketIndex_RADOS::RGWSI_BucketIndex_RADOS(CephContext *cct) : RGWSI_Bucke } void RGWSI_BucketIndex_RADOS::init(RGWSI_Zone *zone_svc, - RGWSI_RADOS *rados_svc) + RGWSI_RADOS *rados_svc, + RGWSI_BILog_RADOS *bilog_svc) { svc.zone = zone_svc; svc.rados = rados_svc; + svc.bilog = bilog_svc; } int RGWSI_BucketIndex_RADOS::open_pool(const rgw_pool& pool, @@ -386,3 +389,33 @@ int RGWSI_BucketIndex_RADOS::get_reshard_status(const RGWBucketInfo& bucket_info return 0; } +int RGWSI_BucketIndex_RADOS::handle_overwrite(const RGWBucketInfo& info, + const RGWBucketInfo& orig_info) +{ + if (orig_info.datasync_flag_enabled() != info.datasync_flag_enabled()) { + int shards_num = info.num_shards? info.num_shards : 1; + int shard_id = info.num_shards? 0 : -1; + + int ret; + if (!info.datasync_flag_enabled()) { + ret = svc.bilog->log_stop(info, -1); + } else { + ret = svc.bilog->log_start(info, -1); + } + if (ret < 0) { + lderr(cct) << "ERROR: failed writing bilog (bucket=" << info.bucket << "); ret=" << ret << dendl; + return ret; + } + + for (int i = 0; i < shards_num; ++i, ++shard_id) { + ret = store->data_log->add_entry(info.bucket, shard_id); + if (ret < 0) { + lderr(cct) << "ERROR: failed writing data log (info.bucket=" << info.bucket << ", shard_id=" << shard_id << ")" << dendl; + return ret; + } + } + } + + return 0; +} + diff --git a/src/rgw/services/svc_bi_rados.h b/src/rgw/services/svc_bi_rados.h index c0f1ea1546a..6f66c3b5a66 100644 --- a/src/rgw/services/svc_bi_rados.h +++ b/src/rgw/services/svc_bi_rados.h @@ -25,6 +25,8 @@ struct rgw_bucket_dir_header; +class RGWSI_BILog_RADOS; + #define RGW_NO_SHARD -1 #define RGW_SHARDS_PRIME_0 7877 @@ -32,6 +34,8 @@ struct rgw_bucket_dir_header; class RGWSI_BucketIndex_RADOS : public RGWSI_BucketIndex { + friend class RGWSI_BILog_RADOS; + int open_pool(const rgw_pool& pool, RGWSI_RADOS::Pool *index_pool, bool mostly_omap); @@ -78,12 +82,14 @@ public: struct Svc { RGWSI_Zone *zone{nullptr}; RGWSI_RADOS *rados{nullptr}; + RGWSI_BILog_RADOS *bilog{nullptr}; } svc; RGWSI_BucketIndex_RADOS(CephContext *cct); void init(RGWSI_Zone *zone_svc, - RGWSI_RADOS *rados_svc); + RGWSI_RADOS *rados_svc, + RGWSI_BILog_RADOS *bilog_svc); static int shards_max() { return RGW_SHARDS_PRIME_1; @@ -108,6 +114,9 @@ public: int get_reshard_status(const RGWBucketInfo& bucket_info, std::list *status); + + int handle_overwrite(const RGWBucketInfo& info, + const RGWBucketInfo& orig_info); }; diff --git a/src/rgw/services/svc_bilog_rados.cc b/src/rgw/services/svc_bilog_rados.cc new file mode 100644 index 00000000000..14770bd2827 --- /dev/null +++ b/src/rgw/services/svc_bilog_rados.cc @@ -0,0 +1,68 @@ + +#include "svc_bilog_rados.h" +#include "svc_bi_rados.h" + +#include "cls/rgw/cls_rgw_client.h" + + +#define dout_subsys ceph_subsys_rgw + +RGWSI_BILog_RADOS::RGWSI_BILog_RADOS(CephContext *cct) : RGWServiceInstance(cct) +{ +} + +void RGWSI_BILog_RADOS::init(RGWSI_BucketIndex_RADOS *bi_rados_svc) +{ + svc.bi = bi_rados_svc; +} + +int RGWSI_BILog_RADOS::log_trim(const RGWBucketInfo& bucket_info, int shard_id, string& start_marker, string& end_marker) +{ + RGWSI_RADOS::Pool index_pool; + map bucket_objs; + + BucketIndexShardsManager start_marker_mgr; + BucketIndexShardsManager end_marker_mgr; + + int r = svc.bi->open_bucket_index(bucket_info, shard_id, &index_pool, &bucket_objs, nullptr); + if (r < 0) { + return r; + } + + r = start_marker_mgr.from_string(start_marker, shard_id); + if (r < 0) { + return r; + } + + r = end_marker_mgr.from_string(end_marker, shard_id); + if (r < 0) { + return r; + } + + return CLSRGWIssueBILogTrim(index_pool.ioctx(), start_marker_mgr, end_marker_mgr, bucket_objs, + cct->_conf->rgw_bucket_index_max_aio)(); +} + +int RGWSI_BILog_RADOS::log_start(const RGWBucketInfo& bucket_info, int shard_id) +{ + RGWSI_RADOS::Pool index_pool; + map bucket_objs; + int r = svc.bi->open_bucket_index(bucket_info, shard_id, &index_pool, &bucket_objs, nullptr); + if (r < 0) + return r; + + return CLSRGWIssueResyncBucketBILog(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); +} + +int RGWSI_BILog_RADOS::log_stop(const RGWBucketInfo& bucket_info, int shard_id) +{ + RGWSI_RADOS::Pool index_pool; + map bucket_objs; + int r = svc.bi->open_bucket_index(bucket_info, shard_id, &index_pool, &bucket_objs, nullptr); + if (r < 0) + return r; + + return CLSRGWIssueBucketBILogStop(index_pool.ioctx(), bucket_objs, cct->_conf->rgw_bucket_index_max_aio)(); +} + + diff --git a/src/rgw/services/svc_bilog_rados.h b/src/rgw/services/svc_bilog_rados.h new file mode 100644 index 00000000000..44a12d7dff9 --- /dev/null +++ b/src/rgw/services/svc_bilog_rados.h @@ -0,0 +1,46 @@ + +// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- +// vim: ts=8 sw=2 smarttab + +/* + * Ceph - scalable distributed file system + * + * Copyright (C) 2019 Red Hat, Inc. + * + * This is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License version 2.1, as published by the Free Software + * Foundation. See file COPYING. + * + */ + + +#pragma once + +#include "rgw/rgw_service.h" + +#include "svc_rados.h" + + + + +class RGWSI_BILog_RADOS : public RGWServiceInstance +{ +public: + struct Svc { + RGWSI_BucketIndex_RADOS *bi{nullptr}; + } svc; + + RGWSI_BILog_RADOS(CephContext *cct); + + void init(RGWSI_BucketIndex_RADOS *bi_rados_svc); + + int log_start(const RGWBucketInfo& bucket_info, int shard_id); + int log_stop(const RGWBucketInfo& bucket_info, int shard_id); + + int log_trim(const RGWBucketInfo& bucket_info, + int shard_id, + std::string& start_marker, + std::string& end_marker); +}; + diff --git a/src/rgw/services/svc_bucket_sobj.cc b/src/rgw/services/svc_bucket_sobj.cc index 07cb7e9d69c..2537582369a 100644 --- a/src/rgw/services/svc_bucket_sobj.cc +++ b/src/rgw/services/svc_bucket_sobj.cc @@ -491,9 +491,9 @@ int RGWSI_Bucket_SObj::store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, } if (orig_info && *orig_info && !exclusive) { - int r = handle_bucket_overwrite(ctx, key, info, *(orig_info.value())); + int r = svc.bi->handle_overwrite(info, *(orig_info.value())); if (r < 0) { - ldout(cct, 0) << "ERROR: " << __func__ << "(): handle_bucket_overwrite() of key=" << key << " returned r=" << r << dendl; + ldout(cct, 0) << "ERROR: " << __func__ << "(): svc.bi->handle_overwrite() of key=" << key << " returned r=" << r << dendl; return r; } } @@ -520,38 +520,6 @@ int RGWSI_Bucket_SObj::store_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, return ret; } -int RGWSI_Bucket_SObj::handle_bucket_overwrite(RGWSI_Bucket_BI_Ctx& ctx, - const string& key, - const RGWBucketInfo& info, - const RGWBucketInfo& orig_info) -{ - if (orig_info.datasync_flag_enabled() != info.datasync_flag_enabled()) { - int shards_num = info.num_shards? info.num_shards : 1; - int shard_id = info.num_shards? 0 : -1; - - int ret; - if (!info.datasync_flag_enabled()) { - ret = store->stop_bi_log_entries(info, -1); - } else { - ret = store->resync_bi_log_entries(info, -1); - } - if (ret < 0) { - lderr(cct) << "ERROR: failed writing bilog (key=" << key << "); ret=" << ret << dendl; - return ret; - } - - for (int i = 0; i < shards_num; ++i, ++shard_id) { - ret = store->data_log->add_entry(info.bucket, shard_id); - if (ret < 0) { - lderr(cct) << "ERROR: failed writing data log (info.bucket=" << info.bucket << ", shard_id=" << shard_id << ")" << dendl; - return ret; - } - } - } - - return 0; -} - int RGWSI_Bucket_SObj::remove_bucket_instance_info(RGWSI_Bucket_BI_Ctx& ctx, const string& key, RGWObjVersionTracker *objv_tracker, diff --git a/src/rgw/services/svc_bucket_sobj.h b/src/rgw/services/svc_bucket_sobj.h index 02a96106847..54962ea2f3f 100644 --- a/src/rgw/services/svc_bucket_sobj.h +++ b/src/rgw/services/svc_bucket_sobj.h @@ -64,10 +64,6 @@ class RGWSI_Bucket_SObj : public RGWSI_Bucket RGWBucketEnt *ent, optional_yield y); - int handle_bucket_overwrite(RGWSI_Bucket_BI_Ctx& ctx, - const string& key, - const RGWBucketInfo& info, - const RGWBucketInfo& orig_info); public: struct Svc { RGWSI_Bucket_SObj *bucket{nullptr}; -- 2.39.5