From 76b736b33708c78cbe4200f1483a8517d8b5b501 Mon Sep 17 00:00:00 2001 From: Yehuda Sadeh Date: Wed, 8 May 2013 10:52:26 -0700 Subject: [PATCH] rgw: metadata log trim Signed-off-by: Yehuda Sadeh --- src/cls/log/cls_log_client.cc | 2 +- src/cls/log/cls_log_client.h | 2 +- src/rgw/rgw_admin.cc | 24 ++++++++++++++++++++++++ src/rgw/rgw_metadata.cc | 22 +++++++++++++++++++++- src/rgw/rgw_metadata.h | 2 ++ src/rgw/rgw_rados.cc | 12 ++++++++++++ src/rgw/rgw_rados.h | 1 + 7 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/cls/log/cls_log_client.cc b/src/cls/log/cls_log_client.cc index c551f40735876..00e37bb43aa2d 100644 --- a/src/cls/log/cls_log_client.cc +++ b/src/cls/log/cls_log_client.cc @@ -55,7 +55,7 @@ void cls_log_trim(librados::ObjectWriteOperation& op, utime_t& from, utime_t& to op.exec("log", "trim", in); } -int cls_log_trim(librados::IoCtx& io_ctx, string& oid, utime_t& from, utime_t& to) +int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, utime_t& from, utime_t& to) { bool done = false; diff --git a/src/cls/log/cls_log_client.h b/src/cls/log/cls_log_client.h index 4171adbda113f..a035e58bde180 100644 --- a/src/cls/log/cls_log_client.h +++ b/src/cls/log/cls_log_client.h @@ -22,6 +22,6 @@ void cls_log_list(librados::ObjectReadOperation& op, utime_t& from, utime_t& to, string *out_marker, bool *truncated); void cls_log_trim(librados::ObjectWriteOperation& op, utime_t& from, utime_t& to); -int cls_log_trim(librados::IoCtx& io_ctx, string& oid, utime_t& from, utime_t& to); +int cls_log_trim(librados::IoCtx& io_ctx, const string& oid, utime_t& from, utime_t& to); #endif diff --git a/src/rgw/rgw_admin.cc b/src/rgw/rgw_admin.cc index bb6f69ed63821..7766e5196e5c1 100644 --- a/src/rgw/rgw_admin.cc +++ b/src/rgw/rgw_admin.cc @@ -89,6 +89,7 @@ void _usage() cerr << " metadata rm remove metadata info\n"; cerr << " metadata list list metadata info\n"; cerr << " mdlog list list metadata log\n"; + cerr << " mdlog trim trim metadata log\n"; cerr << " bilog list list bucket index log\n"; cerr << " datalog list list data log\n"; cerr << "options:\n"; @@ -198,6 +199,7 @@ enum { OPT_METADATA_RM, OPT_METADATA_LIST, OPT_MDLOG_LIST, + OPT_MDLOG_TRIM, OPT_BILOG_LIST, OPT_DATALOG_LIST, }; @@ -366,6 +368,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, bool *need_more) } else if (strcmp(prev_cmd, "mdlog") == 0) { if (strcmp(cmd, "list") == 0) return OPT_MDLOG_LIST; + if (strcmp(cmd, "trim") == 0) + return OPT_MDLOG_TRIM; } else if (strcmp(prev_cmd, "bilog") == 0) { if (strcmp(cmd, "list") == 0) return OPT_BILOG_LIST; @@ -1765,6 +1769,26 @@ next: formatter->flush(cout); } + if (opt_cmd == OPT_MDLOG_TRIM) { + utime_t start_time, end_time; + + int ret = parse_date_str(start_date, start_time); + if (ret < 0) + return -ret; + + ret = parse_date_str(end_date, end_time); + if (ret < 0) + return -ret; + + RGWMetadataLog *meta_log = store->meta_mgr->get_log(); + + ret = meta_log->trim(store, start_time, end_time); + if (ret < 0) { + cerr << "ERROR: meta_log->trim(): " << cpp_strerror(-ret) << std::endl; + return -ret; + } + } + if (opt_cmd == OPT_BILOG_LIST) { if (bucket_name.empty()) { cerr << "ERROR: bucket not specified" << std::endl; diff --git a/src/rgw/rgw_metadata.cc b/src/rgw/rgw_metadata.cc index 695372c639938..423d96bde4fa2 100644 --- a/src/rgw/rgw_metadata.cc +++ b/src/rgw/rgw_metadata.cc @@ -130,7 +130,7 @@ int RGWMetadataLog::list_entries(void *handle, if (!is_truncated) { ++ctx->cur_shard; - if (ctx->cur_shard _conf->rgw_md_log_max_shards) { + if (ctx->cur_shard < cct->_conf->rgw_md_log_max_shards) { get_shard_oid(ctx->cur_shard, ctx->cur_oid); ctx->marker.clear(); } else { @@ -145,6 +145,26 @@ int RGWMetadataLog::list_entries(void *handle, return 0; } +int RGWMetadataLog::trim(RGWRados *store, utime_t& from_time, utime_t& end_time) +{ + string oid; + for (int shard = 0; shard < cct->_conf->rgw_md_log_max_shards; shard++) { + get_shard_oid(shard, oid); + + int ret; + + ret = store->time_log_trim(oid, from_time, end_time); + + if (ret == -ENOENT) + ret = 0; + + if (ret < 0) + return ret; + } + + return 0; +} + obj_version& RGWMetadataObject::get_version() { return objv; diff --git a/src/rgw/rgw_metadata.h b/src/rgw/rgw_metadata.h index 686e41ca116c7..266769e2e9cc5 100644 --- a/src/rgw/rgw_metadata.h +++ b/src/rgw/rgw_metadata.h @@ -88,6 +88,8 @@ public: int max_entries, list& entries, bool *truncated); + + int trim(RGWRados *store, utime_t& from_time, utime_t& end_time); }; class RGWMetadataManager { diff --git a/src/rgw/rgw_rados.cc b/src/rgw/rgw_rados.cc index 0b818cacd5c1b..6d6b6324331a2 100644 --- a/src/rgw/rgw_rados.cc +++ b/src/rgw/rgw_rados.cc @@ -1173,6 +1173,18 @@ int RGWRados::time_log_list(const string& oid, utime_t& start_time, utime_t& end return 0; } +int RGWRados::time_log_trim(const string& oid, utime_t& start_time, utime_t& end_time) +{ + librados::IoCtx io_ctx; + + const char *log_pool = zone.log_pool.name.c_str(); + int r = rados->ioctx_create(log_pool, io_ctx); + if (r < 0) + return r; + + return cls_log_trim(io_ctx, oid, start_time, end_time); +} + int RGWRados::decode_policy(bufferlist& bl, ACLOwner *owner) { bufferlist::iterator i = bl.begin(); diff --git a/src/rgw/rgw_rados.h b/src/rgw/rgw_rados.h index 5d59f9a2a7b65..231026e1f0da3 100644 --- a/src/rgw/rgw_rados.h +++ b/src/rgw/rgw_rados.h @@ -992,6 +992,7 @@ public: int time_log_add(const string& oid, const utime_t& ut, string& section, string& key, bufferlist& bl); int time_log_list(const string& oid, utime_t& start_time, utime_t& end_time, int max_entries, list& entries, string& marker, bool *truncated); + int time_log_trim(const string& oid, utime_t& start_time, utime_t& end_time); /// clean up/process any temporary objects older than given date[/time] int remove_temp_objects(string date, string time); -- 2.39.5