From b6f9aab03713410dd1b50edc7fc1d71242ac3e74 Mon Sep 17 00:00:00 2001 From: Theofilos Mouratidis Date: Wed, 3 Oct 2018 13:42:26 +0200 Subject: [PATCH] librbd: add trash purge api calls Add trash purge api calls and fix the docs of some other rbd trash functions Signed-off-by: Theofilos Mouratidis --- src/include/rbd/librbd.h | 4 + src/include/rbd/librbd.hpp | 3 + src/librbd/api/Trash.cc | 168 +++++++++++++++++++++++++++ src/librbd/api/Trash.h | 2 + src/librbd/internal.cc | 8 +- src/librbd/librbd.cc | 46 ++++++++ src/pybind/rbd/rbd.pyx | 35 +++++- src/test/librbd/test_librbd.cc | 69 +++++++++++ src/test/pybind/test_rbd.py | 32 ++++- src/tools/rbd/action/Trash.cc | 205 ++++++--------------------------- src/tracing/librbd.tp | 21 ++++ 11 files changed, 416 insertions(+), 177 deletions(-) diff --git a/src/include/rbd/librbd.h b/src/include/rbd/librbd.h index 976a6cbeff9be..dc16512dd629b 100644 --- a/src/include/rbd/librbd.h +++ b/src/include/rbd/librbd.h @@ -384,6 +384,10 @@ CEPH_RBD_API int rbd_trash_list(rados_ioctx_t io, size_t *num_entries); CEPH_RBD_API void rbd_trash_list_cleanup(rbd_trash_image_info_t *trash_entries, size_t num_entries); +CEPH_RBD_API int rbd_trash_purge(rados_ioctx_t io, time_t expire_ts, float threshold); +CEPH_RBD_API int rbd_trash_purge_with_progress(rados_ioctx_t io, time_t expire_ts, + float threshold, librbd_progress_fn_t cb, + void* cbdata); CEPH_RBD_API int rbd_trash_remove(rados_ioctx_t io, const char *id, bool force); CEPH_RBD_API int rbd_trash_remove_with_progress(rados_ioctx_t io, const char *id, diff --git a/src/include/rbd/librbd.hpp b/src/include/rbd/librbd.hpp index 926158e1a1432..a186121bca939 100644 --- a/src/include/rbd/librbd.hpp +++ b/src/include/rbd/librbd.hpp @@ -238,6 +238,9 @@ public: int trash_move(IoCtx &io_ctx, const char *name, uint64_t delay); int trash_get(IoCtx &io_ctx, const char *id, trash_image_info_t *info); int trash_list(IoCtx &io_ctx, std::vector &entries); + int trash_purge(IoCtx &io_ctx, time_t expire_ts, float threshold); + int trash_purge_with_progress(IoCtx &io_ctx, time_t expire_ts, float threshold, + ProgressContext &pctx); int trash_remove(IoCtx &io_ctx, const char *image_id, bool force); int trash_remove_with_progress(IoCtx &io_ctx, const char *image_id, bool force, ProgressContext &pctx); diff --git a/src/librbd/api/Trash.cc b/src/librbd/api/Trash.cc index 319be136b5893..fdc2d03a64309 100644 --- a/src/librbd/api/Trash.cc +++ b/src/librbd/api/Trash.cc @@ -18,6 +18,7 @@ #include "librbd/mirror/DisableRequest.h" #include "librbd/mirror/EnableRequest.h" #include "librbd/trash/MoveRequest.h" +#include #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -270,6 +271,173 @@ int Trash::list(IoCtx &io_ctx, vector &entries) { return 0; } +template +int Trash::purge(IoCtx& io_ctx, time_t expire_ts, + float threshold, ProgressContext& pctx) { + auto *cct((CephContext *) io_ctx.cct()); + + librbd::RBD rbd; + + std::vector trash_entries; + int r = librbd::api::Trash<>::list(io_ctx, trash_entries); + if (r < 0) { + return r; + } + + std::remove_if(trash_entries.begin(), trash_entries.end(), + [](librbd::trash_image_info_t info) { + return info.source != RBD_TRASH_IMAGE_SOURCE_USER; + } + ); + + std::vector to_be_removed; + if (threshold != -1) { + if (threshold < 0 || threshold > 1) { + lderr(cct) << "argument 'threshold' is out of valid range" + << dendl; + return -EINVAL; + } + + librados::bufferlist inbl; + librados::bufferlist outbl; + std::string pool_name = io_ctx.get_pool_name(); + + librados::Rados rados(io_ctx); + rados.mon_command(R"({"prefix": "df", "format": "json"})", inbl, + &outbl, nullptr); + + json_spirit::mValue json; + if (!json_spirit::read(outbl.to_str(), json)) { + lderr(cct) << "ceph df json output could not be parsed" + << dendl; + return -EBADMSG; + } + + json_spirit::mArray arr = json.get_obj()["pools"].get_array(); + + double pool_percent_used = 0; + uint64_t pool_total_bytes = 0; + + std::map> datapools; + + std::sort(trash_entries.begin(), trash_entries.end(), + [](librbd::trash_image_info_t a, librbd::trash_image_info_t b) { + return a.deferment_end_time < b.deferment_end_time; + } + ); + + for (const auto &entry : trash_entries) { + librbd::Image image; + std::string data_pool; + r = rbd.open_by_id_read_only(io_ctx, image, entry.id.c_str(), NULL); + if (r < 0) continue; + + int64_t data_pool_id = image.get_data_pool_id(); + if (data_pool_id != io_ctx.get_id()) { + librados::IoCtx data_io_ctx; + r = util::create_ioctx(io_ctx, "image", data_pool_id, + {}, &data_io_ctx); + if (r < 0) { + lderr(cct) << "error accessing data pool" << dendl; + continue; + } + data_pool = data_io_ctx.get_pool_name(); + datapools[data_pool].push_back(entry.id.c_str()); + } else { + datapools[pool_name].push_back(entry.id.c_str()); + } + } + + uint64_t bytes_to_free = 0; + + for (uint8_t i = 0; i < arr.size(); ++i) { + json_spirit::mObject obj = arr[i].get_obj(); + std::string name = obj.find("name")->second.get_str(); + auto img = datapools.find(name); + if (img != datapools.end()) { + json_spirit::mObject stats = arr[i].get_obj()["stats"].get_obj(); + pool_percent_used = stats["percent_used"].get_real(); + if (pool_percent_used <= threshold) continue; + + bytes_to_free = 0; + + pool_total_bytes = stats["max_avail"].get_uint64() + + stats["bytes_used"].get_uint64(); + + auto bytes_threshold = (uint64_t) (pool_total_bytes * + (pool_percent_used - threshold)); + + librbd::Image curr_img; + for (const auto &it : img->second) { + r = rbd.open_by_id_read_only(io_ctx, curr_img, it, NULL); + if (r < 0) continue; + + uint64_t img_size; + curr_img.size(&img_size); + r = curr_img.diff_iterate2(nullptr, 0, img_size, false, true, + [](uint64_t offset, size_t len, int exists, void *arg) { + auto *to_free = reinterpret_cast(arg); + if (exists) + (*to_free) += len; + return 0; + }, &bytes_to_free + ); + if (r < 0) continue; + to_be_removed.push_back(it); + if (bytes_to_free >= bytes_threshold) break; + } + } + } + if (bytes_to_free == 0) { + ldout(cct, 10) << "pool usage is lower than or equal to " + << (threshold * 100) + << "%" << dendl; + return 0; + } + } + + if (expire_ts == 0) { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + expire_ts = now.tv_sec; + } + + for (const auto &entry : trash_entries) { + if (expire_ts >= entry.deferment_end_time) { + to_be_removed.push_back(entry.id.c_str()); + } + } + + NoOpProgressContext remove_pctx; + uint64_t list_size = to_be_removed.size(), i = 0; + for (const auto &entry_id : to_be_removed) { + r = librbd::api::Trash<>::remove(io_ctx, entry_id, true, remove_pctx); + if (r < 0) { + if (r == -ENOTEMPTY) { + ldout(cct, 5) << "image has snapshots - these must be deleted " + << "with 'rbd snap purge' before the image can be removed." + << dendl; + } else if (r == -EBUSY) { + ldout(cct, 5) << "error: image still has watchers" + << std::endl + << "This means the image is still open or the client using " + << "it crashed. Try again after closing/unmapping it or " + << "waiting 30s for the crashed client to timeout." + << dendl; + } else if (r == -EMLINK) { + ldout(cct, 5) << "Remove the image from the group and try again." + << dendl; + } else { + lderr(cct) << "remove error: " << cpp_strerror(r) << dendl; + } + return r; + } + pctx.update_progress(++i, list_size); + } + + return 0; +} + template int Trash::remove(IoCtx &io_ctx, const std::string &image_id, bool force, ProgressContext& prog_ctx) { diff --git a/src/librbd/api/Trash.h b/src/librbd/api/Trash.h index cf7217de8098e..f2e52f6cbeb1d 100644 --- a/src/librbd/api/Trash.h +++ b/src/librbd/api/Trash.h @@ -27,6 +27,8 @@ struct Trash { trash_image_info_t *info); static int list(librados::IoCtx &io_ctx, std::vector &entries); + static int purge(IoCtx& io_ctx, time_t expire_ts, + float threshold, ProgressContext& pctx); static int remove(librados::IoCtx &io_ctx, const std::string &image_id, bool force, ProgressContext& prog_ctx); static int restore(librados::IoCtx &io_ctx, const std::string &image_id, diff --git a/src/librbd/internal.cc b/src/librbd/internal.cc index ccb38aed5e4c9..8b499c283da64 100644 --- a/src/librbd/internal.cc +++ b/src/librbd/internal.cc @@ -1135,8 +1135,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { } int lock_break(ImageCtx *ictx, rbd_lock_mode_t lock_mode, - const std::string &lock_owner) - { + const std::string &lock_owner) { CephContext *cct = ictx->cct; ldout(cct, 20) << __func__ << ": ictx=" << ictx << ", " << "lock_mode=" << lock_mode << ", " @@ -1527,7 +1526,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { if (throttle.pending_error()) { return throttle.wait_for_ret(); } - + { RWLock::RLocker snap_locker(src->snap_lock); if (src->object_map != nullptr) { @@ -1545,7 +1544,7 @@ int validate_pool(IoCtx &io_ctx, CephContext *cct) { } else { object_id += src->stripe_count; } - } + } uint64_t len = min(period, src_size - offset); bufferlist *bl = new bufferlist(); @@ -2006,4 +2005,3 @@ std::ostream &operator<<(std::ostream &os, const librbd::ImageOptions &opts) { return os; } - diff --git a/src/librbd/librbd.cc b/src/librbd/librbd.cc index 30c608737c3c4..142cdec6ea07f 100644 --- a/src/librbd/librbd.cc +++ b/src/librbd/librbd.cc @@ -648,6 +648,26 @@ namespace librbd { return r; } + int RBD::trash_purge(IoCtx &io_ctx, time_t expire_ts, float threshold) { + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, trash_purge_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), expire_ts, threshold); + NoOpProgressContext nop_pctx; + int r = librbd::api::Trash<>::purge(io_ctx, expire_ts, threshold, nop_pctx); + tracepoint(librbd, trash_purge_exit, r); + return r; + } + + int RBD::trash_purge_with_progress(IoCtx &io_ctx, time_t expire_ts, + float threshold, ProgressContext &pctx) { + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, trash_purge_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), expire_ts, threshold); + int r = librbd::api::Trash<>::purge(io_ctx, expire_ts, threshold, pctx); + tracepoint(librbd, trash_purge_exit, r); + return r; + } + int RBD::namespace_create(IoCtx& io_ctx, const char *namespace_name) { return librbd::api::Namespace<>::create(io_ctx, namespace_name); } @@ -3182,6 +3202,32 @@ extern "C" void rbd_trash_list_cleanup(rbd_trash_image_info_t *entries, } } +extern "C" int rbd_trash_purge(rados_ioctx_t io, time_t expire_ts, + float threshold) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(io, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, trash_purge_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), expire_ts, threshold); + librbd::NoOpProgressContext nop_pctx; + int r = librbd::api::Trash<>::purge(io_ctx, expire_ts, threshold, nop_pctx); + tracepoint(librbd, trash_purge_exit, r); + return r; +} + +extern "C" int rbd_trash_purge_with_progress(rados_ioctx_t io, time_t expire_ts, + float threshold, librbd_progress_fn_t cb, void* cbdata) { + librados::IoCtx io_ctx; + librados::IoCtx::from_rados_ioctx_t(io, io_ctx); + TracepointProvider::initialize(get_cct(io_ctx)); + tracepoint(librbd, trash_purge_enter, io_ctx.get_pool_name().c_str(), + io_ctx.get_id(), expire_ts, threshold); + librbd::CProgressContext pctx(cb, cbdata); + int r = librbd::api::Trash<>::purge(io_ctx, expire_ts, threshold, pctx); + tracepoint(librbd, trash_purge_exit, r); + return r; +} + extern "C" int rbd_trash_remove(rados_ioctx_t p, const char *image_id, bool force) { librados::IoCtx io_ctx; diff --git a/src/pybind/rbd/rbd.pyx b/src/pybind/rbd/rbd.pyx index 66c4fe899e41d..d91de1b72963d 100644 --- a/src/pybind/rbd/rbd.pyx +++ b/src/pybind/rbd/rbd.pyx @@ -304,6 +304,7 @@ cdef extern from "rbd/librbd.h" nogil: size_t *num_entries) void rbd_trash_list_cleanup(rbd_trash_image_info_t *trash_entries, size_t num_entries) + int rbd_trash_purge(rados_ioctx_t io, time_t expire_ts, float threshold) int rbd_trash_remove(rados_ioctx_t io, const char *id, int force) int rbd_trash_restore(rados_ioctx_t io, const char *id, const char *name) @@ -1246,10 +1247,39 @@ class RBD(object): if ret != 0: raise make_ex(ret, 'error moving image to trash') + def trash_purge(self, ioctx, expire_ts=datetime.now(), threshold=-1): + """ + Delete RBD images from trash in bulk. + + By default it removes images with deferment end time less than now. + + The timestamp is configurable, e.g. delete images that have expired a + week ago. + + If the threshold is used it deletes images until X% pool usage is met. + + :param ioctx: determines which RADOS pool the image is in + :type ioctx: :class:`rados.Ioctx` + :param expire_ts: timestamp for images to be considered as expired (UTC) + :type expire_ts: datetime + :param threshold: percentage of pool usage to be met (0 to 1) + :type threshold: float + """ + as_time_t = int((expire_ts - datetime.utcfromtimestamp(0)).total_seconds()) + cdef: + rados_ioctx_t _ioctx = convert_ioctx(ioctx) + float _threshold = threshold + time_t _expire_ts = as_time_t + with nogil: + ret = rbd_trash_purge(_ioctx, _expire_ts, _threshold) + if ret != 0: + raise make_ex(ret, 'error purging images from trash') + def trash_remove(self, ioctx, image_id, force=False): """ Delete an RBD image from trash. If image deferment time has not expired :class:`PermissionError` is raised. + :param ioctx: determines which RADOS pool the image is in :type ioctx: :class:`rados.Ioctx` :param image_id: the id of the image to remove @@ -1270,7 +1300,8 @@ class RBD(object): def trash_get(self, ioctx, image_id): """ - Retrieve RBD image info from trash + Retrieve RBD image info from trash. + :param ioctx: determines which RADOS pool the image is in :type ioctx: :class:`rados.Ioctx` :param image_id: the id of the image to restore @@ -1314,6 +1345,7 @@ class RBD(object): def trash_list(self, ioctx): """ List all entries from trash. + :param ioctx: determines which RADOS pool the image is in :type ioctx: :class:`rados.Ioctx` :returns: :class:`TrashIterator` @@ -1323,6 +1355,7 @@ class RBD(object): def trash_restore(self, ioctx, image_id, name): """ Restore an RBD image from trash. + :param ioctx: determines which RADOS pool the image is in :type ioctx: :class:`rados.Ioctx` :param image_id: the id of the image to restore diff --git a/src/test/librbd/test_librbd.cc b/src/test/librbd/test_librbd.cc index 825d2b2e4b36a..770eb3e70f005 100644 --- a/src/test/librbd/test_librbd.cc +++ b/src/test/librbd/test_librbd.cc @@ -6680,6 +6680,75 @@ TEST_F(TestLibRBD, TestTrashMoveAndPurgeNonExpiredDelay) { true, pp2)); } +TEST_F(TestLibRBD, TestTrashPurge) { + librados::IoCtx ioctx; + ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx)); + + librbd::RBD rbd; + std::string name1 = get_temp_image_name(); + std::string name2 = get_temp_image_name(); + + uint64_t size = 1 << 18; + int order = 12; + ASSERT_EQ(0, create_image_pp(rbd, ioctx, name1.c_str(), size, &order)); + ASSERT_EQ(0, create_image_pp(rbd, ioctx, name1.c_str(), size, &order)); + + librbd::Image image1; + ASSERT_EQ(0, rbd.open(ioctx, image1, name1.c_str(), nullptr)); + uint8_t old_format; + ASSERT_EQ(0, image1.old_format(&old_format)); + + if (old_format) { + ASSERT_EQ(-EOPNOTSUPP, rbd.trash_move(ioctx, name1.c_str(), 0)); + image1.close(); + return; + } + std::string image_id1; + ASSERT_EQ(0, image1.get_id(&image_id1)); + image1.close(); + + ASSERT_EQ(0, rbd.trash_move(ioctx, name1.c_str(), 0)); + + librbd::Image image2; + ASSERT_EQ(0, rbd.open(ioctx, image2, name2.c_str(), nullptr)); + ASSERT_EQ(0, image2.old_format(&old_format)); + + if (old_format) { + ASSERT_EQ(-EOPNOTSUPP, rbd.trash_move(ioctx, name2.c_str(), 0)); + image2.close(); + return; + } + std::string image_id2; + ASSERT_EQ(0, image2.get_id(&image_id2)); + image2.close(); + + ASSERT_EQ(0, rbd.trash_move(ioctx, name2.c_str(), 100)); + ASSERT_EQ(0, rbd.trash_purge(ioctx, 0, -1)); + + std::vector entries; + ASSERT_EQ(0, rbd.trash_list(ioctx, entries)); + ASSERT_FALSE(entries.empty()); + bool found = false; + for(auto& entry : entries) { + if (entry.id == image_id1 && entry.name == name1) + found = true; + } + ASSERT_FALSE(found); + entries.clear(); + + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + ASSERT_EQ(0, rbd.trash_purge(ioctx, now.tv_sec+1000, 0)); + ASSERT_EQ(0, rbd.trash_list(ioctx, entries)); + + found = false; + for(auto& entry : entries) { + if (entry.id == image_id2 && entry.name == name2) + found = true; + } + ASSERT_FALSE(found); +} + TEST_F(TestLibRBD, TestTrashMoveAndRestore) { librados::IoCtx ioctx; ASSERT_EQ(0, _rados.ioctx_create(m_pool_name.c_str(), ioctx)); diff --git a/src/test/pybind/test_rbd.py b/src/test/pybind/test_rbd.py index 836992c93a2f4..f858c2b910b86 100644 --- a/src/test/pybind/test_rbd.py +++ b/src/test/pybind/test_rbd.py @@ -5,7 +5,7 @@ import os import time import sys -from datetime import datetime +from datetime import datetime, timedelta from nose import with_setup, SkipTest from nose.tools import eq_ as eq, assert_raises, assert_not_equal from rados import (Rados, @@ -1863,6 +1863,36 @@ class TestTrash(object): RBD().trash_move(ioctx, image_name, 1000) RBD().trash_remove(ioctx, image_id, True) + def test_purge(self): + create_image() + with Image(ioctx, image_name) as image: + image_name1 = image_name + image_id1 = image.id() + + create_image() + with Image(ioctx, image_name) as image: + image_name2 = image_name + image_id2 = image.id() + + create_image() + with Image(ioctx, image_name) as image: + image_name3 = image_name + image_id3 = image.id() + + RBD().trash_move(ioctx, image_name1, 0) + RBD().trash_move(ioctx, image_name2, 1000) + RBD().trash_move(ioctx, image_name3, 1000) + + RBD().trash_purge(ioctx, datetime.now()) + entries = list(RBD().trash_list(ioctx)) + for e in entries: + assert(e['id'] != image_id1) + + RBD.trash_purge(ioctx, datetime.now() + timedelta(seconds=5000), 0) + entries = list(RBD().trash_list(ioctx)) + for e in entries: + assert(e['id'] not in [image_id2, image_id3]) + def test_remove_denied(self): create_image() with Image(ioctx, image_name) as image: diff --git a/src/tools/rbd/action/Trash.cc b/src/tools/rbd/action/Trash.cc index c2b9f98692ff8..e3595a32697db 100644 --- a/src/tools/rbd/action/Trash.cc +++ b/src/tools/rbd/action/Trash.cc @@ -24,7 +24,6 @@ #include #include #include -#include namespace rbd { namespace action { @@ -116,32 +115,6 @@ void get_remove_arguments(po::options_description *positional, ("force", po::bool_switch(), "force remove of non-expired delayed images"); } -void remove_error_check(int r) { - if (r == -ENOTEMPTY) { - std::cerr << "rbd: image has snapshots - these must be deleted" - << " with 'rbd snap purge' before the image can be removed." - << std::endl; - } else if (r == -EBUSY) { - std::cerr << "rbd: error: image still has watchers" - << std::endl - << "This means the image is still open or the client using " - << "it crashed. Try again after closing/unmapping it or " - << "waiting 30s for the crashed client to timeout." - << std::endl; - } else if (r == -EMLINK) { - std::cerr << std::endl - << "Remove the image from the group and try again." - << std::endl; - } else if (r == -EPERM) { - std::cerr << std::endl - << "Deferment time has not expired, please use --force if you " - << "really want to remove the image" - << std::endl; - } else { - std::cerr << "rbd: remove error: " << cpp_strerror(r) << std::endl; - } -} - int execute_remove(const po::variables_map &vm, const std::vector &ceph_global_init_args) { size_t arg_index = 0; @@ -168,7 +141,29 @@ int execute_remove(const po::variables_map &vm, r = rbd.trash_remove_with_progress(io_ctx, image_id.c_str(), vm["force"].as(), pc); if (r < 0) { - remove_error_check(r); + if (r == -ENOTEMPTY) { + std::cerr << "rbd: image has snapshots - these must be deleted" + << " with 'rbd snap purge' before the image can be removed." + << std::endl; + } else if (r == -EBUSY) { + std::cerr << "rbd: error: image still has watchers" + << std::endl + << "This means the image is still open or the client using " + << "it crashed. Try again after closing/unmapping it or " + << "waiting 30s for the crashed client to timeout." + << std::endl; + } else if (r == -EMLINK) { + std::cerr << std::endl + << "Remove the image from the group and try again." + << std::endl; + } else if (r == -EPERM) { + std::cerr << std::endl + << "Deferment time has not expired, please use --force if you " + << "really want to remove the image" + << std::endl; + } else { + std::cerr << "rbd: remove error: " << cpp_strerror(r) << std::endl; + } pc.fail(); return r; } @@ -386,149 +381,36 @@ void get_purge_arguments(po::options_description *positional, (EXPIRED_BEFORE.c_str(), po::value()->value_name("date"), "purges images that expired before the given date"); options->add_options() - (THRESHOLD.c_str(), po::value(), + (THRESHOLD.c_str(), po::value(), "purges images until the current pool data usage is reduced to X%, " "value range: 0.0-1.0"); } - int execute_purge (const po::variables_map &vm, const std::vector &ceph_global_init_args) { size_t arg_index = 0; std::string pool_name = utils::get_pool_name(vm, &arg_index); std::string namespace_name = utils::get_namespace_name(vm, &arg_index); - librados::Rados rados; - librados::IoCtx io_ctx; - int r = utils::init(pool_name, namespace_name, &rados, &io_ctx); - if (r < 0) { - return r; - } - utils::disable_cache(); - io_ctx.set_osdmap_full_try(); librbd::RBD rbd; - std::vector trash_entries; - r = rbd.trash_list(io_ctx, trash_entries); + librados::Rados rados; + librados::IoCtx io_ctx; + int r = utils::init(pool_name, namespace_name, &rados, &io_ctx); if (r < 0) { return r; } - std::remove_if(trash_entries.begin(), trash_entries.end(), - [](librbd::trash_image_info_t info) { - return info.source != RBD_TRASH_IMAGE_SOURCE_USER; - } - ); + io_ctx.set_osdmap_full_try(); - std::vector to_be_removed; + float threshold = -1; + time_t expire_ts = 0; if (vm.find(THRESHOLD) != vm.end()) { - double threshold = vm[THRESHOLD].as(); - if (threshold < 0 || threshold > 1) { - std::cerr << "rbd: argument 'threshold' is out of valid range" - << std::endl; - return -EINVAL; - } - - librados::bufferlist inbl; - librados::bufferlist outbl; - rados.mon_command("{\"prefix\": \"df\", \"format\": \"json\"}", inbl, - &outbl, NULL); - - json_spirit::mValue json; - if(!json_spirit::read(outbl.to_str(), json)) { - std::cerr << "rbd: ceph df json output could not be parsed" - << std::endl; - return -EBADMSG; - } - - json_spirit::mArray arr = json.get_obj()["pools"].get_array(); - - double pool_percent_used = 0; - uint64_t pool_total_bytes = 0; - - std::map> datapools; - - std::sort(trash_entries.begin(), trash_entries.end(), - [](librbd::trash_image_info_t a, librbd::trash_image_info_t b) { - return a.deferment_end_time < b.deferment_end_time; - } - ); - - for (const auto& entry : trash_entries) { - librbd::Image image; - std::string data_pool; - r = utils::open_image_by_id(io_ctx, entry.id, true, &image); - if(r < 0) continue; - - int64_t data_pool_id = image.get_data_pool_id(); - if (data_pool_id != io_ctx.get_id()) { - librados::Rados rados(io_ctx); - librados::IoCtx data_io_ctx; - r = rados.ioctx_create2(data_pool_id, data_io_ctx); - if (r < 0) { - std::cerr << "rbd: error accessing data pool" << std::endl; - continue; - } - data_pool = data_io_ctx.get_pool_name(); - datapools[data_pool].push_back(entry.id.c_str()); - } else { - datapools[pool_name].push_back(entry.id.c_str()); - } - } - - uint64_t bytes_to_free = 0; - - for(uint8_t i = 0; i < arr.size(); ++i) { - json_spirit::mObject obj = arr[i].get_obj(); - std::string name = obj.find("name")->second.get_str(); - auto img = datapools.find(name); - if(img != datapools.end()) { - json_spirit::mObject stats = arr[i].get_obj()["stats"].get_obj(); - pool_percent_used = stats["percent_used"].get_real(); - if(pool_percent_used <= threshold) continue; - - bytes_to_free = 0; - - pool_total_bytes = stats["max_avail"].get_uint64() + - stats["bytes_used"].get_uint64(); - - auto bytes_threshold = (uint64_t)(pool_total_bytes * - (pool_percent_used - threshold)); - - librbd::Image curr_img; - for(const auto &it : img->second){ - r = utils::open_image_by_id(io_ctx, it, true, &curr_img); - if(r < 0) continue; - - uint64_t img_size; curr_img.size(&img_size); - r = curr_img.diff_iterate2(nullptr, 0, img_size, false, true, - [](uint64_t offset, size_t len, int exists, void *arg) { - auto *to_free = reinterpret_cast(arg); - if (exists) (*to_free) += len; - return 0; - }, &bytes_to_free - ); - if(r < 0) continue; - to_be_removed.push_back(it); - if(bytes_to_free >= bytes_threshold) break; - } - } - } - if (bytes_to_free == 0) { - std::cout << "rbd: pool usage is lower than or equal to " - << (threshold*100) - << "%" << endl; - std::cout << "Nothing to do" << std::endl; - return 0; - } + threshold = vm[THRESHOLD].as(); } else { - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - - time_t expire_ts = now.tv_sec; if (vm.find(EXPIRED_BEFORE) != vm.end()) { utime_t new_time; r = utime_t::invoke_date(vm[EXPIRED_BEFORE].as(), &new_time); @@ -539,30 +421,13 @@ int execute_purge (const po::variables_map &vm, } expire_ts = new_time.sec(); } - - for(const auto &entry : trash_entries) { - if (expire_ts >= entry.deferment_end_time) { - to_be_removed.push_back(entry.id.c_str()); - } - } } - uint64_t list_size = to_be_removed.size(), i = 0; - - if(list_size == 0) { - std::cout << "rbd: nothing to remove" << std::endl; + utils::ProgressContext pc("Removing images", vm[at::NO_PROGRESS].as()); + r = rbd.trash_purge_with_progress(io_ctx, expire_ts, threshold, pc); + if (r < 0) { + pc.fail(); } else { - utils::ProgressContext pc("Removing images", - vm[at::NO_PROGRESS].as()); - for(const auto &entry_id : to_be_removed) { - r = rbd.trash_remove(io_ctx, entry_id, true); - if (r < 0) { - remove_error_check(r); - pc.fail(); - return r; - } - pc.update_progress(++i, list_size); - } pc.finish(); } diff --git a/src/tracing/librbd.tp b/src/tracing/librbd.tp index cd13e45fd9aac..8a8d4e4194e67 100644 --- a/src/tracing/librbd.tp +++ b/src/tracing/librbd.tp @@ -552,6 +552,27 @@ TRACEPOINT_EVENT(librbd, trash_list_exit, ) ) +TRACEPOINT_EVENT(librbd, trash_purge_enter, + TP_ARGS( + const char*, pool_name, + int64_t, id, + time_t, expire_ts, + float, threshold), + TP_FIELDS( + ctf_string(pool_name, pool_name) + ctf_integer(int64_t, id, id) + ceph_ctf_time_t(expire_ts, expire_ts) + ctf_float(float, threshold, threshold) + ) +) + +TRACEPOINT_EVENT(librbd, trash_purge_exit, + TP_ARGS(int, retval), + TP_FIELDS( + ctf_integer(int, retval, retval) + ) +) + TRACEPOINT_EVENT(librbd, trash_remove_enter, TP_ARGS( const char*, pool_name, -- 2.39.5