From 0867f0994ab794962aefabe5fd031e0c95adf44c Mon Sep 17 00:00:00 2001 From: Jan Fajerski Date: Thu, 3 Sep 2020 12:46:38 +0200 Subject: [PATCH] snap-schedule: don't prune all on empty retention This commit ensures that an empty retention doesn't prune all snapshots. It will however add a limit of 50 snapshots per directory (as defined by MAX_SNAPS_PER_PATH). Fixes: https://tracker.ceph.com/issues/47268 Signed-off-by: Jan Fajerski --- .../mgr/snap_schedule/fs/schedule_client.py | 78 ++++++++++--------- .../tests/fs/test_schedule_client.py | 20 +++++ 2 files changed, 61 insertions(+), 37 deletions(-) create mode 100644 src/pybind/mgr/snap_schedule/tests/fs/test_schedule_client.py diff --git a/src/pybind/mgr/snap_schedule/fs/schedule_client.py b/src/pybind/mgr/snap_schedule/fs/schedule_client.py index 58a57aef181..5c7cb0bd01d 100644 --- a/src/pybind/mgr/snap_schedule/fs/schedule_client.py +++ b/src/pybind/mgr/snap_schedule/fs/schedule_client.py @@ -56,6 +56,46 @@ def updates_schedule_db(func): return f +def get_prune_set(candidates, retention): + PRUNING_PATTERNS = OrderedDict([ + # n is for keep last n snapshots, uses the snapshot name timestamp + # format for lowest granularity + ("n", SNAPSHOT_TS_FORMAT), + # TODO remove M for release + ("M", '%Y-%m-%d-%H_%M'), + ("h", '%Y-%m-%d-%H'), + ("d", '%Y-%m-%d'), + ("w", '%G-%V'), + ("m", '%Y-%m'), + ("y", '%Y'), + ]) + keep = [] + if not retention: + log.info(f'no retention set, assuming n: {MAX_SNAPS_PER_PATH}') + retention = {'n': MAX_SNAPS_PER_PATH} + for period, date_pattern in PRUNING_PATTERNS.items(): + log.debug(f'compiling keep set for period {period}') + period_count = retention.get(period, 0) + if not period_count: + continue + last = None + for snap in sorted(candidates, key=lambda x: x[0].d_name, + reverse=True): + snap_ts = snap[1].strftime(date_pattern) + if snap_ts != last: + last = snap_ts + if snap not in keep: + log.debug(f'keeping {snap[0].d_name} due to {period_count}{period}') + keep.append(snap) + if len(keep) == period_count: + log.debug(f'found enough snapshots for {period_count}{period}') + break + if len(keep) > MAX_SNAPS_PER_PATH: + log.info(f'Would keep more then {MAX_SNAPS_PER_PATH}, pruning keep set') + keep = keep[:MAX_SNAPS_PER_PATH] + return candidates - set(keep) + + class SnapSchedClient(CephfsClient): def __init__(self, mgr): @@ -175,7 +215,7 @@ class SnapSchedClient(CephfsClient): else: log.debug(f'skipping dir entry {dir_.d_name}') dir_ = fs_handle.readdir(d_handle) - to_prune = self.get_prune_set(prune_candidates, ret) + to_prune = get_prune_set(prune_candidates, ret) for k in to_prune: dirname = k[0].d_name.decode('utf-8') log.debug(f'rmdir on {dirname}') @@ -186,42 +226,6 @@ class SnapSchedClient(CephfsClient): except Exception: self._log_exception('prune_snapshots') - def get_prune_set(self, candidates, retention): - PRUNING_PATTERNS = OrderedDict([ - # n is for keep last n snapshots, uses the snapshot name timestamp - # format for lowest granularity - ("n", SNAPSHOT_TS_FORMAT), - # TODO remove M for release - ("M", '%Y-%m-%d-%H_%M'), - ("h", '%Y-%m-%d-%H'), - ("d", '%Y-%m-%d'), - ("w", '%G-%V'), - ("m", '%Y-%m'), - ("y", '%Y'), - ]) - keep = [] - for period, date_pattern in PRUNING_PATTERNS.items(): - log.debug(f'compiling keep set for period {period}') - period_count = retention.get(period, 0) - if not period_count: - continue - last = None - for snap in sorted(candidates, key=lambda x: x[0].d_name, - reverse=True): - snap_ts = snap[1].strftime(date_pattern) - if snap_ts != last: - last = snap_ts - if snap not in keep: - log.debug(f'keeping {snap[0].d_name} due to {period_count}{period}') - keep.append(snap) - if len(keep) == period_count: - log.debug(f'found enough snapshots for {period_count}{period}') - break - if len(keep) > MAX_SNAPS_PER_PATH: - log.info(f'Would keep more then {MAX_SNAPS_PER_PATH}, pruning keep set') - keep = keep[:MAX_SNAPS_PER_PATH] - return candidates - set(keep) - def get_snap_schedules(self, fs, path): db = self.get_schedule_db(fs) return Schedule.get_db_schedules(path, db, fs) diff --git a/src/pybind/mgr/snap_schedule/tests/fs/test_schedule_client.py b/src/pybind/mgr/snap_schedule/tests/fs/test_schedule_client.py new file mode 100644 index 00000000000..4ebed8d03b8 --- /dev/null +++ b/src/pybind/mgr/snap_schedule/tests/fs/test_schedule_client.py @@ -0,0 +1,20 @@ +from datetime import datetime, timedelta +from unittest.mock import MagicMock +import pytest +from ...fs.schedule_client import get_prune_set, SNAPSHOT_TS_FORMAT + + +class TestScheduleClient(object): + + def test_get_prune_set_empty_retention_no_prune(self): + now = datetime.now() + candidates = set() + for i in range(10): + ts = now - timedelta(minutes=i*5) + fake_dir = MagicMock() + fake_dir.d_name = f'scheduled-{ts.strftime(SNAPSHOT_TS_FORMAT)}' + candidates.add((fake_dir, ts)) + ret = {} + prune_set = get_prune_set(candidates, ret) + assert prune_set == set(), 'candidates are pruned despite empty retention' + -- 2.39.5