mgr/vol: show progress in "fs clone status" output

Output of command "ceph fs clone status" will now show the progress made
by that specific clone. The output will show cloning completed in terms
of percentage and amount of bytes that have been cloned and number of
files that have been cloned.

Fixes: https://tracker.ceph.com/issues/61904
Signed-off-by: Rishabh Dave <ridave@redhat.com>
Signed-off-by: Rishabh Dave <ridave@redhat.com>

diff --git a/src/pybind/mgr/volumes/fs/stats_util.py b/src/pybind/mgr/volumes/fs/stats_util.py
new file mode 100644 (file)
index 0000000..74deb10
--- /dev/null
+++ b/src/pybind/mgr/volumes/fs/stats_util.py
@@ -0,0 +1,56 @@
+'''
+This module contains classes, methods & helpers that are used to get statistics
+(specifically number of files and total size of data present under the source
+and destination directory for the copy operation that is performed for snapshot
+cloning) and pass, print, log and convert them to human readable format
+conveniently.
+'''
+from typing import Optional
+
+from mgr_util import format_bytes, format_dimless
+
+
+def get_size_ratio_str(size1, size2):
+    size1, size2 = format_bytes(size1, 4), format_bytes(size2, 4)
+
+    size_string =  f'{size1}/{size2}'
+    size_string = size_string.replace(' ', '')
+    return size_string
+
+
+def get_num_ratio_str(num1, num2):
+    num1, num2 = format_dimless(num1, 4), format_dimless(num2, 4)
+
+    num_string = f'{num1}/{num2}'
+    num_string = num_string.replace(' ', '')
+    return num_string
+
+
+def get_amount_copied(src_path, dst_path, fs_handle):
+    rbytes = 'ceph.dir.rbytes'
+
+    size_t = int(fs_handle.getxattr(src_path, rbytes))
+    size_c = int(fs_handle.getxattr(dst_path, rbytes))
+
+    percent: Optional[float]
+    if size_t == 0 or size_c == 0:
+        percent = 0
+    else:
+        percent = ((size_c/size_t) * 100)
+        percent = round(percent, 3)
+
+    return size_t, size_c, percent
+
+
+def get_stats(src_path, dst_path, fs_handle):
+    rentries = 'ceph.dir.rentries'
+    rentries_t = int(fs_handle.getxattr(src_path, rentries))
+    rentries_c = int(fs_handle.getxattr(dst_path, rentries))
+
+    size_t, size_c, percent = get_amount_copied(src_path, dst_path, fs_handle)
+
+    return {
+        'percentage cloned': percent,
+        'amount cloned': get_size_ratio_str(size_c, size_t),
+        'files cloned': get_num_ratio_str(rentries_c, rentries_t),
+    }

diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index 2b73900bb62bd7bbff99c3e09c05b2176a48eea6..778817a8013853941e9778a3f8bcd714f880eeb2 100644 (file)
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -12,6 +12,7 @@ import cephfs
 from mgr_util import CephfsClient
 
 from .fs_util import listdir, has_subdir
+from .stats_util import get_stats
 
 from .operations.group import open_group, create_group, remove_group, \
     open_group_unique, set_group_attrs
@@ -19,7 +20,7 @@ from .operations.volume import create_volume, delete_volume, rename_volume, \
     list_volumes, open_volume, get_pool_names, get_pool_ids, \
     get_pending_subvol_deletions_count, get_all_pending_clones_count
 from .operations.subvolume import open_subvol, create_subvol, remove_subvol, \
-    create_clone
+    create_clone, open_subvol_in_group
 
 from .vol_spec import VolSpec
 from .exception import VolumeException, ClusterError, ClusterTimeout, \
@@ -846,6 +847,59 @@ class VolumeClient(CephfsClient["Module"]):
             ret = self.volume_exception_to_retval(ve)
         return ret
 
+    def _get_clone_src_path(self, vol_handle, dst_group, dst_subvol):
+        src_subvol_details = dst_subvol._get_clone_source()
+        # We exercise op type checks on subvolume but not on subvolumegroups and we don't allow
+        # internal directories (including "_nogroup" to be opened). To do that we need to pass
+        # None (instead of "_nogroup") as value of "groupname" which is a parameter accepted by
+        # Group.__init__(). We could've allowed opening "_nogroup" but moving forward with
+        # current convention.
+        src_group_name = src_subvol_details.get('group', None)
+        src_subvol_name = src_subvol_details['subvolume']
+        src_snap_name = src_subvol_details['snapshot']
+
+        try:
+            if src_group_name != dst_group.groupname:
+                with open_subvol_in_group(self.mgr, vol_handle, self.volspec,
+                                          src_group_name, src_subvol_name,
+                                          SubvolumeOpType.CLONE_SOURCE) as src_subvol:
+                    src_path = src_subvol.snapshot_data_path(src_snap_name).decode('utf-8')
+
+            else:
+                with open_subvol(self.mgr, vol_handle, self.volspec, dst_group,
+                                 src_subvol_name, SubvolumeOpType.CLONE_SOURCE) as \
+                                 src_subvol:
+                    src_path = src_subvol.snapshot_data_path(src_snap_name).decode('utf-8')
+        except VolumeException as e:
+            if e.errno != -errno.ENOENT:
+                raise
+
+            log.debug(f'snapshot "{src_snap_name}" which is/was being cloned to create subvolume '
+                      '"{dst_subvol.subvolname}" has become missing. skipping adding progress '
+                      'report to "clone status" output and, likely, cloning will fail.')
+            src_path = None
+
+        return src_path
+
+    def _get_clone_progress_report(self, vol_handle, dst_group, dst_subvol):
+        dst_path = dst_subvol.base_path.decode('utf-8')
+        src_path = self._get_clone_src_path(vol_handle, dst_group, dst_subvol)
+        if not src_path:
+            return None
+
+        stats = get_stats(src_path, dst_path, vol_handle)
+        stats['percentage cloned'] = str(stats['percentage cloned']) + '%'
+        return stats
+
+    def _get_clone_status(self, vol_handle, group, subvol):
+        status = subvol.status
+        if status['state'] == 'in-progress':
+            stats = self._get_clone_progress_report(vol_handle, group, subvol)
+            if stats:
+                status.update({'progress_report': stats})
+
+        return json.dumps({'status' : status}, indent=2)
+
     def clone_status(self, **kwargs):
         ret       = 0, "", ""
         volname   = kwargs['vol_name']
@@ -856,7 +910,8 @@ class VolumeClient(CephfsClient["Module"]):
             with open_volume(self, volname) as fs_handle:
                 with open_group(fs_handle, self.volspec, groupname) as group:
                     with open_subvol(self.mgr, fs_handle, self.volspec, group, clonename, SubvolumeOpType.CLONE_STATUS) as subvolume:
-                        ret = 0, json.dumps({'status' : subvolume.status}, indent=2), ""
+                        status = self._get_clone_status(fs_handle, group, subvolume)
+                        ret = 0, status, ""
         except VolumeException as ve:
             ret = self.volume_exception_to_retval(ve)
         return ret