From: Venky Shankar <vshankar@redhat.com>
Date: Mon, 19 Aug 2019 07:45:25 +0000 (-0400)
Subject: mgr/volumes: cluster log when a purge thread bails out
X-Git-Tag: v15.1.0~1530^2
X-Git-Url: http://git-server-git.apps.pok.os.sepia.ceph.com/?a=commitdiff_plain;h=387746ca7e5a0a529afbb5864900efa3219de3a8;p=ceph.git

mgr/volumes: cluster log when a purge thread bails out

Fixes: http://tracker.ceph.com/issues/41218
Signed-off-by: Venky Shankar <vshankar@redhat.com>
---

diff --git a/src/pybind/mgr/volumes/fs/purge_queue.py b/src/pybind/mgr/volumes/fs/purge_queue.py
index ac99f07418bb..a76065cbab95 100644
--- a/src/pybind/mgr/volumes/fs/purge_queue.py
+++ b/src/pybind/mgr/volumes/fs/purge_queue.py
@@ -19,7 +19,8 @@ class PurgeQueueBase(object):
     MAX_RETRIES_ON_EXCEPTION = 10
 
     class PurgeThread(threading.Thread):
-        def __init__(self, name, purge_fn):
+        def __init__(self, volume_client, name, purge_fn):
+            self.vc = volume_client
             self.purge_fn = purge_fn
             # event object to cancel ongoing purge
             self.cancel_event = threading.Event()
@@ -42,6 +43,7 @@ class PurgeQueueBase(object):
                         traceback.format_exception(exc_type, exc_value, exc_traceback))))
                     time.sleep(1)
             log.error("purge thread [{0}] reached exception limit, bailing out...".format(thread_name))
+            self.vc.cluster_log("purge thread {0} bailing out due to exception".format(thread_name))
 
         def cancel_job(self):
             self.cancel_event.set()
@@ -154,7 +156,7 @@ class ThreadPoolPurgeQueueMixin(PurgeQueueBase):
         self.threads = []
         for i in range(tp_size):
             self.threads.append(
-                PurgeQueueBase.PurgeThread(name="purgejob.{}".format(i), purge_fn=self.run))
+                PurgeQueueBase.PurgeThread(volume_client, name="purgejob.{}".format(i), purge_fn=self.run))
             self.threads[-1].start()
 
     def pick_purge_dir_from_volume(self):
diff --git a/src/pybind/mgr/volumes/fs/volume.py b/src/pybind/mgr/volumes/fs/volume.py
index ea411bce980a..9beb1206e8ae 100644
--- a/src/pybind/mgr/volumes/fs/volume.py
+++ b/src/pybind/mgr/volumes/fs/volume.py
@@ -175,6 +175,14 @@ class VolumeClient(object):
         for fs in fs_map['filesystems']:
             self.purge_queue.queue_purge_job(fs['mdsmap']['fs_name'])
 
+    def cluster_log(self, msg, lvl=None):
+        """
+        log to cluster log with default log level as WARN.
+        """
+        if not lvl:
+            lvl = self.mgr.CLUSTER_LOG_PRIO_WARN
+        self.mgr.cluster_log("cluster", lvl, msg)
+
     def gen_pool_names(self, volname):
         """
         return metadata and data pool name (from a filesystem/volume name) as a tuple