]> git.apps.os.sepia.ceph.com Git - ceph.git/commitdiff
mgr/prometheus: Log collection issues
authorBoris Ranto <branto@redhat.com>
Wed, 25 Nov 2020 09:32:35 +0000 (10:32 +0100)
committerBoris Ranto <branto@redhat.com>
Wed, 25 Nov 2020 15:32:16 +0000 (16:32 +0100)
Log any issues encountered during the data collection and continue to
collect the data anyway (after a sleep).

Signed-off-by: Boris Ranto <branto@redhat.com>
src/pybind/mgr/prometheus/module.py

index 763a65ce1f221834f2ad007faa7d0c14ad39f186..c5ec065853b257ba8d0bfecab1210cd4e77179d4 100644 (file)
@@ -196,7 +196,15 @@ class MetricCollectionThread(threading.Thread):
             self.mod.log.debug('collecting cache in thread')
             if self.mod.have_mon_connection():
                 start_time = time.time()
-                data = self.mod.collect()
+
+                try:
+                    data = self.mod.collect()
+                except Exception as e:
+                    # Log any issues encountered during the data collection and continue
+                    self.mod.log.exception("failed to collect metrics:")
+                    time.sleep(self.mod.scrape_interval)
+                    continue
+
                 duration = time.time() - start_time
 
                 self.mod.log.debug('collecting cache in thread done')