Merge pull request #38277 from b-ranto/wip-prom-fixes

mgr/prometheus: Make module more stable

Reviewed-by: Mykola Golub <mgolub@suse.com>
Reviewed-by: Sebastian Wagner <swagner@suse.com>
Reviewed-by: Kefu Chai <kchai@redhat.com>

diff --cc src/pybind/mgr/prometheus/module.py
index c122ec7e4c5e47cb38f42dbe85fbd80c8a7890b7,359578e5415b3e55ab1357dedb396c0196bdb667..bfac1e5ab29d76a4ad8900b48c60cfc97ac103b7
--- 1/src/pybind/mgr/prometheus/module.py
--- 2/src/pybind/mgr/prometheus/module.py
+++ b/src/pybind/mgr/prometheus/module.py
@@@ -195,10 -197,19 +197,18 @@@ class MetricCollectionThread(threading.
              self.mod.log.debug('collecting cache in thread')
              if self.mod.have_mon_connection():
                  start_time = time.time()
-                 data = self.mod.collect()
+ 
+                 try:
+                     data = self.mod.collect()
+                 except Exception as e:
+                     # Log any issues encountered during the data collection and continue
+                     self.mod.log.exception("failed to collect metrics:")
+                     self.event.wait(self.mod.scrape_interval)
+                     continue
+ 
                  duration = time.time() - start_time
 -
                  self.mod.log.debug('collecting cache in thread done')
 -                
 +
                  sleep_time = self.mod.scrape_interval - duration
                  if sleep_time < 0:
                      self.mod.log.warning(