mgr/callhome: persist operations between mgr restarts

Currently the operations dictionary is only kept in memory. It is lost
when the mgr restarts, and this can cause the module to handle upload
requests which were already processed and registered in the operations
dictionary. To prevent that, we write the operations to the db, and load
them when the module starts.

Resolves: rhbz#2320831

https://bugzilla.redhat.com/show_bug.cgi?id=2320831

Signed-off-by: Yaarit Hatuka <yhatuka@ibm.com>
(cherry picked from commit 9a28b7c97467ede85afdb8d71a19b0d7be124280)
(cherry picked from commit c5a8a8b89b8f9a548ffae072c5bbe85d6bfe77b2)

diff --git a/src/pybind/mgr/call_home_agent/module.py b/src/pybind/mgr/call_home_agent/module.py
index aee68ea095499a674a378aa46fddf162e9e3d648..9769d180f7c01c6b248c3950ee4009b2e478ddcf 100644 (file)
--- a/src/pybind/mgr/call_home_agent/module.py
+++ b/src/pybind/mgr/call_home_agent/module.py
@@ -1153,6 +1153,9 @@ class CallHomeAgent(MgrModule):
         # set up some members to enable the serve() method and shutdown()
         self.run = True
 
+        # Load operations from db, this makes them persistent across mgr restarts
+        self.init_operations()
+
         # Module options
         self.refresh_options()
 
@@ -1167,6 +1170,17 @@ class CallHomeAgent(MgrModule):
         # Prepare reports
         self.prepare_reports()
 
+    def init_operations(self) -> None:
+        # We fetch from db the operations we already processed,
+        # and assign it to the global operations dictionary
+        db_operations = self.get_store('db_operations')
+
+        global operations
+        if db_operations is not None:
+            # We already set_store('db_operations') in the past
+            operations = json.loads(db_operations)
+            self.log.debug(f"operations loaded from db after restart: {operations}")
+
     def refresh_options(self):
         # Env vars (if they exist) have preference over module options
         self.cha_target_url = str(os.environ.get('CHA_TARGET', self.get_module_option('target')))
@@ -1450,6 +1464,11 @@ class CallHomeAgent(MgrModule):
                     self.log.info('Operations: Processing operations finished')
                 except Exception as ex:
                     self.log.error(f"Operations ({operation_key}): error: {ex}")
+
+                # persist operations
+                self.set_store('db_operations', json.dumps(operations))
+                self.log.debug(f"updating operations db: {json.dumps(operations)}")
+
                 await asyncio.sleep(seconds)
         except asyncio.CancelledError:
             return
@@ -1775,6 +1794,11 @@ class CallHomeAgent(MgrModule):
                 operations.clear()
 
             output = json.dumps(operations)
+
+            # persist operations
+            self.set_store('db_operations', json.dumps(operations))
+            self.log.debug(f"updating operations db after cleaning: {json.dumps(operations)}")
+
         except Exception as ex:
             return HandleCommandResult(stderr=str(ex))
         else: