mgr/k8sevents:sanitise kubernetes events

Kubernetes can generate events without a timestamp
or an event count. When this occurs the k8sevents 'ls'
command fails due to None type values. This patch
sanitises the event received before adding to the
internal data structures to account for these
issues.

Signed-off-by: Paul Cuzner <pcuzner@redhat.com>
(cherry picked from commit 48d1781ae2a7d4fcdd834fb3f3052c02a5b8bbc2)

diff --git a/src/pybind/mgr/k8sevents/module.py b/src/pybind/mgr/k8sevents/module.py
index 0c933966da6ff1c1cda55f0a2f9b7f56e2ddfd94..e6568269c8a6512853a7e0497e72ef18aed10d47 100644 (file)
--- a/src/pybind/mgr/k8sevents/module.py
+++ b/src/pybind/mgr/k8sevents/module.py
@@ -250,6 +250,27 @@ class BaseThread(threading.Thread):
     daemon = True
 
 
+def clean_event(event):
+    """ clean an event record """
+    if not event.first_timestamp:
+        log.error("first_timestamp is empty")
+        if event.metadata.creation_timestamp:
+            log.error("setting first_timestamp to the creation timestamp")
+            event.first_timestamp = event.metadata.creation_timestamp
+        else:
+            log.error("defaulting event first timestamp to current datetime")
+            event.first_timestamp = datetime.datetime.now()
+
+    if not event.last_timestamp:
+        log.error("setting event last timestamp to {}".format(event.first_timestamp))
+        event.last_timestamp = event.first_timestamp
+
+    if not event.count:
+        event.count = 1
+
+    return event
+
+
 class NamespaceWatcher(BaseThread):
     """Watch events in a given namespace 
     
@@ -289,7 +310,7 @@ class NamespaceWatcher(BaseThread):
             self.resource_version = resp.metadata.resource_version
             
             for item in resp.items:
-                self.events[item.metadata.name] = item
+                self.events[item.metadata.name] = clean_event(item)
             log.info('Added {} events'.format(len(resp.items)))
 
     def run(self):
@@ -311,7 +332,7 @@ class NamespaceWatcher(BaseThread):
                         with self.lock:
 
                             if item['type'] in ['ADDED', 'MODIFIED']:
-                                self.events[obj.metadata.name] = obj
+                                self.events[obj.metadata.name] = clean_event(obj)
 
                             elif item['type'] == 'DELETED':
                                 del self.events[obj.metadata.name]
@@ -333,6 +354,10 @@ class NamespaceWatcher(BaseThread):
                     log.warning("Restarting namespace watcher")
                     self.fetch()
 
+                except ProtocolError as e:
+                    log.warning("Namespace watcher hit protocolerror ({}) - restarting".format(e))
+                    self.fetch()
+
                 except Exception:
                     self.health = "{} Exception at {}".format(
                         sys.exc_info()[0].__name__,
@@ -1156,10 +1181,10 @@ class Module(MgrModule):
 
             s += fmt.format(
                     datetime.strftime(event.last_timestamp,"%Y/%m/%d %H:%M:%S"),
-                    event.type,
-                    event.count,
-                    event.message,
-                    event_name
+                    str(event.type),
+                    str(event.count),
+                    str(event.message),
+                    str(event_name)
             )
         s += "Total : {:>3}\n".format(len(events))
         return s