Merge pull request #980 from anmolsachan/Fix_multiple_nodes_down_alert

Updated logic for alert notifications and node status watch
Tendrl · May 31, 2018 · af323ba · af323ba
2 parents 153c6e9 + b238d9e
commit af323ba
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 3 deletions.
diff --git a/tendrl/commons/__init__.py b/tendrl/commons/__init__.py
@@ -304,8 +304,9 @@ def setup_common_objects(self):
                        {"message": "Setup NodeContext for namespace."
                                    "%s" % self.ns_name})
             self.current_ns.node_context = \
-                self.current_ns.objects.NodeContext(status="UP")
+                self.current_ns.objects.NodeContext().load()
             NS.node_context = self.current_ns.node_context
+            NS.node_context.status = "UP"
 
         # TendrlContext, if the namespace has implemented its own
         if "TendrlContext" in self.current_ns.objects:

diff --git a/tendrl/commons/objects/cluster_node_context/__init__.py b/tendrl/commons/objects/cluster_node_context/__init__.py
@@ -11,7 +11,7 @@ def __init__(self, integration_id=None,
                  first_sync_done=None, is_managed=None,
                  *args, **kwargs):
         super(ClusterNodeContext, self).__init__(*args, **kwargs)
-        _node_context = NS.node_context.load()
+        _node_context = NS.tendrl.objects.NodeContext(node_id=node_id).load()
         self.integration_id = integration_id
         self.node_id = node_id or _node_context.node_id
         self.fqdn = fqdn or _node_context.fqdn

diff --git a/tendrl/commons/objects/node_context/__init__.py b/tendrl/commons/objects/node_context/__init__.py
@@ -111,10 +111,36 @@ def on_change(self, attr, prev_value, current_value):
                     "WARNING",
                     node_id=self.node_id
                 )
-
                 _tc = NS.tendrl.objects.TendrlContext(
                     node_id=self.node_id
                 ).load()
+                # Load cluster_node_context will load node_context
+                # and it will be updated with latest values
+                cluster_node_context = NS.tendrl.objects.ClusterNodeContext(
+                    node_id=self.node_id,
+                    integration_id=_tc.integration_id
+                )
+                cluster_node_context.save()
+                del cluster_node_context
+                global_details = NS.tendrl.objects.GlobalDetails(
+                    integration_id=_tc.integration_id).load()
+                if global_details.status.lower() == "healthy":
+                    global_details.status = "unhealthy"
+                    global_details.save()
+                    _cluster = NS.tendrl.objects.Cluster(
+                        integration_id=_tc.integration_id
+                    ).load()
+                    msg = "Cluster:%s is %s" % (
+                        _cluster.short_name, "unhealthy")
+                    instance = "cluster_%s" % _tc.integration_id
+                    event_utils.emit_event(
+                        "cluster_health_status",
+                        "unhealthy",
+                        msg,
+                        instance,
+                        'WARNING',
+                        integration_id=_tc.integration_id
+                    )
                 _tag = "provisioner/%s" % _tc.integration_id
                 if _tag in self.tags:
                     _index_key = "/indexes/tags/%s" % _tag
@@ -161,3 +187,13 @@ def on_change(self, attr, prev_value, current_value):
                             )
                         except (etcd.EtcdAlreadyExist, etcd.EtcdKeyNotFound):
                             pass
+            elif current_value == "UP":
+                msg = "{0} is UP".format(self.fqdn)
+                event_utils.emit_event(
+                    "node_status",
+                    "UP",
+                    msg,
+                    "node_{0}".format(self.fqdn),
+                    "INFO",
+                    node_id=self.node_id
+                )
diff --git a/tendrl/commons/utils/central_store/utils.py b/tendrl/commons/utils/central_store/utils.py
@@ -103,6 +103,7 @@ def watch(obj, key):
 
                 is_deleted = prev_val is None and cur_val is None
                 if prev_val != cur_val or is_deleted:
+                    obj = obj.load()
                     attr = key.rstrip("/").split("/")[-1]
                     obj.on_change(attr, prev_val,
                                   cur_val)