Incorrect number of hosts down reported

tendrl-bug-id: Tendrl#1076
bugzilla: 1687333

Signed-off-by: GowthamShanmugasundaram <gshanmug@redhat.com>

tendrl/commons/objects/node_context/__init__.py

@@ -93,6 +93,11 @@ def render(self):
     def save(self, update=True, ttl=None):
         super(NodeContext, self).save(update)
         status = self.value + "/status"
+        if self.status == "UP" and ttl == None:
+            # Set ttl always when node status in up
+            ttl = int(
+                NS.config.data.get("sync_interval", 60)
+            )
         if ttl:
             self._ttl = ttl
             try:
@@ -141,7 +146,10 @@ def on_change(self, attr, prev_value, current_value):
                     _index_key = "/indexes/tags/%s" % _tag
                     self.tags.remove(_tag)
                     self.save()
-                    etcd_utils.delete(_index_key)
+                    try:
+                        etcd_utils.delete(_index_key)
+                    except etcd.EtcdKeyNotFound:
+                        pass
                 if _tc.sds_name in ["gluster", "RHGS"]:
                     bricks = etcd_utils.read(
                         "clusters/{0}/Bricks/all/{1}".format(

tendrl/commons/utils/central_store/utils.py

@@ -107,7 +107,10 @@ def watch(obj, key):
                     attr = key.rstrip("/").split("/")[-1]
                     obj.on_change(attr, prev_val,
                                   cur_val)
-        except etcd.EtcdKeyNotFound:
+        except Exception as ex:
+            if type(ex) is etcd.EtcdEventIndexCleared:
+                continue
+            # When watch crash then clear key from the watchers
             NS._int.watchers.pop(key, None)
             return