Merge pull request #3687 from golemfactory/0.18-node-integration-test…

…s--retry-node-connection

+ reconnect nodes in case the initial connection attempt doesn't resu…

scripts/node_integration_tests/playbooks/base.py

@@ -58,6 +58,10 @@ class NodeTestPlaybook:
     task_settings = 'default'
     task_dict = None
 
+    reconnect_attempts_left = 7
+    reconnect_countdown_initial = 10
+    reconnect_countdown = None
+
     playbook_description = 'Runs a golem node integration test'
 
     @property
@@ -94,6 +98,10 @@ def success(self):
     def next(self):
         self.current_step += 1
 
+    def previous(self):
+        assert (self.current_step > 0), "Cannot move back past step 0"
+        self.current_step -= 1
+
     def print_result(self, result):
         print("Result: {}".format(result))
 
@@ -178,10 +186,10 @@ def on_success(result):
         return self.call_requestor('net.status',
                               on_success=on_success, on_error=self.print_error)
 
-
     def step_connect_nodes(self):
         def on_success(result):
             print("Peer connection initialized.")
+            self.reconnect_countdown = self.reconnect_countdown_initial
             self.next()
         return self.call_requestor('net.peer.connect',
                               ("localhost", self.provider_port, ),
@@ -204,7 +212,19 @@ def on_success(result):
                 print("Requestor connected with provider.")
                 self.next()
             else:
-                print("Waiting for nodes to sync...")
+                if self.reconnect_countdown <= 0:
+                    if self.reconnect_attempts_left > 0:
+                        self.reconnect_attempts_left -= 1
+                        print("Retrying peer connection.")
+                        self.previous()
+                        return
+                    else:
+                        self.fail("Could not sync nodes despite trying hard.")
+                        return
+                else:
+                    self.reconnect_countdown -= 1
+                    print("Waiting for nodes to sync...")
+                    time.sleep(10)
 
         return self.call_requestor('net.peers.connected',
                               on_success=on_success, on_error=self.print_error)