[IMP] pre_resolve: skip pre-resolve healthcheck on load-balancing dns

when we detect a load balancing dns (responding with different ips on
almost every request) we deactivate the resolver healthcheck to not
restart those containers all the time

healthcheck.py

@@ -125,21 +125,37 @@ def process_healthcheck():
 
 def preresolve_healthcheck():
     """
-    Check that the preresolved ip is still valid now for target
+    Check that the pre-resolved ip is still valid now for target
     :return:
     """
-    import subprocess
-    from dns.resolver import Resolver
-    pre_resolved_ips = {line.split(":")[2] for line in subprocess.check_output(
-        ["sh", "-c", "grep -R '\\(udp\\|tcp\\)-connect:' /proc/[0-9]*/cmdline"]
-    ).decode("utf-8").split("\n") if line}
-    resolver = Resolver()
-    resolver.nameservers = os.environ["NAMESERVERS"].split()
-    target = os.environ["TARGET"]
-    resolved_ips = [answer.address for answer in resolver.resolve(target)]
-    for ip in pre_resolved_ips:
-        if ip not in resolved_ips:
-            error(f"{target} no longer resolves to {ip}")
+    from tempfile import gettempdir
+    load_balancing_dns_fs_flag = os.path.join(gettempdir(), 'load_balancing_dns_detected')
+    if not os.path.exists(load_balancing_dns_fs_flag):
+        # only run the resolver check if a previous run didn't flag the target as being dns load-balanced
+        import subprocess
+        from dns.resolver import Resolver
+        pre_resolved_ips = {line.split(":")[2] for line in subprocess.check_output(
+            ["sh", "-c", "grep -R '\\(udp\\|tcp\\)-connect:' /proc/[0-9]*/cmdline"]
+        ).decode("utf-8").split("\n") if line}
+        resolver = Resolver()
+        resolver.nameservers = os.environ["NAMESERVERS"].split()
+        target = os.environ["TARGET"]
+        resolved_ips = [answer.address for answer in resolver.resolve(target)]
+        for ip in pre_resolved_ips:
+            logger.info(f"checking {target} resolves to {ip}")
+            if ip not in resolved_ips:
+                resolved_ips_2 = [answer.address for answer in resolver.resolve(target)]
+                if resolved_ips_2 == resolved_ips:
+                    error(f"{target} no longer resolves to {ip}, {resolved_ips}, {resolved_ips_2}")
+                else:
+                    resolved_ips_3 = [answer.address for answer in resolver.resolve(target)]
+                    # to make sure we didn't just hit the server switch in dns, we check again before deactivating
+                    # the healthcheck permanently (until the container restarts)
+                    if resolved_ips_3 != resolved_ips_2:
+                        logger.info(f"{target} seems to be load-balancing with dns ({resolved_ips} != {resolved_ips_2}), "
+                                    f"deactivating the resolver healthcheck")
+                        with open(f"{load_balancing_dns_fs_flag}", "w") as fp:
+                            fp.write(target)
 
 
 process_healthcheck()