added HTTP Adaptor with random DNS resolution

PandaPkgInfo.py

@@ -1 +1 @@
-release_version = "0.0.17"
+release_version = "0.0.18"

README.txt

@@ -7,7 +7,10 @@ Includes all libraries used by both server and monitor (and others).
 Release Note
 ------------
 
-* 0.0.17 (1/20/2019)
+* 0.0.18 (2/4/2020)
+  * added HTTP Adaptor with random DNS resolution
+
+* 0.0.17 (1/20/2020)
   * added msg stuff 
 
 * 0.0.15 (11/7/2019)

pandacommon/pandautils/net_utils.py

@@ -0,0 +1,70 @@
+import copy
+import random
+import socket
+import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.connection import allowed_gai_family
+try:
+    from urllib.parse import urlparse
+except ImportError:
+    from urlparse import urlparse
+
+from .thread_utils import MapWithLockAndTimeout
+
+
+# DNS cache
+dnsMap = MapWithLockAndTimeout()
+
+
+# HTTP adaptor with randomized DNS resolution
+class HTTPAdapterWithRandomDnsResolver (HTTPAdapter):
+
+    # override to get connection to random host
+    def get_connection(self, url, proxies=None):
+        # parse URL
+        parsed = urlparse(url)
+        host = parsed.hostname
+        port = parsed.port
+        if port is None:
+            if parsed.scheme == 'http':
+                port = 80
+            else:
+                port = 443
+        # check record
+        if parsed.hostname in dnsMap:
+            dnsRecord = dnsMap[parsed.hostname]
+        else:
+            family = allowed_gai_family()
+            dnsRecord = socket.getaddrinfo(host, port, family, socket.SOCK_STREAM)
+            dnsMap[parsed.hostname] = dnsRecord
+        dnsRecord = copy.copy(dnsRecord)
+        random.shuffle(dnsRecord)
+        # loop over all hosts
+        err = None
+        for af, sock_type, proto, canon_name, sa in dnsRecord:
+            if af == socket.AF_INET6:
+                addr = '[' + sa[0] + ']'
+            else:
+                addr = sa[0]
+            if parsed.port is not None:
+                addr += ':{0}'.format(parsed.port)
+            tmp_url = parsed._replace(netloc=addr).geturl()
+            try:
+                con = HTTPAdapter.get_connection(self, tmp_url, proxies=proxies)
+                # return if valid
+                if con is not None:
+                    return con
+            except Exception as e:
+                err = e
+        if err is not None:
+            raise err
+        return None
+
+
+# utility function to get HTTPAdapterWithRandomDnsResolver
+def get_http_adapter_with_random_dns_resolution():
+    session = requests.Session()
+    adapter = HTTPAdapterWithRandomDnsResolver(max_retries=0)
+    session.mount('http://', adapter)
+    session.mount('https://', adapter)
+    return session

pandacommon/pandautils/thread_utils.py

@@ -1,6 +1,7 @@
 import os
 import threading
 import socket
+import datetime
 
 
 class GenericThread(threading.Thread):
@@ -16,3 +17,39 @@ def get_pid(self):
         """
         thread_id = self.ident if self.ident else 0
         return '{0}_{1}-{2}'.format(self.hostname, self.os_pid, format(thread_id, 'x'))
+
+
+# map with lock
+class MapWithLockAndTimeout(dict):
+
+    def __init__(self, *args, **kwargs):
+        # set timeout
+        if 'timeout' in kwargs:
+            self.timeout = kwargs['timeout']
+            del kwargs['timeout']
+        else:
+            self.timeout = 10
+        self.lock = threading.Lock()
+        dict.__init__(self, *args, **kwargs)
+
+    # get item regardless of freshness to avoid race-condition in check->get
+    def __getitem__(self, item):
+        with self.lock:
+            ret = dict.__getitem__(self, item)
+            return ret['data']
+
+    def __setitem__(self, item, value):
+        with self.lock:
+            dict.__setitem__(self, item, {'time_stamp': datetime.datetime.utcnow(),
+                                          'data': value})
+
+    # check data by taking freshness into account
+    def __contains__(self, item):
+        with self.lock:
+            try:
+                ret = dict.__getitem__(self, item)
+                if ret['time_stamp'] > datetime.datetime.utcnow() - datetime.timedelta(minutes=self.timeout):
+                    return True
+            except Exception:
+                pass
+        return False

setup.py

@@ -39,6 +39,7 @@ def finalize_options (self):
     install_requires=['configparser',
                       'pytz',
                       'stomp.py',
+                      'requests',
                       ],
     data_files=[
                 ('etc/panda',