pep8 and python3 cleanup

src/pyff/builtins.py

@@ -19,7 +19,6 @@
 import yaml
 from iso8601 import iso8601
 from lxml.etree import DocumentInvalid
-
 from .constants import NS
 from .decorators import deprecated
 from .logs import log
@@ -433,7 +432,7 @@ def load(req, *opts):
     opts['filter_invalid'] = bool(strtobool(opts['filter_invalid']))
 
     remotes = []
-    store = req.md.store_class() # start the load process by creating a provisional store object
+    store = req.md.store_class()  # start the load process by creating a provisional store object
     req._store = store
     for x in req.args:
         x = x.strip()
@@ -475,7 +474,7 @@ def _null(t):
     log.debug("Refreshing all resources")
     req.md.rm.reload(fail_on_error=bool(opts['fail_on_error']), store=store)
     req._store = None
-    req.md.store = store # commit the store
+    req.md.store = store  # commit the store
 
 
 def _select_args(req):
@@ -788,16 +787,18 @@ def stats(req, *opts):
     if req.t is None:
         raise PipeException("Your pipeline is missing a select statement.")
 
-    print ("---")
-    print ("total size:     {:d}".format(req.store.size()))
+    print("---")
+    print("total size:     {:d}".format(req.store.size()))
     if not hasattr(req.t, 'xpath'):
         raise PipeException("Unable to call stats on non-XML")
 
     if req.t is not None:
-        print ("selected:       {:d}".format(len(req.t.xpath("//md:EntityDescriptor", namespaces=NS))))
-        print ("          idps: {:d}".format(len(req.t.xpath("//md:EntityDescriptor[md:IDPSSODescriptor]", namespaces=NS))))
-        print ("           sps: {:d}".format(len(req.t.xpath("//md:EntityDescriptor[md:SPSSODescriptor]", namespaces=NS))))
-    print ("---")
+        print("selected:       {:d}".format(len(req.t.xpath("//md:EntityDescriptor", namespaces=NS))))
+        print("          idps: {:d}".format(
+            len(req.t.xpath("//md:EntityDescriptor[md:IDPSSODescriptor]", namespaces=NS))))
+        print(
+            "           sps: {:d}".format(len(req.t.xpath("//md:EntityDescriptor[md:SPSSODescriptor]", namespaces=NS))))
+    print("---")
     return req.t
 
 

src/pyff/constants.py

@@ -61,5 +61,6 @@ class Config(object):
     request_timeout = pyconfig.setting("pyff.request_timeout",10)
     request_cache_time = pyconfig.setting("pyff.request_cache_time", 300)
     request_override_encoding = pyconfig.setting("pyff.request_override_encoding", "utf8") # set to non to enable chardet guessing
+    devel_memory_profile = pyconfig.setting("pyff.devel_memory_profile", False)
 
 config = Config()

src/pyff/fetch.py

@@ -18,7 +18,6 @@
 from itertools import chain
 from requests_cache.core import CachedSession
 from copy import deepcopy
-from six import StringIO
 
 requests.packages.urllib3.disable_warnings()
 
@@ -90,8 +89,8 @@ def reload(self, url=None, fail_on_error=False, store=None):
                             for nr in res:
                                 new_resources.append(nr)
                     except Exception as ex:
-                        #from traceback import print_exc
-                        #print_exc()
+                        # from traceback import print_exc
+                        # print_exc()
                         print("caught fetch thread exception")
                         print(ex)
                         log.error(ex)
@@ -214,4 +213,3 @@ def fetch(self, store=None):
         self.add_info(info)
 
         return self.children
-

src/pyff/logs.py

@@ -4,14 +4,16 @@
 import syslog
 import cherrypy
 
+
 def printable(s):
-    if isinstance(s,unicode):
+    if isinstance(s, unicode):
         return s.encode('utf8', errors='ignore').decode('utf8')
-    elif isinstance(s,str):
+    elif isinstance(s, str):
         return s.decode("utf8", errors="ignore").encode('utf8')
     else:
         return repr(s)
 
+
 class PyFFLogger(object):
     def __init__(self):
         self._loggers = {logging.WARN: logging.warn,
@@ -53,6 +55,7 @@ def isEnabledFor(self, lvl):
 
 log = PyFFLogger()
 
+
 # http://www.aminus.org/blogs/index.php/2008/07/03/writing-high-efficiency-large-python-sys-1?blog=2
 # blog post explicitly gives permission for use
 
@@ -80,4 +83,4 @@ def __init__(self, facility):
         logging.Handler.__init__(self)
 
     def emit(self, record):
-        syslog.syslog(self.facility | self.priority_map[record.levelno], self.format(record))
+        syslog.syslog(self.facility | self.priority_map[record.levelno], self.format(record))

src/pyff/mdrepo.py

@@ -187,9 +187,11 @@ def lookup(self, member, xp=None, store=None):
 :type member: basestring
 :param xp: An optional xpath filter
 :type xp: basestring
+:param store: the store to operate on
 :return: An interable of EntityDescriptor elements
 :rtype: etree.Element
 
+
 **Selector Syntax**
 
     - selector "+" selector

src/pyff/mdx.py

@@ -49,7 +49,6 @@
 
 import pkg_resources
 
-
 from six import StringIO
 import getopt
 import urlparse
@@ -152,13 +151,13 @@ def dispatch(self, path_info):
         vpath = path_info
         for prefix in self.prefixes:
             if vpath.startswith(prefix):
-                #log.debug("EncodingDispatcher (%s) called with %s" % (",".join(self.prefixes), path_info))
+                # log.debug("EncodingDispatcher (%s) called with %s" % (",".join(self.prefixes), path_info))
                 vpath = path_info.replace("%2F", "/")
                 plen = len(prefix)
                 vpath = vpath[plen + 1:]
                 npath = "%s/%s" % (prefix, self.enc(vpath))
-                #log.debug("EncodingDispatcher %s" % npath)
-                return self.next_dispatcher(npath.encode('ascii',errors='ignore'))
+                # log.debug("EncodingDispatcher %s" % npath)
+                return self.next_dispatcher(npath.encode('ascii', errors='ignore'))
         return self.next_dispatcher(vpath)
 
 
@@ -306,12 +305,12 @@ def __init__(self, server):
     stats = MDStats()
     discovery = SHIBDiscovery()
 
-    try:  # pragma: nocover
-        import dowser
-
-        memory = dowser.Root()
-    except ImportError:
-        memory = NotImplementedFunction('Memory profiling needs dowser')
+    if config.devel_memory_profile:
+        try:  # pragma: nocover
+            import dowser
+            memory = dowser.Root()
+        except ImportError:
+            memory = NotImplementedFunction('Memory profiling needs dowser')
 
     _well_known = WellKnown()
     static = cherrypy.tools.staticdir.handler("/static", os.path.join(site_dir, "static"))
@@ -441,7 +440,7 @@ def default(self, *args, **kwargs):
         """The default request processor unpacks base64-encoded reuqests and passes them onto the MDServer.request
         handler.
         """
-        #log.debug("ROOT default args: %s, kwargs: %s" % (repr(args), repr(kwargs)))
+        # log.debug("ROOT default args: %s, kwargs: %s" % (repr(args), repr(kwargs)))
         if len(args) > 0 and args[0] in self.server.aliases:
             kwargs['pfx'] = args[0]
             if len(args) > 1:
@@ -511,12 +510,12 @@ def request(self, **kwargs):
         path = kwargs.get('path', None)
         content_type = kwargs.get('content_type', None)
 
-        #log.debug("MDServer pfx=%s, path=%s, content_type=%s" % (pfx, path, content_type))
+        # log.debug("MDServer pfx=%s, path=%s, content_type=%s" % (pfx, path, content_type))
 
         def _d(x, do_split=True):
             if x is not None:
                 x = x.strip()
-            #log.debug("_d(%s,%s)" % (x, do_split))
+            # log.debug("_d(%s,%s)" % (x, do_split))
             if x is None or len(x) == 0:
                 return None, None
 

src/pyff/parse.py

@@ -1,4 +1,3 @@
-
 import os
 from .utils import parse_xml, root
 from .constants import NS
@@ -9,6 +8,7 @@
 
 __author__ = 'leifj'
 
+
 class ParserException(Exception):
     def __init__(self, msg, wrapped=None, data=None):
         self._wraped = wrapped
@@ -20,7 +20,6 @@ def raise_wraped(self):
 
 
 class NoParser():
-
     def magic(self, content):
         return True
 
@@ -29,7 +28,6 @@ def parse(self, resource, content):
 
 
 class DirectoryParser():
-
     def __init__(self, ext):
         self.ext = ext
 
@@ -52,7 +50,7 @@ def parse(self, resource, content):
         resource.children = []
         n = 0
         for fn in self._find_matching_files(content):
-            resource.add_child("file://"+fn)
+            resource.add_child("file://" + fn)
             n += 1
 
         if n == 0:
@@ -62,7 +60,6 @@ def parse(self, resource, content):
 
 
 class XRDParser():
-
     def __init__(self):
         pass
 
@@ -86,18 +83,17 @@ def parse(self, resource, content):
                 resource.add_child(link_href, verify=fp)
         resource.last_seen = datetime.now
         resource.expire_time = None
-
         return info
 
 
 _parsers = [DirectoryParser('.xml'), XRDParser(), NoParser()]
 
 
 def add_parser(parser):
-    _parsers.insert(0,parser)
+    _parsers.insert(0, parser)
 
 
 def parse_resource(resource, content):
     for parser in _parsers:
         if parser.magic(content):
-            return parser.parse(resource, content)
+            return parser.parse(resource, content)

src/pyff/pipes.py

@@ -236,6 +236,7 @@ def process(self, md, state=None, t=None, store=None):
 :param md: The current metadata repository
 :param state: The active request state
 :param t: The active working document
+:param store: The store object to operate on
 :return: The result of applying the processing pipeline to t.
         """
         if not state:
@@ -251,15 +252,15 @@ def iprocess(self, req):
         log.debug("Processing pipeline... {}".format(self.pipeline))
         for p in self.pipeline:
             try:
-                pipe, opts, name, args = load_pipe(p)
+                pipefn, opts, name, args = load_pipe(p)
                 # log.debug("traversing pipe %s,%s,%s using %s" % (pipe,name,args,opts))
                 if type(args) is str or type(args) is unicode:
                     args = [args]
                 if args is not None and type(args) is not dict and type(args) is not list and type(args) is not tuple:
                     raise PipeException("Unknown argument type %s" % repr(args))
                 req.args = args
                 req.name = name
-                ot = pipe(req, *opts)
+                ot = pipefn(req, *opts)
                 if ot is not None:
                     req.t = ot
                 if req.done:

src/pyff/samlmd.py

@@ -44,7 +44,7 @@ def find_merge_strategy(strategy_name):
     if '.' not in strategy_name:
         strategy_name = "pyff.merge_strategies:%s" % strategy_name
     if ':' not in strategy_name:
-        strategy_name = rreplace(strategy_name, '.', ':') # backwards compat for old way of specifying these
+        strategy_name = rreplace(strategy_name, '.', ':')  # backwards compat for old way of specifying these
     return load_callable(strategy_name)
 
 
@@ -115,7 +115,6 @@ def parse_saml_metadata(source,
 
 
 class SAMLMetadataResourceParser():
-
     def __init__(self):
         pass
 
@@ -146,6 +145,7 @@ def parse(self, resource, content):
 
 
 from .parse import add_parser
+
 add_parser(SAMLMetadataResourceParser())
 
 
@@ -561,9 +561,11 @@ def discojson(e, langs=None):
 
     return d
 
+
 def sha1_id(e):
     return hash_id(e, 'sha1')
 
+
 def entity_simple_summary(e):
     if e is None:
         return dict()
@@ -613,7 +615,8 @@ def entity_service_description(entity, langs=None):
 
 
 def entity_requested_attributes(entity, langs=None):
-    return [(a.get('Name'),bool(a.get('isRequired'))) for a in filter_lang(entity.iter("{%s}RequestedAttribute" % NS['md']), langs=langs)]
+    return [(a.get('Name'), bool(a.get('isRequired'))) for a in
+            filter_lang(entity.iter("{%s}RequestedAttribute" % NS['md']), langs=langs)]
 
 
 def entity_idp(entity):
@@ -634,18 +637,20 @@ def entity_contacts(entity):
     def _contact_dict(contact):
         first_name = first_text(contact, "{%s}GivenName" % NS['md'])
         last_name = first_text(contact, "{%s}SurName" % NS['md'])
-        org = first_text(entity,"{%s}OrganizationName" % NS['md']) or first_text(entity,"{%s}OrganizationDisplayName" % NS['md'])
-        company = first_text(entity,"{%s}Company" % NS['md'])
+        org = first_text(entity, "{%s}OrganizationName" % NS['md']) or first_text(entity,
+                                                                                  "{%s}OrganizationDisplayName" % NS[
+                                                                                      'md'])
+        company = first_text(entity, "{%s}Company" % NS['md'])
         mail = first_text(contact, "{%s}EmailAddress" % NS['md'])
         display_name = "Unknown"
         if first_name and last_name:
-            display_name = ' '.join([first_name,last_name])
+            display_name = ' '.join([first_name, last_name])
         elif first_name:
             display_name = first_name
         elif last_name:
             display_name = last_name
         elif mail:
-            _,_,display_name = mail.partition(':')
+            _, _, display_name = mail.partition(':')
 
         return dict(type=contact.get('contactType'),
                     first_name=first_name,
@@ -682,12 +687,14 @@ def entity_info(e, langs=None):
     d['is_idp'] = is_idp(e)
     d['is_sp'] = is_sp(e)
     d['is_aa'] = is_aa(e)
-    d['xml'] = dumptree(e, xml_declaration=False, pretty_print=True).decode('utf8').replace('<','&lt;').replace('>','&gt;')
+    d['xml'] = dumptree(e, xml_declaration=False, pretty_print=True).decode('utf8').replace('<', '&lt;').replace('>',
+                                                                                                                 '&gt;')
     if d['is_idp']:
-        d['protocols'] = entity_idp(e).get('protocolSupportEnumeration',"").split()
+        d['protocols'] = entity_idp(e).get('protocolSupportEnumeration', "").split()
 
     return d
 
+
 def entity_extensions(e):
     """Return a list of the Extensions elements in the EntityDescriptor
 
@@ -760,7 +767,7 @@ def set_entity_attributes(e, d, nf=NF_URI):
     if e.tag != "{%s}EntityDescriptor" % NS['md']:
         raise MetadataException("I can only add EntityAttribute(s) to EntityDescriptor elements")
 
-    for attr, value in d.iteritems():
+    for attr, value in d.items():
         a = _eattribute(e, attr, nf)
         velt = etree.Element("{%s}AttributeValue" % NS['saml'])
         velt.text = value
@@ -804,7 +811,7 @@ def set_reginfo(e, policy=None, authority=None):
     ri = etree.Element("{%s}RegistrationInfo" % NS['mdrpi'])
     ext.append(ri)
     ri.set('registrationAuthority', authority)
-    for lang, policy_url in policy.iteritems():
+    for lang, policy_url in policy.items():
         rp = etree.Element("{%s}RegistrationPolicy" % NS['mdrpi'])
         rp.text = policy_url
         rp.set('{%s}lang' % NS['xml'], lang)