From 033d890826652406178b10628ec0040ae0278e86 Mon Sep 17 00:00:00 2001 From: Leif Johansson Date: Wed, 13 Dec 2017 20:48:18 +0100 Subject: [PATCH] pep8 and python3 cleanup --- src/pyff/builtins.py | 19 ++++++++++--------- src/pyff/constants.py | 1 + src/pyff/fetch.py | 6 ++---- src/pyff/logs.py | 9 ++++++--- src/pyff/mdrepo.py | 2 ++ src/pyff/mdx.py | 25 ++++++++++++------------- src/pyff/parse.py | 12 ++++-------- src/pyff/pipes.py | 5 +++-- src/pyff/samlmd.py | 29 ++++++++++++++++++----------- src/pyff/store.py | 6 +++--- src/pyff/test/test_utils.py | 4 ++-- 11 files changed, 63 insertions(+), 55 deletions(-) diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 03ed1dbe..862f2d72 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -19,7 +19,6 @@ import yaml from iso8601 import iso8601 from lxml.etree import DocumentInvalid - from .constants import NS from .decorators import deprecated from .logs import log @@ -433,7 +432,7 @@ def load(req, *opts): opts['filter_invalid'] = bool(strtobool(opts['filter_invalid'])) remotes = [] - store = req.md.store_class() # start the load process by creating a provisional store object + store = req.md.store_class() # start the load process by creating a provisional store object req._store = store for x in req.args: x = x.strip() @@ -475,7 +474,7 @@ def _null(t): log.debug("Refreshing all resources") req.md.rm.reload(fail_on_error=bool(opts['fail_on_error']), store=store) req._store = None - req.md.store = store # commit the store + req.md.store = store # commit the store def _select_args(req): @@ -788,16 +787,18 @@ def stats(req, *opts): if req.t is None: raise PipeException("Your pipeline is missing a select statement.") - print ("---") - print ("total size: {:d}".format(req.store.size())) + print("---") + print("total size: {:d}".format(req.store.size())) if not hasattr(req.t, 'xpath'): raise PipeException("Unable to call stats on non-XML") if req.t is not None: - print ("selected: {:d}".format(len(req.t.xpath("//md:EntityDescriptor", namespaces=NS)))) - print (" idps: {:d}".format(len(req.t.xpath("//md:EntityDescriptor[md:IDPSSODescriptor]", namespaces=NS)))) - print (" sps: {:d}".format(len(req.t.xpath("//md:EntityDescriptor[md:SPSSODescriptor]", namespaces=NS)))) - print ("---") + print("selected: {:d}".format(len(req.t.xpath("//md:EntityDescriptor", namespaces=NS)))) + print(" idps: {:d}".format( + len(req.t.xpath("//md:EntityDescriptor[md:IDPSSODescriptor]", namespaces=NS)))) + print( + " sps: {:d}".format(len(req.t.xpath("//md:EntityDescriptor[md:SPSSODescriptor]", namespaces=NS)))) + print("---") return req.t diff --git a/src/pyff/constants.py b/src/pyff/constants.py index 4231b332..e76513a8 100644 --- a/src/pyff/constants.py +++ b/src/pyff/constants.py @@ -61,5 +61,6 @@ class Config(object): request_timeout = pyconfig.setting("pyff.request_timeout",10) request_cache_time = pyconfig.setting("pyff.request_cache_time", 300) request_override_encoding = pyconfig.setting("pyff.request_override_encoding", "utf8") # set to non to enable chardet guessing + devel_memory_profile = pyconfig.setting("pyff.devel_memory_profile", False) config = Config() diff --git a/src/pyff/fetch.py b/src/pyff/fetch.py index 017a7c48..32fecc5c 100644 --- a/src/pyff/fetch.py +++ b/src/pyff/fetch.py @@ -18,7 +18,6 @@ from itertools import chain from requests_cache.core import CachedSession from copy import deepcopy -from six import StringIO requests.packages.urllib3.disable_warnings() @@ -90,8 +89,8 @@ def reload(self, url=None, fail_on_error=False, store=None): for nr in res: new_resources.append(nr) except Exception as ex: - #from traceback import print_exc - #print_exc() + # from traceback import print_exc + # print_exc() print("caught fetch thread exception") print(ex) log.error(ex) @@ -214,4 +213,3 @@ def fetch(self, store=None): self.add_info(info) return self.children - diff --git a/src/pyff/logs.py b/src/pyff/logs.py index 2dfedd7a..89349bed 100644 --- a/src/pyff/logs.py +++ b/src/pyff/logs.py @@ -4,14 +4,16 @@ import syslog import cherrypy + def printable(s): - if isinstance(s,unicode): + if isinstance(s, unicode): return s.encode('utf8', errors='ignore').decode('utf8') - elif isinstance(s,str): + elif isinstance(s, str): return s.decode("utf8", errors="ignore").encode('utf8') else: return repr(s) + class PyFFLogger(object): def __init__(self): self._loggers = {logging.WARN: logging.warn, @@ -53,6 +55,7 @@ def isEnabledFor(self, lvl): log = PyFFLogger() + # http://www.aminus.org/blogs/index.php/2008/07/03/writing-high-efficiency-large-python-sys-1?blog=2 # blog post explicitly gives permission for use @@ -80,4 +83,4 @@ def __init__(self, facility): logging.Handler.__init__(self) def emit(self, record): - syslog.syslog(self.facility | self.priority_map[record.levelno], self.format(record)) \ No newline at end of file + syslog.syslog(self.facility | self.priority_map[record.levelno], self.format(record)) diff --git a/src/pyff/mdrepo.py b/src/pyff/mdrepo.py index 5ca77eaa..43dcc472 100644 --- a/src/pyff/mdrepo.py +++ b/src/pyff/mdrepo.py @@ -187,9 +187,11 @@ def lookup(self, member, xp=None, store=None): :type member: basestring :param xp: An optional xpath filter :type xp: basestring +:param store: the store to operate on :return: An interable of EntityDescriptor elements :rtype: etree.Element + **Selector Syntax** - selector "+" selector diff --git a/src/pyff/mdx.py b/src/pyff/mdx.py index bbcbaa03..b754c195 100644 --- a/src/pyff/mdx.py +++ b/src/pyff/mdx.py @@ -49,7 +49,6 @@ import pkg_resources - from six import StringIO import getopt import urlparse @@ -152,13 +151,13 @@ def dispatch(self, path_info): vpath = path_info for prefix in self.prefixes: if vpath.startswith(prefix): - #log.debug("EncodingDispatcher (%s) called with %s" % (",".join(self.prefixes), path_info)) + # log.debug("EncodingDispatcher (%s) called with %s" % (",".join(self.prefixes), path_info)) vpath = path_info.replace("%2F", "/") plen = len(prefix) vpath = vpath[plen + 1:] npath = "%s/%s" % (prefix, self.enc(vpath)) - #log.debug("EncodingDispatcher %s" % npath) - return self.next_dispatcher(npath.encode('ascii',errors='ignore')) + # log.debug("EncodingDispatcher %s" % npath) + return self.next_dispatcher(npath.encode('ascii', errors='ignore')) return self.next_dispatcher(vpath) @@ -306,12 +305,12 @@ def __init__(self, server): stats = MDStats() discovery = SHIBDiscovery() - try: # pragma: nocover - import dowser - - memory = dowser.Root() - except ImportError: - memory = NotImplementedFunction('Memory profiling needs dowser') + if config.devel_memory_profile: + try: # pragma: nocover + import dowser + memory = dowser.Root() + except ImportError: + memory = NotImplementedFunction('Memory profiling needs dowser') _well_known = WellKnown() static = cherrypy.tools.staticdir.handler("/static", os.path.join(site_dir, "static")) @@ -441,7 +440,7 @@ def default(self, *args, **kwargs): """The default request processor unpacks base64-encoded reuqests and passes them onto the MDServer.request handler. """ - #log.debug("ROOT default args: %s, kwargs: %s" % (repr(args), repr(kwargs))) + # log.debug("ROOT default args: %s, kwargs: %s" % (repr(args), repr(kwargs))) if len(args) > 0 and args[0] in self.server.aliases: kwargs['pfx'] = args[0] if len(args) > 1: @@ -511,12 +510,12 @@ def request(self, **kwargs): path = kwargs.get('path', None) content_type = kwargs.get('content_type', None) - #log.debug("MDServer pfx=%s, path=%s, content_type=%s" % (pfx, path, content_type)) + # log.debug("MDServer pfx=%s, path=%s, content_type=%s" % (pfx, path, content_type)) def _d(x, do_split=True): if x is not None: x = x.strip() - #log.debug("_d(%s,%s)" % (x, do_split)) + # log.debug("_d(%s,%s)" % (x, do_split)) if x is None or len(x) == 0: return None, None diff --git a/src/pyff/parse.py b/src/pyff/parse.py index 51977fce..d64e7a9f 100644 --- a/src/pyff/parse.py +++ b/src/pyff/parse.py @@ -1,4 +1,3 @@ - import os from .utils import parse_xml, root from .constants import NS @@ -9,6 +8,7 @@ __author__ = 'leifj' + class ParserException(Exception): def __init__(self, msg, wrapped=None, data=None): self._wraped = wrapped @@ -20,7 +20,6 @@ def raise_wraped(self): class NoParser(): - def magic(self, content): return True @@ -29,7 +28,6 @@ def parse(self, resource, content): class DirectoryParser(): - def __init__(self, ext): self.ext = ext @@ -52,7 +50,7 @@ def parse(self, resource, content): resource.children = [] n = 0 for fn in self._find_matching_files(content): - resource.add_child("file://"+fn) + resource.add_child("file://" + fn) n += 1 if n == 0: @@ -62,7 +60,6 @@ def parse(self, resource, content): class XRDParser(): - def __init__(self): pass @@ -86,7 +83,6 @@ def parse(self, resource, content): resource.add_child(link_href, verify=fp) resource.last_seen = datetime.now resource.expire_time = None - return info @@ -94,10 +90,10 @@ def parse(self, resource, content): def add_parser(parser): - _parsers.insert(0,parser) + _parsers.insert(0, parser) def parse_resource(resource, content): for parser in _parsers: if parser.magic(content): - return parser.parse(resource, content) \ No newline at end of file + return parser.parse(resource, content) diff --git a/src/pyff/pipes.py b/src/pyff/pipes.py index 5dce9fb2..b8b54eb1 100644 --- a/src/pyff/pipes.py +++ b/src/pyff/pipes.py @@ -236,6 +236,7 @@ def process(self, md, state=None, t=None, store=None): :param md: The current metadata repository :param state: The active request state :param t: The active working document +:param store: The store object to operate on :return: The result of applying the processing pipeline to t. """ if not state: @@ -251,7 +252,7 @@ def iprocess(self, req): log.debug("Processing pipeline... {}".format(self.pipeline)) for p in self.pipeline: try: - pipe, opts, name, args = load_pipe(p) + pipefn, opts, name, args = load_pipe(p) # log.debug("traversing pipe %s,%s,%s using %s" % (pipe,name,args,opts)) if type(args) is str or type(args) is unicode: args = [args] @@ -259,7 +260,7 @@ def iprocess(self, req): raise PipeException("Unknown argument type %s" % repr(args)) req.args = args req.name = name - ot = pipe(req, *opts) + ot = pipefn(req, *opts) if ot is not None: req.t = ot if req.done: diff --git a/src/pyff/samlmd.py b/src/pyff/samlmd.py index 52ec3d15..1ae5f2e7 100644 --- a/src/pyff/samlmd.py +++ b/src/pyff/samlmd.py @@ -44,7 +44,7 @@ def find_merge_strategy(strategy_name): if '.' not in strategy_name: strategy_name = "pyff.merge_strategies:%s" % strategy_name if ':' not in strategy_name: - strategy_name = rreplace(strategy_name, '.', ':') # backwards compat for old way of specifying these + strategy_name = rreplace(strategy_name, '.', ':') # backwards compat for old way of specifying these return load_callable(strategy_name) @@ -115,7 +115,6 @@ def parse_saml_metadata(source, class SAMLMetadataResourceParser(): - def __init__(self): pass @@ -146,6 +145,7 @@ def parse(self, resource, content): from .parse import add_parser + add_parser(SAMLMetadataResourceParser()) @@ -561,9 +561,11 @@ def discojson(e, langs=None): return d + def sha1_id(e): return hash_id(e, 'sha1') + def entity_simple_summary(e): if e is None: return dict() @@ -613,7 +615,8 @@ def entity_service_description(entity, langs=None): def entity_requested_attributes(entity, langs=None): - return [(a.get('Name'),bool(a.get('isRequired'))) for a in filter_lang(entity.iter("{%s}RequestedAttribute" % NS['md']), langs=langs)] + return [(a.get('Name'), bool(a.get('isRequired'))) for a in + filter_lang(entity.iter("{%s}RequestedAttribute" % NS['md']), langs=langs)] def entity_idp(entity): @@ -634,18 +637,20 @@ def entity_contacts(entity): def _contact_dict(contact): first_name = first_text(contact, "{%s}GivenName" % NS['md']) last_name = first_text(contact, "{%s}SurName" % NS['md']) - org = first_text(entity,"{%s}OrganizationName" % NS['md']) or first_text(entity,"{%s}OrganizationDisplayName" % NS['md']) - company = first_text(entity,"{%s}Company" % NS['md']) + org = first_text(entity, "{%s}OrganizationName" % NS['md']) or first_text(entity, + "{%s}OrganizationDisplayName" % NS[ + 'md']) + company = first_text(entity, "{%s}Company" % NS['md']) mail = first_text(contact, "{%s}EmailAddress" % NS['md']) display_name = "Unknown" if first_name and last_name: - display_name = ' '.join([first_name,last_name]) + display_name = ' '.join([first_name, last_name]) elif first_name: display_name = first_name elif last_name: display_name = last_name elif mail: - _,_,display_name = mail.partition(':') + _, _, display_name = mail.partition(':') return dict(type=contact.get('contactType'), first_name=first_name, @@ -682,12 +687,14 @@ def entity_info(e, langs=None): d['is_idp'] = is_idp(e) d['is_sp'] = is_sp(e) d['is_aa'] = is_aa(e) - d['xml'] = dumptree(e, xml_declaration=False, pretty_print=True).decode('utf8').replace('<','<').replace('>','>') + d['xml'] = dumptree(e, xml_declaration=False, pretty_print=True).decode('utf8').replace('<', '<').replace('>', + '>') if d['is_idp']: - d['protocols'] = entity_idp(e).get('protocolSupportEnumeration',"").split() + d['protocols'] = entity_idp(e).get('protocolSupportEnumeration', "").split() return d + def entity_extensions(e): """Return a list of the Extensions elements in the EntityDescriptor @@ -760,7 +767,7 @@ def set_entity_attributes(e, d, nf=NF_URI): if e.tag != "{%s}EntityDescriptor" % NS['md']: raise MetadataException("I can only add EntityAttribute(s) to EntityDescriptor elements") - for attr, value in d.iteritems(): + for attr, value in d.items(): a = _eattribute(e, attr, nf) velt = etree.Element("{%s}AttributeValue" % NS['saml']) velt.text = value @@ -804,7 +811,7 @@ def set_reginfo(e, policy=None, authority=None): ri = etree.Element("{%s}RegistrationInfo" % NS['mdrpi']) ext.append(ri) ri.set('registrationAuthority', authority) - for lang, policy_url in policy.iteritems(): + for lang, policy_url in policy.items(): rp = etree.Element("{%s}RegistrationPolicy" % NS['mdrpi']) rp.text = policy_url rp.set('{%s}lang' % NS['xml'], lang) diff --git a/src/pyff/store.py b/src/pyff/store.py index 1d8fc4d0..21246b83 100644 --- a/src/pyff/store.py +++ b/src/pyff/store.py @@ -85,7 +85,7 @@ def _m(idx, vv): _m(self.index[hn], hash_id(entity, hn, False)) attr_idx = self.index.setdefault('attr', {}) - for attr, values in entity_attribute_dict(entity).iteritems(): + for attr, values in entity_attribute_dict(entity).items(): vidx = attr_idx.setdefault(attr, {}) for v in values: _m(vidx, v) @@ -113,7 +113,7 @@ def _get_index(self, a, v): else: m = re.compile(v) entities = [] - for value, ents in idx.iteritems(): + for value, ents in idx.items(): if m.match(value): entities.extend(ents) return entities @@ -278,7 +278,7 @@ def update(self, t, tid=None, ts=None, merge_strategy=None): # TODO: merge ? entity_id = relt.get("entityID") if entity_id is not None: self.membership("entities", entity_id, ts, p) - for ea, eav in entity_attribute_dict(relt).iteritems(): + for ea, eav in entity_attribute_dict(relt).items(): for v in eav: # log.debug("%s=%s" % (ea, v)) self.membership("{%s}%s" % (ea, v), tid, ts, p) diff --git a/src/pyff/test/test_utils.py b/src/pyff/test/test_utils.py index 1df32574..73bf2a24 100644 --- a/src/pyff/test/test_utils.py +++ b/src/pyff/test/test_utils.py @@ -74,7 +74,7 @@ def test_resource_filename(self): assert(resource_filename(tmp) == tmp) (d, fn) = os.path.split(tmp) assert(resource_filename(fn, d) == tmp) - except IOError, ex: + except IOError as ex: raise ex finally: try: @@ -95,7 +95,7 @@ def test_resource_string(self): assert(resource_string(tmp) == "test") (d, fn) = os.path.split(tmp) assert(resource_string(fn, d) == "test") - except IOError, ex: + except IOError as ex: raise ex finally: try: