From 9408e4ed5b24b51eb7e2d5345897c18648433104 Mon Sep 17 00:00:00 2001 From: Leif Johansson Date: Wed, 9 Mar 2016 21:49:48 +0100 Subject: [PATCH] pep8 and import cleanup --- bootstrap.py | 4 +- docs/conf.py | 1 + scripts/csv2xrd.py | 4 +- scripts/yamltest.py | 3 +- setup.py | 5 +- src/pyff/__init__.py | 3 +- src/pyff/builtins.py | 78 ++++--- src/pyff/decorators.py | 2 +- src/pyff/i18n.py | 5 +- src/pyff/locks.py | 3 +- src/pyff/logs.py | 5 +- src/pyff/md.py | 6 +- src/pyff/mdrepo.py | 7 +- src/pyff/mdx.py | 1 + src/pyff/schema/MetadataExchange.xsd | 12 +- ...oasis-200401-wss-wssecurity-secext-1.0.xsd | 7 +- src/pyff/schema/shibboleth.xsd | 15 +- src/pyff/schema/sstc-saml-idp-discovery.xsd | 10 +- src/pyff/schema/ws-federation.xsd | 18 +- src/pyff/stats.py | 2 +- src/pyff/store.py | 6 +- src/pyff/test/__init__.py | 7 +- src/pyff/test/data/metadata/test01.xml | 3 +- .../test/data/metadata/test02-invalid.xml | 3 +- .../test/data/simple-pipeline/idp.aco.net.xml | 4 - .../simple-pipeline/skriptenforum.net.xml | 5 +- src/pyff/test/test_cherrypy.py | 2 +- src/pyff/test/test_decorators.py | 4 +- src/pyff/test/test_log.py | 4 +- src/pyff/test/test_md_main.py | 36 ++-- src/pyff/test/test_observable.py | 1 + src/pyff/test/test_pipeline.py | 15 +- src/pyff/test/test_repo.py | 4 +- src/pyff/test/test_rwlock.py | 1 + src/pyff/test/test_simple_pipeline.py | 2 +- src/pyff/test/test_store.py | 5 +- src/pyff/test/test_time.py | 4 +- src/pyff/test/test_utils.py | 7 +- src/pyff/utils.py | 34 +-- src/pyff/xslt/atom.xsl | 6 +- src/pyff/xslt/atom2html.xsl | 2 +- src/pyff/xslt/discojson.xsl | 7 +- src/pyff/xslt/entity2html.xsl | 193 +++++++++--------- src/pyff/xslt/kalmar2.xsl | 6 +- src/pyff/xslt/tidy.xsl | 6 +- src/pyff/xslt/unsign.xsl | 6 +- 46 files changed, 303 insertions(+), 261 deletions(-) diff --git a/bootstrap.py b/bootstrap.py index a4599211..ff82bebe 100644 --- a/bootstrap.py +++ b/bootstrap.py @@ -18,13 +18,13 @@ use the -c option to specify an alternate configuration file. """ -import os import shutil import sys import tempfile - from optparse import OptionParser +import os + __version__ = '2015-07-01' # See zc.buildout's changelog if this version is up to date. diff --git a/docs/conf.py b/docs/conf.py index 77fddcda..db82c10e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # serve to show the default. import sys + import os # If extensions (or modules to document with autodoc) are in another directory, diff --git a/scripts/csv2xrd.py b/scripts/csv2xrd.py index aac05c0a..5264c8cb 100755 --- a/scripts/csv2xrd.py +++ b/scripts/csv2xrd.py @@ -1,8 +1,8 @@ #!/usr/bin/env python -import os -import sys import io +import sys + from lxml import etree ns = {None: "http://docs.oasis-open.org/ns/xri/xrd-1.0"} diff --git a/scripts/yamltest.py b/scripts/yamltest.py index 3fc450e6..5b29119d 100755 --- a/scripts/yamltest.py +++ b/scripts/yamltest.py @@ -1,6 +1,7 @@ #!/usr/bin/env python -import yaml import sys +import yaml + with open(sys.argv[1]) as fd: print yaml.safe_load(fd) diff --git a/setup.py b/setup.py index 83402db9..0af8e682 100755 --- a/setup.py +++ b/setup.py @@ -2,11 +2,12 @@ # -*- encoding: utf-8 -*- from distutils.core import setup -from os.path import abspath, dirname, join from platform import python_implementation -from setuptools import find_packages from sys import version_info +from os.path import abspath, dirname, join +from setuptools import find_packages + __author__ = 'Leif Johansson' __version__ = '0.10.0dev' diff --git a/src/pyff/__init__.py b/src/pyff/__init__.py index f66ff3de..9d9a43c0 100644 --- a/src/pyff/__init__.py +++ b/src/pyff/__init__.py @@ -3,10 +3,11 @@ """ import pkg_resources +from . import builtins __author__ = 'Leif Johansson' __copyright__ = "Copyright 2009-2014 SUNET" __license__ = "BSD" __maintainer__ = "leifj@sunet.se" __status__ = "Production" -__version__ = pkg_resources.require("pyFF")[0].version +__version__ = pkg_resources.require("pyFF")[0].version \ No newline at end of file diff --git a/src/pyff/builtins.py b/src/pyff/builtins.py index 24e509d6..0cb06fd0 100644 --- a/src/pyff/builtins.py +++ b/src/pyff/builtins.py @@ -1,27 +1,29 @@ """Package that contains the basic set of pipes - functions that can be used to put together a processing pipeling for pyFF. """ -from distutils.util import strtobool +import base64 +import hashlib import json +import sys import traceback +from copy import deepcopy +from datetime import datetime +from distutils.util import strtobool + +import os +import re +import xmlsec +import yaml from iso8601 import iso8601 from lxml.etree import DocumentInvalid -import yaml + +from pyff.constants import NS from pyff.decorators import deprecated +from pyff.logs import log +from pyff.pipes import Plumbing, PipeException, PipelineCallback, pipe from pyff.stats import set_metadata_info from pyff.utils import total_seconds, dumptree, safe_write, root, duration2timedelta, xslt_transform, \ iter_entities, validate_document -from pyff.constants import NS -from pyff.pipes import Plumbing, PipeException, PipelineCallback, pipe -from copy import deepcopy -import sys -import os -import re -from pyff.logs import log -import hashlib -import xmlsec -import base64 -from datetime import datetime try: from cStringIO import StringIO @@ -48,6 +50,7 @@ def dump(req, *opts): else: print "" % NS['md'] + @pipe def end(req, *opts): """ @@ -144,7 +147,7 @@ def fork(req, *opts): nt = deepcopy(req.t) ip = Plumbing(pipeline=req.args, pid="%s.fork" % req.plumbing.pid) - #ip.process(req.md,t=nt) + # ip.process(req.md,t=nt) ireq = Plumbing.Request(ip, req.md, nt) ip._process(ireq) @@ -233,7 +236,7 @@ def _pipe(req, *opts): - two """ - #req.process(Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.pid)) + # req.process(Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.pid)) ot = Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.id)._process(req) req.done = False return ot @@ -265,9 +268,9 @@ def when(req, condition, *values): The condition operates on the state: if 'foo' is present in the state (with any value), then the something branch is followed. If 'bar' is present in the state with the value 'bill' then the other branch is followed. """ - #log.debug("condition key: %s" % repr(condition)) + # log.debug("condition key: %s" % repr(condition)) c = req.state.get(condition, None) - #log.debug("condition %s" % repr(c)) + # log.debug("condition %s" % repr(c)) if c is not None: if not values or _any(values, c): return Plumbing(pipeline=req.args, pid="%s.when" % req.plumbing.id)._process(req) @@ -343,6 +346,7 @@ def publish(req, *opts): req.md.store.update(req.t, tid=resource_name) # TODO maybe this is not the right thing to do anymore return req.t + @pipe def loadstats(req, *opts): """ @@ -365,6 +369,7 @@ def loadstats(req, *opts): log.info("pyff loadstats: %s" % _stats) + @pipe @deprecated def remote(req, *opts): @@ -372,6 +377,7 @@ def remote(req, *opts): """ return load(req, opts) + @pipe @deprecated def local(req, *opts): @@ -379,11 +385,13 @@ def local(req, *opts): """ return load(req, opts) + @pipe @deprecated def _fetch(req, *opts): return load(req, *opts) + @pipe def load(req, *opts): """ @@ -436,7 +444,8 @@ def load(req, *opts): log.debug("load parsing '%s'" % x) r = x.split() - assert len(r) in range(1, 7), PipeException("Usage: load resource [as url] [[verify] verification] [via pipeline]") + assert len(r) in range(1, 7), PipeException( + "Usage: load resource [as url] [[verify] verification] [via pipeline]") url = r.pop(0) params = dict() @@ -466,17 +475,19 @@ def load(req, *opts): elif os.path.exists(url): if os.path.isdir(url): log.debug("directory %s verify %s as %s via %s" % (url, params['verify'], params['as'], params['via'])) - req.md.load_dir(url, url=params['as'], validate=opts['validate'], post=post, fail_on_error=opts['fail_on_error'], filter_invalid=opts['filter_invalid']) + req.md.load_dir(url, url=params['as'], validate=opts['validate'], post=post, + fail_on_error=opts['fail_on_error'], filter_invalid=opts['filter_invalid']) elif os.path.isfile(url): log.debug("file %s verify %s as %s via %s" % (url, params['verify'], params['as'], params['via'])) remote.append(("file://%s" % url, params['verify'], params['as'], post)) else: - error="Unknown file type for load: '%s'" % url + error = "Unknown file type for load: '%s'" % url if opts['fail_on_error']: raise PipeException(error) log.error(error) else: - error="Don't know how to load '%s' as %s verify %s via %s (file does not exist?)" % (url, params['as'], params['verify'], params['via']) + error = "Don't know how to load '%s' as %s verify %s via %s (file does not exist?)" % ( + url, params['as'], params['verify'], params['via']) if opts['fail_on_error']: raise PipeException(error) log.error(error) @@ -498,6 +509,7 @@ def _select_args(req): return args + @pipe def select(req, *opts): """ @@ -584,6 +596,7 @@ def select(req, *opts): return ot + @pipe(name="filter") def _filter(req, *opts): """ @@ -634,9 +647,10 @@ def _find(member): if ot is None: raise PipeException("empty filter - stop") - #print "filter returns %s" % [e for e in iter_entities(ot)] + # print "filter returns %s" % [e for e in iter_entities(ot)] return ot + @pipe def pick(req, *opts): """ @@ -654,6 +668,7 @@ def pick(req, *opts): raise PipeException("empty select '%s' - stop" % ",".join(args)) return ot + @pipe def first(req, *opts): """ @@ -697,6 +712,7 @@ def discojson(req, *opts): return json.dumps([req.md.discojson(e) for e in iter_entities(req.t)]) + @pipe def sign(req, *opts): """ @@ -795,6 +811,7 @@ def stats(req, *opts): print "---" return req.t + @pipe def store(req, *opts): """ @@ -833,6 +850,7 @@ def store(req, *opts): safe_write("%s.xml" % os.path.join(target_dir, d), dumptree(e, pretty_print=True)) return req.t + @pipe def xslt(req, *opts): """ @@ -866,11 +884,12 @@ def xslt(req, *opts): del params['stylesheet'] try: return xslt_transform(req.t, stylesheet, params) - #log.debug(ot) + # log.debug(ot) except Exception, ex: traceback.print_exc(ex) raise ex + @pipe def validate(req, *opts): """ @@ -888,6 +907,7 @@ def validate(req, *opts): return req.t + @pipe def certreport(req, *opts): """ @@ -998,6 +1018,7 @@ def certreport(req, *opts): except Exception, ex: log.error(ex) + @pipe def emit(req, ctype="application/xml", *opts): """ @@ -1043,6 +1064,7 @@ def emit(req, ctype="application/xml", *opts): req.state['headers']['Content-Type'] = ctype return unicode(d.decode('utf-8')).encode("utf-8") + @pipe def signcerts(req, *opts): """ @@ -1067,6 +1089,7 @@ def signcerts(req, *opts): log.info("found signing cert with fingerprint %s" % fp) return req.t + @pipe def finalize(req, *opts): """ @@ -1136,13 +1159,13 @@ def finalize(req, *opts): elif valid_until is not None: try: dt = iso8601.parse_date(valid_until) - dt = dt.replace(tzinfo=None) # make dt "naive" (tz-unaware) + dt = dt.replace(tzinfo=None) # make dt "naive" (tz-unaware) offset = dt - now e.set('validUntil', dt.strftime("%Y-%m-%dT%H:%M:%SZ")) except ValueError, ex: log.error("Unable to parse validUntil: %s (%s)" % (valid_until, ex)) - # set a reasonable default: 50% of the validity + # set a reasonable default: 50% of the validity # we replace this below if we have cacheDuration set req.state['cache'] = int(total_seconds(offset) / 50) @@ -1157,6 +1180,7 @@ def finalize(req, *opts): return req.t + @pipe(name='reginfo') def _reginfo(req, *opts): """ @@ -1186,6 +1210,7 @@ def _reginfo(req, *opts): return req.t + @pipe(name='pubinfo') def _pubinfo(req, *opts): """ @@ -1212,6 +1237,7 @@ def _pubinfo(req, *opts): return req.t + @pipe(name='setattr') def _setattr(req, *opts): """ @@ -1240,7 +1266,7 @@ def _setattr(req, *opts): raise PipeException("Your pipeline is missing a select statement.") for e in iter_entities(req.t): - #log.debug("setting %s on %s" % (req.args,e.get('entityID'))) + # log.debug("setting %s on %s" % (req.args,e.get('entityID'))) req.md.set_entity_attributes(e, req.args) return req.t diff --git a/src/pyff/decorators.py b/src/pyff/decorators.py index 01aa86ef..c7de4321 100644 --- a/src/pyff/decorators.py +++ b/src/pyff/decorators.py @@ -1,8 +1,8 @@ """ Various decorators used in pyFF. """ -from collections import namedtuple import functools +from collections import namedtuple __author__ = 'leifj' diff --git a/src/pyff/i18n.py b/src/pyff/i18n.py index c5fa85a2..6c6cf5aa 100644 --- a/src/pyff/i18n.py +++ b/src/pyff/i18n.py @@ -1,10 +1,11 @@ __author__ = 'leifj' # -*- coding: utf-8 -*- -import os -import locale import gettext + import cherrypy +import locale +import os # Change this variable to your app name! # The translation files will be under diff --git a/src/pyff/locks.py b/src/pyff/locks.py index e0b6da2c..d62e1bf2 100644 --- a/src/pyff/locks.py +++ b/src/pyff/locks.py @@ -8,9 +8,10 @@ Released under the BSD-license. """ +from contextlib import contextmanager from threading import Condition, Lock, currentThread from time import time -from contextlib import contextmanager + # Read write lock # --------------- diff --git a/src/pyff/logs.py b/src/pyff/logs.py index a7de6e3f..77f2b080 100644 --- a/src/pyff/logs.py +++ b/src/pyff/logs.py @@ -1,8 +1,9 @@ __author__ = 'leifj' -import cherrypy -import syslog import logging +import syslog + +import cherrypy class PyFFLogger(object): diff --git a/src/pyff/md.py b/src/pyff/md.py index 8daaf400..cc69269b 100644 --- a/src/pyff/md.py +++ b/src/pyff/md.py @@ -7,16 +7,16 @@ [--logfile=] [--version] """ +import getopt import importlib +import logging import sys -import getopt import traceback -import logging +from . import __version__ from .mdrepo import MDRepository from .pipes import plumbing from .store import MemoryStore -from . import __version__ def main(): diff --git a/src/pyff/mdrepo.py b/src/pyff/mdrepo.py index c6fbb67b..c5db1706 100644 --- a/src/pyff/mdrepo.py +++ b/src/pyff/mdrepo.py @@ -4,6 +4,7 @@ """ import traceback + from pyff.stats import set_metadata_info, get_metadata_info try: @@ -29,7 +30,7 @@ from .logs import log from .utils import schema, filter_lang, root, duration2timedelta, \ hash_id, MetadataException, find_merge_strategy, entities_list, url2host, subdomains, avg_domain_distance, \ - iter_entities, validate_document, load_url, iso2datetime, xml_error, dumptree, find_entity + iter_entities, validate_document, load_url, iso2datetime, xml_error, find_entity from .constants import NS, NF_URI, EVENT_DROP_ENTITY, EVENT_IMPORT_FAIL @@ -534,7 +535,7 @@ def _process_url(rurl, verifier, tid, post, enable_cache=True): resource = load_url(rurl, timeout=timeout, enable_cache=enable_cache) except Exception, ex: raise MetadataException(ex, "Exception fetching '%s': %s" % (rurl, str(ex)) ) - if (not resource.result): + if not resource.result: raise MetadataException("error fetching '%s'" % rurl) xml = resource.result.strip() retry_resources = [] @@ -576,7 +577,6 @@ def _process_url(rurl, verifier, tid, post, enable_cache=True): expiration=self.expiration, post=post) - if t is None: self.fire(type=EVENT_IMPORT_FAIL, url=rurl) raise MetadataException("no valid metadata found at '%s'" % rurl) @@ -686,6 +686,7 @@ def parse_metadata(self, :param filter_invalid: (default True) remove invalid EntityDescriptor elements rather than raise an errror :param validate: (default: True) set to False to turn off all XML schema validation :param post: A callable that will be called to modify the parse-tree after schema validation +:param validation_errors: A dict that will be used to return validation errors to the caller (but after xinclude processing and signature validation) """ diff --git a/src/pyff/mdx.py b/src/pyff/mdx.py index 0c2b0f5c..75c8b2d1 100644 --- a/src/pyff/mdx.py +++ b/src/pyff/mdx.py @@ -45,6 +45,7 @@ """ import importlib + import pkg_resources try: diff --git a/src/pyff/schema/MetadataExchange.xsd b/src/pyff/schema/MetadataExchange.xsd index 8a54b853..69811e1c 100644 --- a/src/pyff/schema/MetadataExchange.xsd +++ b/src/pyff/schema/MetadataExchange.xsd @@ -41,13 +41,11 @@ No other rights are granted by implication, estoppel or otherwise. --> + targetNamespace='http://schemas.xmlsoap.org/ws/2004/09/mex' + xmlns:tns='http://schemas.xmlsoap.org/ws/2004/09/mex' + xmlns:xs='http://www.w3.org/2001/XMLSchema' + elementFormDefault='qualified' + blockDefault='#all'> diff --git a/src/pyff/schema/oasis-200401-wss-wssecurity-secext-1.0.xsd b/src/pyff/schema/oasis-200401-wss-wssecurity-secext-1.0.xsd index 0195d356..3b7862da 100644 --- a/src/pyff/schema/oasis-200401-wss-wssecurity-secext-1.0.xsd +++ b/src/pyff/schema/oasis-200401-wss-wssecurity-secext-1.0.xsd @@ -7,7 +7,12 @@ This document and translations of it may be copied and furnished to others, and The limited permissions granted above are perpetual and will not be revoked by OASIS or its successors or assigns. This document and the information contained herein is provided on an “AS IS” basis and OASIS DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. --> - + diff --git a/src/pyff/schema/shibboleth.xsd b/src/pyff/schema/shibboleth.xsd index 392fed45..d96330b3 100644 --- a/src/pyff/schema/shibboleth.xsd +++ b/src/pyff/schema/shibboleth.xsd @@ -1,13 +1,12 @@ + xmlns="http://www.w3.org/2001/XMLSchema" + xmlns:ds="http://www.w3.org/2000/09/xmldsig#" + xmlns:xml="http://www.w3.org/XML/1998/namespace" + xmlns:shib="urn:mace:shibboleth:1.0" + elementFormDefault="qualified" + attributeFormDefault="unqualified" + version="1.2"> diff --git a/src/pyff/schema/sstc-saml-idp-discovery.xsd b/src/pyff/schema/sstc-saml-idp-discovery.xsd index 00da6a38..0b6fbf05 100644 --- a/src/pyff/schema/sstc-saml-idp-discovery.xsd +++ b/src/pyff/schema/sstc-saml-idp-discovery.xsd @@ -1,11 +1,11 @@ - diff --git a/src/pyff/schema/ws-federation.xsd b/src/pyff/schema/ws-federation.xsd index 1337dc3c..fe1ae2fe 100644 --- a/src/pyff/schema/ws-federation.xsd +++ b/src/pyff/schema/ws-federation.xsd @@ -21,16 +21,14 @@ INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. --> + xmlns:sp='http://docs.oasis-open.org/ws-sx/ws-securitypolicy/200702' + xmlns:tns='http://docs.oasis-open.org/wsfed/federation/200706' + xmlns:wsa='http://www.w3.org/2005/08/addressing' + xmlns:wsu='http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd' + xmlns:md='urn:oasis:names:tc:SAML:2.0:metadata' + xmlns:auth='http://docs.oasis-open.org/wsfed/authorization/200706' + targetNamespace='http://docs.oasis-open.org/wsfed/federation/200706' + elementFormDefault='qualified'> diff --git a/src/pyff/stats.py b/src/pyff/stats.py index d744e758..a20e31da 100644 --- a/src/pyff/stats.py +++ b/src/pyff/stats.py @@ -4,8 +4,8 @@ """ -import time import logging +import time __author__ = 'leifj' diff --git a/src/pyff/store.py b/src/pyff/store.py index 0df8167e..877a5b81 100644 --- a/src/pyff/store.py +++ b/src/pyff/store.py @@ -4,15 +4,17 @@ print(" *** install cStringIO for better performance") from StringIO import StringIO +import time from copy import deepcopy + import re from redis import Redis -import time + from pyff.constants import NS, ATTRS from pyff.decorators import cached +from pyff.logs import log from pyff.utils import root, dumptree, parse_xml, hex_digest, hash_id, EntitySet, \ url2host, subdomains, has_tag, iter_entities, valid_until_ts -from pyff.logs import log def is_idp(entity): diff --git a/src/pyff/test/__init__.py b/src/pyff/test/__init__.py index 10c6098a..090c9e7b 100644 --- a/src/pyff/test/__init__.py +++ b/src/pyff/test/__init__.py @@ -5,14 +5,15 @@ from StringIO import StringIO import logging -import os import subprocess +import sys import tempfile from unittest import TestCase + +import os import pkg_resources -import sys + from pyff import __version__ as pyffversion -import pyff.builtins # this is so the tests don't have to import builtins class ExitException(Exception): diff --git a/src/pyff/test/data/metadata/test01.xml b/src/pyff/test/data/metadata/test01.xml index 2d04f7db..4637de51 100644 --- a/src/pyff/test/data/metadata/test01.xml +++ b/src/pyff/test/data/metadata/test01.xml @@ -1,5 +1,6 @@ - + example.com diff --git a/src/pyff/test/data/metadata/test02-invalid.xml b/src/pyff/test/data/metadata/test02-invalid.xml index e7305c32..022cb8ba 100644 --- a/src/pyff/test/data/metadata/test02-invalid.xml +++ b/src/pyff/test/data/metadata/test02-invalid.xml @@ -1,5 +1,6 @@ - + example.com diff --git a/src/pyff/test/data/simple-pipeline/idp.aco.net.xml b/src/pyff/test/data/simple-pipeline/idp.aco.net.xml index 06142851..1b6c05d7 100644 --- a/src/pyff/test/data/simple-pipeline/idp.aco.net.xml +++ b/src/pyff/test/data/simple-pipeline/idp.aco.net.xml @@ -1,10 +1,6 @@ diff --git a/src/pyff/test/data/simple-pipeline/skriptenforum.net.xml b/src/pyff/test/data/simple-pipeline/skriptenforum.net.xml index 64c3e7cc..9a3186b2 100644 --- a/src/pyff/test/data/simple-pipeline/skriptenforum.net.xml +++ b/src/pyff/test/data/simple-pipeline/skriptenforum.net.xml @@ -1,5 +1,8 @@ - + diff --git a/src/pyff/test/test_cherrypy.py b/src/pyff/test/test_cherrypy.py index 54add675..6f15b3ff 100644 --- a/src/pyff/test/test_cherrypy.py +++ b/src/pyff/test/test_cherrypy.py @@ -1,5 +1,5 @@ -from unittest import TestCase import logging +from unittest import TestCase class TestStats(TestCase): diff --git a/src/pyff/test/test_decorators.py b/src/pyff/test/test_decorators.py index e1068b22..6ea2b721 100644 --- a/src/pyff/test/test_decorators.py +++ b/src/pyff/test/test_decorators.py @@ -1,8 +1,10 @@ -from StringIO import StringIO import logging +from StringIO import StringIO from time import sleep from unittest import TestCase, skip + from mock import patch + from pyff.decorators import retry, deprecated, cached diff --git a/src/pyff/test/test_log.py b/src/pyff/test/test_log.py index d8f04420..99aac493 100644 --- a/src/pyff/test/test_log.py +++ b/src/pyff/test/test_log.py @@ -1,8 +1,10 @@ import logging +from StringIO import StringIO from unittest import TestCase + from mock import patch + from pyff.logs import log, SysLogLibHandler -from StringIO import StringIO class TestLog(TestCase): diff --git a/src/pyff/test/test_md_main.py b/src/pyff/test/test_md_main.py index e2fc5de8..87925af7 100644 --- a/src/pyff/test/test_md_main.py +++ b/src/pyff/test/test_md_main.py @@ -15,18 +15,17 @@ from pyff.test import SignerTestCase, run_pyffd, run_pyff from pyff.test.test_pipeline import PipeLineTest import os -import random from pyff.md import __doc__ as pyffdoc from pyff import __version__ as pyffversion from pyff.utils import parse_xml, root, validate_document - import random import socket # range of ports where available ports can be found -PORT_RANGE = [33000,60000] +PORT_RANGE = [33000, 60000] MAX_PORT_TRIES = 100 + def find_unbound_port(i=0): """ Returns an unbound port number on 127.0.0.1. @@ -38,9 +37,10 @@ def find_unbound_port(i=0): try: sock.bind(("127.0.0.1", port)) except socket.error: - port = find_unbound_port(i+1) + port = find_unbound_port(i + 1) return port + class PyFFDTest(PipeLineTest): """ Runs twill tests using the pyffd cmdline - only mocks exit @@ -79,7 +79,7 @@ def setUpClass(cls): cls.mdx]) cls.pyffd_thread.start() sleep(1) - for i in range(0,120): + for i in range(0, 120): try: r = requests.get("http://127.0.0.1:%s/status" % cls.port) if r.json() and 'running' in r.json()['status']: @@ -89,7 +89,6 @@ def setUpClass(cls): sleep(1) raise ValueError("unable to start test pyffd server on port %d" % cls.port) - def test_is_running(self): assert (os.path.exists(self.pidfile)) with open(PyFFDTest.pidfile) as pidf: @@ -112,27 +111,30 @@ def test_stats(self): def test_alias_ndn(self): r = requests.get("http://127.0.0.1:%s/ndn.xml" % self.port) assert (r.status_code == 200) - #assert (r.encoding == 'utf8') + # assert (r.encoding == 'utf8') t = parse_xml(StringIO(r.content)) assert (t is not None) assert (root(t).get('entityID') == 'https://idp.nordu.net/idp/shibboleth') validate_document(t) def test_metadata_html(self): - r = requests.get("http://localhost:%s/metadata/%%7Bsha1%%7Dc50752ce1d12c2b37da13a1a396b8e3895d35dd9.html" % self.port) + r = requests.get( + "http://127.0.0.1:%s/metadata/%%7Bsha1%%7Dc50752ce1d12c2b37da13a1a396b8e3895d35dd9.html" % self.port) assert (r.status_code == 200) assert ('text/html' in r.headers['Content-Type']) def test_metadata_xml(self): - r = requests.get("http://localhost:%s/metadata/%%7Bsha1%%7Dc50752ce1d12c2b37da13a1a396b8e3895d35dd9.xml" % self.port) + r = requests.get( + "http://127.0.0.1:%s/metadata/%%7Bsha1%%7Dc50752ce1d12c2b37da13a1a396b8e3895d35dd9.xml" % self.port) assert (r.status_code == 200) assert ('application/xml' in r.headers['Content-Type']) def test_metadata_json(self): - r = requests.get("http://localhost:%s/metadata/%%7Bsha1%%7Dc50752ce1d12c2b37da13a1a396b8e3895d35dd9.json" % self.port) + r = requests.get( + "http://127.0.0.1:%s/metadata/%%7Bsha1%%7Dc50752ce1d12c2b37da13a1a396b8e3895d35dd9.json" % self.port) assert (r.status_code == 200) info = r.json()[0] - assert(type(info) == dict) + assert (type(info) == dict) assert (info['title'] == 'NORDUnet') assert ('nordu.net' in info['scope']) @@ -149,7 +151,7 @@ def test_md_query_single(self): def test_all_entities_parses(self): r = requests.get("http://127.0.0.1:%s/entities" % self.port) assert (r.status_code == 200) - #assert (r.encoding == 'utf8') + # assert (r.encoding == 'utf8') t = parse_xml(StringIO(r.content)) assert (t is not None) validate_document(t) @@ -159,7 +161,8 @@ def test_webfinger_bad_protocol(self): assert (r.status_code == 400) def test_webfinger(self): - r = requests.get("http://127.0.0.1:%s/.well-known/webfinger?resource=http://127.0.0.1:%s" % (self.port, self.port)) + r = requests.get( + "http://127.0.0.1:%s/.well-known/webfinger?resource=http://127.0.0.1:%s" % (self.port, self.port)) assert (r.status_code == 200) assert r.json() @@ -189,7 +192,8 @@ def test_ds_bad_request(self): assert (r.status_code == 400) def test_ds_request(self): - r = requests.get("http://127.0.0.1:%s/role/idp.ds?entityID=https://idp.nordu.net/idp/shibboleth&return=#" % self.port) + r = requests.get( + "http://127.0.0.1:%s/role/idp.ds?entityID=https://idp.nordu.net/idp/shibboleth&return=#" % self.port) assert (r.status_code == 200) def test_ds_search(self): @@ -197,7 +201,6 @@ def test_ds_search(self): assert (r.status_code == 200) assert len(r.json()) == 0 - @classmethod def tearDownClass(cls): SignerTestCase.tearDownClass() @@ -218,6 +221,7 @@ class PyFFTest(PipeLineTest): """ Runs tests through the pyff cmdline - only mocks exit """ + def setUp(self): super(PyFFTest, self).setUp() self.templates = TemplateLookup(directories=[os.path.join(self.datadir, 'simple-pipeline')]) @@ -275,4 +279,4 @@ def tear_down(self): super(PyFFTest, self).tearDown() os.unlink(self.signer) os.unlink(self.output) - os.unlink(self.logfile) \ No newline at end of file + os.unlink(self.logfile) diff --git a/src/pyff/test/test_observable.py b/src/pyff/test/test_observable.py index b2fe6fd7..9f76359b 100644 --- a/src/pyff/test/test_observable.py +++ b/src/pyff/test/test_observable.py @@ -1,4 +1,5 @@ from unittest import TestCase + from pyff.mdrepo import Observable from pyff.utils import debug_observer diff --git a/src/pyff/test/test_pipeline.py b/src/pyff/test/test_pipeline.py index 151c8cea..58aa2432 100644 --- a/src/pyff/test/test_pipeline.py +++ b/src/pyff/test/test_pipeline.py @@ -1,18 +1,19 @@ -import os import shutil -import mock import sys import tempfile +from StringIO import StringIO + +import os +import yaml from mako.lookup import TemplateLookup +from mock import patch from nose.plugins.skip import Skip -import yaml + from pyff.mdrepo import MDRepository, MetadataException from pyff.pipes import plumbing, Plumbing, PipeException from pyff.test import ExitException -from StringIO import StringIO -from pyff.utils import hash_id, parse_xml, resource_filename, root from pyff.test import SignerTestCase -from mock import patch +from pyff.utils import hash_id, parse_xml, resource_filename, root __author__ = 'leifj' @@ -34,7 +35,7 @@ def run_pipeline(self, pl_name, ctx=None, md=MDRepository()): def exec_pipeline(self, pstr): md = MDRepository() p = yaml.load(StringIO(pstr)) - #print p + # print p res = Plumbing(p, pid="test").process(md, state={'batch': True, 'stats': {}}) return res, md diff --git a/src/pyff/test/test_repo.py b/src/pyff/test/test_repo.py index 3895c2ea..d0e3fd44 100644 --- a/src/pyff/test/test_repo.py +++ b/src/pyff/test/test_repo.py @@ -1,10 +1,12 @@ from copy import deepcopy from unittest import TestCase + +import os + from pyff.constants import NS from pyff.mdrepo import MDRepository from pyff.store import MemoryStore from pyff.utils import resource_filename, parse_xml, root, hash_id, MetadataException -import os class TestRepo(TestCase): diff --git a/src/pyff/test/test_rwlock.py b/src/pyff/test/test_rwlock.py index dc901cd9..478645f4 100644 --- a/src/pyff/test/test_rwlock.py +++ b/src/pyff/test/test_rwlock.py @@ -1,6 +1,7 @@ from threading import Thread, current_thread from time import sleep from unittest import TestCase + from pyff.locks import ReadWriteLock diff --git a/src/pyff/test/test_simple_pipeline.py b/src/pyff/test/test_simple_pipeline.py index 932cc74b..596661fe 100644 --- a/src/pyff/test/test_simple_pipeline.py +++ b/src/pyff/test/test_simple_pipeline.py @@ -1,6 +1,6 @@ -import os import tempfile +import os from mako.lookup import TemplateLookup from pyff.constants import NS diff --git a/src/pyff/test/test_store.py b/src/pyff/test/test_store.py index 249713e1..dfcf307a 100644 --- a/src/pyff/test/test_store.py +++ b/src/pyff/test/test_store.py @@ -1,10 +1,11 @@ from unittest import TestCase -from mock import patch + import mockredis +import os + from pyff.constants import ATTRS from pyff.store import MemoryStore, StoreBase, entity_attribute_dict, RedisStore from pyff.utils import resource_filename, parse_xml, root -import os class TestRedisStore(TestCase): diff --git a/src/pyff/test/test_time.py b/src/pyff/test/test_time.py index 39f853d9..528f6617 100644 --- a/src/pyff/test/test_time.py +++ b/src/pyff/test/test_time.py @@ -1,6 +1,8 @@ +from datetime import datetime from unittest import TestCase + from pyff.utils import duration2timedelta, total_seconds, iso_now, iso2datetime, iso_fmt, totimestamp -from datetime import datetime + class TestDuration(TestCase): diff --git a/src/pyff/test/test_utils.py b/src/pyff/test/test_utils.py index aedb8c1b..c94096cd 100644 --- a/src/pyff/test/test_utils.py +++ b/src/pyff/test/test_utils.py @@ -1,11 +1,12 @@ +import copy import tempfile from unittest import TestCase + +import os + from pyff import utils from pyff.constants import NS from pyff.utils import resource_filename, parse_xml, find_entity, root, resource_string, entities_list -import os -import copy - from ..merge_strategies import replace_existing, remove diff --git a/src/pyff/utils.py b/src/pyff/utils.py index 8988f0e2..abb852ce 100644 --- a/src/pyff/utils.py +++ b/src/pyff/utils.py @@ -3,27 +3,29 @@ This module contains various utilities. """ +import hashlib +import io +import tempfile from collections import namedtuple from datetime import timedelta, datetime -import tempfile +from email.utils import parsedate +from threading import local +from time import gmtime, strftime, clock from traceback import print_exc +from urlparse import urlparse + import cherrypy +import httplib2 +import iso8601 import os -import io import pkg_resources import re +from jinja2 import Environment, PackageLoader from lxml import etree -from time import gmtime, strftime, clock + from .constants import NS from .decorators import retry from .logs import log -import httplib2 -import hashlib -from email.utils import parsedate -from urlparse import urlparse -from threading import local -import iso8601 -from jinja2 import Environment, PackageLoader __author__ = 'leifj' @@ -375,8 +377,16 @@ def xslt_transform(t, stylesheet, params=None): xsl = etree.fromstring(resource_string(stylesheet, "xslt")) thread_data.xslt[stylesheet] = etree.XSLT(xsl) transform = thread_data.xslt[stylesheet] - res = transform(t, **params) - return res + try: + return transform(t, **params) + except etree.XSLTApplyError, ex: + for entry in transform.error_log: + log.error('\tmessage from line %s, col %s: %s' % (entry.line, entry.column, entry.message)) + log.error('\tdomain: %s (%d)' % (entry.domain_name, entry.domain)) + log.error('\ttype: %s (%d)' % (entry.type_name, entry.type)) + log.error('\tlevel: %s (%d)' % (entry.level_name, entry.level)) + log.error('\tfilename: %s' % entry.filename) + raise ex def valid_until_ts(elt, default_ts): diff --git a/src/pyff/xslt/atom.xsl b/src/pyff/xslt/atom.xsl index 06393a77..79453320 100644 --- a/src/pyff/xslt/atom.xsl +++ b/src/pyff/xslt/atom.xsl @@ -1,13 +1,9 @@ +> diff --git a/src/pyff/xslt/atom2html.xsl b/src/pyff/xslt/atom2html.xsl index 00defa77..82a09d90 100644 --- a/src/pyff/xslt/atom2html.xsl +++ b/src/pyff/xslt/atom2html.xsl @@ -2,7 +2,7 @@ +> diff --git a/src/pyff/xslt/discojson.xsl b/src/pyff/xslt/discojson.xsl index b3c2a230..59fa3e56 100644 --- a/src/pyff/xslt/discojson.xsl +++ b/src/pyff/xslt/discojson.xsl @@ -2,13 +2,10 @@ + extension-element-prefixes="str" +> diff --git a/src/pyff/xslt/entity2html.xsl b/src/pyff/xslt/entity2html.xsl index bbd213c5..61d6345e 100644 --- a/src/pyff/xslt/entity2html.xsl +++ b/src/pyff/xslt/entity2html.xsl @@ -2,7 +2,6 @@ -
-
-
- -
-
-
-
- - - -
-
-
+ + + @@ -74,7 +70,6 @@ - external @@ -88,7 +83,7 @@ -
+
@@ -102,8 +97,8 @@
-
-
+ + -
+ @@ -124,36 +119,36 @@
-
-

-

-
Unable to display map
-
- - - $(function() { $("#map_canvas").gmap({'zoom': 12, - 'center':' - - - - ', - 'zoomControl': true, - 'zoomControlOptions': {'style': google.maps.ZoomControlStyle.LARGE}, - 'callback': function() { - var self = this; - $('#locationtab').on('shown.bs.tab',function(ev) { - self.refresh(); - }); - - - }})}); - -

-
+ + + + Unable to display map + + + + $(function() { $("#map_canvas").gmap({'zoom': 12, + 'center':' + + + + ', + 'zoomControl': true, + 'zoomControlOptions': {'style': google.maps.ZoomControlStyle.LARGE}, + 'callback': function() { + var self = this; + $('#locationtab').on('shown.bs.tab',function(ev) { + self.refresh(); + }); + + + }})}); + + + -
-
-
+ + + - + @@ -180,20 +175,20 @@ -
-
+ + Protocols -
+ -

Entity Attributes

+ Entity Attributes -
+ @@ -225,9 +220,9 @@ -
-
-
+ + + @@ -238,27 +233,27 @@ -
-
+ + -
+ - + mailto: - + -
-

-

+ + + Protocols @@ -271,10 +266,10 @@ NameID format(s) -
+ -

-
+ + @@ -299,7 +294,7 @@ - + margin-left: 2px; @@ -307,16 +302,16 @@ http://www.alvestrand.no/objectid/.html - - + + margin-left: 2px; http://oid-info.com/get/ - - + + @@ -325,9 +320,9 @@ -
+ -
+ diff --git a/src/pyff/xslt/kalmar2.xsl b/src/pyff/xslt/kalmar2.xsl index aa1dc315..50f40835 100644 --- a/src/pyff/xslt/kalmar2.xsl +++ b/src/pyff/xslt/kalmar2.xsl @@ -2,13 +2,9 @@ +> diff --git a/src/pyff/xslt/tidy.xsl b/src/pyff/xslt/tidy.xsl index f4634f25..9d2db7d4 100644 --- a/src/pyff/xslt/tidy.xsl +++ b/src/pyff/xslt/tidy.xsl @@ -1,12 +1,8 @@ +> diff --git a/src/pyff/xslt/unsign.xsl b/src/pyff/xslt/unsign.xsl index 35c0071d..01eb5976 100644 --- a/src/pyff/xslt/unsign.xsl +++ b/src/pyff/xslt/unsign.xsl @@ -1,12 +1,8 @@ +>