pep8 and import cleanup

bootstrap.py

@@ -18,13 +18,13 @@
 use the -c option to specify an alternate configuration file.
 """
 
-import os
 import shutil
 import sys
 import tempfile
-
 from optparse import OptionParser
 
+import os
+
 __version__ = '2015-07-01'
 # See zc.buildout's changelog if this version is up to date.
 

docs/conf.py

@@ -12,6 +12,7 @@
 # serve to show the default.
 
 import sys
+
 import os
 
 # If extensions (or modules to document with autodoc) are in another directory,

scripts/csv2xrd.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
-import os
-import sys
 import io
+import sys
+
 from lxml import etree
 
 ns = {None: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}

scripts/yamltest.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
-import yaml
 import sys
 
+import yaml
+
 with open(sys.argv[1]) as fd:
     print yaml.safe_load(fd)

setup.py

@@ -2,11 +2,12 @@
 # -*- encoding: utf-8 -*-
 
 from distutils.core import setup
-from os.path import abspath, dirname, join
 from platform import python_implementation
-from setuptools import find_packages
 from sys import version_info
 
+from os.path import abspath, dirname, join
+from setuptools import find_packages
+
 __author__ = 'Leif Johansson'
 __version__ = '0.10.0dev'
 

src/pyff/__init__.py

@@ -3,10 +3,11 @@
 """
 
 import pkg_resources
+from . import builtins
 
 __author__ = 'Leif Johansson'
 __copyright__ = "Copyright 2009-2014 SUNET"
 __license__ = "BSD"
 __maintainer__ = "leifj@sunet.se"
 __status__ = "Production"
-__version__ = pkg_resources.require("pyFF")[0].version
+__version__ = pkg_resources.require("pyFF")[0].version

src/pyff/builtins.py

@@ -1,27 +1,29 @@
 """Package that contains the basic set of pipes - functions that can be used to put together a processing pipeling
 for pyFF.
 """
-from distutils.util import strtobool
+import base64
+import hashlib
 import json
+import sys
 import traceback
+from copy import deepcopy
+from datetime import datetime
+from distutils.util import strtobool
+
+import os
+import re
+import xmlsec
+import yaml
 from iso8601 import iso8601
 from lxml.etree import DocumentInvalid
-import yaml
+
+from pyff.constants import NS
 from pyff.decorators import deprecated
+from pyff.logs import log
+from pyff.pipes import Plumbing, PipeException, PipelineCallback, pipe
 from pyff.stats import set_metadata_info
 from pyff.utils import total_seconds, dumptree, safe_write, root, duration2timedelta, xslt_transform, \
     iter_entities, validate_document
-from pyff.constants import NS
-from pyff.pipes import Plumbing, PipeException, PipelineCallback, pipe
-from copy import deepcopy
-import sys
-import os
-import re
-from pyff.logs import log
-import hashlib
-import xmlsec
-import base64
-from datetime import datetime
 
 try:
     from cStringIO import StringIO
@@ -48,6 +50,7 @@ def dump(req, *opts):
     else:
         print "<EntitiesDescriptor xmlns=\"%s\"/>" % NS['md']
 
+
 @pipe
 def end(req, *opts):
     """
@@ -144,7 +147,7 @@ def fork(req, *opts):
         nt = deepcopy(req.t)
 
     ip = Plumbing(pipeline=req.args, pid="%s.fork" % req.plumbing.pid)
-    #ip.process(req.md,t=nt)
+    # ip.process(req.md,t=nt)
     ireq = Plumbing.Request(ip, req.md, nt)
     ip._process(ireq)
 
@@ -233,7 +236,7 @@ def _pipe(req, *opts):
     - two
 
     """
-    #req.process(Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.pid))
+    # req.process(Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.pid))
     ot = Plumbing(pipeline=req.args, pid="%s.pipe" % req.plumbing.id)._process(req)
     req.done = False
     return ot
@@ -265,9 +268,9 @@ def when(req, condition, *values):
 The condition operates on the state: if 'foo' is present in the state (with any value), then the something branch is
 followed. If 'bar' is present in the state with the value 'bill' then the other branch is followed.
     """
-    #log.debug("condition key: %s" % repr(condition))
+    # log.debug("condition key: %s" % repr(condition))
     c = req.state.get(condition, None)
-    #log.debug("condition %s" % repr(c))
+    # log.debug("condition %s" % repr(c))
     if c is not None:
         if not values or _any(values, c):
             return Plumbing(pipeline=req.args, pid="%s.when" % req.plumbing.id)._process(req)
@@ -343,6 +346,7 @@ def publish(req, *opts):
         req.md.store.update(req.t, tid=resource_name)  # TODO maybe this is not the right thing to do anymore
     return req.t
 
+
 @pipe
 def loadstats(req, *opts):
     """
@@ -365,25 +369,29 @@ def loadstats(req, *opts):
 
     log.info("pyff loadstats: %s" % _stats)
 
+
 @pipe
 @deprecated
 def remote(req, *opts):
     """Deprecated. Calls :py:mod:`pyff.pipes.builtins.load`.
     """
     return load(req, opts)
 
+
 @pipe
 @deprecated
 def local(req, *opts):
     """Deprecated. Calls :py:mod:`pyff.pipes.builtins.load`.
     """
     return load(req, opts)
 
+
 @pipe
 @deprecated
 def _fetch(req, *opts):
     return load(req, *opts)
 
+
 @pipe
 def load(req, *opts):
     """
@@ -436,7 +444,8 @@ def load(req, *opts):
         log.debug("load parsing '%s'" % x)
         r = x.split()
 
-        assert len(r) in range(1, 7), PipeException("Usage: load resource [as url] [[verify] verification] [via pipeline]")
+        assert len(r) in range(1, 7), PipeException(
+            "Usage: load resource [as url] [[verify] verification] [via pipeline]")
 
         url = r.pop(0)
         params = dict()
@@ -466,17 +475,19 @@ def load(req, *opts):
         elif os.path.exists(url):
             if os.path.isdir(url):
                 log.debug("directory %s verify %s as %s via %s" % (url, params['verify'], params['as'], params['via']))
-                req.md.load_dir(url, url=params['as'], validate=opts['validate'], post=post, fail_on_error=opts['fail_on_error'], filter_invalid=opts['filter_invalid'])
+                req.md.load_dir(url, url=params['as'], validate=opts['validate'], post=post,
+                                fail_on_error=opts['fail_on_error'], filter_invalid=opts['filter_invalid'])
             elif os.path.isfile(url):
                 log.debug("file %s verify %s as %s via %s" % (url, params['verify'], params['as'], params['via']))
                 remote.append(("file://%s" % url, params['verify'], params['as'], post))
             else:
-                error="Unknown file type for load: '%s'" % url
+                error = "Unknown file type for load: '%s'" % url
                 if opts['fail_on_error']:
                     raise PipeException(error)
                 log.error(error)
         else:
-            error="Don't know how to load '%s' as %s verify %s via %s (file does not exist?)" % (url, params['as'], params['verify'], params['via'])
+            error = "Don't know how to load '%s' as %s verify %s via %s (file does not exist?)" % (
+            url, params['as'], params['verify'], params['via'])
             if opts['fail_on_error']:
                 raise PipeException(error)
             log.error(error)
@@ -498,6 +509,7 @@ def _select_args(req):
 
     return args
 
+
 @pipe
 def select(req, *opts):
     """
@@ -584,6 +596,7 @@ def select(req, *opts):
 
     return ot
 
+
 @pipe(name="filter")
 def _filter(req, *opts):
     """
@@ -634,9 +647,10 @@ def _find(member):
     if ot is None:
         raise PipeException("empty filter - stop")
 
-    #print "filter returns %s" % [e for e in iter_entities(ot)]
+    # print "filter returns %s" % [e for e in iter_entities(ot)]
     return ot
 
+
 @pipe
 def pick(req, *opts):
     """
@@ -654,6 +668,7 @@ def pick(req, *opts):
         raise PipeException("empty select '%s' - stop" % ",".join(args))
     return ot
 
+
 @pipe
 def first(req, *opts):
     """
@@ -697,6 +712,7 @@ def discojson(req, *opts):
 
     return json.dumps([req.md.discojson(e) for e in iter_entities(req.t)])
 
+
 @pipe
 def sign(req, *opts):
     """
@@ -795,6 +811,7 @@ def stats(req, *opts):
     print "---"
     return req.t
 
+
 @pipe
 def store(req, *opts):
     """
@@ -833,6 +850,7 @@ def store(req, *opts):
             safe_write("%s.xml" % os.path.join(target_dir, d), dumptree(e, pretty_print=True))
     return req.t
 
+
 @pipe
 def xslt(req, *opts):
     """
@@ -866,11 +884,12 @@ def xslt(req, *opts):
     del params['stylesheet']
     try:
         return xslt_transform(req.t, stylesheet, params)
-        #log.debug(ot)
+        # log.debug(ot)
     except Exception, ex:
         traceback.print_exc(ex)
         raise ex
 
+
 @pipe
 def validate(req, *opts):
     """
@@ -888,6 +907,7 @@ def validate(req, *opts):
 
     return req.t
 
+
 @pipe
 def certreport(req, *opts):
     """
@@ -998,6 +1018,7 @@ def certreport(req, *opts):
             except Exception, ex:
                 log.error(ex)
 
+
 @pipe
 def emit(req, ctype="application/xml", *opts):
     """
@@ -1043,6 +1064,7 @@ def emit(req, ctype="application/xml", *opts):
     req.state['headers']['Content-Type'] = ctype
     return unicode(d.decode('utf-8')).encode("utf-8")
 
+
 @pipe
 def signcerts(req, *opts):
     """
@@ -1067,6 +1089,7 @@ def signcerts(req, *opts):
         log.info("found signing cert with fingerprint %s" % fp)
     return req.t
 
+
 @pipe
 def finalize(req, *opts):
     """
@@ -1136,13 +1159,13 @@ def finalize(req, *opts):
         elif valid_until is not None:
             try:
                 dt = iso8601.parse_date(valid_until)
-                dt = dt.replace(tzinfo=None) # make dt "naive" (tz-unaware)
+                dt = dt.replace(tzinfo=None)  # make dt "naive" (tz-unaware)
                 offset = dt - now
                 e.set('validUntil', dt.strftime("%Y-%m-%dT%H:%M:%SZ"))
             except ValueError, ex:
                 log.error("Unable to parse validUntil: %s (%s)" % (valid_until, ex))
 
-            # set a reasonable default: 50% of the validity
+                # set a reasonable default: 50% of the validity
         # we replace this below if we have cacheDuration set
         req.state['cache'] = int(total_seconds(offset) / 50)
 
@@ -1157,6 +1180,7 @@ def finalize(req, *opts):
 
     return req.t
 
+
 @pipe(name='reginfo')
 def _reginfo(req, *opts):
     """
@@ -1186,6 +1210,7 @@ def _reginfo(req, *opts):
 
     return req.t
 
+
 @pipe(name='pubinfo')
 def _pubinfo(req, *opts):
     """
@@ -1212,6 +1237,7 @@ def _pubinfo(req, *opts):
 
     return req.t
 
+
 @pipe(name='setattr')
 def _setattr(req, *opts):
     """
@@ -1240,7 +1266,7 @@ def _setattr(req, *opts):
         raise PipeException("Your pipeline is missing a select statement.")
 
     for e in iter_entities(req.t):
-        #log.debug("setting %s on %s" % (req.args,e.get('entityID')))
+        # log.debug("setting %s on %s" % (req.args,e.get('entityID')))
         req.md.set_entity_attributes(e, req.args)
 
     return req.t

src/pyff/decorators.py

@@ -1,8 +1,8 @@
 """
 Various decorators used in pyFF.
 """
-from collections import namedtuple
 import functools
+from collections import namedtuple
 
 __author__ = 'leifj'