Merge pull request #92 from fgpv-vpgf/develop

Update Master for the release of v3.0.0

.travis.yml

@@ -12,7 +12,7 @@ notifications:
     urls:
     - https://webhooks.gitter.im/e/7bd6d69bcfa59bddf1e2
 python:
-- '2.7'
+- '3.6.4'
 script:
 - make check
 - source docs/docdeploy.sh

Makefile

@@ -1,8 +1,8 @@
 check:
-	flake8 && py.test
+	flake8 && PYTHONPATH=/vagrant pytest
 
 build:
-	python setup.py sdist --formats=bztar,zip
+	python3.6 setup.py sdist --formats=bztar,zip
 
 serve:
-	python run.py --listen-all
+	python3.6 run.py --listen-all

Vagrantfile

@@ -8,7 +8,7 @@
 Vagrant.configure(2) do |config|
   # Every Vagrant development environment requires a box. You can search for
   # boxes at https://atlas.hashicorp.com/search.
-  config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "ubuntu/xenial64"
 
   # Create a forwarded port mapping which allows access to a specific port
   # within the machine from a port on the host machine. In the example below,
@@ -44,18 +44,19 @@ Vagrant.configure(2) do |config|
   # end
 
   config.vm.provision "shell", inline: <<-SHELL
+    sudo add-apt-repository ppa:deadsnakes/ppa
     sudo apt-get update
-    sudo apt-get install -y couchdb python-virtualenv
-    echo 'CouchDB installed'
+    sudo apt-get install -y python3.6 couchdb python3.6-venv
     netstat -antp
     sleep 5
     curl -X PUT http://127.0.0.1:5984/rcs_cache
     curl -X PUT http://127.0.0.1:5984/rcs_auth
     cd /vagrant
-    virtualenv --always-copy -p python2 .
+    python3.6 -m venv .
     . bin/activate
     pip install -r requirements.txt
     pip install -r requirements-dev.txt
-    python seed_qa_keys.py
+    python3.6 seed_qa_keys.py
+    echo 'Provision finished'
   SHELL
 end

config.py

@@ -10,7 +10,7 @@
 ACCESS_LOG = 'access.log'
 LOG_ROTATE_BYTES = 200000
 LOG_BACKUPS = 10
-# https://docs.python.org/2/library/logging.html#levels
+# https://docs.python.org/3/library/logging.html#logging-levels
 # common options DEBUG 10; ERROR 40
 LOG_LEVEL = 10
 

docs/admin/configuration.rst

@@ -6,7 +6,7 @@ RCS Configuration Options
 LOG_FILE
     Name of the log file
 LOG_LEVEL
-    Follows `Python log levels <https://docs.python.org/2/library/logging.html#levels>`_
+    Follows `Python log levels <https://docs.python.org/3/library/logging.html#logging-levels>`_
 LOG_BACKUPS
     Number of backups (rotated log files) to keep
 LOG_ROTATE_BYTES

docs/admin/deployment.rst

@@ -36,7 +36,7 @@ Install `CouchDB <http://couchdb.apache.org/>`_
 Configure Python Environment
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-#. Ensure python is a 2.7.x release
+#. Ensure python is a 3.6.x release
 #. Get an RCS release package ``rcs-X.Y.Z.zip``
 #. Extract the release package, it should be somewhere IIS can be configured to read from ``c:\inetpub\rcs``
 #. [Optional] Get prepackaged dependencies (should be a directory full of ``.whl`` files)
@@ -68,8 +68,8 @@ Configure Python Environment
     * **HTTP_PROXY**
     * **URL_PREFIX** A general prefix for the application, useful if you want to have side by side installs of RCS
 
-#. Test the installation ``python run.py`` (this will run a test server on localhost)
-#. Seed the database ``python seed_qa_keys.py`` (**change these values before running on Production!**)
+#. Test the installation ``python3.6 run.py`` (this will run a test server on localhost)
+#. Seed the database ``python3.6 seed_qa_keys.py`` (**change these values before running on Production!**)
 
 IIS Integration
 ^^^^^^^^^^^^^^^
@@ -133,7 +133,7 @@ The following steps can be used in lieu of :ref:`pyvenvconfig` :
     * **HTTP_PROXY**
     * **URL_PREFIX** A general prefix for the application, useful if you want to have side by side installs of RCS
 
-#. Test the installation ``python rcs.py`` (this will run a test server on localhost)
+#. Test the installation ``python3.6 rcs.py`` (this will run a test server on localhost)
 #. Follow any other version specific upgrade notes in this section.
 #. Update IIS's FastCGI Environment variables collection: change WSGI_HANDLER's value from ``rcs.app`` to ``run.app``.
 

docs/conf.py

@@ -58,9 +58,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.3'
+version = '3.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.3.1'
+release = '3.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/dev/building.rst

@@ -14,7 +14,7 @@ If addtional dependencies are not required a source distribution will likely be
 #. Initialize the virtual environment ``scripts\activate``
 #. Edit ``setup.py`` if version or packaged dependencies have changed
 #. Edit ``MANIFEST.in`` if any other dependencies (e.g. docs, static test files) have changed
-#. Execute ``python setup.py sdist --formats=bztar,zip``
+#. Execute ``python3.6 setup.py sdist --formats=bztar,zip``
 #. A distributable source package should be available in the ``dist`` folder
 
 Packaging RCS and Dependencies

docs/dev/setup.rst

@@ -14,7 +14,7 @@ Install `CouchDB <http://couchdb.apache.org/>`_
 
 Configure Python Environment
 ----------------------------
-#. Ensure python is a 2.7.x release
+#. Ensure python is a 3.6.x release
 #. Install pip (https://pip.pypa.io/en/latest/installing.html)
 #. Install virtualenv ``pip install virtualenv``
 #. Clone the repo ``git clone git@github.com:RAMP-PCAR/RCS.git rcs``
@@ -23,17 +23,17 @@ Configure Python Environment
 #. Activate the virtualenv ``scripts\activate``
 #. Install the project dependencies ``pip install -r requirements.txt``
 #. Make any path changes necessary to ``config.py`` (if you followed the above there should be none)
-#. Test the installation ``python rcs.py`` (this will run a test server on localhost)
-#. Seed the database ``python seed_qa_keys.py``
+#. Test the installation ``python3.6 rcs.py`` (this will run a test server on localhost)
+#. Seed the database ``python3.6 seed_qa_keys.py``
 
 Set up Development Environment and run RCS locally
---------------------------------------------------------
+--------------------------------------------------
 #. Install Vagrant (https://www.vagrantup.com/downloads.html)
 #. Clone RCS repo ``git clone https://github.com/fgpv-vpgf/rcs.git``
 #. Activate Vagrant ``vagrant up`` (Note: Please ensure a Vagrant file exits)
 #. Establish a connection ``vagrant ssh``
 #. Change to the Vagrant folder ``cd /vagrant``
 #. Activate the programmes required ``. bin/activate``
 #. If vagrant failed at any point, run ``vagrant destroy`` then repeat the previous steps again
-#. Run RCS ``python run.py --listen-all``
+#. Run RCS ``python3.6 run.py --listen-all``
 #. Go to http://localhost:6101/static/test.html for testing (Note: The port number might be different)

readme.rst

@@ -17,4 +17,4 @@ Python with a fairly minimal store and retrieve API.
 RCS is developed by Environment and Climate Change Canada as part of the RAMP project and the Federal Geospatial Platform and is licensed
 under the MIT license.
 
-**RAMP Configuration Service Docs:** http://fgpv-vpgf.github.io/rcs/v2.3.1/
+**RAMP Configuration Service Docs:** http://fgpv-vpgf.github.io/rcs/v3.0.0/

requirements-dev.txt

@@ -1,3 +1,3 @@
-pytest==2.8.2
+pytest==3.4.2
 Sphinx==1.6.6
-flake8==2.5.2
+flake8==3.5.0

run.py

@@ -14,8 +14,6 @@
 # FIXME clean this up
 app = Flask(__name__)
 
-reload(sys)
-sys.setdefaultencoding('utf8')
 app.config.from_object(config)
 if os.environ.get('RCS_CONFIG'):
     app.config.from_envvar('RCS_CONFIG')
@@ -64,6 +62,7 @@ def before_request():
     flask.g.get_validator = lambda: jsonschema.validators.Draft4Validator(json.load(open(schema_path)))
     flask.g.proxies = {'http': app.config['HTTP_PROXY'], 'https': app.config['HTTP_PROXY']}
 
+
 if app.config.get('DEBUG_ENDPOINTS'):
     @app.after_request
     def after_request(response):

services/__init__.py

@@ -1,2 +1,2 @@
-import db, regparse
+from . import db, regparse
 __all__ = ['db', 'regparse']

services/db/__init__.py

@@ -1,4 +1,4 @@
-from doc import init_doc_db, get_doc, get_raw, put_doc, delete_doc, query, get_all
-from auth import init_auth_db, get_key
+from .doc import init_doc_db, get_doc, get_raw, put_doc, delete_doc, query, get_all
+from .auth import init_auth_db, get_key
 
 __all__ = ['init_doc_db', 'get_doc', 'get_raw', 'put_doc', 'delete_doc', 'query', 'init_auth_db', 'get_key', 'get_all']

services/debug.py

@@ -1,6 +1,7 @@
+import json
 from flask import current_app, Response
 from flask.ext.restful import Resource
-import db, json
+from . import db
 
 
 class FetchFile(Resource):

services/registration.py

@@ -1,4 +1,5 @@
-import regparse, db, json, flask, pycouchdb
+import json, flask, pycouchdb
+from . import regparse, db
 
 from flask import Response, current_app
 from flask.ext.restful import request, abort, Resource
@@ -48,7 +49,7 @@ def refresh_records(day_limit, limit, config):
         req = r['value']['request']
         try:
             v2_node, v1_node = regparse.make_node(key, req, config)
-            db.put_doc(key, v2_node.values()[0]['layerType'], req, layer_config=v2_node, v1_config=v1_node)
+            db.put_doc(key, list(v2_node.values())[0]['layerType'], req, layer_config=v2_node, v1_config=v1_node)
             valid.append(key)
         except Exception as e:
             current_app.logger.warning('Error in refresh', exc_info=e)
@@ -73,9 +74,9 @@ def put(self, key):
         :returns: JSON Response -- 201 on success; 400 with JSON payload of an errors array on failure
         """
         try:
-            req = json.loads(request.data)
+            req = json.loads(request.data.decode("utf-8"))
         except Exception as e:
-            current_app.logger.error(e.message)
+            current_app.logger.error(e)
             return '{"errors":["Unparsable json"]}', 400
         errors = get_registration_errors(req)
         if errors:
@@ -94,7 +95,7 @@ def put(self, key):
 
         current_app.logger.debug(v2_node)
         current_app.logger.debug(v1_node)
-        db.put_doc(key, v2_node.values()[0]['layerType'], req, layer_config=v2_node, v1_config=v1_node)
+        db.put_doc(key, list(v2_node.values())[0]['layerType'], req, layer_config=v2_node, v1_config=v1_node)
         current_app.logger.info('added a key %s' % key)
         return Response(json.dumps(dict(key=key)), mimetype='application/json', status=201)
 
@@ -137,15 +138,15 @@ def post(self, arg, limit=None):
 
         try:
             day_limit = int(arg)
-        except:
+        except Exception:
             pass
         if day_limit is None and arg != 'all' or day_limit is not None and day_limit < 1:
             return '{"error":"argument should be either \'all\' or a positive integer"}', 400
 
         if limit is not None:
             try:
                 rec_limit = int(limit)
-            except:
+            except Exception:
                 return '{"error":"limit must be positive integer if specified"}', 400
         return Response(json.dumps(refresh_records(day_limit, rec_limit, current_app.config)),
                         mimetype='application/json')

services/regparse/__init__.py

@@ -1,4 +1,4 @@
-import ogc, esri, sigcheck
-from universal import ServiceTypes, ServiceEndpointException, make_node, make_id
+from . import ogc, esri, sigcheck
+from .universal import ServiceTypes, ServiceEndpointException, make_node, make_id
 __all__ = ['esri', 'sigcheck', 'ogc', 'make_id',
            'ServiceTypes', 'ServiceEndpointException', 'make_node']

services/regparse/esri.py

@@ -170,7 +170,7 @@ def test_small_layer(svc_url, svc_data):
             r = requests.get(get_base_url(svc_url) + id_query, proxies=flask.g.proxies)
             if 'objectIds' in r.json():
                 return len(r.json()['objectIds']) <= 2000
-    except:
+    except Exception:
         pass
     return False
 

services/regparse/sigcheck.py

@@ -23,11 +23,10 @@ def sign(key, *parts):
     :type key: str
     :returns: str -- a URL safe base64 encoded signature
     """
-    u8parts = [p.encode('utf8') for p in parts]
-    msg = str('').join(u8parts)
+    msg = str('').join(parts)
     logging.debug(msg)
-    h = hmac.new(str(key), msg, digestmod=hashlib.sha256)
-    return base64.urlsafe_b64encode(h.digest()).replace('=', '')
+    h = hmac.new(bytes(key, 'utf-8'), bytes(msg, 'utf-8'), digestmod=hashlib.sha256)
+    return str(base64.urlsafe_b64encode(h.digest())).replace('=', '')
 
 
 def test_request(request):
@@ -50,12 +49,12 @@ def test_request(request):
         logger.warning('Missing data from headers, sig check failed')
         return False
     rqpath = request.path
-    rqbody = request.data
+    rqbody = request.data.decode("utf-8")
     psk = services.db.auth.get_key(cid)
     if psk and flask.current_app.config.get('PROD') and cid in ['jstest', 'ecdmpdev']:
         raise Exception('Production mode should never be enabled while connected to a database containing test keys')
 
-    ref_sig = sign(psk, rqpath, cid, dt, rqbody)
+    ref_sig = sign(psk, rqpath, cid, dt, str(rqbody))
     logger.info('Signature received: {0}  ##  Signature generated: {1}'.format(msg_sig, ref_sig))
     return ref_sig is not None and ref_sig == msg_sig
 
@@ -103,7 +102,7 @@ def check_time(request):
     logger = get_logger()
     sent = request.headers.get('TimeStamp')
     dt = iso8601.parse_date(sent)
-    now = datetime.datetime.now(iso8601.iso8601.Utc())
+    now = datetime.datetime.now(iso8601.UTC)
     two_min = datetime.timedelta(minutes=2)
     logger.info('Header date: {0}  ##  Current timestamp: {1}'.format(sent, now))
     return -two_min < now-dt < two_min

services/regparse/universal.py

@@ -1,4 +1,5 @@
-import metadata, requests, ogc, esri, re, flask
+import requests, re, flask
+from . import metadata, ogc, esri
 
 
 remapped_types = {'esriMapServer': 'esriDynamic', 'esriFeatureServer': 'esriDynamic'}
@@ -52,8 +53,6 @@ def get_endpoint_type(endpoint, type_hint=None):
             # probably isn't an ESRI endpoint so try GetCapabilities
             endpoint += '?VERSION=1.1.1&REQUEST=GetCapabilities&SERVICE=wms'
         r = requests.get(endpoint)
-        print r.status_code
-        print r.headers
         if ('content-type' in r.headers and xml_regex.search(r.headers['content-type'])):
             # XML response means WMS or WMTS (latter is not implemented)
             # FIXME type detection should be much more robust, add proper XML parsing, ...
@@ -107,7 +106,7 @@ def make_node(key, json_request, config):
                  for lang in langs}
     if len(set(svc_types.values())) > 1:
         raise ServiceEndpointException('Mismatched service types across languages {0}'.format(svc_types.values()))
-    if svc_types.values()[0] in [ServiceTypes.WMS, ServiceTypes.FEATURE]:
+    if list(svc_types.values())[0] in [ServiceTypes.WMS, ServiceTypes.FEATURE]:
         v1 = {lang: {} for lang in langs}
     for lang in langs:
         n = node[lang]

services/retrieval.py

@@ -1,6 +1,7 @@
-import db, json, re
+import json, re
 
-from utils import jsonp
+from . import db
+from .utils import jsonp
 from flask import Response
 from flask.ext.restful import Resource
 

services/update.py

@@ -1,5 +1,6 @@
-import regparse, db, json
+import json
 
+from . import regparse, db
 from flask import Response, current_app
 from flask.ext.restful import request, Resource
 

services/upgrade.py

@@ -1,5 +1,6 @@
-import regparse, db, json
+import json
 
+from . import regparse, db
 from flask import Response, current_app
 from flask.ext.restful import Resource
 
@@ -49,7 +50,6 @@ def post(self, key):
             v1_request = dbdata['data']['request']
             upgrade_method = wms_upgrade if v1_request['payload_type'] == 'wms' else feat_upgrade
             v2_request = {lang: upgrade_method(v1_request[lang]) for lang in current_app.config['LANGS']}
-            print v2_request
             v2_node, v1_node = regparse.make_node(key, v2_request, current_app.config)
             db.put_doc(key, v2_node.values()[0]['layerType'], v2_request, layer_config=v2_node, v1_config=v1_node)
         except Exception as e:

services/v1.py

@@ -1,7 +1,7 @@
 import json, numbers
 
-import regparse, db, registration
-from retrieval import DocV1, DocsV1
+from . import regparse, db, registration
+from .retrieval import DocV1, DocsV1
 from flask import Blueprint, Response, current_app
 from flask.ext.restful import request, Api, Resource
 

services/v2.py

@@ -1,8 +1,8 @@
-from registration import Register, Refresh
-from upgrade import Upgrade
-from update import Update
-from retrieval import DocV2, DocsV2, Version
-from debug import AccessLog, Log, AllKeys
+from .registration import Register, Refresh
+from .upgrade import Upgrade
+from .update import Update
+from .retrieval import DocV2, DocsV2, Version
+from .debug import AccessLog, Log, AllKeys
 
 
 from flask import Blueprint