Connects to #1287. Snovault conversion

.travis.yml

@@ -49,7 +49,7 @@ before_script:
         --pidfile $SC_PIDFILE \
         --tunnel-identifier $TRAVIS_JOB_NUMBER \
         --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY > /dev/null &
-      while test -f "$SC_PIDFILE" && test ! -f "$SC_READYFILE"; do 
+      while test -f "$SC_PIDFILE" && test ! -f "$SC_READYFILE"; do
         sleep 0.5
       done
     fi
@@ -67,7 +67,9 @@ script:
         --browser-arg-int build  "$TRAVIS_BUILD_NUMBER" \
         --browser-arg-int idleTimeout 600 \
         --browser-arg name "$TRAVIS_REPO_SLUG $TRAVIS_BRANCH $TRAVIS_COMMIT" \
-        --browser-arg browser "$BROWSER"
+        --browser-arg browser "$BROWSER" \
+        --wsgi-arg port_range.min 65525 \
+        --wsgi-arg port_range.max 65535
     fi
 after_script:
   - >

README.rst

@@ -77,7 +77,7 @@ Note: These will all be installed locally for the application and should never c
 
 Step 1b: Run buildout::
 
-    $ python3.4 bootstrap.py -v 2.3.1 --setuptools-version 15.2
+    $ python3 bootstrap.py -v 2.4.1 --setuptools-version 18.5
     $ bin/buildout -c buildout-dev.cfg
 
 If you see a clang error like this::
@@ -132,7 +132,7 @@ Run the Pyramid tests with::
 
 Run the Browser tests with::
 
-    $ bin/test -m bdd -v -v
+    $ bin/test -m bdd -v -v --wsgi-arg port_range.min 65525 --wsgi-arg port_range.max 65535
 
 Run the Javascript tests with::
 

buildout.cfg

@@ -28,7 +28,7 @@ parts =
 
 develop = .
 sources-dir = develop
-auto-checkout =
+auto-checkout = snovault
 
 [sources]
 behave = git https://github.com/behave/behave.git
@@ -47,19 +47,16 @@ webtest = git https://github.com/Pylons/webtest.git
 WSGIProxy2 = git https://github.com/lrowe/WSGIProxy2.git
 zope.sqlalchemy = git https://github.com/zopefoundation/zope.sqlalchemy.git
 pytest-bdd = git https://github.com/lrowe/pytest-bdd.git branch=allow-any-step-order
+snovault = git https://github.com/ENCODE-DCC/snovault.git rev=0.24
 
 [versions]
 # Hand set versions
-# XXX set a unique jsonschema version per tag.
-jsonschema = 2.1.0.dev20150513
-pyramid = 1.6a1
+pyramid = 1.6a2
 # Update .travis.yml and cloud-config.yml when updating buildout
-zc.buildout = 2.3.1
-setuptools = 15.2
+zc.buildout = 2.4.1
+setuptools = 18.5
 # https://github.com/Pylons/venusuian/issues/40
 venusian = 1.0.1.dev40
-# https://github.com/cobrateam/splinter/pull/410
-splinter = 0.7.3.dev20150610
 
 [clincoded]
 recipe = zc.recipe.egg

cloud-config.yml

@@ -56,6 +56,7 @@ packages:
 - graphviz
 - libapache2-mod-wsgi-py3
 - libevent-dev
+- libffi-dev
 - libfreetype6-dev
 - libjpeg-dev
 - liblcms2-dev
@@ -143,7 +144,7 @@ runcmd:
 -   sudo -u clincoded psql clincoded < DEV_TEST_DB_DUMP_9.4
 -   sudo -u clincoded psql -c 'update transactions set xid = NULL'
 - fi
-- sudo -u clincoded python3.4 bootstrap.py -v 2.3.1 --setuptools-version 15.2
+- sudo -u clincoded python3.4 bootstrap.py -v 2.4.1 --setuptools-version 18.5
 - sudo -u clincoded LANG=en_US.UTF-8 bin/buildout -c %(ROLE)s.cfg
 - until sudo -u postgres psql postgres -c ""; do sleep 10; done
 - sudo -u clincoded sh -c 'cat /dev/urandom | head -c 256 | base64 > session-secret.b64'

development.ini

@@ -5,7 +5,7 @@
 
 [app:app]
 use = config:base.ini#app
-sqlalchemy.url = postgresql://postgres@:5432/postgres?host=/tmp/clincoded/pgdata
+sqlalchemy.url = postgresql://postgres@:5432/postgres?host=/tmp/snovault/pgdata
 load_test_only = true
 create_tables = true
 testing = true
@@ -17,8 +17,11 @@ pyramid.debug_notfound = true
 pyramid.debug_routematch = false
 pyramid.default_locale_name = en
 
+snovault.load_test_data = clincoded.loadxl:load_test_data
+
 [pipeline:debug]
 pipeline =
+    egg:PasteDeploy#prefix
     egg:repoze.debug#pdbpm
     app
 set pyramid.includes =

docs/database-dumping.rst

@@ -11,28 +11,28 @@ How?
 1. Start up a local test server with the test data
 2. Dump the test data database using this command::
 
-    pg_dump --no-owner postgresql://postgres@:5432/postgres?host=/tmp/clincoded/pgdata > DEV_TEST_DB_DUMP_9.4
+    pg_dump --no-owner postgresql://postgres@:5432/postgres?host=/tmp/snovault/pgdata > DEV_TEST_DB_DUMP_9.4
 
 3. Stop the server
-4. Move the test ``gene.json`` and ``orphaPhenotype.json`` files out of ``src/clincoded/tests/data/inserts``, and replace them with the ones in ``src/clincoded/tests/data/insertsComplete``::
+4. Move the test ``gene.json`` and ``orphaphenotype.json`` files out of ``src/clincoded/tests/data/inserts``, and replace them with the ones in ``src/clincoded/tests/data/insertsComplete``::
 
     mv src/clincoded/tests/data/inserts/gene.json src/clincoded/tests/data/
-    mv src/clincoded/tests/data/inserts/orphaPhenotype.json src/clincoded/tests/data/
+    mv src/clincoded/tests/data/inserts/orphaphenotype.json src/clincoded/tests/data/
     cp src/clincoded/tests/data/insertsComplete/gene.json src/clincoded/tests/data/inserts/gene.json
-    cp src/clincoded/tests/data/insertsComplete/orphaPhenotype.json src/clincoded/tests/data/inserts/orphaPhenotype.json
+    cp src/clincoded/tests/data/insertsComplete/orphaphenotype.json src/clincoded/tests/data/inserts/orphaphenotype.json
 
 5. Start the server again; this time the process will take significantly longer as its loading the complete Gene and Disease datasets
 6. Dump the complete data database using this command::
 
-    pg_dump --no-owner postgresql://postgres@:5432/postgres?host=/tmp/clincoded/pgdata > DEV_TEST_DB_DUMP_GENES_9.4
+    pg_dump --no-owner postgresql://postgres@:5432/postgres?host=/tmp/snovault/pgdata > DEV_TEST_DB_DUMP_GENES_9.4
 
 7. Stop the server
 8. Restore the test data files so you you don't have to go through loading the huge dataset every time you spin up a server for testing purposes::
 
     rm src/clincoded/tests/data/inserts/gene.json
-    rm src/clincoded/tests/data/inserts/orphaPhenotype.json
+    rm src/clincoded/tests/data/inserts/orphaphenotype.json
     mv src/clincoded/tests/data/gene.json src/clincoded/tests/data/inserts/
-    mv src/clincoded/tests/data/orphaPhenotype.json src/clincoded/tests/data/inserts/
+    mv src/clincoded/tests/data/orphaphenotype.json src/clincoded/tests/data/inserts/
 
 9. Commit and push your newly updated database dumps.
 

package.json

@@ -23,6 +23,7 @@
     "jest-cli": "^18.0.0"
   },
   "dependencies": {
+    "auth0-lock": "^10.4.0",
     "babel-core": "^6.18.2",
     "babel-jest": "^18.0.0",
     "babel-plugin-transform-runtime": "^6.6.0",

production.ini.in

@@ -48,7 +48,7 @@ propagate = 0
 [logger_clincoded_listener]
 level = INFO
 handlers = console
-qualname = clincoded.commands.es_index_listener
+qualname = snovault.elasticsearch.es_index_listener
 propagate = 0
 
 [handler_console]

pytest.ini

@@ -1,9 +1,9 @@
 [pytest]
 addopts =
     --pyargs clincoded.tests
-    --pyargs contentbase.tests
     -p clincoded.tests
     --instafail
     --splinter-make-screenshot-on-failure=false
     --splinter-implicit-wait=5
     --splinter-webdriver=chrome
+    --basetemp=/tmp/pytest

setup.py

@@ -7,17 +7,23 @@
 CHANGES = open(os.path.join(here, 'CHANGES.rst')).read()
 
 requires = [
+    'snovault',
     'Pillow',
     'PyBrowserID',
     'SQLAlchemy>=1.0.0b1',
     'WSGIProxy2',
     'WebTest',
     'boto',
+    'botocore',
+    'jmespath',
+    'boto3',
     'elasticsearch',
+    'lucenequery',
     'future',
     'humanfriendly',
-    'jsonschema',
+    'jsonschema_serialize_fork',
     'loremipsum',
+    'netaddr',
     'passlib',
     'psutil',
     'pyramid',
@@ -46,7 +52,9 @@
 tests_require = [
     'pytest>=2.4.0',
     'pytest-bdd',
+    'pytest-mock',
     'pytest-splinter',
+    'pytest_exact_fixtures',
 ]
 
 setup(
@@ -69,17 +77,17 @@
     },
     entry_points='''
         [console_scripts]
-        batchupgrade = contentbase.batchupgrade:main
-        create-mapping = contentbase.elasticsearch.create_mapping:main
+        batchupgrade = snovault.batchupgrade:main
+        create-mapping = snovault.elasticsearch.create_mapping:main
+        dev-servers = snovault.dev_servers:main
+        es-index-listener = snovault.elasticsearch.es_index_listener:main
 
         add-date-created = clincoded.commands.add_date_created:main
         check-files = clincoded.commands.check_files:main
         check-rendering = clincoded.commands.check_rendering:main
         deploy = clincoded.commands.deploy:main
-        dev-servers = clincoded.commands.dev_servers:main
         extract_test_data = clincoded.commands.extract_test_data:main
         es-index-data = clincoded.commands.es_index_data:main
-        es-index-listener = clincoded.commands.es_index_listener:main
         import-data = clincoded.commands.import_data:main
         jsonld-rdf = clincoded.commands.jsonld_rdf:main
         migrate-files-aws = clincoded.commands.migrate_files_aws:main
@@ -91,7 +99,7 @@
         main = clincoded:main
 
         [paste.composite_factory]
-        indexer = clincoded.commands.es_index_listener:composite
+        indexer = snovault.elasticsearch.es_index_listener:composite
 
         [paste.filter_app_factory]
         memlimit = clincoded.memlimit:filter_app

src/clincoded/__init__.py

@@ -17,10 +17,12 @@
 from pyramid.settings import asbool
 from sqlalchemy import engine_from_config
 from webob.cookies import JSONSerializer
-from contentbase.storage import (
-    Base,
-    DBSession,
+from snovault.elasticsearch import (
+    PyramidJSONSerializer,
+    TimedUrllib3HttpConnection,
 )
+from snovault.json_renderer import json_renderer
+from elasticsearch import Elasticsearch
 STATIC_MAX_AGE = 0
 
 
@@ -57,12 +59,8 @@ def changelogs(config):
         'profiles/changelogs', 'schemas/changelogs', cache_max_age=STATIC_MAX_AGE)
 
 
-def configure_engine(settings, test_setup=False):
-    from contentbase.json_renderer import json_renderer
-    engine_url = settings.get('sqlalchemy.url')
-    if not engine_url:
-        # Already setup by test fixture
-        return None
+def configure_engine(settings):
+    engine_url = settings['sqlalchemy.url']
     engine_opts = {}
     if engine_url.startswith('postgresql'):
         if settings.get('indexer_worker'):
@@ -82,11 +80,6 @@ def configure_engine(settings, test_setup=False):
         if timeout:
             timeout = int(timeout) * 1000
             set_postgresql_statement_timeout(engine, timeout)
-    if test_setup:
-        return engine
-    if asbool(settings.get('create_tables', False)):
-        Base.metadata.create_all(engine)
-    DBSession.configure(bind=engine)
     return engine
 
 
@@ -108,6 +101,28 @@ def connect(dbapi_connection, connection_record):
             dbapi_connection.commit()
 
 
+def configure_dbsession(config):
+    from snovault import DBSESSION
+    settings = config.registry.settings
+    DBSession = settings.pop(DBSESSION, None)
+    if DBSession is None:
+        engine = configure_engine(settings)
+
+        if asbool(settings.get('create_tables', False)):
+            from snovault.storage import Base
+            Base.metadata.create_all(engine)
+
+        import snovault.storage
+        import zope.sqlalchemy
+        from sqlalchemy import orm
+
+        DBSession = orm.scoped_session(orm.sessionmaker(bind=engine))
+        zope.sqlalchemy.register(DBSession)
+        snovault.storage.register(DBSession)
+
+    config.registry[DBSESSION] = DBSession
+
+
 def load_workbook(app, workbook_filename, docsdir, test=False):
     from .loadxl import load_all
     from webtest import TestApp
@@ -153,19 +168,34 @@ def session(config):
     config.set_session_factory(session_factory)
 
 
+def app_version(config):
+    import hashlib
+    import os
+    import subprocess
+    version = subprocess.check_output(
+        ['git', '-C', os.path.dirname(__file__), 'describe']).decode('utf-8').strip()
+    diff = subprocess.check_output(
+        ['git', '-C', os.path.dirname(__file__), 'diff', '--no-ext-diff'])
+    if diff:
+        version += '-patch' + hashlib.sha1(diff).hexdigest()[:7]
+    config.registry.settings['snovault.app_version'] = version
+
+
 def main(global_config, **local_config):
     """ This function returns a Pyramid WSGI application.
     """
     settings = global_config
     settings.update(local_config)
 
-    settings['contentbase.jsonld.namespaces'] = json_asset('clincoded:schemas/namespaces.json')
-    settings['contentbase.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/'
-    settings['contentbase.jsonld.terms_prefix'] = 'encode'
-    settings['contentbase.elasticsearch.index'] = 'clincoded'
+    settings['snovault.jsonld.namespaces'] = json_asset('clincoded:schemas/namespaces.json')
+    settings['snovault.jsonld.terms_namespace'] = 'https://www.encodeproject.org/terms/'
+    settings['snovault.jsonld.terms_prefix'] = 'encode'
+    settings['snovault.elasticsearch.index'] = 'clincoded'
 
     config = Configurator(settings=settings)
-    config.registry['app_factory'] = main  # used by mp_indexer
+    from snovault.elasticsearch import APP_FACTORY
+    config.registry[APP_FACTORY] = main  # used by mp_indexer
+    config.include(app_version)
 
     config.include('pyramid_multiauth')  # must be before calling set_authorization_policy
     from pyramid_localroles import LocalRolesAuthorizationPolicy
@@ -174,8 +204,8 @@ def main(global_config, **local_config):
     config.include(session)
     config.include('.auth0')
 
-    configure_engine(settings)
-    config.include('contentbase')
+    config.include(configure_dbsession)
+    config.include('snovault')
     config.commit()  # commit so search can override listing
 
     # Render an HTML page to browsers and a JSON document for API clients
@@ -188,7 +218,7 @@ def main(global_config, **local_config):
     config.include('.visualization')
 
     if 'elasticsearch.server' in config.registry.settings:
-        config.include('contentbase.elasticsearch')
+        config.include('snovault.elasticsearch')
         config.include('.search')
 
     config.include(static_resources)

src/clincoded/audit/__init__.py

@@ -1,2 +1,3 @@
 def includeme(config):
-    config.scan()
+    pass
+    #config.scan()

src/clincoded/audit/antibody_characterization.py

@@ -1,4 +1,4 @@
-from contentbase.auditor import (
+from snovault import (
     AuditFailure,
     audit_checker,
 )

src/clincoded/audit/antibody_lot.py

@@ -1,4 +1,4 @@
-from contentbase.auditor import (
+from snovault import (
     AuditFailure,
     audit_checker,
 )

src/clincoded/audit/biosample.py

@@ -1,4 +1,4 @@
-from contentbase.auditor import (
+from snovault import (
     AuditFailure,
     audit_checker,
 )

src/clincoded/audit/dataset.py

@@ -1,4 +1,4 @@
-from contentbase.auditor import (
+from snovault import (
     AuditFailure,
     audit_checker,
 )