Merge branch 'AnalyticsAutomated:main' into main

analytics_automated/api.py

@@ -1,6 +1,6 @@
 import ast
 import uuid
-from ipware.ip import get_ip
+from ipware.ip import get_client_ip
 from collections import defaultdict
 import pprint
 import logging
@@ -181,7 +181,7 @@ def __prepare_data(self, request):
             data['submission_name'] = request_contents.pop('submission_name')
             data['email'] = request_contents.pop('email')
             data['job'] = request_contents.pop('job')
-            data['ip'] = get_ip(request)
+            data['ip'] = get_client_ip(request)[0]
             # data['UUID'] = str(uuid.uuid1())
         except MultiValueDictKeyError:
             raise MultiValueDictKeyError

analytics_automated/tests/model_factories.py

@@ -27,7 +27,7 @@ def random_string(length=10):
     return u''.join(random.choice(string.ascii_letters) for x in range(length))
 
 
-class QueueTypeFactory(factory.DjangoModelFactory):
+class QueueTypeFactory(factory.django.DjangoModelFactory):
     name = "localhost"
     execution_behaviour = QueueType.LOCALHOST
 
@@ -36,7 +36,7 @@ class Meta:
         django_get_or_create = ('name',)
 
 
-class BackendFactory(factory.DjangoModelFactory):
+class BackendFactory(factory.django.DjangoModelFactory):
     name = factory.Sequence(lambda n: 'test_{}'.format(n))
     queue_type = factory.SubFactory(QueueTypeFactory)
     # ip = ".".join(map(str, (random.randint(0, 255) for _ in range(4))))
@@ -48,7 +48,7 @@ class Meta:
         django_get_or_create = ('name',)
 
 
-class TaskFactory(factory.DjangoModelFactory):
+class TaskFactory(factory.django.DjangoModelFactory):
     backend = factory.SubFactory(BackendFactory)
     name = factory.Sequence(lambda n: 'task_{}'.format(n))
     in_glob = factory.LazyAttribute(lambda t: random_string())
@@ -63,7 +63,7 @@ class Meta:
         django_get_or_create = ('name',)
 
 
-class ConfigurationFactory(factory.DjangoModelFactory):
+class ConfigurationFactory(factory.django.DjangoModelFactory):
     task = factory.SubFactory(TaskFactory)
     type = random.randint(0, 2)
     name = factory.LazyAttribute(lambda t: random_string())
@@ -74,7 +74,7 @@ class Meta:
         model = Configuration
 
 
-class JobFactory(factory.DjangoModelFactory):
+class JobFactory(factory.django.DjangoModelFactory):
     name = factory.Sequence(lambda n: 'job_{}'.format(n))
     runnable = True
 
@@ -83,7 +83,7 @@ class Meta:
         django_get_or_create = ('name',)
 
 
-class StepFactory(factory.DjangoModelFactory):
+class StepFactory(factory.django.DjangoModelFactory):
     job = factory.SubFactory(JobFactory)
     task = factory.SubFactory(TaskFactory)
     ordering = random.randint(0, 20)
@@ -92,15 +92,15 @@ class Meta:
         model = Step
 
 
-class BatchFactory(factory.DjangoModelFactory):
+class BatchFactory(factory.django.DjangoModelFactory):
     UUID = factory.LazyAttribute(lambda t: str(uuid.uuid1()))
     status = random.randint(0, 4)
 
     class Meta:
         model = Batch
 
 
-class SubmissionFactory(factory.DjangoModelFactory):
+class SubmissionFactory(factory.django.DjangoModelFactory):
     job = factory.SubFactory(JobFactory)
     submission_name = factory.Sequence(lambda n: 'submission_{}'.format(n))
     UUID = factory.LazyAttribute(lambda t: str(uuid.uuid1()))
@@ -115,7 +115,7 @@ class Meta:
         django_get_or_create = ('submission_name',)
 
 
-class ParameterFactory(factory.DjangoModelFactory):
+class ParameterFactory(factory.django.DjangoModelFactory):
     task = factory.SubFactory(TaskFactory)
     flag = factory.LazyAttribute(lambda t: random_string())
     default = factory.LazyAttribute(lambda t: random_string())
@@ -126,7 +126,7 @@ class Meta:
         model = Parameter
 
 
-class EnvironmentFactory(factory.DjangoModelFactory):
+class EnvironmentFactory(factory.django.DjangoModelFactory):
     task = factory.SubFactory(TaskFactory)
     env = factory.LazyAttribute(lambda t: random_string())
     value = factory.LazyAttribute(lambda t: random_string())
@@ -135,22 +135,22 @@ class Meta:
         model = Environment
 
 
-class ValidatorTypesFactory(factory.DjangoModelFactory):
+class ValidatorTypesFactory(factory.django.DjangoModelFactory):
     name = factory.LazyAttribute(lambda t: random_string())
 
     class Meta:
         model = ValidatorTypes
 
 
-class ValidatorFactory(factory.DjangoModelFactory):
+class ValidatorFactory(factory.django.DjangoModelFactory):
     job = factory.SubFactory(JobFactory)
     validation_type = factory.SubFactory(ValidatorTypesFactory)
 
     class Meta:
         model = Validator
 
 
-class ResultFactory(factory.DjangoModelFactory):
+class ResultFactory(factory.django.DjangoModelFactory):
     submission = factory.SubFactory(SubmissionFactory)
     task = factory.SubFactory(TaskFactory)
     step = step_value

analytics_automated_project/settings/base.py

@@ -136,7 +136,7 @@ def get_secret(setting, secrets):
 # Application definition
 
 INSTALLED_APPS = (
-    'django_admin_bootstrapped',
+    'bootstrap_admin',
     'django.contrib.admin',
     'django.contrib.auth',
     'django.contrib.contenttypes',
@@ -306,3 +306,5 @@ def get_secret(setting, secrets):
         }
     }
 }
+
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'

analytics_automated_project/settings/dev.py

@@ -20,8 +20,8 @@
 }
 
 CORS_ORIGIN_WHITELIST = (
-        '127.0.0.1:4000',
-        '127.0.0.1:8000',
+        'http://127.0.0.1:4000',
+        'http://127.0.0.1:8000',
     )
 
 SECRET_KEY = get_secret("SECRET_KEY", secrets)

example_scripts/send_fasta.py

@@ -2,10 +2,11 @@
 import requests
 import json
 
-url = 'http://bioinf.cs.ucl.ac.uk/psipred_beta/api/submission.json'
+# url = 'http://bioinf.cs.ucl.ac.uk/psipred_beta/api/submission.json'
+url = 'http://127.0.0.1:8000/analytics_automated/submission.json'
 
-payload = {'input_data': ('prot.txt', open('../submissions/files/prot.txt', 'rb'))}
-data = {'job': 'disopred',
+payload = {'input_data': ('prot.txt', open('./submissions/files/prot.txt', 'rb'))}
+data = {'job': 'psipred',
         'submission_name': 'testing',
         'email': 'daniel.buchan@ucl.ac.uk', }
 r = requests.post(url, data=data, files=payload)

requirements/base.txt

@@ -1,64 +1,94 @@
-alabaster==0.7.6
-amqp==2.4.1
+alabaster==0.7.12
+amqp==2.6.1
 anyjson==0.3.3
-Babel==2.3.4
-billiard==3.5.0.2
-bugsnag==2.3.1
-celery==4.2.1
-certifi==2019.3.9
-chardet==3.0.4
+asgiref==3.4.1
+attrs==21.2.0
+Babel==2.9.1
+billiard==3.6.4.0
+bootstrap-admin==0.4.4
+bugsnag==4.1.0
+celery==4.4.7
+certifi==2021.5.30
+cffi==1.14.6
+chardet==4.0.0
+charset-normalizer==2.0.3
 commandRunner==0.8.6
-coverage==3.7.1
-defusedxml==0.4.1
-Django==2.2
+coverage==5.5
+defusedxml==0.7.1
+Django==3.2.5
 django-admin-bootstrapped==2.5.6
-django-bootstrap3==11.0.0
-django-celery-beat==1.4.0
-django-cors-headers==2.5.2
-django-debug-toolbar==1.10.0
-django-filter==1.0.4
-django-guardian==1.3
-django-ipware==1.0.0
-django-registration-redux==1.2
-django-smuggler==0.7.0
-django-timezone-field==3.0
-djangorestframework==3.9.2
-djangorestframework-xml==1.4.0
+django-bootstrap3==15.0.0
+django-celery-beat==1.6.0
+django-cors-headers==3.7.0
+django-debug-toolbar==3.2.1
+django-filter==2.4.0
+django-guardian==2.4.0
+django-hstore==1.4.2
+django-ipware==3.0.2
+django-registration-redux==2.9
+django-smuggler==1.0.2
+django-timezone-field==3.1
+django-utils-six==2.0
+djangorestframework==3.12.4
+djangorestframework-hstore==1.3
+djangorestframework-xml==2.0.0
 DjangoRestless==0.0.10
-docutils==0.12
-dr-dump==0.2.5
-drmaa==0.7.6
-factory-boy==2.8.1
-Faker==0.7.7
-flower==0.9.1
-idna==2.8
-Jinja2==2.7.3
-kombu==4.3.0
-MarkupSafe==0.23
-mod-wsgi==4.6.5
-mod-wsgi-httpd==2.4.35.1
-numpy==1.10.1
-Pillow==2.8.1
-psycopg2==2.7.3.2
-pyaml==18.11.0
-Pygments==2.0.2
-pyRserve==0.8.1
-python-crontab==2.3.6
-python-dateutil==2.6.0
-pytz==2017.2
-PyYAML==3.12
-redis==2.10.5
-requests==2.21.0
-rpy2==2.8.5
-scipy==0.19.0
-six==1.10.0
-snowballstemmer==1.2.0
-Sphinx==1.3.1
-sphinx-rtd-theme==0.1.8
-sqlparse==0.2.2
-tornado==4.2
+docutils==0.16
+dr-dump==1.1.4
+drmaa==0.7.9
+factory-boy==3.2.0
+Faker==8.10.3
+flower==0.9.7
+humanize==3.10.0
+idna==3.2
+imagesize==1.2.0
+importlib-metadata==2.1.1
+iniconfig==1.1.1
+Jinja2==3.0.1
+kombu==4.6.11
+MarkupSafe==2.0.1
+mod-wsgi==4.8.0
+mod-wsgi-httpd==2.4.46.1
+numpy==1.19.5
+packaging==21.0
+Pillow==8.3.1
+pluggy==0.13.1
+prometheus-client==0.8.0
+psycopg2==2.9.1
+py==1.10.0
+pyaml==20.4.0
+pycparser==2.20
+Pygments==2.9.0
+pyparsing==2.4.7
+pyRserve==0.9.2
+pytest==6.2.4
+python-crontab==2.5.1
+python-dateutil==2.8.2
+pytz==2021.1
+PyYAML==5.4.1
+redis==3.5.3
+requests==2.26.0
+rpy2==3.4.5
+scipy==1.5.4
+six==1.16.0
+snowballstemmer==2.1.0
+Sphinx==4.1.2
+sphinx-rtd-theme==0.5.2
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==2.0.0
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.5
+sqlparse==0.4.1
+text-unidecode==1.3
+toml==0.10.2
+tornado==6.1
+typing-extensions==3.10.0.0
+tzlocal==2.1
 Unipath==1.1
-urllib3==1.24.1
-vine==1.1.3
-WebOb==1.5.0
-xmltodict==0.9.2
+urllib3==1.26.6
+vine==1.3.0
+WebOb==1.8.7
+xmltodict==0.12.0
+zipp==3.5.0

utility_scripts/aa.hist

@@ -1,3 +1,6 @@
 python manage.py runserver --settings=analytics_automated_project.settings.dev
 celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python
 workon analytics_automated
+exit
+celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python,blast_backend,high_blast_backend,low_blast_backend
+celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python,blast,high_blast,low_blast

utility_scripts/spawn_dev_processes.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+#possibly more the postmaster.pid if the machine rebooted
+gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'postmaster -D /home/dbuchan/Databases/postgres_96_db/;'
+gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'sleep 2; redis-server;'
+gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash --rcfile ~/.bashrc -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; python manage.py runserver --settings=analytics_automated_project.settings.dev; exec bash"'
+gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source  virtualenvwrapper.sh; sleep 2; workon analytics_automated; celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python; exec bash"'

utility_scripts/spawn_dev_processes_dbuchan.sh

@@ -4,4 +4,4 @@
 gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'postmaster -D /home/dbuchan/Databases/postgres_96_db/;'
 gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'sleep 2; redis-server;'
 gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash --rcfile ~/.bashrc -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source virtualenvwrapper.sh; sleep 2; workon analytics_automated; python manage.py runserver --settings=analytics_automated_project.settings.dev; exec bash"'
-gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source  virtualenvwrapper.sh; sleep 2; workon analytics_automated; celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python; exec bash"'
+gnome-terminal --working-directory=/home/dbuchan/Code/analytics_automated --tab -- /bin/sh -c 'bash -c "export HISTFILE=~/Code/analytics_automated/utility_scripts/aa.hist; source  virtualenvwrapper.sh; sleep 2; workon analytics_automated; celery --app=analytics_automated_project.celery:app worker --loglevel=INFO -Q low_localhost,localhost,high_localhost,low_GridEngine,GridEngine,high_GridEngine,low_R,R,high_R,low_Python,Python,high_Python,blast,high_blast,low_blast; exec bash"'