Merge pull request #50 from RockefellerArchiveCenter/v0.5

v0.5

requirements.txt

@@ -1,6 +1,5 @@
 amqp==2.3.2
-anyjson==0.3.3
-billiard==3.5.0.4
+billiard==3.5.0.5
 celery==4.2.1
 certifi==2018.4.16
 chardet==3.0.4
@@ -10,17 +9,13 @@ django-celery-results==1.0.4
 django-crispy-forms==1.7.2
 django-datatables-view==1.17.0
 django-timezone-field==3.0
-djangorestframework==3.9.1
-get==1.0.3
+djangorestframework==3.10.2
 idna==2.7
 kombu==4.2.1
-post==1.0.2
 psycopg2-binary==2.7.4
-public==1.0.3
-python-crontab==2.3.5
+python-crontab==2.3.8
 python-dateutil==2.7.5
 pytz==2018.4
-query-string==1.0.2
 requests==2.20.0
 six==1.11.0
 urllib3==1.24.2

setup_services.py

@@ -3,11 +3,11 @@
 from gateway.models import Application, ServiceRegistry, Source
 
 APPLICATIONS = [
-    {'name': 'Ursa Major', 'host': 'ursa-major-web', 'port': 8005},
-    {'name': 'Fornax', 'host': 'fornax-web', 'port': 8003},
-    {'name': 'Gemini', 'host': 'gemini-web', 'port': 8006},
-    {'name': 'Aquarius', 'host': 'aquarius-web', 'port': 8002},
-    {'name': 'Aurora', 'host': 'localhost', 'port': 8000},
+    {'name': 'Ursa Major', 'host': 'ursa-major-web', 'port': 8005, 'health_check_path': '/status'},
+    {'name': 'Fornax', 'host': 'fornax-web', 'port': 8003, 'health_check_path': '/status'},
+    {'name': 'Gemini', 'host': 'gemini-web', 'port': 8006, 'health_check_path': '/status'},
+    {'name': 'Aquarius', 'host': 'aquarius-web', 'port': 8002, 'health_check_path': '/status'},
+    {'name': 'Aurora', 'host': 'localhost', 'port': 8000, 'health_check_path': None},
 ]
 
 SERVICES = [
@@ -16,6 +16,11 @@
      'external_uri': 'api/transfers/', 'service_route': 'api/transfers/',
      'plugin': 0, 'method': 'POST', 'callback_service': None, 'post_service': None,
      'sources': None,},
+    {'name': 'Update Accessions', 'application': 'Aurora',
+     'description': 'Updates accession data.',
+     'external_uri': 'api/accessions/', 'service_route': 'api/acccessions/',
+     'plugin': 0, 'method': 'POST', 'callback_service': None, 'post_service': None,
+     'sources': None,},
     {'name': 'Store Accessions', 'application': 'Ursa Major',
      'description': 'Stores incoming accession data and creates associated transfer objects.',
      'external_uri': 'store-accessions', 'service_route': 'accessions',
@@ -94,8 +99,13 @@
     {'name': 'Process Accessions', 'application': 'Aquarius',
      'description': 'Transforms and delivers accession data to ArchivesSpace',
      'external_uri': 'process-accessions/', 'service_route': 'accessions/',
-     'plugin': 0, 'method': 'POST', 'callback_service': 'Aquarius.Process Grouping Components',
+     'plugin': 0, 'method': 'POST', 'callback_service': 'Aquarius.Update Accession Status',
      'post_service': None, 'sources': None,},
+    {'name': 'Update Accession Status', 'application': 'Aquarius',
+     'description': 'Sends information about updated accessions',
+     'external_uri': 'update-accessions/', 'service_route': 'send-accession-update/',
+     'plugin': 0, 'method': 'POST', 'callback_service': 'Aquarius.Process Grouping Components',
+     'post_service': 'Aurora.Update Accessions', 'sources': None,},
     {'name': 'Process Grouping Components', 'application': 'Aquarius',
      'description': 'Transforms and delivers grouping component data to ArchivesSpace',
      'external_uri': 'process-grouping-components/', 'service_route': 'grouping-components/',
@@ -140,6 +150,7 @@
             is_active=True,
             app_host=application['host'],
             app_port=application['port'],
+            health_check_path=application['health_check_path'],
         )
         print("Created application: {}".format(application['name']))
 

zodiac/gateway/migrations/0021_requestlog_task_result_status.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.13 on 2019-09-15 15:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gateway', '0020_auto_20190215_2215'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='requestlog',
+            name='task_result_status',
+            field=models.CharField(blank=True, choices=[('success', 'Success'), ('error', 'Error'), ('idle', 'Idle')], max_length=100, null=True),
+        ),
+    ]

zodiac/gateway/migrations/0022_application_health_check_path.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.13 on 2019-10-20 21:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gateway', '0021_requestlog_task_result_status'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='application',
+            name='health_check_path',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

zodiac/gateway/models.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 
 from django.db import models
@@ -19,12 +18,23 @@ class Source(models.Model):
     def __str__(self):
         return self.user.username
 
+    def get_absolute_url(self):
+        return reverse('sources-detail', args=[self.pk])
+
+    def get_update_url(self):
+        return reverse('sources-update', args=[self.pk])
+
+    @classmethod
+    def get_list_url(self):
+        return reverse('sources-list')
+
 
 class Application(models.Model):
     name = models.CharField(max_length=64, unique=True)
     is_active = models.BooleanField(default=True)
     app_host = models.CharField(max_length=40)
     app_port = models.PositiveSmallIntegerField(null=True, blank=True)
+    health_check_path = models.CharField(max_length=255, null=True, blank=True)
     created_time = models.DateTimeField(auto_now_add=True)
     modified_time = models.DateTimeField(auto_now=True)
 
@@ -37,6 +47,10 @@ def get_update_url(self):
     def get_absolute_url(self):
         return reverse('applications-detail', args=[self.pk])
 
+    @classmethod
+    def get_list_url(self):
+        return reverse('applications-list')
+
 
 class ServiceRegistry(models.Model):
     REMOTE_AUTH = 0
@@ -93,6 +107,13 @@ def get_update_url(self):
     def get_absolute_url(self):
         return reverse('services-detail', args=[self.pk])
 
+    @classmethod
+    def get_list_url(self):
+        return reverse('services-list')
+
+    def get_clear_errors_url(self):
+        return reverse('services-clear-errors', args=[self.pk])
+
     def get_trigger_url(self):
         return reverse('services-trigger', args=[self.pk])
 
@@ -113,6 +134,7 @@ class RequestLog(models.Model):
     async_result_id = models.CharField(max_length=36, blank=True, null=True)
     created_time = models.DateTimeField(auto_now_add=True)
     task_result = models.ForeignKey(TaskResult, on_delete=models.CASCADE, blank=True, null=True, related_name='request_log')
+    task_result_status = models.CharField(max_length=100, choices=(('success', 'Success'), ('error', 'Error'), ('idle', 'Idle')), blank=True, null=True)
 
     def error_messages(self):
         errors = []
@@ -123,14 +145,3 @@ def error_messages(self):
                 emess = e
             errors.append(emess)
         return errors
-
-    @classmethod
-    def create(cls, service, status_code, request_url, async_result_id=None, task_result=None):
-        record = cls(
-            service=service,
-            status_code=status_code,
-            request_url=request_url,
-            async_result_id=async_result_id,
-            task_result=task_result
-        ).save()
-        return record

zodiac/gateway/serializers.py

@@ -1,6 +1,3 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from rest_framework import serializers
 from django_celery_results.models import TaskResult
 

zodiac/gateway/signals.py

@@ -1,39 +1,47 @@
+import json
+
 from celery.signals import task_prerun, task_postrun
 from django_celery_results.models import TaskResult
 from .models import ServiceRegistry, RequestLog
 
-@task_prerun.connect
-def on_task_prerun(task_id=None, task=None, *args, **kwargs):
-
-    print("{}[{}] started".format(task, task_id))
 
-    # Mark service as active
+def update_service_status(kwargs, status):
+    """Sets `has_active_task` property on Service"""
     if 'service_id' in kwargs['kwargs']:
         service = ServiceRegistry.objects.get(pk=kwargs['kwargs']['service_id'])
-        service.has_active_task = True
+        service.has_active_task = status
         service.save()
+        return service
+    return None
 
-@task_postrun.connect
-def on_task_postrun(task_id=None, task=None, retval=None, state=None, *args, **kwargs):
 
-    print("{}[{}] finished".format(task, task_id))
+def get_task_result_status(task_result):
+    """Returns custom category of task result status"""
+    task_result_status = 'Error'
+    if task_result.status == 'SUCCESS':
+        task_result_status = 'Idle'
+        if json.loads(task_result.result).get('count', 0) > 0:
+            task_result_status = 'Success'
+    return task_result_status
 
-    # Mark service as inactive
-    def update_service(kwargs):
-        if 'service_id' in kwargs['kwargs']:
-            service = ServiceRegistry.objects.get(pk=kwargs['kwargs']['service_id'])
-            service.has_active_task = False
-            service.save()
-            return service
-        return None
 
-    # Add result to request log
+@task_prerun.connect
+def on_task_prerun(task_id=None, task=None, *args, **kwargs):
+    """Marks service as active"""
+    update_service_status(kwargs, True)
+
+
+@task_postrun.connect
+def on_task_postrun(task_id=None, task=None, retval=None, state=None, *args, **kwargs):
+    """Marks service as inactive and saves TaskResult"""
+    service = update_service_status(kwargs, False)
     if len(kwargs['args']) > 1:
         task_result = TaskResult.objects.get(task_id=task_id)
-        request_log = RequestLog.create(
-            service=update_service(kwargs),
+        request_log = RequestLog.objects.create(
+            service=service,
             status_code=None,
             request_url=kwargs['args'][1],
             async_result_id=task_id,
-            task_result=task_result
+            task_result=task_result,
+            task_result_status=get_task_result_status(task_result),
         )

zodiac/gateway/static/css/styles.css

@@ -113,6 +113,10 @@ pre {
 	background-color: rgba(0,0,0,.06);
 }
 
+.service-title {
+	font-weight: bold;
+}
+
 .graph {
 	overflow-x: scroll;
 }

zodiac/gateway/tasks.py

@@ -23,34 +23,41 @@
 @shared_task()
 def queue_callbacks():
     completed = {'detail': {'callbacks': []}}
+
     for registry in ServiceRegistry.objects.filter(callback_service__isnull=False,
-                                                   has_active_task=False, is_active=True,
-                                                   application__is_active=True).order_by('callback_service__modified_time')[:settings.MAX_SERVICES]:
+                                                   callback_service__is_active=True,
+                                                   callback_service__has_active_task=False,
+                                                   callback_service__application__is_active=True).order_by('callback_service__modified_time')[:settings.MAX_SERVICES]:
         callback = ServiceRegistry.objects.get(pk=registry.callback_service.pk)
-        if not callback.has_active_task:
-            url = render_service_path(callback, '')
-            r = queue_request.delay(
-                'post',
-                url,
-                headers={'Content-Type': 'application/json'},
-                data=None,
-                files=None,
-                params={'post_service_url': render_service_path(callback.post_service)},
-                service_id=callback.id
-            )
-            if r:
-                completed['detail']['callbacks'].append({callback.name: r.id})
+        url = render_service_path(callback, '')
+        r = queue_request.delay(
+            'post',
+            url,
+            headers={'Content-Type': 'application/json'},
+            data=None,
+            files=None,
+            params={'post_service_url': render_service_path(callback.post_service)},
+            service_id=callback.id
+        )
+        if r:
+            completed['detail']['callbacks'].append({callback.name: r.id})
     return completed
 
 
 @shared_task()
 def queue_request(method, url, headers, data, files, params, service_id):
     r = method_map[method](url, headers=headers, data=data, files=files, params=params)
-    if r.status_code == 200:
+    if r.status_code in [200, 201]:
         return r.json()
-    raise Exception(r.json())
+    else:
+        try:
+            message = r.json()
+        except:
+            message = str(r)
+        raise Exception(message)
 
 
 @shared_task()
 def delete_successful():
-    TaskResult.objects.filter(status="SUCCESS", date_done__lte=timezone.now()-timezone.timedelta(days=1)).delete()
+    TaskResult.objects.filter(status="SUCCESS",
+                              date_done__lte=timezone.now()-timezone.timedelta(hours=settings.DELETE_SUCCESSFUL_AFTER)).delete()

zodiac/gateway/templates/gateway/applications_add.html → zodiac/gateway/templates/gateway/add.html

@@ -1,8 +1,9 @@
 {% extends "gateway/base.html" %}
 {% load crispy_forms_tags %}
+{% load util %}
 
 {% block h1_title %}
-  Add Application
+  Add {{form | form_to_class_name }}
 {% endblock %}
 
 {% block content %}
@@ -16,7 +17,7 @@
         </div>
         <div class="box-footer">
           <input class="btn btn-primary" type="submit" value="Save" />
-          <a href="{% url 'applications-list' %}" class="btn btn-danger">Cancel</a>
+          <a href="{{form | form_to_list_url }}" class="btn btn-danger">Cancel</a>
         </div>
       </div>
     </form>