Merge pull request #194 from NREL/dev

Release/v1.1.1

calliope_app/account/urls.py

@@ -1,7 +1,6 @@
 from django.contrib.auth import views as auth_views
 from django.conf import settings
-from django.conf.urls import url
-from django.urls import path
+from django.urls import path, re_path
 
 from account import views
 
@@ -26,8 +25,8 @@
         views.set_timezone,
         name='set_timezone'
     ),
-    
-    url(
+
+    re_path(
         r'^password_reset/$',
         auth_views.PasswordResetView.as_view(
             template_name='registration/pw_reset_form.html',
@@ -37,14 +36,14 @@
         ),
         name='password_reset'
     ),
-    url(
+    re_path(
         r'^password_reset/done/$',
         auth_views.PasswordResetDoneView.as_view(
             template_name='registration/pw_reset_done.html'
         ),
         name='password_reset_done'
     ),
-    url(
+    re_path(
         r'^reset/\
         (?P<uidb64>[0-9A-Za-z_\-]+)/\
         (?P<token>[0-9A-Za-z]{1,13}-[0-9A-Za-z]{1,40})/$',
@@ -53,7 +52,7 @@
         ),
         name='password_reset_confirm'
     ),
-    url(
+    re_path(
         r'^reset/done/$',
         auth_views.PasswordResetCompleteView.as_view(
             template_name='registration/pw_reset_complete.html'

calliope_app/api/admin.py

@@ -9,11 +9,13 @@
     Scenario_Loc_Tech, Scenario_Param, Job_Meta, Carrier
 from api.models.outputs import Run
 from api.models.engage import User_Profile, ComputeEnvironment
+from api.forms import ComputeEnvironmentModelForm
 
 
 class ComputeEnvironmentAdmin(admin.ModelAdmin):
+    form = ComputeEnvironmentModelForm
     filter_horizontal = ("users",)
-    list_display = ['id',  'name', 'full_name', 'is_default', 'solver', 'ncpu', 'memory', 'type', '_users']
+    list_display = ['id',  'name', 'full_name', 'is_default', 'solvers', 'ncpu', 'memory', 'type', '_users']
 
     @staticmethod
     def _users(instance):

calliope_app/api/engage.py

@@ -6,6 +6,28 @@
 from django.core.mail.message import sanitize_address
 
 
+ENGAGE_SOLVERS = [
+    {
+        "name": "appsi_highs",
+        "pretty_name": "HiGHS",
+        "order": 1,
+        "is_active": True
+    },
+    {
+        "name": "cbc",
+        "pretty_name": "CBC",
+        "order": 2,
+        "is_active": True
+    },
+    {
+        "name": "amplxpress",
+        "pretty_name": "Xpress",
+        "order": 3,
+        "is_active": False
+    }
+]
+
+
 def aws_ses_configured():
     """
     Check the configuration of AWS SES settings

calliope_app/api/forms.py

@@ -0,0 +1,15 @@
+
+from django import forms
+
+from client.widgets import JSONEditorWidget
+
+from api.models.engage import ComputeEnvironment
+
+
+class ComputeEnvironmentModelForm(forms.ModelForm):
+    class Meta:
+        model = ComputeEnvironment
+        fields = '__all__'
+        widgets = {
+            'solvers': JSONEditorWidget()
+        }

calliope_app/api/migrations/0068_remove_computeenvironment_solver_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.15 on 2024-09-17 03:26
+
+import api.models.engage
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0067_auto_20240613_1706'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='computeenvironment',
+            name='solver',
+        ),
+        migrations.AddField(
+            model_name='computeenvironment',
+            name='solvers',
+            field=models.JSONField(default=api.models.engage.default_solvers),
+        ),
+    ]

calliope_app/api/models/engage.py

@@ -9,6 +9,8 @@
 from django.urls import reverse
 from django.utils.html import mark_safe
 
+from api.engage import ENGAGE_SOLVERS
+
 logger = logging.getLogger(__name__)
 
 
@@ -102,6 +104,10 @@ def activate(cls, activation_uuid):
         return True
 
 
+def default_solvers():
+    return ENGAGE_SOLVERS
+
+
 class ComputeEnvironment(models.Model):
 
     ENV_TYPES = [
@@ -114,11 +120,11 @@ class ComputeEnvironment(models.Model):
     full_name = models.CharField(max_length=120)
     is_default = models.BooleanField(default=False)
     type = models.CharField(max_length=60, choices=ENV_TYPES)
-    solver = models.CharField(max_length=60, null=True, blank=True)
     ncpu = models.PositiveSmallIntegerField(null=True, blank=True)
     memory = models.PositiveSmallIntegerField(null=True, blank=True)
     cmd = models.TextField(blank=True, null=True)
     users = models.ManyToManyField(User, related_name="compute_environments", blank=True)
+    solvers = models.JSONField(default=default_solvers)
 
     class Meta:
         db_table = "compute_environments"
@@ -141,4 +147,4 @@ class Meta:
         verbose_name_plural = "[Admin] Request Rate Limits"
 
     def __str__(self):
-        return f"{self.year}, {self.month}, {self.total}"
+        return f"{self.year}, {self.month}, {self.total}"

calliope_app/api/models/outputs.py

@@ -272,18 +272,25 @@ def get_static_values(self, meta, metric, location,
         if not df.empty:
             df = df.groupby('techs').sum()
         df = df['values'].to_dict()
+
         # Process Max Bounds Context (ctx)
         if ctx is not None:
             ctx = ctx.replace(np.inf, np.nan).dropna()
             if location:
                 ctx = ctx[ctx['locs'] == location]
             if etx is not None:
-                etx = etx[etx['techs'].isin(ctx['techs'])]
-                ctx['values'] = ctx['values'].add(etx['values'],fill_value=0)
-            ctx = ctx.groupby('techs').sum()
-            ctx = ctx['values'].to_dict()
+                ctx_g = ctx.groupby('techs').sum(["values"])
+                etx_g = etx.groupby('techs').sum(["values"])
+                merged = pd.merge(ctx_g, etx_g, on='techs', how='left', suffixes=('_ctx_g', '_etx_g'))
+                merged.fillna(0, inplace=True)
+                merged['values'] = merged["values_ctx_g"] + merged["values_etx_g"]
+                ctx = merged['values'].to_dict()
+            else:
+                ctx = ctx.groupby('techs').sum()
+                ctx = ctx['values'].to_dict()
         else:
             ctx = {}
+
         # Viz Layers
         layers = [{'key': key,
                    'name': meta['names'][key] if key in meta['names'] else key,

calliope_app/api/urls.py

@@ -133,6 +133,8 @@
     path('upload_outputs/',
          outputs_views.upload_outputs,
          name='upload_outputs'),
+    path('solvers/', outputs_views.solvers,
+         name="solvers"),
 
     # Bulk Data
     path('upload_locations/',

calliope_app/api/views/outputs.py

@@ -3,16 +3,13 @@
 import io
 import logging
 import os
-import shutil
 import zipfile
-import sys
 from re import match
 
 from datetime import datetime, timedelta
 from urllib.parse import urljoin
 import requests
 import pandas as pd
-import pint
 
 from celery import current_app,chain
 from django.views.decorators.csrf import csrf_protect
@@ -30,19 +27,45 @@
 from api.models.calliope import Abstract_Tech, Abstract_Tech_Param, Parameter, Run_Parameter
 from api.models.configuration import (
     Model, ParamsManager, User_File, Location, Technology,
-    Tech_Param, Loc_Tech, Loc_Tech_Param, Timeseries_Meta, Carrier, Scenario_Param
+    Tech_Param, Loc_Tech, Loc_Tech_Param, Timeseries_Meta, Carrier
 )
 
 from api.models.engage import ComputeEnvironment
+from api.engage import ENGAGE_SOLVERS
 from api.utils import zip_folder, initialize_units, convert_units, noconv_units
 from batch.managers import AWSBatchJobManager
 from taskmeta.models import CeleryTask, BatchTask, batch_task_status
 
-from calliope_app.celery import app
-
 logger = logging.getLogger(__name__)
 
 
+@csrf_protect
+def solvers(request):
+    env_name = request.GET.get("env_name", None)
+    if not env_name:
+        env_name = "default"
+
+    flag = True
+    try:
+        env = ComputeEnvironment.objects.get(name=env_name)
+    except ComputeEnvironment.DoesNotExist:
+        flag = False
+
+    if (not flag) or (not env.solvers) or (not isinstance(env.solvers, list)):
+        solvers = ENGAGE_SOLVERS
+    else:
+        solvers = env.solvers
+
+    candidates = []
+    for solver in solvers:
+        is_active = solver.get("is_active", "false")
+        if (is_active is True) or (is_active == "true"):
+            candidates.append(solver)
+    payload = sorted(candidates, key=lambda x: x["order"])
+
+    return HttpResponse(json.dumps(payload), content_type="application/json")
+
+
 @csrf_protect
 def build(request):
     """
@@ -156,13 +179,13 @@ def build(request):
                 )
             inputs_path = inputs_path.lower().replace(" ", "-")
             os.makedirs(inputs_path, exist_ok=True)
-            
+
             run.run_options = []
             for id in parameters.keys():
                 run_parameter= Run_Parameter.objects.get(pk=int(id))
                 run.run_options.append({'root':run_parameter.root,'name':run_parameter.name,'value':parameters[id]})
-           
-            # Celery task        
+
+            # Celery task
             async_result = build_model.apply_async(
                 kwargs={
                     "inputs_path": inputs_path,
@@ -300,7 +323,7 @@ def optimize(request):
                 r.batch_job.status = batch_task_status.FAILED
             r.batch_job.save()
             r.save()
-        
+
         if not all_complete:
             payload = {
                 "status": "BLOCKED",
@@ -344,7 +367,7 @@ def optimize(request):
                     else:
                         logger.info("Found a subsequent gradient model for year %s but it was not built.",next_run.year)
                         break
-    
+
     # Unknown environment, not supported
     else:
         raise Exception("Failed to submit job, unknown compute environment")

calliope_app/calliope_app/settings/base.py

@@ -47,7 +47,8 @@
     'client.apps.ClientConfig',
     'geophires.apps.GeophiresConfig',
     'taskmeta.apps.TaskmetaConfig',
-    'template.apps.TemplateConfig'
+    'template.apps.TemplateConfig',
+    'notification.apps.NotificationConfig'
 ]
 
 INSTALLED_APPS = DJANGO_APPS + LOCAL_APPS

calliope_app/calliope_app/settings/local.py

@@ -39,7 +39,7 @@
 # --------------------------------------------------------------------------------
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'HOST': env.str('POSTGRES_HOST'),
         'PORT': env.str('POSTGRES_PORT'),
         'USER': env.str('POSTGRES_USER'),

calliope_app/calliope_app/settings/prod.py

@@ -17,7 +17,7 @@
 # --------------------------------------------------------------------------------
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'HOST': env.str('POSTGRES_HOST'),
         'PORT': env.str('POSTGRES_PORT'),
         'USER': env.str('POSTGRES_USER'),

calliope_app/calliope_app/settings/test.py

@@ -23,7 +23,7 @@
 # --------------------------------------------------------------------------------
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'ENGINE': 'django.db.backends.postgresql',
         'HOST': env.str('POSTGRES_HOST'),
         'PORT': env.str('POSTGRES_PORT'),
         'USER': env.str('POSTGRES_USER'),