diff --git a/python/requirements.csv b/python/requirements.csv index 222fe06b7..8e3fecdf7 100644 --- a/python/requirements.csv +++ b/python/requirements.csv @@ -21,7 +21,8 @@ ipykernel,,,5.5.5 ipyplot,,,1.1.1 ipython,3,,8.10.0 ipython,,,8.10.0 -matplotlib,2,,3.3.3 +marimo,,, +matplotlib,2,,3.8.2 matplotlib_inline,,,0.1.6 metar,,,1.9.0 networkx,,,2.8.8 @@ -32,6 +33,7 @@ pandas,,,2.0.0 pandas_access,,,0.0.1 Pillow,2,,9.3.0 Pillow,,,9.3.0 +psutil,,, PyGithub,2,,1.54.1 pymysql,,,1.0.2 pyproj,,,3.4.0 diff --git a/source/load.cpp b/source/load.cpp index 5154023b3..47ec2feb4 100755 --- a/source/load.cpp +++ b/source/load.cpp @@ -8404,7 +8404,14 @@ STATUS GldLoader::loadall_glm(const char *fname) /**< a pointer to the first cha int move = 0; errno = 0; - fp = fopen(file,"rt"); + if ( file[0] == '.' ) // format only for /dev/stdin + { + fp = stdin; + } + else + { + fp = fopen(file,"rt"); + } if (fp==NULL) goto Failed; if (fstat(fileno(fp),&stat)==0) diff --git a/source/save.cpp b/source/save.cpp index bc98879fc..092c2e924 100644 --- a/source/save.cpp +++ b/source/save.cpp @@ -37,7 +37,7 @@ int saveall(const char *filename) /* identify output format */ if (ext==NULL) { /* no extension given */ - if (filename[0]=='-') /* stdout */ + if (filename[0]=='.' || filename[0]=='-') /* stdout */ { ext=filename+1; /* format is specified after - */ } @@ -145,7 +145,7 @@ int saveall(const char *filename) } /* setup output stream */ - if (filename[0]=='-') + if (filename[0]=='.' || filename[0]=='-') { fp = stdout; } diff --git a/subcommands/Makefile.mk b/subcommands/Makefile.mk index 9ff5ba902..701d8d8a0 100644 --- a/subcommands/Makefile.mk +++ b/subcommands/Makefile.mk @@ -14,6 +14,7 @@ bin_SCRIPTS += subcommands/gridlabd-library bin_SCRIPTS += subcommands/gridlabd-loaddata bin_SCRIPTS += subcommands/gridlabd-lock bin_SCRIPTS += subcommands/gridlabd-manual +bin_SCRIPTS += subcommands/gridlabd-marimo bin_SCRIPTS += subcommands/gridlabd-matrix bin_SCRIPTS += subcommands/gridlabd-model bin_SCRIPTS += subcommands/gridlabd-openfido diff --git a/subcommands/gridlabd-marimo b/subcommands/gridlabd-marimo new file mode 100644 index 000000000..995099ea3 --- /dev/null +++ b/subcommands/gridlabd-marimo @@ -0,0 +1,215 @@ +#!/bin/bash +## Syntax: gridlabd marimo COMMAND [OPTIONS ...] +## +## Access GridLAB-D marimo apps. +## +## Commands: +## +## index - get a list of available marimo apps +## +## list - get a list of active marimo apps +## +## stop all|PORT [...] - stop marimo apps +## +## open [--edit] NAME [...] - open marimo page +## +## reopen all|PORT [...] - reopen marimo pages +## +FIRST=2718 +LAST=2800 + +E_OK=0 +E_SYNTAX=1 +E_INVALID=2 +E_FAILED=3 +E_NOTFOUND=4 + +function error() +{ + CODE=$1 + shift 1 + echo "ERROR [gridlabd-marimo]: $* (code $CODE)" >/dev/stderr + exit $CODE +} + +function warning() +{ + echo "WARNING [gridlabd-marimo]: $*" >/dev/stderr +} + +function get_active() +{ + PORT=$FIRST + while [ $PORT -lt $LAST ]; do + FILENAME=$(get_filename $PORT) + if [ ! -z "$FILENAME" -a "$FILENAME" != "NAN" ]; then + echo $PORT + fi + PORT=$(($PORT+1)) + done +} + +function get_filename() # PORT +{ + FILENAME=$(curl -s http://localhost:$1 | grep ']*>' | sed 's/<[^>]*>//g' || echo "NAN") + if [ ! -z "$FILENAME" -a "$FILENAME" != "NAN" ]; then + echo $FILENAME + fi +} + +function get_pid() # FILENAME +{ + if [ ! -z "$1" -a "$1" != "NAN" ]; then + ps ax | grep "marimo run $1\|marimo edit $1" | grep -v grep | cut -f1 -d' ' + fi +} + +function get_name() # FILENAME +{ + NAME=$1 + echo $(basename ${NAME/marimo_//}) | sed 's/\.py$//' +} + +# de profundis clamo ad te +if [ $# -eq 0 ]; then + grep '^## Syntax:' $0 | cut -c4- + exit $E_SYNTAX +fi + +# the helping hand obviously worked +case $1 in + +help) + + grep '^## ' $0 | cut -c4- + ;; + +list) + echo "PORT PID FILENAME" + echo "----- ----- ---------------" + for PORT in $(get_active); do + FILENAME=$(get_filename $PORT) + PID=$(get_pid $FILENAME) + printf '%5s %5s %s\n' $PORT ${PID:--} $(get_name $FILENAME) + done + ;; + +stop) + + shift 1 + if [ $# -eq 0 ]; then + error $E_SYNTAX "missing port number" + elif [ "$1" == "all" ]; then # kill them all + for PORT in $(get_active); do + FILENAME=$(get_filename $PORT) + PID=$(get_pid $FILENAME) + if [ -z "$PID" ]; then + echo -n "[$$] Killing $PID..." + kill $PID && echo "ok" || echo "failed" + fi + done + else # kill specific ports + for PORT in $*; do + FILENAME=$(get_filename $PORT) + if [ "$FILENAME" == "NAN" ]; then + error $E_INVALID "server on port $PORT does not appear to a marimo notebook" + elif [ ! -z "$FILENAME" ]; then + PID=$(get_pid $FILENAME) + if [ ! -z "$PID" ]; then + kill $PID + else + error $E_NOTFOUND "no valid PID for $FILENAME on port $PORT" + fi + else + error $E_NOTFOUND "no server running on port $PORT" + fi + done + fi + ;; + +open) + + shift 1 + if [ $# -eq 0 ]; then + error $E_SYNTAX "missing app name" + fi + if [ $1 == '--edit' ]; then + shift 1 + COMMAND="edit" + else + COMMAND="run" + fi + for PORT in $(get_active); do + FILENAME=$(get_filename $PORT) + NAME=$(get_name $FILENAME) + if [ "$1" == "$NAME" ]; then + open http://localhost:$PORT/ + if [ "$COMMAND" == "edit" ]; then + warning "cannot edit running app" + fi + echo $PORT + exit $E_OK + fi + done + for NAME in $*; do + APP=${GLD_ETC}/marimo_$NAME.py + if [ ! -f $APP ]; then + error $E_NOTFOUND "$APP not found" + fi + LOG=/tmp/gridlabd-marimo-$NAME.log + marimo $COMMAND $APP 1>$LOG 2>&1 & + sleep 1 + for PORT in $(get_active); do + FILENAME=$(get_filename $PORT) + if [ $NAME == "$(get_name $FILENAME)" ]; then + echo $PORT + break; + fi + done + done + ;; + +reopen) + + # automatically open any notebook without an open browser tab + shift 1 + if [ $# -eq 0 ]; then # search + PORT=$FIRST + while [ $PORT -lt $LAST ]; do + FILENAME=$(get_filename $PORT) + if [ "$FILENAME" == "NAN" ]; then + error $E_INVALID "server on port $PORT does not appear to a marimo notebook" + elif [ ! -z "$FILENAME" ]; then + open http://localhost:$PORT/ & + else + error $E_NOTFOUND "no server running on port $PORT" + fi + PORT=$(($PORT+1)) + done + else # reopen specific ports + for PORT in $*; do + FILENAME=$(get_filename $PORT) + if [ "$FILENAME" == "NAN" ]; then + error $E_INVALID "server on port $PORT does not appear to a marimo notebook" + elif [ ! -z "$FILENAME" ]; then + open http://localhost:$PORT/ & + else + error $E_NOTFOUND "no server running on port $PORT" + fi + done + fi + ;; + +index) + + for FILENAME in $GLD_ETC/marimo_*.py; do + get_name $FILENAME + done + ;; + +*) + + error $E_SYNTAX "$1 is not a valid command" + ;; + +esac diff --git a/tools/Makefile.mk b/tools/Makefile.mk index ce50f2058..0738fbd7a 100644 --- a/tools/Makefile.mk +++ b/tools/Makefile.mk @@ -13,10 +13,15 @@ dist_pkgdata_DATA += tools/fit_filter.py dist_pkgdata_DATA += tools/gldserver.py dist_pkgdata_DATA += tools/gridlabd-editor.png dist_pkgdata_DATA += tools/gridlabd-editor.py +dist_pkgdata_DATA += tools/gridlabd_model.py +dist_pkgdata_DATA += tools/gridlabd_runner.py dist_pkgdata_DATA += tools/group.py dist_pkgdata_DATA += tools/insights.py dist_pkgdata_DATA += tools/install.py dist_pkgdata_DATA += tools/isone.py +dist_pkgdata_DATA += tools/marimo_admin.py +dist_pkgdata_DATA += tools/marimo_nsrdb_weather.py +dist_pkgdata_DATA += tools/marimo_viewer.py dist_pkgdata_DATA += tools/market_data.py dist_pkgdata_DATA += tools/market_model.py dist_pkgdata_DATA += tools/mdb_info.py diff --git a/tools/gridlabd_model.py b/tools/gridlabd_model.py new file mode 100644 index 000000000..6a0e4394f --- /dev/null +++ b/tools/gridlabd_model.py @@ -0,0 +1,175 @@ +"""GridLAB-D JSON file accessor + +""" + +import os, sys +import json +import re +import io +from gridlabd_runner import gridlabd +import pandas as pd + +VERSION = "4.3.1" # oldest version supported by this module +ENCODING = "utf-8" # data encode to use for JSON files + +class GridlabdModelException(Exception): + pass + +class GridlabdModel: + + def __init__(self, + data = None, # GLM filename, JSON data, or JSON filename + name = None, # filename to use (overrides default filename) + force = False, # force overwrite of existing JSON + initialize = False, # initialize GLM first + ): + + if name: + self.filename = name + + if data is None: # new file + if not name: + self.autoname(force) + gridlabd("-o",self.filename) + self.read_json(self.filename) + + elif type(data) is str and os.path.splitext(data)[1] == ".glm": # glm file + + self.read_glm(data,force=force,initialize=initialize) + + elif type(data) is str and os.path.splitext(data)[1] == ".json": # json file + + self.read_json(data) + + else: # raw data + + if not name: + self.autoname(force) + self.load(data) + + def autoname(self,force=False): + """Generate a new filename + + Arguments: + - force (bool): use the first name generate regardless of whether the file exists + """ + num = 0 + def _name(num): + return f"untitled-{num}.json" + while os.path.exists(_name(num)) and not force: + num += 1 + self.filename = _name(num) + + def read_glm(self,filename,force=True,initialize=False): + """Read GLM file + + Arguments: + - filename (str): name of GLM file to read + - force (bool): force overwrite of JSON + - initialize (bool): initialize GLM first + """ + with open(filename,"r") as fh: + self.filename = filename + self.load(data) + + def read_json(self,filename): + """Read JSON file + + Arguments: + - filename (str): name of JSON file to read + """ + with open(filename,"r") as fh: + data = json.load(fh) + self.filename = filename + self.load(data) + + def load(self,data): + """Load data from dictionary + + Arguments: + - data (dict|str|bytes): data to load (must be a valid GLM model) + """ + if type(data) is bytes: + data = data.decode(ENCODING) + if type(data) is str: + if data[0] in ["[","{"]: + data = json.loads(data) + else: + data = json.loads(gridlabd("-C",".glm","-o","-json",binary=True,source=io.StringIO(data))) + print(data) + if not type(data) is dict: + raise GridlabdModelException("data is not a dictionary") + if data["application"] != "gridlabd": + raise GridlabdModelException("data is not a gridlabd model") + if data["version"] < VERSION: + raise GridlabdModelException("data is from an outdated version of gridlabd") + for key,values in data.items(): + setattr(self,key,values) + self.is_modified = False + + def get_objects(self,classes=None,as_type=dict,**kwargs): + """Find objects belonging to specified classes + + Arguments: + - classes (str|list of str): patterns of class names to search (default is '.*') + - as_type (class): type to use for return value + - kwargs (dict): arguments for as_type + + Return: + - as_type: object data + """ + if type(classes) is str: + match = classes.split(",") + else: + match = [] + for y in classes if classes else '.*': + match.extend([x for x in self.classes if re.match(y,x)]) + data = dict([(obj,data) for obj,data in self.objects.items() if data["class"] in match]) + return as_type(data,**kwargs) + + def rename(self,name): + self.filename = name + self.is_modified = True + +if __name__ == "__main__": + + import unittest + + if not os.path.exists("/tmp/13.glm"): + gridlabd("model","get","IEEE/13") + os.system("mv 13.glm /tmp") + gridlabd("-C","/tmp/13.glm","-o","/tmp/13.json") + + class TestModel(unittest.TestCase): + + # def test_glm(self): + # glm = GridlabdModel("/tmp/13.glm",force=True) + # loads = glm.get_objects(classes=[".*load"]) + # self.assertEqual(len(loads),10) + + # def test_json(self): + # glm = GridlabdModel("/tmp/13.json") + # loads = glm.get_objects(classes=["load","triplex_load"]) + # self.assertEqual(len(loads),10) + + # def test_new(self): + + # glm = GridlabdModel("/tmp/test.json",force=True) + # self.assertEqual(glm.objects,{}) + # self.assertEqual(glm.classes,{}) + + def test_str_glm(self): + with open("/tmp/13.glm","r") as fh: + data = fh.read() + glm = GridlabdModel(data) + loads = glm.get_objects(classes=["load","triplex_load"]) + self.assertEqual(len(loads),10) + + # def test_str_json(self): + # with open("/tmp/13.json","r") as fh: + # data = json.load(fh) + # glm = GridlabdModel(data) + # loads = glm.get_objects(classes=["load","triplex_load"]) + # self.assertEqual(len(loads),10) + + unittest.main() \ No newline at end of file diff --git a/tools/gridlabd_runner.py b/tools/gridlabd_runner.py new file mode 100644 index 000000000..acd3fcd2a --- /dev/null +++ b/tools/gridlabd_runner.py @@ -0,0 +1,199 @@ +"""GridLAB-D runner""" +import os, sys +import subprocess +import time +import shutil + +class GridlabdException(Exception): + pass + +_exitcode = { + -1 : "FAILED", + 0 : "SUCCESS", + 1 : "ARGERR", + 2 : "ENVERR", + 3 : "TSTERR", + 4 : "USRERR", + 5 : "RUNERR", + 6 : "INIERR", + 7 : "PRCERR", + 8 : "SVRKLL", + 9 : "IOERR", + 10 : "LDERR", + 11 : "TMERR", + 127 : "SHFAILED", + 128 : "SIGNAL", + 128+1 : "SIGHUP", + 128+2 : "SIGINT", + 128+9 : "SIGKILL", +128+15 : "SIGTERM", + 255 : "EXCEPTION", +} + +def gridlabd(*args,split=None,**kwargs): + """Run gridlabd and return the output + + Arguments: + - *args: gridlabd command line arguments + - **kwargs: gridlabd global definitions + + Returns: + - str: stdout + + Exceptions: + - GridlabdException(code,stderr) + """ + gld = Gridlabd(*args,**kwargs) + return gld.result.stdout if not split else gld.result.stdout.strip().split(split if type(split) is str else "\n") + +class Gridlabd: + + def __init__(self,*args, + binary = False, + start = True, + wait = True, + timeout = None, + source = None, + **kwargs, + ): + """Construct a runner + + Arguments: + - *args=[]: gridlabd command line arguments + - binary=True (bool=True): use the gridlabd binary if possible + - start=True (bool): start gridlabd immediately + - wait=True (bool): wait for gridlabd to complete + - timeout=None (float): seconds to wait for completion before failing + - source=None (bufferedio): input data source + - **kwargs: gridlabd global definitions + + Exceptions: + - GridlabdException(code,stderr) + """ + cmd = shutil.which("gridlabd.bin") + cmd = cmd if cmd and binary else shutil.which("gridlabd") + if not cmd: + raise GridlabdException(-1,"gridlabd not found") + self.command = [cmd] + for name,value in kwargs.items(): + self.command.extend(["-D",f"{name}={value}"]) + self.command.extend(args) + self.process = None + self.result = None + if not start: + raise NotImplementedError("Gridlabd.start") + elif not wait: + raise NotImplementedError("Gridlabd.wait") + else: + self.run(timeout=timeout,source=source) + + def run(self,timeout=None,source=None): + """Run gridlabd + + Arguments: + - timeout=None (float): seconds to wait before for completion failing + + Returns: + - str: output + + Exceptions: + - GridlabdException(code,message) + - subprocess.TimeoutExpired + """ + try: + self.result = subprocess.run(self.command, + capture_output = True, + text = True, + timeout = timeout, + stdin = source, + ) + except subprocess.TimeoutExpired: + raise + except: + raise + if self.result.returncode != 0: + raise GridlabdException(f"gridlabd.{_exitcode[self.result.returncode]} -- {self.result.stderr}" + if self.result.returncode in _exitcode + else f"gridlabd.EXITCODE {self.result.returncode}") + return self.result.stdout + + def is_started(self): + """Check if gridlabd is started + + Returns: + bool: gridlabd is started + """ + return not self.process is None + + def is_running(self): + """Check if gridlabd is running + + Returns: + bool: gridlabd is running + """ + return not self.process is None and self.result is None + + def is_completed(self): + """Check if gridlabd is done + + Returns: + bool: process is completed + """ + return not self.result is None + + def start(self,wait=True): + """Start gridlabd""" + if self.is_completed(): + raise GridlabdException("already completed") + if self.is_started(): + raise GridlabdException("already started") + + def wait(self,timeout=None): + """Wait for gridlabd to complete""" + if self.is_completed(): + raise GridlabdException("already completed") + +if __name__ == '__main__': + + import unittest + + class TestGridlabd(unittest.TestCase): + + def test_ok(self): + output = gridlabd("--version") + self.assertTrue(output.startswith("HiPAS GridLAB-D")) + + def test_err(self): + try: + output = gridlabd("--nogood") + msg = None + except GridlabdException as err: + msg = err.args + self.assertEqual(msg[0],5) + + # def test_start(self): + # try: + # proc = Gridlabd("--version",start=False).start() + # msg = None + # except GridlabdException as err: + # msg = err + # self.assertEqual(msg.args[0],"already completed") + + # def test_wait(self): + # gld = Gridlabd("--version",wait=False) + # try: + # proc = gld.wait() + # msg = None + # except GridlabdException as err: + # msg = err + # self.assertEqual(msg.args[0],"already completed") + + # run = Gridlabd("8500.glm","clock.glm","recorders.glm") + # run.start(wait=False) + # while not run.result: + # print("STDOUT",run.output,file=sys.stdout,flush=True) + # print("STDERR",run.errors,file=sys.stderr,flush=True) + # time.sleep(1) + # run.wait() + + unittest.main() \ No newline at end of file diff --git a/tools/marimo_admin.py b/tools/marimo_admin.py new file mode 100644 index 000000000..18b9fb5f4 --- /dev/null +++ b/tools/marimo_admin.py @@ -0,0 +1,139 @@ +import marimo + +__generated_with = "0.1.59" +app = marimo.App() + + +@app.cell +def __(mo): + mo.md("# GridLAB-D Marimo Dashboard") + return + + +@app.cell +def __(actions, editor, mo, refresh): + refresh + mo.tabs({ + "Marimo Tools" : actions, + # "Settings" : mo.vstack([ + # mo.hstack([mo.md("Use editor"),editor],justify = 'start'), + # mo.hstack([mo.md("Refresh rate"),refresh],justify = 'start'), + # ]) + "Settings" : mo.md(""" + + + +
Use editor{editor}
Refresh rate>{refresh}
+ """).batch(editor=editor,refresh=refresh) + }) + + return + + +@app.cell +def __(gridlabd): + available = gridlabd("marimo","index",split=True) + return available, + + +@app.cell +def __(mo): + refresh = mo.ui.refresh(options = ["1s","2s","5s","10s","20s","30s","1min"], + default_interval = "1s", + ) + return refresh, + + +@app.cell +def __(mo): + editor = mo.ui.switch() + return editor, + + +@app.cell +def __(available, editor, get_active, gridlabd, inactive, mo, psutil): + def _action(x): + if x in get_active(): + proc = psutil.Process(get_active()[x]["pid"]) + proc.terminate() + elif editor.value: + gridlabd("marimo","open","--edit",x) + else: + gridlabd("marimo","open",x) + + def _url(x): + port = get_active()[x]["port"] + return f"""{x.replace("_"," ").title()} """ + + _active = get_active() + _args = dict([(x,mo.ui.button(label="Stop" if x in _active else "Start",on_click=lambda _,x=x:_action(x))) for x in available]) + actions = mo.md(""+"\n".join([f"""""" for x in available])+"
ActionTool namePortProcess
{{{x}}}{x.replace("_"," ").title() if x in inactive else _url(x)}{_active[x]["port"] if x in _active else ""}{_active[x]["pid"] if x in _active else ""}
").batch(**_args) + return actions, + + +@app.cell +def __(available, mo, os, psutil, re, refresh, requests): + refresh + + def _getfile(body): + """Get the filename from the marimo reply""" + for item in body.strip().split("\n"): + if item.startswith("