diff --git a/converters/Makefile.mk b/converters/Makefile.mk index 096413cd1..4d071b29b 100644 --- a/converters/Makefile.mk +++ b/converters/Makefile.mk @@ -51,6 +51,7 @@ dist_pkgdata_DATA += converters/glm2omd.py # json -> csv dist_pkgdata_DATA += converters/json2csv.py dist_pkgdata_DATA += converters/json2csv-profile.py +dist_pkgdata_DATA += converters/json2csv-pandas.py # json -> html dist_pkgdata_DATA += converters/json2html.py diff --git a/converters/json2csv-pandas.py b/converters/json2csv-pandas.py new file mode 100644 index 000000000..69be534b3 --- /dev/null +++ b/converters/json2csv-pandas.py @@ -0,0 +1,74 @@ +"""Convert JSON to CSV + +SYNTAX +------ + + gridlabd -C input.glm -D csv_save_options="-t pandas -f PROPERTY=VALUE ..." -o output.csv + +DESCRIPTION +----------- + +The filter option can be used to limit the output to object with matching properties, e.g., + + -f class=CLASS + +or + + -f class="CLASS1|CLASS2". + +Multiple filters may be specified in which case, all objects will match all +filters, with an "and" operation, r.h., + + -f class=CLASS name=NAME + +Filter values are interpreted using regular expressions, e.g., + + -f name=EXPRESSION + +Be careful to quote expressions that can be interpreted by the shell. + +EXAMPLE +------- + +The following command downloads the IEEE 13-bus model and saves all the PQ bus nodes and loads: + + gridlabd model get IEEE/13 + gridlabd -C 13.glm -D csv_save_options='-t pandas -f class="node|load" -f bustype=PQ' -o 13.csv +""" +import json +import os +import sys, getopt +from datetime import datetime +import csv +import io +import pandas as pd +import re + +def convert(input_file,output_file=None, options={}): + + if output_file == '': + if input_file[-5:] == ".json": + output_file = input_file[:-5] + ".csv" + else: + output_file = input_file + ".csv" + + with open(input_file,"r") as f : + data = json.load(f) + assert(data['application']=='gridlabd') + assert(data['version'] >= '4.2.0') + + if "filter" in options and len(options["filter"]) > 0: + result = {} + for name,properties in data["objects"].items(): + ok = 0 + for key,value in options["filter"].items(): + if key == "name" and re.match(value,name): + ok += 1 + elif key in properties and re.match(value,properties[key]): + ok += 1 + if ok == len(options["filter"]): + result[name] = properties + else: + result = data["objects"] + df = pd.DataFrame(result).transpose() + df.to_csv(output_file,header=True,index=False) diff --git a/converters/json2csv.py b/converters/json2csv.py index a03979a78..8899c9f8b 100644 --- a/converters/json2csv.py +++ b/converters/json2csv.py @@ -9,25 +9,28 @@ "input" : "json", "output" : "csv", "type" : { - "profile" : "voltage profile" + "profile" : "voltage profile", + "pandas" : "pandas dataframe", } } def help(): print(f'Syntax:') - print(f'{config["input"]}2{config["output"]}.py -i|--ifile [,[,...]] -o|--ofile -t|--type ') + print(f'{config["input"]}2{config["output"]}.py -i|--ifile [,[,...]] -o|--ofile -t|--type -f|--filter ') print(f' -c|--config : [OPTIONAL] output converter configuration') print(f' -i|--ifile : [REQUIRED] {config["input"]} input file name') print(f' -o|--ofile : [REQUIRED] {config["output"]} output file name') print(f' -t|--type : [REQUIRED] specify output type') + print(f' -f|--filter : [OPTIONAL] specify object filter (all objects if none)') print(f'Output types') print(f' profile : voltage profile'); input_file = None output_file = None output_type = None +options = {} -opts, args = getopt.getopt(sys.argv[1:],"hci:o:t:",["help","config","ifile=","ofile=","type="]) +opts, args = getopt.getopt(sys.argv[1:],"hci:o:t:f:",["help","config","ifile=","ofile=","type=","filter="]) if not opts : help() @@ -43,11 +46,16 @@ def help(): input_file = arg.strip() elif opt in ("-o", "--ofile"): output_file = arg.strip() - elif opt in ("-t","--type"): + elif opt in ["-t","--type"]: if arg in config['type'].keys(): output_type = arg.strip() else: raise Exception(f"type '{arg}' is not recognized") + elif opt in ["-f","--filter"]: + spec = arg.strip().split("=") + if not "filter" in options: + options["filter"] = {} + options["filter"][spec[0]] = "=".join(spec[1:]) else: raise Exception(f"'{opt}' is an invalid command line option") @@ -56,8 +64,8 @@ def help(): modspec = util.spec_from_file_location(output_type, modname) mod = importlib.import_module(f"{config['input']}2{config['output']}-{output_type}") - mod.convert(input_file=input_file,output_file=output_file) + mod.convert(input_file=input_file,output_file=output_file,options=options) else: - raise Exception(f"txt2glm-{output_type}.py not found") \ No newline at end of file + raise Exception(f"json2csv-{output_type}.py not found") \ No newline at end of file