Add JSON to CSV converter (#158)

Signed-off-by: David P. Chassin <dchassin@slac.stanford.edu>
Signed-off-by: Duncan Ragsdale <88173870+Thistleman@users.noreply.github.com>
Co-authored-by: Duncan Ragsdale <88173870+Thistleman@users.noreply.github.com>

converters/Makefile.mk

@@ -51,6 +51,7 @@ dist_pkgdata_DATA += converters/glm2omd.py
 # json -> csv
 dist_pkgdata_DATA += converters/json2csv.py
 dist_pkgdata_DATA += converters/json2csv-profile.py
+dist_pkgdata_DATA += converters/json2csv-pandas.py
 
 # json -> html
 dist_pkgdata_DATA += converters/json2html.py

converters/json2csv-pandas.py

@@ -0,0 +1,74 @@
+"""Convert JSON to CSV
+
+SYNTAX
+------
+
+    gridlabd -C input.glm -D csv_save_options="-t pandas -f PROPERTY=VALUE ..." -o output.csv
+
+DESCRIPTION
+-----------
+
+The filter option can be used to limit the output to object with matching properties, e.g.,
+
+	-f class=CLASS
+
+or
+
+	-f class="CLASS1|CLASS2".
+
+Multiple filters may be specified in which case, all objects will match all
+filters, with an "and" operation, r.h.,
+
+	-f class=CLASS name=NAME
+
+Filter values are interpreted using regular expressions, e.g.,
+
+	-f name=EXPRESSION
+
+Be careful to quote expressions that can be interpreted by the shell.
+
+EXAMPLE
+-------
+
+The following command downloads the IEEE 13-bus model and saves all the PQ bus nodes and loads:
+
+	gridlabd model get IEEE/13
+	gridlabd -C 13.glm -D csv_save_options='-t pandas -f class="node|load" -f bustype=PQ' -o 13.csv
+"""
+import json 
+import os 
+import sys, getopt
+from datetime import datetime 
+import csv
+import io
+import pandas as pd
+import re
+
+def convert(input_file,output_file=None, options={}):
+
+	if output_file == '':
+		if input_file[-5:] == ".json":
+			output_file = input_file[:-5] + ".csv" 
+		else: 
+			output_file = input_file + ".csv"
+
+	with open(input_file,"r") as f :
+		data = json.load(f)
+		assert(data['application']=='gridlabd')
+		assert(data['version'] >= '4.2.0')
+
+	if "filter" in options and len(options["filter"]) > 0:
+		result = {}
+		for name,properties in data["objects"].items():
+			ok = 0
+			for key,value in options["filter"].items():
+				if key == "name" and re.match(value,name):
+					ok += 1
+				elif key in properties and re.match(value,properties[key]):
+					ok += 1
+			if ok == len(options["filter"]):
+				result[name] = properties
+	else:
+		result = data["objects"]
+	df = pd.DataFrame(result).transpose()
+	df.to_csv(output_file,header=True,index=False)	

converters/json2csv.py

@@ -9,25 +9,28 @@
 	"input" :	"json",
 	"output" :  "csv",
 	"type" : {
-		"profile" : "voltage profile"
+		"profile" : "voltage profile",
+		"pandas" : "pandas dataframe",
 	}
 }
 
 def help():
 	print(f'Syntax:')
-	print(f'{config["input"]}2{config["output"]}.py -i|--ifile <input-file>[,<input-file>[,...]] -o|--ofile <output-file> -t|--type <input-type>')
+	print(f'{config["input"]}2{config["output"]}.py -i|--ifile <input-file>[,<input-file>[,...]] -o|--ofile <output-file> -t|--type <input-type> -f|--filter <filter-spec>')
 	print(f'  -c|--config    : [OPTIONAL] output converter configuration')
 	print(f'  -i|--ifile     : [REQUIRED] {config["input"]} input file name')
 	print(f'  -o|--ofile     : [REQUIRED] {config["output"]} output file name')
 	print(f'  -t|--type      : [REQUIRED] specify output type')
+	print(f'  -f|--filter    : [OPTIONAL] specify object filter (all objects if none)')
 	print(f'Output types')
 	print(f'  profile        : voltage profile');
 
 input_file = None
 output_file = None
 output_type = None
+options = {}
 
-opts, args = getopt.getopt(sys.argv[1:],"hci:o:t:",["help","config","ifile=","ofile=","type="])
+opts, args = getopt.getopt(sys.argv[1:],"hci:o:t:f:",["help","config","ifile=","ofile=","type=","filter="])
 
 if not opts : 
 	help()
@@ -43,11 +46,16 @@ def help():
 		input_file = arg.strip()
 	elif opt in ("-o", "--ofile"):
 		output_file = arg.strip()
-	elif opt in ("-t","--type"):
+	elif opt in ["-t","--type"]:
 		if arg in config['type'].keys():
 			output_type = arg.strip()
 		else:
 			raise Exception(f"type '{arg}' is not recognized")
+	elif opt in ["-f","--filter"]:
+		spec = arg.strip().split("=")
+		if not "filter" in options:
+			options["filter"] = {}
+		options["filter"][spec[0]] = "=".join(spec[1:])
 	else:
 		raise Exception(f"'{opt}' is an invalid command line option")
 
@@ -56,8 +64,8 @@ def help():
 
 	modspec = util.spec_from_file_location(output_type, modname)
 	mod = importlib.import_module(f"{config['input']}2{config['output']}-{output_type}")
-	mod.convert(input_file=input_file,output_file=output_file)
+	mod.convert(input_file=input_file,output_file=output_file,options=options)
 
 else:
 
-	raise Exception(f"txt2glm-{output_type}.py not found")
+	raise Exception(f"json2csv-{output_type}.py not found")