Fixing poor implementation of phase integration for AMI (#163)

Signed-off-by: Alyona Teyber <Ivanova.alyona5@gmail.com>
Signed-off-by: David P. Chassin <dchassin@slac.stanford.edu>
Co-authored-by: David P. Chassin <dchassin@slac.stanford.edu>

converters/csv-ami2glm-player.py

@@ -46,6 +46,13 @@ def string_clean(input_str):
 network = False
 ami_key = False
 
+def write_player(file, obj, node_ID, phase) : 
+	file.write('object player {\n')
+	file.write('\tparent "' + obj + '";\n')
+	file.write('\tfile "' + os.path.join(folder_name,str(node_ID)) + '.csv";\n')
+	file.write(f'\tproperty constant_power_{phase};\n')
+	file.write('}\n')
+
 def convert(input_files, output_file, options={}):
 
 	if type(input_files) is dict:
@@ -98,22 +105,20 @@ def convert(input_files, output_file, options={}):
 	if os.path.splitext(output_file)[1]=='.csv' : 
 		df.to_csv(os.path.join(folder_name,os.path.basename(output_file)), index=False)
 	elif os.path.splitext(output_file)[1]=='.glm' :
+		phase_dict = {}
 		with open(output_file, mode='w') as file :  
 			file.write('module tape;\n')
 
 			for node_ID in node_ID_set : 
 				if isinstance(node_ID, float) and math.isnan(node_ID) :
 					continue 
-				file.write('object player {\n')
 				for obj,val in network["objects"].items() : 
 					if "load" in val["class"] and node_ID in obj: 
-						node_phase = val["phases"]
+						node_phase = ''.join([x for x in 'ABC' if x in val['phases']])
 						parent = val["parent"]
-				file.write('\tparent "' + str(parent) + '";\n')
-				file.write('\tfile "' + os.path.join(folder_name,str(node_ID)) + '.csv";\n')
-				file.write('\tphases "' + node_phase  + '";\n')
-				file.write('}\n')
-
+						phase_dict[node_ID]=node_phase
+				for p in node_phase : 
+					write_player(file, obj, node_ID, p)
 
 	new_column_names = {
 		'reading_dttm': 'timestamp',
@@ -123,12 +128,14 @@ def convert(input_files, output_file, options={}):
 	df_ami.rename(columns=new_column_names,inplace=True)
 	df_ami.drop(['interval_pcfc_date','interval_pcfc_hour'],axis=1,inplace=True)
 	df_ami.sort_index(inplace=True)
-
 	# Iterate over unique customer IDs
 	for customer_id in df_ami['customer_id'].unique():
 	    # Create a new DataFrame for each customer ID
+		if isinstance(customer_id, float) and math.isnan(customer_id) : 
+			continue
 		customer_df = df_ami[df_ami['customer_id'] == customer_id].drop(columns='customer_id')
 		customer_df = customer_df.sort_values(by='timestamp')
+		customer_df['power[kW]'] = customer_df['power[kW]']/len(phase_dict[customer_id])*1000
 	    # Save the DataFrame to a CSV file
 		output_file = f"{folder_name}/{customer_id}.csv"
 		customer_df.to_csv(output_file, index=False, header=False)

converters/json2csv-pandas.py

@@ -25,7 +25,17 @@
 
 	-f name=EXPRESSION
 
-Be careful to quote expressions that can be interpreted by the shell.
+The output type can include a column list to limit the fields that are
+included in the output CSV file. For example,
+
+	gridlabd -C input.glm -D csv_save_options="-t pandas:name,phases" -o output.csv
+
+will only output the `name` and `phases` columns. 
+
+Be careful to quote expressions that can be interpreted by the shell. Also
+note that regular expressions match the beginning of any string. For an exact
+match you must use a `$` to stop the parse, e.g., `overhead_line$` will not
+match `overhead_line_conductor`.
 
 EXAMPLE
 -------
@@ -45,7 +55,11 @@
 import re
 
 def convert(input_file,output_file=None, options={}):
-
+	"""
+	Valid options are:
+	- output-columns (list of str): a list of columns to output
+	- filter (dict of regex): patterns to match for properties to filter objects
+	"""
 	if output_file == '':
 		if input_file[-5:] == ".json":
 			output_file = input_file[:-5] + ".csv" 
@@ -71,4 +85,10 @@ def convert(input_file,output_file=None, options={}):
 	else:
 		result = data["objects"]
 	df = pd.DataFrame(result).transpose()
-	df.to_csv(output_file,header=True,index=False)	
+	df.index.name = "name"
+	if "output-columns" in options:
+		for field in df.columns:
+			if not re.match("|".join(options["output-columns"]),field):
+				df.drop(field,inplace=True,axis=1)
+	keep_index = ("output-columns" in options and re.match("|".join(options["output-columns"]),"name"))
+	df.to_csv(output_file,header=True,index=keep_index)

converters/json2csv.py

@@ -16,7 +16,7 @@
 
 def help():
 	print(f'Syntax:')
-	print(f'{config["input"]}2{config["output"]}.py -i|--ifile <input-file>[,<input-file>[,...]] -o|--ofile <output-file> -t|--type <input-type> -f|--filter <filter-spec>')
+	print(f'{config["input"]}2{config["output"]}.py -i|--ifile <input-file>[,<input-file>[,...]] -o|--ofile <output-file> -t|--type <output-type>[:<column-list>] -f|--filter <filter-spec>')
 	print(f'  -c|--config    : [OPTIONAL] output converter configuration')
 	print(f'  -i|--ifile     : [REQUIRED] {config["input"]} input file name')
 	print(f'  -o|--ofile     : [REQUIRED] {config["output"]} output file name')
@@ -47,8 +47,12 @@ def help():
 	elif opt in ("-o", "--ofile"):
 		output_file = arg.strip()
 	elif opt in ["-t","--type"]:
-		if arg in config['type'].keys():
+		if arg.split(":")[0] in config['type'].keys():
 			output_type = arg.strip()
+			if ":" in output_type:
+				spec = output_type.split(":")
+				output_type = spec[0]
+				options["output-columns"] = spec[1].split(",")
 		else:
 			raise Exception(f"type '{arg}' is not recognized")
 	elif opt in ["-f","--filter"]: