From ba3fae02a2ddcf9fe38b1d2f1019f019841864aa Mon Sep 17 00:00:00 2001 From: "David P. Chassin" Date: Thu, 1 Feb 2024 05:54:45 -0800 Subject: [PATCH 1/3] Update version.h Signed-off-by: David P. Chassin --- source/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/version.h b/source/version.h index 863af9c9b..f241ad539 100644 --- a/source/version.h +++ b/source/version.h @@ -11,7 +11,7 @@ #define REV_MAJOR 4 #define REV_MINOR 3 -#define REV_PATCH 3 +#define REV_PATCH 4 #ifdef HAVE_CONFIG_H #include "config.h" From 9b54ff0b79bb3fc2d90c23837cb7a2c6427d8ad1 Mon Sep 17 00:00:00 2001 From: Alyona Teyber Date: Fri, 2 Feb 2024 16:00:24 -0800 Subject: [PATCH 2/3] Fixing poor implementation of phase integration for AMI (#163) Signed-off-by: Alyona Teyber Signed-off-by: David P. Chassin Co-authored-by: David P. Chassin --- converters/csv-ami2glm-player.py | 23 +++++++++++++++-------- converters/json2csv-pandas.py | 26 +++++++++++++++++++++++--- converters/json2csv.py | 8 ++++++-- 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/converters/csv-ami2glm-player.py b/converters/csv-ami2glm-player.py index 6774af29b..50625f238 100644 --- a/converters/csv-ami2glm-player.py +++ b/converters/csv-ami2glm-player.py @@ -46,6 +46,13 @@ def string_clean(input_str): network = False ami_key = False +def write_player(file, obj, node_ID, phase) : + file.write('object player {\n') + file.write('\tparent "' + obj + '";\n') + file.write('\tfile "' + os.path.join(folder_name,str(node_ID)) + '.csv";\n') + file.write(f'\tproperty constant_power_{phase};\n') + file.write('}\n') + def convert(input_files, output_file, options={}): if type(input_files) is dict: @@ -98,22 +105,20 @@ def convert(input_files, output_file, options={}): if os.path.splitext(output_file)[1]=='.csv' : df.to_csv(os.path.join(folder_name,os.path.basename(output_file)), index=False) elif os.path.splitext(output_file)[1]=='.glm' : + phase_dict = {} with open(output_file, mode='w') as file : file.write('module tape;\n') for node_ID in node_ID_set : if isinstance(node_ID, float) and math.isnan(node_ID) : continue - file.write('object player {\n') for obj,val in network["objects"].items() : if "load" in val["class"] and node_ID in obj: - node_phase = val["phases"] + node_phase = ''.join([x for x in 'ABC' if x in val['phases']]) parent = val["parent"] - file.write('\tparent "' + str(parent) + '";\n') - file.write('\tfile "' + os.path.join(folder_name,str(node_ID)) + '.csv";\n') - file.write('\tphases "' + node_phase + '";\n') - file.write('}\n') - + phase_dict[node_ID]=node_phase + for p in node_phase : + write_player(file, obj, node_ID, p) new_column_names = { 'reading_dttm': 'timestamp', @@ -123,12 +128,14 @@ def convert(input_files, output_file, options={}): df_ami.rename(columns=new_column_names,inplace=True) df_ami.drop(['interval_pcfc_date','interval_pcfc_hour'],axis=1,inplace=True) df_ami.sort_index(inplace=True) - # Iterate over unique customer IDs for customer_id in df_ami['customer_id'].unique(): # Create a new DataFrame for each customer ID + if isinstance(customer_id, float) and math.isnan(customer_id) : + continue customer_df = df_ami[df_ami['customer_id'] == customer_id].drop(columns='customer_id') customer_df = customer_df.sort_values(by='timestamp') + customer_df['power[kW]'] = customer_df['power[kW]']/len(phase_dict[customer_id])*1000 # Save the DataFrame to a CSV file output_file = f"{folder_name}/{customer_id}.csv" customer_df.to_csv(output_file, index=False, header=False) diff --git a/converters/json2csv-pandas.py b/converters/json2csv-pandas.py index 69be534b3..c8678875c 100644 --- a/converters/json2csv-pandas.py +++ b/converters/json2csv-pandas.py @@ -25,7 +25,17 @@ -f name=EXPRESSION -Be careful to quote expressions that can be interpreted by the shell. +The output type can include a column list to limit the fields that are +included in the output CSV file. For example, + + gridlabd -C input.glm -D csv_save_options="-t pandas:name,phases" -o output.csv + +will only output the `name` and `phases` columns. + +Be careful to quote expressions that can be interpreted by the shell. Also +note that regular expressions match the beginning of any string. For an exact +match you must use a `$` to stop the parse, e.g., `overhead_line$` will not +match `overhead_line_conductor`. EXAMPLE ------- @@ -45,7 +55,11 @@ import re def convert(input_file,output_file=None, options={}): - + """ + Valid options are: + - output-columns (list of str): a list of columns to output + - filter (dict of regex): patterns to match for properties to filter objects + """ if output_file == '': if input_file[-5:] == ".json": output_file = input_file[:-5] + ".csv" @@ -71,4 +85,10 @@ def convert(input_file,output_file=None, options={}): else: result = data["objects"] df = pd.DataFrame(result).transpose() - df.to_csv(output_file,header=True,index=False) + df.index.name = "name" + if "output-columns" in options: + for field in df.columns: + if not re.match("|".join(options["output-columns"]),field): + df.drop(field,inplace=True,axis=1) + keep_index = ("output-columns" in options and re.match("|".join(options["output-columns"]),"name")) + df.to_csv(output_file,header=True,index=keep_index) diff --git a/converters/json2csv.py b/converters/json2csv.py index 8899c9f8b..c0a8b568a 100644 --- a/converters/json2csv.py +++ b/converters/json2csv.py @@ -16,7 +16,7 @@ def help(): print(f'Syntax:') - print(f'{config["input"]}2{config["output"]}.py -i|--ifile [,[,...]] -o|--ofile -t|--type -f|--filter ') + print(f'{config["input"]}2{config["output"]}.py -i|--ifile [,[,...]] -o|--ofile -t|--type [:] -f|--filter ') print(f' -c|--config : [OPTIONAL] output converter configuration') print(f' -i|--ifile : [REQUIRED] {config["input"]} input file name') print(f' -o|--ofile : [REQUIRED] {config["output"]} output file name') @@ -47,8 +47,12 @@ def help(): elif opt in ("-o", "--ofile"): output_file = arg.strip() elif opt in ["-t","--type"]: - if arg in config['type'].keys(): + if arg.split(":")[0] in config['type'].keys(): output_type = arg.strip() + if ":" in output_type: + spec = output_type.split(":") + output_type = spec[0] + options["output-columns"] = spec[1].split(",") else: raise Exception(f"type '{arg}' is not recognized") elif opt in ["-f","--filter"]: From 5b1b3abaf6ca2103cea9c7f8f592471575ef2e14 Mon Sep 17 00:00:00 2001 From: "David P. Chassin" Date: Fri, 2 Feb 2024 19:57:00 -0800 Subject: [PATCH 3/3] Update version.h Signed-off-by: David P. Chassin --- source/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/version.h b/source/version.h index f241ad539..483eed1fc 100644 --- a/source/version.h +++ b/source/version.h @@ -11,7 +11,7 @@ #define REV_MAJOR 4 #define REV_MINOR 3 -#define REV_PATCH 4 +#define REV_PATCH 5 #ifdef HAVE_CONFIG_H #include "config.h"