From 5ed1d9d1a7ead49d9e1ce28fb81984bbe3fb115b Mon Sep 17 00:00:00 2001 From: "David P. Chassin" Date: Fri, 26 Jan 2024 19:45:54 -0800 Subject: [PATCH] Fix XLSX converter (#157) Signed-off-by: David P. Chassin Signed-off-by: David P. Chassin Signed-off-by: Duncan Ragsdale <88173870+Thistleman@users.noreply.github.com> Signed-off-by: Mitchell Victoriano Signed-off-by: Mitchell Victoriano <47313912+MitchellAV@users.noreply.github.com> Co-authored-by: Duncan Ragsdale <88173870+Thistleman@users.noreply.github.com> Co-authored-by: Mitchell Victoriano <47313912+MitchellAV@users.noreply.github.com> Co-authored-by: aivanova5 --- converters/csv-ami2glm-player.py | 73 +++++++++++++++++++--------- converters/csv2glm.py | 2 +- third_party/superLU_MT/sp_colorder.c | 2 +- tools/create_meters.py | 10 +--- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/converters/csv-ami2glm-player.py b/converters/csv-ami2glm-player.py index 7f1edad94..6774af29b 100644 --- a/converters/csv-ami2glm-player.py +++ b/converters/csv-ami2glm-player.py @@ -4,7 +4,7 @@ Shell: $ gridlabd convert -i ami:AMI.csv,ami_key:AMI_KEYS.csv, network:NETWORK.csv - -o PLAYERS.csv -f xlsx-spida -t csv-geodata [OPTIONS ...] + -o PLAYERS.csv -f csv-ami -t glm-player [OPTIONS ...] GLM: #convert ami:AMI.csv,ami_key:AMI_KEYS.csv @@ -29,10 +29,12 @@ import re import numpy as np import os +import csv +import json default_options = { - # "include_network" : None, + "folder_name" : "./player/", } def string_clean(input_str): @@ -41,11 +43,10 @@ def string_clean(input_str): output_str = output_str.replace('"', "") return output_str -include_network = False +network = False ami_key = False def convert(input_files, output_file, options={}): - print('test') if type(input_files) is dict: for key in input_files: @@ -65,9 +66,10 @@ def convert(input_files, output_file, options={}): ami_key = True if "network" in input_files: - global include_network - include_network = input_files["network"] - + global network + network_path = input_files["network"] + with open(network_path,'r') as network_json: + network = json.load(network_json) elif type(input_files) is str: input_ami_file = input_files @@ -88,18 +90,45 @@ def convert(input_files, output_file, options={}): node_ID_set = set(df_ami['transformer_structure']) - with open(output_file, mode='w', newline='') as file : - writer = csv.writer(file) - writer.writerow(['module tape;']) - - for node_ID in node_ID_set : - writer.writerow(['\n']) - writer.writerow(['object player {\n']) - writer.writerow(['\tproperty measured_real_energy;\n']) - writer.writerow(['\tparent ' + node_ID + '\n']) - writer.writerow(['\tfile ./player/' + node_ID + '.csv\n']) - writer.writerow(['}\n']) - - - - + df = pd.DataFrame({'class': ['player']*len(node_ID_set), 'parent' : list(node_ID_set), 'file' : ['player_' + str(node) + '.csv' for node in node_ID_set]}) + + if not os.path.exists(folder_name): + os.makedirs(folder_name) + + if os.path.splitext(output_file)[1]=='.csv' : + df.to_csv(os.path.join(folder_name,os.path.basename(output_file)), index=False) + elif os.path.splitext(output_file)[1]=='.glm' : + with open(output_file, mode='w') as file : + file.write('module tape;\n') + + for node_ID in node_ID_set : + if isinstance(node_ID, float) and math.isnan(node_ID) : + continue + file.write('object player {\n') + for obj,val in network["objects"].items() : + if "load" in val["class"] and node_ID in obj: + node_phase = val["phases"] + parent = val["parent"] + file.write('\tparent "' + str(parent) + '";\n') + file.write('\tfile "' + os.path.join(folder_name,str(node_ID)) + '.csv";\n') + file.write('\tphases "' + node_phase + '";\n') + file.write('}\n') + + + new_column_names = { + 'reading_dttm': 'timestamp', + 'net_usage': 'power[kW]', + 'transformer_structure': 'customer_id' + } + df_ami.rename(columns=new_column_names,inplace=True) + df_ami.drop(['interval_pcfc_date','interval_pcfc_hour'],axis=1,inplace=True) + df_ami.sort_index(inplace=True) + + # Iterate over unique customer IDs + for customer_id in df_ami['customer_id'].unique(): + # Create a new DataFrame for each customer ID + customer_df = df_ami[df_ami['customer_id'] == customer_id].drop(columns='customer_id') + customer_df = customer_df.sort_values(by='timestamp') + # Save the DataFrame to a CSV file + output_file = f"{folder_name}/{customer_id}.csv" + customer_df.to_csv(output_file, index=False, header=False) diff --git a/converters/csv2glm.py b/converters/csv2glm.py index e72af21a0..0084d27e4 100644 --- a/converters/csv2glm.py +++ b/converters/csv2glm.py @@ -51,7 +51,7 @@ def error(msg): elif opt in ("-i", "--ifile"): try: if not arg.startswith("http"): - input_file = dict([x.split(":") for x in arg.strip().split(",")]) + input_file = dict([x.strip().split(":") for x in arg.strip().split(",")]) else: input_file = arg.strip() except: diff --git a/third_party/superLU_MT/sp_colorder.c b/third_party/superLU_MT/sp_colorder.c index 930ffd87b..d5a8afb5f 100644 --- a/third_party/superLU_MT/sp_colorder.c +++ b/third_party/superLU_MT/sp_colorder.c @@ -212,7 +212,7 @@ int dPrintSuperPart(char *pname, int n, int *part_super) { register int i; - FILE *fopen(), *fp; + FILE *fp; char fname[20]; strcpy(fname, pname); strcat(fname, ".dat"); diff --git a/tools/create_meters.py b/tools/create_meters.py index 8c14a4828..5c655b339 100644 --- a/tools/create_meters.py +++ b/tools/create_meters.py @@ -54,7 +54,6 @@ INPUTFILE = None OUTPUTFILE = None WITHAMI = False -MODIFY = False # modify defaults (GLM only) @@ -140,17 +139,13 @@ def get_parent(name): def fix_parent(parent,name,oclass): pclass = get_data(parent,'class') - # change parent nodes to meters if pclass and pclass == oclass.replace('load','node'): - # change node to meter nclass = pclass.replace('node','meter') set_data(parent,'class',nclass) - else: - - # change parent + # change parent parent = get_parent(parent) if parent: fix_parent(parent,name,oclass) @@ -186,7 +181,6 @@ def main(args): if oclass in ['load','triplex_load']: links = get_links(name) if parent: - fix_parent(parent,name,oclass) elif links == None: @@ -221,7 +215,7 @@ def main(args): elif OUTPUTFILE.endswith(".glm"): - if MODIFY: + if MODIFY_LIST: with open(OUTPUTFILE,"wt") as fh: print(f"// generated by '{' '.join(sys.argv)}' on {datetime.datetime.now()}",file=fh) print(f"// NEW OBJECTS",file=fh)