diff --git a/docker/metplus/Dockerfile b/docker/metplus/Dockerfile index 405a2e4..dba4c02 100644 --- a/docker/metplus/Dockerfile +++ b/docker/metplus/Dockerfile @@ -2,8 +2,7 @@ ARG METPLUS_TAG=6.0.0 FROM dtcenter/metplus-analysis:${METPLUS_TAG} -# install wrf python package from branch in PR NCAR/wrf-python#242 -# because wrf will not install from pip +# NCAR/wrf-python commit: 69fd6a458ab60b417f1696d163ac7e9ab8363a69 RUN python3 -m pip install --upgrade pip \ - && python3 -m pip install git+https://github.com/DWesl/wrf-python.git@cmake-build \ + && python3 -m pip install git+https://github.com/NCAR/wrf-python.git@develop \ && python3 -m pip install cartopy diff --git a/docs/Users_Guide/running.rst b/docs/Users_Guide/running.rst index 6dda4dd..327e44c 100644 --- a/docs/Users_Guide/running.rst +++ b/docs/Users_Guide/running.rst @@ -17,11 +17,9 @@ Load the apptainer module:: module load apptainer -Change directory to scratch and pull the images from DockerHub. -This will create a `.sif` file in the current directory:: +Create a working directory in the scratch area:: - apptainer pull ${SCRATCH}/iwrf-metplus.sif docker://ncar/iwrf:metplus-latest - apptainer pull ${SCRATCH}/data-matthew-input-obs.sif oras://registry-1.docker.io/ncar/iwrf:data-matthew-input-obs + IWRF_WORK_DIR=${SCRATCH}/iwrf_work .. note:: @@ -37,11 +35,24 @@ This will create a `.sif` file in the current directory:: Create a directory to store the output:: - mkdir ${SCRATCH}/metplus_out + LOCAL_OUTPUT_DIR=${IWRF_WORK_DIR}/metplus_out + mkdir -p ${LOCAL_OUTPUT_DIR} + +Create a directory to store temporary Apptainer files +($TMPDIR is set automatically for all users on NCAR HPC machines):: + + export APPTAINER_TMPDIR=${TMPDIR} + mkdir -p ${APPTAINER_TMPDIR} + +Change directory to working directory and pull the containers from DockerHub. +This will create a `.sif` file in the current directory:: + + apptainer pull ${IWRF_WORK_DIR}/iwrf-metplus.sif docker://ncar/iwrf:metplus-latest + apptainer pull ${IWRF_WORK_DIR}/data-matthew-input-obs.sif oras://registry-1.docker.io/ncar/iwrf:data-matthew-input-obs Clone the I-WRF GitHub repository to get the configuration files:: - git clone https://github.com/NCAR/i-wrf ${SCRATCH}/i-wrf + git clone https://github.com/NCAR/i-wrf ${IWRF_WORK_DIR}/i-wrf Set environment variable to bind directories to container (note: this can also be accomplished by passing the value on the command line @@ -58,31 +69,33 @@ using the --bind argument) * Local: From data-matthew-input-obs.sif * Container: /data/input/obs/metar * Config directory containing METplus use case configuration file - * Local: ${SCRATCH}/i-wrf/use_cases/Hurricane_Matthew/METplus + * Local: ${IWRF_WORK_DIR}/i-wrf/use_cases/Hurricane_Matthew/METplus * Container: /config * Plot script directory containing WRF plotting scripts - * Local: ${SCRATCH}/i-wrf/use_cases/Hurricane_Matthew/Visualization + * Local: ${IWRF_WORK_DIR}/i-wrf/use_cases/Hurricane_Matthew/Visualization * Container: /plot_scripts * Output directory to write output - * Local: ${SCRATCH}/metplus_out + * Local: ${IWRF_WORK_DIR}/metplus_out * Container: /data/output +* Apptainer temp directory + * Local: ${APPTAINER_TMPDIR} + * Container: ${APPTAINER_TMPDIR} :: - LOCAL_METPLUS_CONFIG_DIR=${SCRATCH}/i-wrf/use_cases/Hurricane_Matthew/METplus - LOCAL_PLOT_SCRIPT_DIR=${SCRATCH}/i-wrf/use_cases/Hurricane_Matthew/Visualization + LOCAL_METPLUS_CONFIG_DIR=${IWRF_WORK_DIR}/i-wrf/use_cases/Hurricane_Matthew/METplus + LOCAL_PLOT_SCRIPT_DIR=${IWRF_WORK_DIR}/i-wrf/use_cases/Hurricane_Matthew/Visualization LOCAL_FCST_INPUT_DIR=/glade/derecho/scratch/jaredlee/nsf_i-wrf/matthew - LOCAL_OUTPUT_DIR=${SCRATCH}/metplus_out - export APPTAINER_BIND="${SCRATCH}/data-matthew-input-obs.sif:/data/input/obs:image-src=/,${LOCAL_METPLUS_CONFIG_DIR}:/config,${LOCAL_FCST_INPUT_DIR}:/data/input/wrf,${LOCAL_OUTPUT_DIR}:/data/output,${LOCAL_PLOT_SCRIPT_DIR}:/plot_scripts" + export APPTAINER_BIND="${IWRF_WORK_DIR}/data-matthew-input-obs.sif:/data/input/obs:image-src=/,${LOCAL_METPLUS_CONFIG_DIR}:/config,${LOCAL_FCST_INPUT_DIR}:/data/input/wrf,${LOCAL_OUTPUT_DIR}:/data/output,${LOCAL_PLOT_SCRIPT_DIR}:/plot_scripts,${APPTAINER_TMPDIR}:${APPTAINER_TMPDIR}" Execute the run_metplus.py command inside the container to run the use case:: - apptainer exec ${SCRATCH}/iwrf-metplus.sif /metplus/METplus/ush/run_metplus.py /config/PointStat_matthew.conf + apptainer exec ${IWRF_WORK_DIR}/iwrf-metplus.sif /metplus/METplus/ush/run_metplus.py /config/PointStat_matthew.conf Check that the output data was created locally:: - ls ${SCRATCH}/metplus_out/point_stat -1 + ls ${IWRF_WORK_DIR}/metplus_out/point_stat -1 Visualization diff --git a/use_cases/Hurricane_Matthew/METplus/PointStat_matthew.conf b/use_cases/Hurricane_Matthew/METplus/PointStat_matthew.conf index 931b7aa..73f3511 100644 --- a/use_cases/Hurricane_Matthew/METplus/PointStat_matthew.conf +++ b/use_cases/Hurricane_Matthew/METplus/PointStat_matthew.conf @@ -8,8 +8,7 @@ # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list ### -PROCESS_LIST = MADIS2NC(metar), MADIS2NC(raob), PointStat(surface), PointStat(upper_air), UserScript(wrf_plot) - +PROCESS_LIST = MADIS2NC(metar), MADIS2NC(raob), PointStat(surface), PointStat(upper_air), UserScript(wrf_plot), UserScript(metplotpy) ### # Time Info @@ -164,6 +163,17 @@ POINT_STAT_OUTPUT_PREFIX = {instance} POINT_STAT_MASK_GRID = FULL + +[user_env_vars] +PYTHONPATH = /metplus/METplotpy:/metplus/METcalcpy + + +[metplotpy] + +USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE +USER_SCRIPT_COMMAND = python /metplus/METdataio/METreformat/write_stat_ascii.py /plot_scripts/reformat_cnt.yaml; python /metplus/METplotpy/metplotpy/plots/line/line.py /plot_scripts/custom_line.yaml + + [wrf_plot] USER_SCRIPT_RUNTIME_FREQ = RUN_ONCE diff --git a/use_cases/Hurricane_Matthew/Visualization/custom_line.yaml b/use_cases/Hurricane_Matthew/Visualization/custom_line.yaml new file mode 100644 index 0000000..3d8f7b6 --- /dev/null +++ b/use_cases/Hurricane_Matthew/Visualization/custom_line.yaml @@ -0,0 +1,206 @@ +alpha: 0.05 +box_avg: 'False' +box_boxwex: 0.2 +box_notch: 'False' +box_outline: 'True' +box_pts: 'False' +caption_align: 0.0 +caption_col: '#333333' +caption_offset: 3.0 +caption_size: 0.8 +caption_weight: 1 +cex: 1 +colors: +- '#ff0000' +- '#0000ff' +con_series: +- 1 +- 1 +create_html: 'False' +derived_series_1: [] +derived_series_2: [] +dump_points_1: 'False' +dump_points_2: 'False' +event_equal: 'True' +fcst_var_val_1: + T2: + - RMSE + - BCMSE +fcst_var_val_2: [] +fixed_vars_vals_input: + fcst_lev: + - Z2 +grid_col: '#cccccc' +grid_lty: 3 +grid_lwd: 1 +grid_on: 'True' +grid_x: listX +indy_label: +- '0' +- '3' +- '6' +- '9' +- '12' +- '15' +- '18' +- '21' +- '24' +- '27' +- '30' +- '33' +- '36' +- '39' +- '42' +- '45' +- '48' +indy_stagger_1: 'True' +indy_stagger_2: 'False' +indy_vals: +- '00000' +- '03000' +- '60000' +- '90000' +- '120000' +- '150000' +- '180000' +- '210000' +- '240000' +- '270000' +- '300000' +- '330000' +- '360000' +- '390000' +- '420000' +- '450000' +- '480000' +indy_var: fcst_lead +legend_box: o +legend_inset: + x: 0.0 + y: -0.25 +legend_ncol: 3 +legend_size: 0.8 +line_type: None +list_stat_1: +- RMSE +- BCMSE +list_stat_2: [] +#- BCMSE +list_static_val: +mar: +- 8 +- 4 +- 5 +- 4 +method: bca +mgp: +- 1 +- 1 +- 0 +num_iterations: 1 +num_threads: -1 +plot_caption: Caption +plot_ci: +- std +- std +plot_disp: +- 'True' +- 'True' +plot_filename: '/data/output/met_plot/line_T2_RMSE_BCMSE.png' +plot_height: 8.5 +plot_res: 72 +plot_stat: median +plot_type: png16m +plot_units: in +plot_width: 11.0 + +# Optional, uncomment and set to directory to store the .points1 file +# that is used by METviewer (created when dump_points_1 is set to True) +# if dump_points_1 is True and this is uncommented, the points1 file +# will be saved in the default location (i.e. where the input data file is stored). +#points_path: /dir_to_save_points1_file +random_seed: null +series_line_style: +- '-' +- '-' +series_line_width: +- 1 +- 1 +series_order: +- 1 +- 2 +series_symbols: +- . +- ^ +series_type: +- b +- b +series_val_1: + model: + - WRF +series_val_2: {} +show_nstats: 'True' +show_signif: +- 'False' +stat_input: '/data/output/met_plot/cnt_reformatted.data' +sync_yaxes: 'False' +title: CNT linetype for T2 RMSE and BCMSE +title_align: 0.5 +title_offset: -2 +title_size: 1.4 +title_weight: 2.0 +user_legend: [] +variance_inflation_factor: 'False' +vert_plot: 'False' +x2lab_align: 0.5 +x2lab_offset: -0.5 +x2lab_size: 0.8 +x2lab_weight: 1 +x2tlab_horiz: 0.5 +x2tlab_orient: 1 +x2tlab_perp: 1 +x2tlab_size: 0.8 +xaxis: fcst lead time (hr) +xaxis_reverse: 'False' +xlab_align: 0.5 +xlab_offset: 2 +xlab_size: 1 +xlab_weight: 1 +xlim: [] +xtlab_decim: 0 +xtlab_horiz: 0.5 +xtlab_orient: 1 +xtlab_perp: -0.75 +xtlab_size: 1 +y2lab_align: 0.5 +y2lab_offset: 1 +y2lab_size: 1 +y2lab_weight: 1 +y2lim: [] +y2tlab_horiz: 0.5 +y2tlab_orient: 1 +y2tlab_perp: 1 +y2tlab_size: 1.0 +yaxis_1: RMSE and BCMSE for T2 WRF Model data +yaxis_2: '' +ylab_align: 0.5 +ylab_offset: -2 +ylab_size: 1 +ylab_weight: 1 +ylim: [] +ytlab_horiz: 0.5 +ytlab_orient: 1 +ytlab_perp: 0.5 +ytlab_size: 1 + + +# To save your log output to a file, specify a path and filename and uncomment the line below. Make sure you have +# permissions to the directory you specify. The default, as specified in the default config file is stdout. +log_filename: /data/output/met_plot/cnt_rmse_bme_line.log + +# To change the log level, specify a log level: debug, info, warning, error and uncomment the line below. +# Debug and info log level will produce more log output. +log_level: INFO +show_legend: +- 'True' +- 'True' diff --git a/use_cases/Hurricane_Matthew/Visualization/plot_wrf.py b/use_cases/Hurricane_Matthew/Visualization/plot_wrf.py index 8ca7e34..42a00d7 100755 --- a/use_cases/Hurricane_Matthew/Visualization/plot_wrf.py +++ b/use_cases/Hurricane_Matthew/Visualization/plot_wrf.py @@ -10,11 +10,9 @@ import sys import argparse import pathlib -import warnings import datetime as dt import numpy as np import pandas as pd -import xarray as xr import netCDF4 import wrf import matplotlib as mpl diff --git a/use_cases/Hurricane_Matthew/Visualization/reformat_cnt.yaml b/use_cases/Hurricane_Matthew/Visualization/reformat_cnt.yaml new file mode 100644 index 0000000..c1123e3 --- /dev/null +++ b/use_cases/Hurricane_Matthew/Visualization/reformat_cnt.yaml @@ -0,0 +1,20 @@ +# Configuration settings for reformatting MET .stat output to format for METplotpy + +# if .stat file input already contains aggregated statistics (i.e. output from MET stat-analysis +# has been applied to MET .stat output from point-stat, grid-stat, or ensemble-stat). +input_stats_aggregated: True + +output_dir: /data/output/met_plot +output_filename: cnt_reformatted.data + +input_data_dir: /data/output/point_stat + +# Set to log_filename to STDOUT/stdout if no log file is to be saved +log_directory: /data/output/met_plot +log_filename: reformatting_cnt_log.txt + +# most verbose is info, less verbose is error +log_level: info + +# Currently support FHO, CTC, CTS, CNT, SL1L2, VL1L2, PCT, MCTC, VCNT, ECNT, and RHIST line types +line_type: CNT