From 317ce5277666627b91eca4fc3b346760ca664525 Mon Sep 17 00:00:00 2001 From: LenkaNovak Date: Fri, 21 Jul 2023 14:52:48 -0700 Subject: [PATCH] add current target longer slabplanet fix InexactError-DiffEq won't allow dt_save=Inf helem 6 test sponge helem 6 test sponge helem 6 test bring back all long runs modular fix modular fix wait clean up reduce unthreaded --- .buildkite/longruns/pipeline.yml | 63 ++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/.buildkite/longruns/pipeline.yml b/.buildkite/longruns/pipeline.yml index 2e4d26e1cb..72d0d3ada3 100644 --- a/.buildkite/longruns/pipeline.yml +++ b/.buildkite/longruns/pipeline.yml @@ -27,9 +27,9 @@ steps: - julia -e 'using Pkg; Pkg.add("MPIPreferences"); using MPIPreferences; use_system_binary()' - echo "--- Instantiate AMIP env" - - "julia --project=experiments/AMIP/moist_mpi_earth/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" - - "julia --project=experiments/AMIP/moist_mpi_earth/ -e 'using Pkg; Pkg.precompile()'" - - "julia --project=experiments/AMIP/moist_mpi_earth/ -e 'using Pkg; Pkg.status()'" + - "julia --project=experiments/AMIP/modular/ -e 'using Pkg; Pkg.instantiate(;verbose=true)'" + - "julia --project=experiments/AMIP/modular/ -e 'using Pkg; Pkg.precompile()'" + - "julia --project=experiments/AMIP/modular/ -e 'using Pkg; Pkg.status()'" - echo "--- Download artifacts" - "julia --project=artifacts -e 'using Pkg; Pkg.instantiate(;verbose=true)'" @@ -54,10 +54,10 @@ steps: steps: - - label: "Moist earth with slab surface - notmono + modular: bulk gray no_sponge idealinsol freq_dt_cpl" - key: "slabplanet" - command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name default_modular_long --coupled true --moist equil --vert_diff true --rad gray --energy_check true --anim true --mode_name slabplanet --t_end 10days --dt_save_to_sol 1days --dt_cpl 200 --dt 200secs --mono_surface false --h_elem 4 --precip_model 0M" - artifact_paths: "experiments/AMIP/modular/output/slabplanet/default_modular_long_artifacts/total_energy*.png" + - label: "Slabplanet: default" + key: "slabplanet_default_longrun" + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name slabplanet_default --FLOAT_TYPE Float64 --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --energy_check true --mode_name slabplanet --t_end 60days --dt_save_to_sol 10days --dt_cpl 200 --dt 200secs --mono_surface true --h_elem 6 --precip_model 0M --anim true --job_id slabplanet_default_longrun" + artifact_paths: "experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -65,9 +65,20 @@ steps: slurm_mem_per_cpu: 16G - label: "MPI AMIP FINE: longrun" - key: "amip_longrun" - command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name target_amip_n64_longrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 180days --dt_save_to_sol 10days --mono_surface false --precip_model 0M" - artifact_paths: "experiments/AMIP/modular/output/amip/target_amip_n64_longrun_artifacts/*" + key: "amip_longrun_fine" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_longrun_fine --coupled true --anim true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 140days --dt_save_to_sol 10days --mono_surface false --precip_model 0M" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_fine_artifacts/*" + env: + CLIMACORE_DISTRIBUTED: "MPI" + BUILD_HISTORY_HANDLE: "" + agents: + slurm_ntasks: 64 + slurm_mem_per_cpu: 16G + + - label: "MPI AMIP FINE: target longrun" + key: "amip_longrun_target" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_longrun_target --coupled true --anim true --surface_setup PrescribedSurface --dt_cpl 150 --energy_check false --mode_name amip --mono_surface false --vert_diff true --moist equil --rad clearsky --precip_model 0M --z_elem 35 --dz_bottom 50 --h_elem 12 --kappa_4 3e16 --rayleigh_sponge true --alpha_rayleigh_uh 0 --dt 150secs --t_end 140days --job_id amip_longrun2 --dt_save_to_sol 5days --dt_save_to_disk 1days --FLOAT_TYPE Float64" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -78,8 +89,8 @@ steps: # MPI performance scaling (10 days) - label: "MPI AMIP FINE: n64" key: "mpi_amip_fine_n64" - command: "mpiexec julia --color=yes --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --run_name target_amip_n64_shortrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" - artifact_paths: "experiments/AMIP/moist_mpi_earth/output/amip/target_amip_n64_shortrun_artifacts/*" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_n64_shortrun --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_n64_shortrun_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -90,8 +101,8 @@ steps: - label: "MPI AMIP FINE: n32" key: "mpi_amip_fine_n32" - command: "mpiexec julia --color=yes --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --run_name target_amip_n32_shortrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" - artifact_paths: "experiments/AMIP/moist_mpi_earth/output/amip/target_amip_n32_shortrun_artifacts/*" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_n32_shortrun --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_n32_shortrun_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -102,8 +113,8 @@ steps: - label: "MPI AMIP FINE: n8" key: "mpi_amip_fine_n8" - command: "mpiexec julia --color=yes --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --run_name target_amip_n8_shortrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" - artifact_paths: "experiments/AMIP/moist_mpi_earth/output/amip/target_amip_n8_shortrun_artifacts/*" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_n8_shortrun --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_n8_shortrun_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -114,8 +125,8 @@ steps: - label: "MPI AMIP FINE: n2" key: "mpi_amip_fine_n2" - command: "mpiexec julia --color=yes --project=experiments/AMIP/moist_mpi_earth/ experiments/AMIP/moist_mpi_earth/coupler_driver.jl --run_name target_amip_n2_shortrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" - artifact_paths: "experiments/AMIP/moist_mpi_earth/output/amip/target_amip_n2_shortrun_artifacts/*" + command: "mpiexec julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_n2_shortrun --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_n2_shortrun_artifacts/*" env: CLIMACORE_DISTRIBUTED: "MPI" BUILD_HISTORY_HANDLE: "" @@ -124,10 +135,10 @@ steps: slurm_mem_per_cpu: 16G slurm_tasks_per_node: 2 - - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph + - label: "Unthreaded AMIP FINE" # also reported by longruns with a flame graph; 10d take 21h, so reducing to 1d key: "unthreaded_amip_fine" - command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name target_amip_n1_shortrun --coupled true --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 10days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" - artifact_paths: "experiments/AMIP/modular/output/amip/target_amip_n1_shortrun_artifacts/*" + command: "julia --color=yes --project=experiments/AMIP/modular/ experiments/AMIP/modular/coupler_driver_modular.jl --run_name amip_n1_shortrun --coupled true --surface_setup PrescribedSurface --moist equil --vert_diff true --rad gray --z_elem 50 --dz_top 3000 --dz_bottom 30 --h_elem 16 --kappa_4 1e16 --z_stretch false --rayleigh_sponge true --alpha_rayleigh_uh 0 --alpha_rayleigh_w 10 --dt_cpl 150 --dt 150secs --dt_rad 1hours --FLOAT_TYPE Float64 --energy_check false --mode_name amip --t_end 1days --dt_save_to_sol 100days --mono_surface false --precip_model 0M" + artifact_paths: "experiments/AMIP/modular/output/amip/amip_n1_shortrun_artifacts/*" env: BUILD_HISTORY_HANDLE: "" agents: @@ -157,12 +168,16 @@ steps: - label: ":envelope: Slack report: Slabplanet" command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/default_modular_long_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/default_modular_long_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_log_bucket.png -m png -n slab_coarse_log -x "Slabplanet energy conservation (log error)" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/slabplanet/slabplanet_default_longrun_artifacts/total_energy_bucket.png -m png -n slab_coarse -x "Slabplanet energy conservation" + + - label: ":envelope: Slack report: AMIP fine benchmark" + command: + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_fine_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Longrun" - label: ":envelope: Slack report: target AMIP" command: - - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/target_amip_n64_longrun_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Longrun" + - slack-upload -c "#coupler-report" -f experiments/AMIP/modular/output/amip/amip_longrun_target_artifacts/amip_paperplots.png -m png -n amip_fine -x "AMIP Longrun" - label: ":envelope: Slack report: Flame Diff" command: