Merge branch 'main' into sk/ke_kernel_opt

.buildkite/dss_scaling/pipeline.yml

@@ -1,5 +1,5 @@
 env:
-  JULIA_VERSION: "1.8.3"
+  JULIA_VERSION: "1.9.2"
   MPICH_VERSION: "4.0.0"
   OPENMPI_VERSION: "4.1.1"
   CUDA_VERSION: "11.2"

.buildkite/pipeline.yml

@@ -1,13 +1,15 @@
 agents:
   queue: central
   slurm_mem: 8G
-  modules: julia/1.8.5 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
+  modules: julia/1.9.2 cuda/11.8 ucx/1.14.1_cuda-11.8 openmpi/4.1.5_cuda-11.8 hdf5/1.12.2-ompi415 nsight-systems/2023.2.1
 
 env:
   JULIA_LOAD_PATH: "${JULIA_LOAD_PATH}:${BUILDKITE_BUILD_CHECKOUT_PATH}/.buildkite"
   OPENBLAS_NUM_THREADS: 1
   JULIA_NVTX_CALLBACKS: gc
   OMPI_MCA_opal_warn_on_missing_libcuda: 0
+  # JULIA_CPU_TARGET: broadwell # arch used by GPU nodes
+  JULIA_CPU_TARGET: 'broadwell;skylake'
 
 steps:
   - label: "initialize"
@@ -20,6 +22,7 @@ steps:
 
       - echo "--- Instantiate test"
       - "julia --project=test -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
+      - "julia --project=test --check-bounds=yes -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
 
       - echo "--- Instantiate lib/ClimaCoreTempestRemap"
       - "julia --project=lib/ClimaCoreTempestRemap -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
@@ -29,6 +32,7 @@ steps:
 
       - echo "--- Instantiate examples"
       - "julia --project=examples -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
+      - "julia --project=examples --check-bounds=yes -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
 
       - echo "--- Instantiate benchmarks"
       - "julia --project=benchmarks/bickleyjet -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
@@ -37,7 +41,8 @@ steps:
       - "julia --project -e 'using Pkg; Pkg.status()'"
 
     agents:
-      slurm_mem: 32G
+      slurm_mem: 80G
+      slurm_gpus: 1
       slurm_cpus_per_task: 8
     env:
       JULIA_NUM_PRECOMPILE_TASKS: 8
@@ -330,10 +335,6 @@ steps:
         agents:
           slurm_gpus: 1
 
-      - label: "Unit: fielddiffeq"
-        key: unit_fielddiffeq
-        command: "julia --color=yes --check-bounds=yes --project=test test/Fields/fielddiffeq.jl"
-
       - label: "Unit: reduction cuda"
         key: unit_reduction_cuda
         command:
@@ -517,6 +518,10 @@ steps:
         key: unit_field2arrays
         command: "julia --color=yes --check-bounds=yes --project=test test/MatrixFields/field2arrays.jl"
 
+      - label: "Unit: matrix multiplication at boundaries"
+        key: unit_matrix_multiplication_at_boundaries
+        command: "julia --color=yes --check-bounds=yes --project=test test/MatrixFields/matrix_multiplication_at_boundaries.jl"
+
       - label: "Unit: matrix field broadcasting (CPU)"
         key: unit_matrix_field_broadcasting_cpu
         command: "julia --color=yes --check-bounds=yes --project=test test/MatrixFields/matrix_field_broadcasting.jl"
@@ -526,7 +531,7 @@ steps:
         command: "julia --color=yes --project=test test/MatrixFields/matrix_field_broadcasting.jl"
         agents:
           slurm_gpus: 1
-          slurm_mem: 20GB
+          slurm_mem: 40GB
 
   - group: "Unit: Hypsography"
     steps:
@@ -1175,6 +1180,8 @@ steps:
           - "julia --color=yes --project=examples examples/hybrid/sphere/hadley_circulation.jl"
         artifact_paths:
           - "examples/hybrid/sphere/output/hadley_circulation/*"
+        agents:
+          slurm_mem: 20GB
 
       - label: ":computer: Float 64 3D sphere baroclinic wave (ρe)"
         key: "cpu_baroclinic_wave_rho_e_float64"
@@ -1255,6 +1262,7 @@ steps:
           - "examples/hybrid/sphere/output/held_suarez_rhoe/Float32/*"
         env:
           TEST_NAME: "sphere/held_suarez_rhoe"
+        soft_fail: true
 
       - label: ":computer: Float64 3D sphere dry Held-Suarez (ρθ)"
         key: "cpu_held_suarez_rho_theta_float64"

.buildkite/scaling/pipeline.yml

@@ -1,5 +1,5 @@
 env:
-  JULIA_VERSION: "1.8.3"
+  JULIA_VERSION: "1.9.2"
   MPICH_VERSION: "4.0.0"
   OPENMPI_VERSION: "4.1.1"
   CUDA_VERSION: "11.2"

.dev/Manifest.toml

@@ -1,6 +1,6 @@
 # This file is machine-generated - editing it directly is not advised
 
-julia_version = "1.8.5"
+julia_version = "1.9.2"
 manifest_format = "2.0"
 project_hash = "30b405be1c677184b7703a9bfb3d2100029ccad0"
 
@@ -27,15 +27,19 @@ uuid = "a80b9123-70ca-4bc0-993e-6e3bcb318db6"
 version = "0.8.12"
 
 [[deps.Compat]]
-deps = ["Dates", "LinearAlgebra", "UUIDs"]
+deps = ["UUIDs"]
 git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957"
 uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
 version = "4.6.1"
+weakdeps = ["Dates", "LinearAlgebra"]
+
+    [deps.Compat.extensions]
+    CompatLinearAlgebraExt = "LinearAlgebra"
 
 [[deps.CompilerSupportLibraries_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
-version = "1.0.1+0"
+version = "1.0.5+0"
 
 [[deps.Crayons]]
 git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
@@ -104,7 +108,7 @@ version = "1.10.2+0"
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
 [[deps.LinearAlgebra]]
-deps = ["Libdl", "libblastrampoline_jll"]
+deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
 uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 
 [[deps.Logging]]
@@ -117,14 +121,14 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 [[deps.MbedTLS_jll]]
 deps = ["Artifacts", "Libdl"]
 uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
-version = "2.28.0+0"
+version = "2.28.2+0"
 
 [[deps.Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [[deps.MozillaCACerts_jll]]
 uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
-version = "2022.2.1"
+version = "2022.10.11"
 
 [[deps.NetworkOptions]]
 uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
@@ -133,7 +137,7 @@ version = "1.2.0"
 [[deps.OpenBLAS_jll]]
 deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
 uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
-version = "0.3.20+0"
+version = "0.3.21+4"
 
 [[deps.OrderedCollections]]
 git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282"
@@ -147,9 +151,9 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
 version = "2.7.1"
 
 [[deps.Pkg]]
-deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
-version = "1.8.0"
+version = "1.9.2"
 
 [[deps.PrecompileTools]]
 deps = ["Preferences"]
@@ -188,12 +192,12 @@ uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
 [[deps.TOML]]
 deps = ["Dates"]
 uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
-version = "1.0.0"
+version = "1.0.3"
 
 [[deps.Tar]]
 deps = ["ArgTools", "SHA"]
 uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
-version = "1.10.1"
+version = "1.10.0"
 
 [[deps.Tokenize]]
 git-tree-sha1 = "90538bf898832b6ebd900fa40f223e695970e3a5"
@@ -215,12 +219,12 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 [[deps.Zlib_jll]]
 deps = ["Libdl"]
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.12+3"
+version = "1.2.13+0"
 
 [[deps.libblastrampoline_jll]]
-deps = ["Artifacts", "Libdl", "OpenBLAS_jll"]
+deps = ["Artifacts", "Libdl"]
 uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
-version = "5.1.1+0"
+version = "5.8.0+0"
 
 [[deps.nghttp2_jll]]
 deps = ["Artifacts", "Libdl"]

.github/workflows/ClimaCoreMakie.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.8'
+          version: '1.9'
       - uses: actions/cache@v1
         env:
           cache-name: cache-artifacts

.github/workflows/ClimaCorePlots.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.8'
+          version: '1.9'
       - uses: actions/cache@v1
         env:
           cache-name: cache-artifacts

.github/workflows/ClimaCoreSpectra.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.8'
+          version: '1.9'
       - uses: actions/cache@v1
         env:
           cache-name: cache-artifacts

.github/workflows/ClimaCoreTempestRemap.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.8'
+          version: '1.9'
       - uses: actions/cache@v1
         env:
           cache-name: cache-artifacts

.github/workflows/ClimaCoreVTK.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.8'
+          version: '1.9'
       - uses: actions/cache@v1
         env:
           cache-name: cache-artifacts

.github/workflows/CodeCov.yml

@@ -19,7 +19,7 @@ jobs:
     - name: Set up Julia
       uses: julia-actions/setup-julia@latest
       with:
-        version: 1.8
+        version: 1.9
 
     - name: Test with coverage
       run: |

.github/workflows/JuliaFormatter.yml

@@ -36,7 +36,7 @@ jobs:
     - uses: julia-actions/setup-julia@latest
       if: steps.filter.outputs.julia_file_change == 'true'
       with:
-        version: 1.8
+        version: 1.9
 
     - name: Apply JuliaFormatter
       if: steps.filter.outputs.julia_file_change == 'true'

.github/workflows/Linux-UnitTests.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        julia-version: ['1.8']
+        julia-version: ['1.9']
 
     env:
       CLIMATEMACHINE_SETTINGS_FIX_RNG_SEED: "true"

.github/workflows/OS-UnitTests.yml

@@ -45,7 +45,7 @@ jobs:
       uses: julia-actions/setup-julia@v1
       if: steps.filter.outputs.run_test == 'true'
       with:
-        version: 1.8
+        version: 1.9
 
     - name: Cache artifacts
       uses: julia-actions/cache@v1

.github/workflows/docs.yml

@@ -16,7 +16,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: 1.8
+          version: 1.9
       - name: Install dependencies
         run: julia --project=docs -e 'using Pkg; Pkg.instantiate(;verbose=true)'
       - name: Build and deploy

Project.toml

@@ -1,7 +1,7 @@
 name = "ClimaCore"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
 authors = ["CliMA Contributors <clima-software@caltech.edu>"]
-version = "0.10.46"
+version = "0.10.47"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
@@ -11,7 +11,6 @@ CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 ClimaComms = "3a4d1b5c-c61d-41fd-a00a-5873ba7a1b0d"
 CubedSphere = "7445602f-e544-4518-8976-18f8e8ae6cdb"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
 GaussQuadrature = "d54b0c1a-921d-58e0-8e36-89d8069c0969"
@@ -34,11 +33,10 @@ UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
 Adapt = "3"
 BandedMatrices = "0.17"
 BlockArrays = "0.16"
-ClimaComms = "0.5"
 CUDA = "3, 4.2.0"
+ClimaComms = "0.5"
 CubedSphere = "0.2"
 DataStructures = "0.18"
-DiffEqBase = "6.75"
 DocStringExtensions = "0.8, 0.9"
 ForwardDiff = "0.10"
 GaussQuadrature = "0.5"