HeldSuarez run on Metal, Advection parallel on GPU

Author: OsKnoth

Jobs/JobAdvectionSlottedCylinder

@@ -1,6 +1,6 @@
-mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
+mpirun -n 6 julia --project Examples/testAdvectionSphere.jl \
   --Problem="AdvectionSphereSlottedCylinder" \
-  --Device="" \
+  --Device="CPU" \
   --FloatTypeBackend="Float64" \
   --NumV=5 \
   --NumTr=1 \
@@ -13,7 +13,7 @@ mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
   --HorLimit=true \
   --Decomp="EqualArea" \
   --vtkFileName="SlottedCylinderSquad" \
-  --SimTime=5 \
+  --SimTime=5.0 \
   --PrintTime=0.25\
   --Flat=false \
   --dtau=0.0004 \
@@ -28,8 +28,8 @@ mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
   --H=1.0 \
   --OrdPoly=4 \
   --HyperVisc=true \
-  --HyperDCurl=0.e13 \
-  --HyperDGrad=0.e13 \
-  --HyperDDiv=0.e13
+  --HyperDCurl=1.e-5 \
+  --HyperDGrad=1.e-5 \
+  --HyperDDiv=1.e-5
 
 

Jobs/JobAdvectionSlottedCylinderC1

@@ -0,0 +1,35 @@
+mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
+  --Problem="AdvectionSphereSlottedCylinder" \
+  --Device="CPU" \
+  --FloatTypeBackend="Float64" \
+  --NumV=5 \
+  --NumTr=1 \
+  --ProfRho="" \
+  --ProfTheta="" \
+  --ProfVel="Divergent" \
+  --ProfVelW="" \
+  --ProfTr="AdvectionSphereSlottedCylinder" \
+  --Upwind=false \
+  --HorLimit=false \
+  --Decomp="EqualArea" \
+  --vtkFileName="SlottedCylinderSquad" \
+  --SimTime=0.05 \
+  --PrintTime=0.01\
+  --Flat=false \
+  --dtau=0.01 \
+  --IntMethod="SSPRungeKutta" \
+  --Table="SSPRK1" \
+  --TopoS="" \
+  --GridType="CubedSphere" \
+  --nz=1 \
+  --RadEarth=1.0 \
+  --nPanel=2 \
+  --RadEarth=1 \
+  --H=1.0 \
+  --OrdPoly=1 \
+  --HyperVisc=true \
+  --HyperDCurl=0.e-5 \
+  --HyperDGrad=0.e-5 \
+  --HyperDDiv=0.e-5
+
+

Jobs/JobAdvectionSlottedCylinderC2

@@ -0,0 +1,35 @@
+mpirun -n 2 julia --project Examples/testAdvectionSphere.jl \
+  --Problem="AdvectionSphereSlottedCylinder" \
+  --Device="CPU" \
+  --FloatTypeBackend="Float64" \
+  --NumV=5 \
+  --NumTr=1 \
+  --ProfRho="" \
+  --ProfTheta="" \
+  --ProfVel="Divergent" \
+  --ProfVelW="" \
+  --ProfTr="AdvectionSphereSlottedCylinder" \
+  --Upwind=false \
+  --HorLimit=false \
+  --Decomp="EqualArea" \
+  --vtkFileName="SlottedCylinderSquad" \
+  --SimTime=0.05 \
+  --PrintTime=0.01\
+  --Flat=false \
+  --dtau=0.01 \
+  --IntMethod="SSPRungeKutta" \
+  --Table="SSPRK1" \
+  --TopoS="" \
+  --GridType="CubedSphere" \
+  --nz=1 \
+  --RadEarth=1.0 \
+  --nPanel=2 \
+  --RadEarth=1 \
+  --H=1.0 \
+  --OrdPoly=1 \
+  --HyperVisc=true \
+  --HyperDCurl=0.e-5 \
+  --HyperDGrad=0.e-5 \
+  --HyperDDiv=0.e-5
+
+

Jobs/JobAdvectionSlottedCylinderCPU

@@ -1,4 +1,4 @@
-mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
+mpirun -n 2 julia --project Examples/testAdvectionSphere.jl \
   --Problem="AdvectionSphereSlottedCylinder" \
   --Device="CPU" \
   --FloatTypeBackend="Float64" \
@@ -10,11 +10,11 @@ mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
   --ProfVelW="" \
   --ProfTr="AdvectionSphereSlottedCylinder" \
   --Upwind=false \
-  --HorLimit=true \
+  --HorLimit=false \
   --Decomp="EqualArea" \
   --vtkFileName="SlottedCylinderSquad" \
-  --SimTime=5 \
-  --PrintTime=0.25\
+  --SimTime=0.05 \
+  --PrintTime=0.01\
   --Flat=false \
   --dtau=0.0004 \
   --IntMethod="SSPRungeKutta" \
@@ -26,10 +26,10 @@ mpirun -n 1 julia --project Examples/testAdvectionSphere.jl \
   --nPanel=32 \
   --RadEarth=1 \
   --H=1.0 \
-  --OrdPoly=4 \
+  --OrdPoly=1 \
   --HyperVisc=true \
-  --HyperDCurl=0.e13 \
-  --HyperDGrad=0.e13 \
-  --HyperDDiv=0.e13
+  --HyperDCurl=0.e-5 \
+  --HyperDGrad=0.e-5 \
+  --HyperDDiv=0.e-5
 
 

Jobs/JobNHHeldSuarezMoistSphere

@@ -1,6 +1,6 @@
 mpirun -n 6 julia --project Examples/testNHSphere.jl \
   --Problem="HeldSuarezMoistSphere" \
-  --Device="" \
+  --Device="CPU" \
   --GPUType="Metal" \
   --NumberThreadGPU=512 \
   --FloatTypeBackend="Float32" \

ToDo

@@ -15,3 +15,11 @@ NVTX.@range "KHyperViscKernel!" let
     KHyperViscKernel!(CacheF,MRho,U,DS,DW,dXdxI,J,M,Glob,ndrange=ndrange)
     KernelAbstractions.synchronize(backend)
 end
+
+
+4 A100
+Simulation time 10 days
+110 km Resolution Timestep  150s  163s
+ 55 km Resolution Timestep   75s  759s
+
+Test parallel Advection with horizontal limiter

src/DyCore/FcnTracer.jl

@@ -35,8 +35,8 @@ function FcnTracer!(F,U,time,CG,Metric,Phys,Cache,Exchange,Global,Param,Profile)
 
   if HorLimit
     @views Limit!(qMin,qMax,U[:,:,NumV+1:NumV+NumTr],U[:,:,RhoPos],CG,Global)
-    ExchangeDataFSend(qMin,qMax,Exchange)
-    ExchangeDataFRecv!(qMin,qMax,Exchange)
+    Parallels.ExchangeDataFSend(qMin,qMax,Exchange)
+    Parallels.ExchangeDataFRecv!(qMin,qMax,Exchange)
   end  
 
   # Hyperdiffusion 

src/DyCore/MassCG.jl

@@ -41,9 +41,9 @@ function MassCGGPU!(CG,J,Glob,Exchange,Global)
   MMass = CG.MMass
   MW = CG.MW
 
-  @. M = FT(0)
-  @. MMass = FT(0)
-  @. MW = FT(0)
+  M .= FT(0)
+  MMass .= FT(0)
+  MW .= FT(0)
 
   NumberThreadGPU = Global.ParallelCom.NumberThreadGPU
 
@@ -71,7 +71,7 @@ end
     ID = I + (J - 1) * N  
     @inbounds ind = Glob[ID,IF]  
     @inbounds @atomic M[Iz,ind] += (JJ[ID,1,Iz,IF] + JJ[ID,2,Iz,IF])
-    @inbounds @atomic MMass[Iz,ind] += 0.5 * (JJ[ID,1,Iz,IF] + JJ[ID,2,Iz,IF]) * w[I] * w[J]
+    @inbounds @atomic MMass[Iz,ind] += eltype(M)(0.5) * (JJ[ID,1,Iz,IF] + JJ[ID,2,Iz,IF]) * w[I] * w[J]
   end  
   if Iz < Nz && IF <= NF
     ID = I + (J - 1) * N  

src/GPU/FcnGPU.jl

@@ -59,6 +59,8 @@ function FcnAdvectionGPU!(F,U,time,FE,Metric,Phys,Cache,Exchange,Global,Param,Pr
   if Global.Model.HorLimit
     @views KLimitKernel!(DoF,qMin,qMax,U[:,:,NumV+1:NumV+NumTr],Rho,Glob,ndrange=ndrangeL)
       KernelAbstractions.synchronize(backend)
+    Parallels.ExchangeDataFSend(qMin,qMax,Exchange)
+    Parallels.ExchangeDataFRecv!(qMin,qMax,Exchange)  
   end
 
 
@@ -74,7 +76,7 @@ function FcnAdvectionGPU!(F,U,time,FE,Metric,Phys,Cache,Exchange,Global,Param,Pr
   Parallels.ExchangeData3DSendGPU(CacheTr,Exchange)
   Parallels.ExchangeData3DRecvGPU!(CacheTr,Exchange)
 
-  @. F = 0
+  F .= FT(0)
 # @views KHyperViscTracerKoeffKernel!(F[:,:,1+NumV],CacheTr,Rho,DS,DW,dXdxI,J,M,Glob,
 #   KoeffDiv,ndrange=ndrange)
 # KernelAbstractions.synchronize(backend)
@@ -98,22 +100,17 @@ function FcnAdvectionGPU!(F,U,time,FE,Metric,Phys,Cache,Exchange,Global,Param,Pr
     @views KDivRhoTrUpwind3Kernel!(F[:,:,1+NumV],U[:,:,1+NumV],U,DS,
       dXdxI,J,M,Glob,ndrange=ndrange)
     KernelAbstractions.synchronize(backend)  
-
-    KDivRhoKernel!(F,U,DS,dXdxI,J,M,Glob,ndrange=ndrange)
-    KernelAbstractions.synchronize(backend)  
   end
 
 # Data exchange  
-  Parallels.ExchangeData3DSendGPU(F[:,:,1+NumV],Exchange)
-  Parallels.ExchangeData3DRecvGPU!(F[:,:,1+NumV],Exchange)
-
+  @views Parallels.ExchangeData3DSendGPU(F[:,:,1:1+NumV],Exchange)
+  @views Parallels.ExchangeData3DRecvGPU!(F[:,:,1:1+NumV],Exchange)
 end
 
 function FcnGPU!(F,U,FE,Metric,Phys,Cache,Exchange,Global,Param,DiscType)
 
   backend = get_backend(F)
   FT = eltype(F)
-  @. F = 0
   Glob = FE.Glob
   DS = FE.DS
   DW = FE.DW
@@ -204,7 +201,7 @@ function FcnGPU!(F,U,FE,Metric,Phys,Cache,Exchange,Global,Param,DiscType)
 ####
 # First phase  
 ####
-  @. Temp1 = FT(0)
+  Temp1 .= FT(0)
   @views MRho = CacheF[:,:,6]
   KHyperViscKernel!(CacheF,MRho,U,DS,DW,dXdxI,J,M,Glob,ndrange=ndrangeB)
   for iT = 1 : NumTr
@@ -227,7 +224,7 @@ function FcnGPU!(F,U,FE,Metric,Phys,Cache,Exchange,Global,Param,DiscType)
 # Second phase  
 ####
 
-  @. F = 0
+  F .= FT(0)
   KHyperViscKoeffKernel!(F,U,CacheF,DS,DW,dXdxI,J,M,Glob,KoeffCurl,KoeffGrad,KoeffDiv,ndrange=ndrangeB)
   for iT = 1 : NumTr
     @views CacheTr = Temp1[:,:,iT + 6]  

src/GPU/HorLimiterKernel.jl

@@ -165,7 +165,7 @@ end
     end
   end  
   @synchronize
-  for iTer = 1 : 5
+  for iTer = 1 : 8
     if Iz <= Nz && conv[iz]
       ID = I + (J - 1) * N  
       @inbounds q[I,J,iz] = medianGPU(qMinS[iz], RhoTrColS[I,J,iz] / RhoColS[I,J,iz] +

src/GPU/OperatorKernel.jl

@@ -1346,14 +1346,14 @@ end
     Rm = Phys.Rd * RhoD + Phys.Rv * RhoV
     Cpml = Phys.Cpd * RhoD + Phys.Cpv * RhoV
     @inbounds T = p[1,ind] / Rm
-    @inbounds LatFlux = - 2.0 * CT[ID,IF] * uStar[ID,IF] * dXdxI[3,3,1,ID,1,IF] * 
+    @inbounds LatFlux = - eltype(F)(2) * CT[ID,IF] * uStar[ID,IF] * dXdxI[3,3,1,ID,1,IF] * 
       (RhoV[1,ind] - RhoVSurf[ID,IF]) / M[1,ind]
-    @inbounds SensFlux = - 2.0 * CH[ID,IF] * uStar[ID,IF] * dXdxI[3,3,1,ID,1,IF] * 
+    @inbounds SensFlux = - eltype(F)(2) * CH[ID,IF] * uStar[ID,IF] * dXdxI[3,3,1,ID,1,IF] * 
       (T - TSurf[ID,IF]) / M[1,ind]
     FRho = LatFlux
     FRhoV = LatFlux
     PrePi=(p[1,ind] / Phys.p0)^(Rm / Cpml)
-    FRhoTh = RhoTh * (SensFlux / T + ((Phys.Rv / Rm) - 1.0 / Rho - 
+    FRhoTh = RhoTh * (SensFlux / T + ((Phys.Rv / Rm) - eltype(F)(1) / Rho - 
       log(PrePi)*(Phys.Rv / Rm - Phys.Cpv / Cpml)) *  LatFlux)
     @inbounds @atomic F[1,ind,1] += FRho 
     @inbounds @atomic F[1,ind,5] += FRhoTh