(4 in pipeline) Faster RDTSC-based timers and new timer/counter APIs

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/CHANGELOG.md

@@ -1,3 +1,10 @@
+<--
+Copyright (C) 2020-2024 CERN and UCLouvain.
+Licensed under the GNU Lesser General Public License (version 3 or later).
+Created by: A. Valassi (Oct 2023) for the MG5aMC CUDACPP plugin.
+Further modified by: A. Valassi (2024) for the MG5aMC CUDACPP plugin.
+-->
+
 # Changelog
 
 All notable changes to this project will be documented in this file.
@@ -11,10 +18,11 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com).
 ### Added
 
 - Infrastructure issues
-  - AV ([#945]) Add cudacpp_bldall runcard to produce multi-backend gridpacks.
-  - AV ([#700]) Add cudacpp_helinl and cudacpp_hrdcod runcards to support HELINL=1 and HRDCOD=1 builds.
-  - AV ([#957]) In internal "tlau" tests, instrument python code in gridpacks to provide timing profiles for event generation.
-  - AV Enhance internal "tlau" tests and add a few test logs and various scripts to analyse them.
+  - AV ([#945]) Added cudacpp_bldall runcard to produce multi-backend gridpacks.
+  - AV ([#700]) Added cudacpp_helinl and cudacpp_hrdcod runcards to support HELINL=1 and HRDCOD=1 builds.
+  - AV ([#957]) In internal "tlau" tests, instrumented python code in gridpacks to provide timing profiles for event generation.
+  - AV Enhanced internal "tlau" tests and added a few test logs and various scripts to analyse them.
+  - AV ([#972]) Added new RDTSC-based timers (faster than the legacy chrono-based timers) and adapted all internal APIs.
 
 ### Changed
 
@@ -29,10 +37,10 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com).
   - AV ([#1011]) Added workaround for Floating Point Exceptions in vxxxxx in the HIP backend.
 
 - Infrastructure issues
-  - AV ([#1013]) Fix release scripts to create 'v1.00.01' tags from a '(1,0,1)' python tuple.
-  - AV ([#1015]) Remove add_input_for_banner from output.py (plugin_run_card is not needed in cudacpp).
-  - AV ([#995]) In cudacpp_config.mk move default FPTYPE from 'd' to 'm' (already the default cudacpp_backend in run_card.dat).
-	
+  - AV ([#1013]) Fixed release scripts to create 'v1.00.01' tags from a '(1,0,1)' python tuple.
+  - AV ([#1015]) Removed add_input_for_banner from output.py (plugin_run_card is not needed in cudacpp).
+  - AV ([#995]) In cudacpp_config.mk moved default FPTYPE from 'd' to 'm' (already the default cudacpp_backend in run_card.dat).
+
 --------------------------------------------------------------------------------
 
 ## [1.00.00] - 2024-10-03
@@ -74,6 +82,7 @@ The format is loosely based on [Keep a Changelog](https://keepachangelog.com).
 [#945]: https://github.com/madgraph5/madgraph4gpu/issues/945
 [#957]: https://github.com/madgraph5/madgraph4gpu/issues/957
 [#959]: https://github.com/madgraph5/madgraph4gpu/issues/959
+[#972]: https://github.com/madgraph5/madgraph4gpu/issues/972
 [#993]: https://github.com/madgraph5/madgraph4gpu/issues/993
 [#995]: https://github.com/madgraph5/madgraph4gpu/issues/995
 [#1011]: https://github.com/madgraph5/madgraph4gpu/issues/1011

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/MG5aMC_patches/PROD/patch.P1

@@ -1,8 +1,8 @@
 diff --git b/epochX/cudacpp/gg_tt.mad/SubProcesses/P1_gg_ttx/driver.f a/epochX/cudacpp/gg_tt.mad/SubProcesses/P1_gg_ttx/driver.f
-index ecd11b239..ec5722702 100644
+index ecd11b239..67d1c2218 100644
 --- b/epochX/cudacpp/gg_tt.mad/SubProcesses/P1_gg_ttx/driver.f
 +++ a/epochX/cudacpp/gg_tt.mad/SubProcesses/P1_gg_ttx/driver.f
-@@ -76,13 +76,77 @@ c      common/to_colstats/ncols,ncolflow,ncolalt,ic
+@@ -76,13 +76,80 @@ c      common/to_colstats/ncols,ncolflow,ncolalt,ic
 
        include 'coupl.inc' ! needs VECSIZE_MEMMAX (defined in vector.inc)
        INTEGER VECSIZE_USED
@@ -27,6 +27,9 @@ index ecd11b239..ec5722702 100644
 +      CALL OMPNUMTHREADS_NOT_SET_MEANS_ONE_THREAD()
 +#endif
 +      CALL COUNTERS_INITIALISE()
++      CALL COUNTERS_REGISTER_COUNTER( 1, 'Fortran MEs'//char(0) )
++      CALL COUNTERS_REGISTER_COUNTER( 2, 'CudaCpp MEs'//char(0) )
++      CALL COUNTERS_REGISTER_COUNTER( 3, 'CudaCpp HEL'//char(0) )
 +
 +#ifdef MG5AMC_MEEXPORTER_CUDACPP
 +      fbridge_mode = 1 ! CppOnly=1, default for CUDACPP
@@ -81,7 +84,7 @@ index ecd11b239..ec5722702 100644
  c
  c     Read process number
  c
-@@ -216,8 +280,33 @@ c      call sample_result(xsec,xerr)
+@@ -216,8 +283,33 @@ c      call sample_result(xsec,xerr)
  c      write(*,*) 'Final xsec: ',xsec
 
        rewind(lun)
@@ -116,7 +119,7 @@ index ecd11b239..ec5722702 100644
        end
 
  c     $B$ get_user_params $B$ ! tag for MadWeight
-@@ -400,7 +489,7 @@ c
+@@ -400,7 +492,7 @@ c
        fopened=.false.
        tempname=filename 	 
        fine=index(tempname,' ') 	 

epochX/cudacpp/CODEGEN/PLUGIN/CUDACPP_SA_OUTPUT/madgraph/iolibs/template_files/gpu/check_sa.cc

@@ -433,10 +433,10 @@ main( int argc, char** argv )
   DeviceBufferSelectedColor devSelCol( nevt );
 #endif
 
-  std::unique_ptr<double[]> genrtimes( new double[niter] );
-  std::unique_ptr<double[]> rambtimes( new double[niter] );
-  std::unique_ptr<double[]> wavetimes( new double[niter] );
-  std::unique_ptr<double[]> wv3atimes( new double[niter] );
+  std::unique_ptr<uint64_t[]> genrcounts( new uint64_t[niter] );
+  std::unique_ptr<uint64_t[]> rambcounts( new uint64_t[niter] );
+  std::unique_ptr<uint64_t[]> wavecounts( new uint64_t[niter] );
+  std::unique_ptr<uint64_t[]> wv3acounts( new uint64_t[niter] );
 
   // --- 0c. Create curand, hiprand or common generator
   const std::string cgenKey = "0c GenCreat";
@@ -540,7 +540,7 @@ main( int argc, char** argv )
     // === STEP 1 OF 3
 
     // *** START THE OLD-STYLE TIMER FOR RANDOM GEN ***
-    double genrtime = 0;
+    uint64_t genrcount = 0;
 
     // --- 1a. Seed rnd generator (to get same results on host and device in curand/hiprand)
     // [NB This should not be necessary using the host API: "Generation functions
@@ -551,7 +551,7 @@ main( int argc, char** argv )
     const std::string sgenKey = "1a GenSeed ";
     timermap.start( sgenKey );
     prnk->seedGenerator( seed + iiter );
-    genrtime += timermap.stop();
+    genrcount += timermap.stop();
 
     // --- 1b. Generate all relevant numbers to build nevt events (i.e. nevt phase space points) on the host
     const std::string rngnKey = "1b GenRnGen";
@@ -566,19 +566,19 @@ main( int argc, char** argv )
     {
       // --- 1c. Copy rndmom from host to device
       const std::string htodKey = "1c CpHTDrnd";
-      genrtime += timermap.start( htodKey );
+      genrcount += timermap.start( htodKey );
       copyDeviceFromHost( devRndmom, hstRndmom );
     }
 #endif
 
     // *** STOP THE OLD-STYLE TIMER FOR RANDOM GEN ***
-    genrtime += timermap.stop();
+    genrcount += timermap.stop();
 
     // === STEP 2 OF 3
     // Fill in particle momenta for each of nevt events on the device
 
     // *** START THE OLD-STYLE TIMER FOR RAMBO ***
-    double rambtime = 0;
+    uint64_t rambcount = 0;
 
     // --- 2a. Fill in momenta of initial state particles on the device
     const std::string riniKey = "2a RamboIni";
@@ -589,7 +589,7 @@ main( int argc, char** argv )
     // --- 2b. Fill in momenta of final state particles using the RAMBO algorithm on the device
     // (i.e. map random numbers to final-state particle momenta for each of nevt events)
     const std::string rfinKey = "2b RamboFin";
-    rambtime += timermap.start( rfinKey );
+    rambcount += timermap.start( rfinKey );
     prsk->getMomentaFinal();
     //std::cout << "Got final momenta" << std::endl;
 
@@ -598,30 +598,30 @@ main( int argc, char** argv )
     {
       // --- 2c. CopyDToH Weights
       const std::string cwgtKey = "2c CpDTHwgt";
-      rambtime += timermap.start( cwgtKey );
+      rambcount += timermap.start( cwgtKey );
       copyHostFromDevice( hstWeights, devWeights );
 
       // --- 2d. CopyDToH Momenta
       const std::string cmomKey = "2d CpDTHmom";
-      rambtime += timermap.start( cmomKey );
+      rambcount += timermap.start( cmomKey );
       copyHostFromDevice( hstMomenta, devMomenta );
     }
     else // only if ( ! bridge ) ???
     {
       // --- 2c. CopyHToD Weights
       const std::string cwgtKey = "2c CpHTDwgt";
-      rambtime += timermap.start( cwgtKey );
+      rambcount += timermap.start( cwgtKey );
       copyDeviceFromHost( devWeights, hstWeights );
 
       // --- 2d. CopyHToD Momenta
       const std::string cmomKey = "2d CpHTDmom";
-      rambtime += timermap.start( cmomKey );
+      rambcount += timermap.start( cmomKey );
       copyDeviceFromHost( devMomenta, hstMomenta );
     }
 #endif
 
     // *** STOP THE OLD-STYLE TIMER FOR RAMBO ***
-    rambtime += timermap.stop();
+    rambcount += timermap.stop();
 
     // === STEP 3 OF 3
     // Evaluate matrix elements for all nevt events
@@ -641,7 +641,7 @@ main( int argc, char** argv )
 #ifdef MGONGPUCPP_GPUIMPL
     // --- 2d. CopyHToD Momenta
     const std::string gKey = "0.. CpHTDg";
-    rambtime += timermap.start( gKey ); // FIXME! NOT A RAMBO TIMER!
+    rambcount += timermap.start( gKey ); // FIXME! NOT A RAMBO TIMER!
     copyDeviceFromHost( devGs, hstGs );
 #endif
 
@@ -654,8 +654,8 @@ main( int argc, char** argv )
     }
 
     // *** START THE OLD-STYLE TIMERS FOR MATRIX ELEMENTS (WAVEFUNCTIONS) ***
-    double wavetime = 0; // calc plus copy
-    double wv3atime = 0; // calc only
+    uint64_t wavecount = 0; // calc plus copy
+    uint64_t wv3acount = 0; // calc only
 
     // --- 3a. SigmaKin
     const std::string skinKey = "3a SigmaKin";
@@ -664,8 +664,8 @@ main( int argc, char** argv )
     pmek->computeMatrixElements( useChannelIds );
 
     // *** STOP THE NEW OLD-STYLE TIMER FOR MATRIX ELEMENTS (WAVEFUNCTIONS) ***
-    wv3atime += timermap.stop(); // calc only
-    wavetime += wv3atime;        // calc plus copy
+    wv3acount += timermap.stop(); // calc only
+    wavecount += wv3acount;       // calc plus copy
 
 #ifdef MGONGPUCPP_GPUIMPL
     if( !bridge )
@@ -675,7 +675,7 @@ main( int argc, char** argv )
       timermap.start( cmesKey );
       copyHostFromDevice( hstMatrixElements, devMatrixElements );
       // *** STOP THE OLD OLD-STYLE TIMER FOR MATRIX ELEMENTS (WAVEFUNCTIONS) ***
-      wavetime += timermap.stop(); // calc plus copy
+      wavecount += timermap.stop(); // calc plus copy
     }
 #endif
 
@@ -688,16 +688,16 @@ main( int argc, char** argv )
     // --- 4a Dump within the loop
     const std::string loopKey = "4a DumpLoop";
     timermap.start( loopKey );
-    genrtimes[iiter] = genrtime;
-    rambtimes[iiter] = rambtime;
-    wavetimes[iiter] = wavetime;
-    wv3atimes[iiter] = wv3atime;
+    genrcounts[iiter] = genrcount;
+    rambcounts[iiter] = rambcount;
+    wavecounts[iiter] = wavecount;
+    wv3acounts[iiter] = wv3acount;
 
     if( verbose )
     {
       std::cout << std::string( SEP79, '*' ) << std::endl
                 << "Iteration #" << iiter + 1 << " of " << niter << std::endl;
-      if( perf ) std::cout << "Wave function time: " << wavetime << std::endl;
+      if( perf ) std::cout << "Wave function time: " << wavecount * timermap.secondsPerCount() << std::endl;
     }
 
     for( unsigned int ievt = 0; ievt < nevt; ++ievt ) // Loop over all events in this iteration
@@ -736,6 +736,20 @@ main( int argc, char** argv )
   // *** END MAIN LOOP ON #ITERATIONS ***
   // **************************************
 
+  // Calibrate seconds per count
+  float secPerCount = timermap.secondsPerCount();
+  std::unique_ptr<double[]> genrtimes( new double[niter] );
+  std::unique_ptr<double[]> rambtimes( new double[niter] );
+  std::unique_ptr<double[]> wavetimes( new double[niter] );
+  std::unique_ptr<double[]> wv3atimes( new double[niter] );
+  for( unsigned int iiter = 0; iiter < niter; ++iiter )
+  {
+    genrtimes[iiter] = genrcounts[iiter] * secPerCount;
+    rambtimes[iiter] = rambcounts[iiter] * secPerCount;
+    wavetimes[iiter] = wavecounts[iiter] * secPerCount;
+    wv3atimes[iiter] = wv3acounts[iiter] * secPerCount;
+  }
+
   // === STEP 8 ANALYSIS
   // --- 8a Analysis: compute stats after the loop
   const std::string statKey = "8a CompStat";