Draft implementation of a CPU vs. GPU workflow and infrastructure for Phase 2 HLT tracking

Configuration/EventContent/python/EventContent_cff.py

@@ -281,6 +281,7 @@ def SwapKeepAndDrop(l):
 from Configuration.Eras.Modifier_phase2_timing_layer_cff import phase2_timing_layer
 from Configuration.Eras.Modifier_run2_GEM_2017_cff import run2_GEM_2017
 from Configuration.Eras.Modifier_run3_GEM_cff import run3_GEM
+from Configuration.ProcessModifiers.alpakaValidationHLTTRK_cff import alpakaValidationHLTTRK
 from Configuration.ProcessModifiers.pp_on_AA_cff import pp_on_AA
 from RecoLocalFastTime.Configuration.RecoLocalFastTime_EventContent_cff import *
 from RecoMTD.Configuration.RecoMTD_EventContent_cff import *
@@ -719,6 +720,16 @@ def SwapKeepAndDrop(l):
                             'keep *_hltOfflinePrimaryVertices_*_*',
                         ])
 
+alpakaValidationHLTTRK.toModify(FEVTDEBUGHLTEventContent,
+    outputCommands = FEVTDEBUGHLTEventContent.outputCommands+[
+        'keep *_hltInitialStepTracks_*_*',
+        'keep *_hltInitialStepTracksSerialSync_*_*',
+        'keep *_hltPhase2PixelTracks_*_*',
+        'keep *_hltPhase2PixelTracksSerialSync_*_*',
+        'keep *_hltPhase2PixelVertices_*_*',
+        'keep *_hltPhase2PixelVerticesSerialSync_*_*',
+    ])
+
 phase2_common.toModify(FEVTDEBUGHLTEventContent,
                        outputCommands = FEVTDEBUGHLTEventContent.outputCommands+[
                            'keep *_hltHGCalRecHit_*_*',

Configuration/HLT/python/autoHLT.py

@@ -25,5 +25,6 @@
   'relvalRun4_trk'      : '75e33_trackingOnly',
   'relvalRun4_scouting' : 'NGTScouting',
   'relvalRun4_offload'  : 'HeterogeneousOnly',
+  'relvalRun4_GPUValid' : 'HeterogeneousValid',
   'test'                : 'GRun',
 }

Configuration/ProcessModifiers/python/alpakaValidationHLTTRK_cff.py

@@ -0,0 +1,4 @@
+import FWCore.ParameterSet.Config as cms
+
+# This modifier performs the host/device validation for the Phase 2 HLT Tracking
+alpakaValidationHLTTRK = cms.Modifier()

Configuration/PyReleaseValidation/python/upgradeWorkflowComponents.py

@@ -1930,6 +1930,19 @@ def condition(self, fragment, stepList, key, hasHarvest):
     '--eventcontent':'FEVTDEBUGHLT,DQMIO'
 }
 
+upgradeWFs['HLTHeterogeneousValid'] = deepcopy(upgradeWFs['HLTTiming75e33'])
+upgradeWFs['HLTHeterogeneousValid'].suffix = '_HLTHeterogeneousValid'
+upgradeWFs['HLTHeterogeneousValid'].offset = 0.7503
+upgradeWFs['HLTHeterogeneousValid'].step2 = {
+    '-s':'DIGI:pdigi_valid,DIGI2RAW,L1TrackTrigger,L1,L1P2GT,HLT:HeterogeneousValid,VALIDATION:hltMultiTrackValidation',
+    '--procModifiers': 'alpakaValidationHLTTRK',
+    '--datatier':'GEN-SIM-DIGI-RAW,DQMIO',
+    '--eventcontent':'FEVTDEBUGHLT,DQMIO'
+}
+upgradeWFs['HLTHeterogeneousValid'].step3 = {
+    '-s':'HARVESTING:@hltValidation'
+}
+
 upgradeWFs['HLTTiming75e33Alpaka'] = deepcopy(upgradeWFs['HLTTiming75e33'])
 upgradeWFs['HLTTiming75e33Alpaka'].suffix = '_HLT75e33TimingAlpaka'
 upgradeWFs['HLTTiming75e33Alpaka'].offset = 0.751

HLTrigger/Configuration/python/HLT_75e33/paths/DQM_TRKHeterogeneousValidation_cfi.py

@@ -0,0 +1,156 @@
+import FWCore.ParameterSet.Config as cms
+from HeterogeneousCore.AlpakaCore.functions import makeSerialClone
+
+
+from ..modules.hltPhase2OtRecHitsSoA_cfi import *
+from ..modules.hltPhase2PixelFitterByHelixProjections_cfi import *
+from ..modules.hltPhase2PixelRecHitsExtendedSoA_cfi import *
+from ..modules.hltPhase2PixelTrackFilterByKinematics_cfi import *
+from ..modules.hltPhase2PixelTracksAndHighPtStepTrackingRegions_cfi import *
+from ..modules.hltSiPhase2RecHits_cfi import *
+from ..modules.hltTrackerClusterCheck_cfi import *
+from ..sequences.HLTBeamSpotSequence_cfi import *
+from ..sequences.HLTBeginSequence_cfi import *
+from ..sequences.HLTItLocalRecoSequence_cfi import *
+from ..sequences.HLTMkFitInputSequence_cfi import *
+from ..sequences.HLTOtLocalRecoSequence_cfi import *
+
+HLTTrackingSequenceCommon_HeterogeneousValidation = cms.Sequence(
+    HLTItLocalRecoSequence
+    + HLTOtLocalRecoSequence
+    + hltTrackerClusterCheck
+    + HLTBeamSpotSequence
+    + hltPhase2PixelTracksAndHighPtStepTrackingRegions
+    + hltPhase2PixelFitterByHelixProjections
+    + hltPhase2PixelTrackFilterByKinematics
+    + hltPhase2OtRecHitsSoA
+    + hltPhase2PixelRecHitsExtendedSoA
+    + hltSiPhase2RecHits
+    + HLTMkFitInputSequence
+)
+
+
+hltBackend = cms.EDProducer( "AlpakaBackendProducer@alpaka"
+)
+
+hltStatusOnGPUFilter = cms.EDFilter( "AlpakaBackendFilter",
+    producer = cms.InputTag( 'hltBackend','backend' ),
+    backends = cms.vstring( 'CudaAsync','ROCmAsync' )
+)
+
+
+from ..modules.hltPhase2PixelTracksSoA_cfi import *
+from ..modules.hltPhase2PixelTracksCAExtension_cfi import *
+from ..modules.hltPhase2PixelVertices_cfi import *
+from ..modules.hltPhase2PixelTracksCutClassifier_cfi import *
+from ..modules.hltPhase2PixelTracks_cfi import *
+
+HLTPhase2PixelTracksAndVerticesSequence_HeterogeneousValidation = cms.Sequence(
+    hltPhase2PixelTracksSoA
+    + hltPhase2PixelTracksCAExtension
+    + hltPhase2PixelVertices
+    + hltPhase2PixelTracksCutClassifier
+    + hltPhase2PixelTracks
+)
+
+hltPhase2PixelTracksSoASerialSync = makeSerialClone(hltPhase2PixelTracksSoA)
+hltPhase2PixelTracksCAExtensionSerialSync = hltPhase2PixelTracksCAExtension.clone(
+    trackSrc = "hltPhase2PixelTracksSoASerialSync"
+)
+hltPhase2PixelVerticesSerialSync = hltPhase2PixelVertices.clone(
+    TrackCollection = "hltPhase2PixelTracksCAExtensionSerialSync"
+)
+hltPhase2PixelTracksCutClassifierSerialSync = hltPhase2PixelTracksCutClassifier.clone(
+    src = "hltPhase2PixelTracksCAExtensionSerialSync",
+    vertices = "hltPhase2PixelVerticesSerialSync"
+)
+hltPhase2PixelTracksSerialSync = hltPhase2PixelTracks.clone(
+    originalMVAVals = cms.InputTag("hltPhase2PixelTracksCutClassifierSerialSync","MVAValues"),
+    originalQualVals = cms.InputTag("hltPhase2PixelTracksCutClassifierSerialSync","QualityMasks"),
+    originalSource = cms.InputTag("hltPhase2PixelTracksCAExtensionSerialSync")
+)
+
+HLTPhase2PixelTracksAndVerticesSequence_HeterogeneousValidationSerialSync = cms.Sequence(
+    hltPhase2PixelTracksSoASerialSync
+    + hltPhase2PixelTracksCAExtensionSerialSync
+    + hltPhase2PixelVerticesSerialSync
+    + hltPhase2PixelTracksCutClassifierSerialSync
+    + hltPhase2PixelTracksSerialSync
+)
+
+
+from ..modules.hltInitialStepSeeds_cfi import *
+from ..modules.hltInitialStepSeedTracksLST_cfi import *
+from ..modules.hltInputLST_cfi import *
+from ..modules.hltLST_cfi import *
+from ..modules.hltInitialStepTrajectorySeedsLST_cfi import *
+from ..modules.hltInitialStepMkFitSeeds_cfi import *
+from ..modules.hltInitialStepTrackCandidatesMkFit_cfi import *
+from ..modules.hltInitialStepTrackCandidates_cfi import *
+from ..modules.hltInitialStepTracks_cfi import *
+
+HLTInitialStepSequence_HeterogeneousValidation = cms.Sequence(
+    hltInitialStepSeeds
+    + hltInitialStepSeedTracksLST
+    + hltInputLST
+    + hltLST
+    + hltInitialStepTrajectorySeedsLST
+    + hltInitialStepMkFitSeeds
+    + hltInitialStepTrackCandidatesMkFit
+    + hltInitialStepTrackCandidates
+    + hltInitialStepTracks
+)
+
+hltInitialStepSeedsSerialSync = hltInitialStepSeeds.clone(
+    InputCollection = cms.InputTag("hltPhase2PixelTracksSerialSync")
+)
+hltInitialStepSeedTracksLSTSerialSync = hltInitialStepSeedTracksLST.clone(
+    src = "hltInitialStepSeedsSerialSync"
+)
+hltInputLSTSerialSync = makeSerialClone(hltInputLST)
+hltLSTSerialSync = makeSerialClone(hltLST,
+    lstInput = "hltInputLSTSerialSync"
+)
+hltInitialStepTrajectorySeedsLSTSerialSync = hltInitialStepTrajectorySeedsLST.clone(
+    lstOutput = "hltLSTSerialSync",
+    lstInput = "hltInputLSTSerialSync",
+    lstPixelSeeds = "hltInputLSTSerialSync"
+)
+hltInitialStepMkFitSeedsSerialSync = hltInitialStepMkFitSeeds.clone(
+    seeds = "hltInitialStepTrajectorySeedsLSTSerialSync"
+)
+hltInitialStepTrackCandidatesMkFitSerialSync = hltInitialStepTrackCandidatesMkFit.clone(
+    seeds = "hltInitialStepMkFitSeedsSerialSync"
+)
+hltInitialStepTrackCandidatesSerialSync = hltInitialStepTrackCandidates.clone(
+    mkFitSeeds = "hltInitialStepMkFitSeedsSerialSync",
+    seeds = "hltInitialStepTrajectorySeedsLSTSerialSync",
+    tracks = "hltInitialStepTrackCandidatesMkFitSerialSync",
+)
+hltInitialStepTracksSerialSync = hltInitialStepTracks.clone(
+    src = "hltInitialStepTrackCandidatesSerialSync",
+)
+
+HLTInitialStepSequence_HeterogeneousValidationSerialSync = cms.Sequence(
+    hltInitialStepSeedsSerialSync
+    +hltInitialStepSeedTracksLSTSerialSync
+    +hltInputLSTSerialSync
+    +hltLSTSerialSync
+    +hltInitialStepTrajectorySeedsLSTSerialSync
+    +hltInitialStepMkFitSeedsSerialSync
+    +hltInitialStepTrackCandidatesMkFitSerialSync
+    +hltInitialStepTrackCandidatesSerialSync
+    +hltInitialStepTracksSerialSync
+)
+
+
+DQM_TRKHeterogeneousValidation = cms.Path(
+    HLTBeginSequence
+    + HLTTrackingSequenceCommon_HeterogeneousValidation
+    + hltBackend
+    + hltStatusOnGPUFilter
+    + HLTPhase2PixelTracksAndVerticesSequence_HeterogeneousValidation
+    + HLTInitialStepSequence_HeterogeneousValidation
+    + HLTPhase2PixelTracksAndVerticesSequence_HeterogeneousValidationSerialSync
+    + HLTInitialStepSequence_HeterogeneousValidationSerialSync
+)

HLTrigger/Configuration/python/HLT_HeterogeneousValid_cff.py

@@ -0,0 +1,16 @@
+import FWCore.ParameterSet.Config as cms
+
+from .HLT_75e33_cff import fragment
+
+for p in dir(fragment):
+    att = getattr(fragment, p)
+    if isinstance(att, cms.Path) and p not in [ "HLTriggerFinalPath", "HLTAnalyzerEndpath"]:
+        delattr(fragment, p)
+    del att
+
+fragment.load("HLTrigger/Configuration/HLT_75e33/paths/DQM_TRKHeterogeneousValidation_cfi")
+fragment.schedule = cms.Schedule(*[
+    fragment.DQM_TRKHeterogeneousValidation,
+    fragment.HLTriggerFinalPath,
+    fragment.HLTAnalyzerEndpath,
+])

Validation/RecoTrack/python/HLTmultiTrackValidator_cff.py

@@ -69,6 +69,39 @@ def _modifyForPhase2(trackvalidator):
     )
 )
 
+from DQM.TrackingMonitorSource.TrackToTrackComparisonHists_cfi import TrackToTrackComparisonHists as _TrackToTrackComparisonHists
+hltPixelTrackToTrackSerialSync = _TrackToTrackComparisonHists.clone(
+    requireValidHLTPaths = False,
+    #ignoreLumiPUPlots = True,
+    monitoredTrack = "hltPhase2PixelTracks",
+    monitoredBeamSpot = "hltOnlineBeamSpot",
+    monitoredPrimaryVertices = "hltPhase2PixelVertices",
+    referenceTrack = "hltPhase2PixelTracksSerialSync",
+    referenceBeamSpot = "hltOnlineBeamSpot",
+    referencePrimaryVertices = "hltPhase2PixelVerticesSerialSync",
+    topDirName = cms.string('HLT/Tracking/ValidationWRTSerialSync/pixelTracks'),
+)
+
+hltInitialStepTrackToTrackSerialSync = _TrackToTrackComparisonHists.clone(
+    requireValidHLTPaths = False,
+    #ignoreLumiPUPlots = True,
+    monitoredTrack = "hltInitialStepTracks",
+    monitoredBeamSpot = "hltOnlineBeamSpot",
+    monitoredPrimaryVertices = "hltPhase2PixelVertices",
+    referenceTrack = "hltInitialStepTracksSerialSync",
+    referenceBeamSpot = "hltOnlineBeamSpot",
+    referencePrimaryVertices = "hltPhase2PixelVerticesSerialSync",
+    topDirName = cms.string('HLT/Tracking/ValidationWRTSerialSync/initialStepTracks'),
+)
+
+from Configuration.ProcessModifiers.alpakaValidationHLTTRK_cff import alpakaValidationHLTTRK
+alpakaValidationHLTTRK.toReplaceWith(hltMultiTrackValidation, cms.Sequence(
+    hltMultiTrackValidation.copy()
+    +hltInitialStepTrackToTrackSerialSync
+    +hltPixelTrackToTrackSerialSync
+    )
+)
+
 from Configuration.ProcessModifiers.ngtScouting_cff import ngtScouting
 from Configuration.ProcessModifiers.hltPhase2LegacyTracking_cff import hltPhase2LegacyTracking
 from Configuration.ProcessModifiers.trackingLST_cff import trackingLST