Merge pull request #1738 from NNPDF/qed-tests

QED tests

n3fit/runcards/examples/Basic_runcard_qed.yml

@@ -95,7 +95,7 @@ datacuts:
 
 ############################################################
 theory:
-  theoryid: 522     # database id
+  theoryid: 523     # database id
 
 ############################################################
 trvlseed: 1551864071
@@ -169,3 +169,4 @@ fiatlux:
   luxset: NNPDF40_nnlo_as_01180
   additional_errors: true # should be set to true only for the last iteration
   luxseed: 1234567890
+  eps_base: 1e-2

n3fit/src/n3fit/tests/regressions/quickcard_qed.yml

@@ -0,0 +1,93 @@
+#
+# Configuration file for n3fit regression tests
+# This runcard includes two DIS datasets, one Hadronic dataset
+# and two positivity datasets
+#
+
+############################################################
+description: n3fit regression test
+
+############################################################
+# frac: training fraction
+# ewk: apply ewk k-factors
+# sys: systematics treatment (see systypes)
+dataset_inputs:
+- { dataset: NMC, frac: 0.5 }
+- { dataset: SLACP_dwsh, frac: 0.5}
+- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
+- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+
+############################################################
+datacuts:
+  t0pdfset: NNPDF40_nnlo_as_01180     # PDF set to generate t0 covmat
+  q2min        : 3.49                # Q2 minimum
+  w2min        : 12.5                # W2 minimum
+  combocuts    : NNPDF31             # NNPDF3.0 final kin. cuts
+  jetptcut_tev : 0                   # jet pt cut for tevatron
+  jetptcut_lhc : 0                   # jet pt cut for lhc
+  wptcut_lhc   : 30.0                # Minimum pT for W pT diff distributions
+  jetycut_tev  : 1e30                # jet rap. cut for tevatron
+  jetycut_lhc  : 1e30                # jet rap. cut for lhc
+  dymasscut_min: 0                   # dy inv.mass. min cut
+  dymasscut_max: 1e30                # dy inv.mass. max cut
+  jetcfactcut  : 1e30                # jet cfact. cut
+
+############################################################
+theory:
+  theoryid: 398        # database id
+
+############################################################
+genrep: True    # on = generate MC replicas, False = use real data
+trvlseed: 3
+nnseed: 2
+mcseed: 1  
+
+load: "weights.h5"
+
+parameters: # This defines the parameter dictionary that is passed to the Model Trainer
+  nodes_per_layer: [15, 10, 8]
+  activation_per_layer: ['sigmoid', 'sigmoid', 'linear']
+  initializer: 'glorot_normal'
+  optimizer:
+    optimizer_name: 'RMSprop'
+    learning_rate: 0.00001
+    clipnorm: 1.0
+  epochs: 1100
+  positivity:
+    multiplier: 1.05
+    initial: 1.5
+  stopping_patience: 0.10 # percentage of the number of epochs
+  layer_type: 'dense'
+  dropout: 0.0
+  threshold_chi2: 10.0
+
+fitting:
+  fitbasis: NN31IC # EVOL (7), EVOLQED (8), etc.
+  basis:
+      - { fl: sng, smallx: [1.05,1.19], largex: [1.47,2.70] }
+      - { fl: g,   smallx: [0.94,1.25], largex: [0.11,5.87] }
+      - { fl: v,   smallx: [0.54,0.75], largex: [1.15,2.76] }
+      - { fl: v3,  smallx: [0.21,0.57], largex: [1.35,3.08] }
+      - { fl: v8,  smallx: [0.52,0.76], largex: [0.77,3.56] }
+      - { fl: t3,  smallx: [-0.37,1.52], largex: [1.74,3.39] }
+      - { fl: t8,  smallx: [0.56,1.29], largex: [1.45,3.03] }
+      - { fl: cp,  smallx: [0.12,1.19], largex: [1.83,6.70] }
+
+############################################################
+positivity:
+  posdatasets:
+    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
+    - { dataset: POSDYS, maxlambda: 1e5 }
+
+integrability:
+  integdatasets:
+    - {dataset: INTEGXT8, maxlambda: 1e2}
+
+############################################################
+debug: true
+
+fiatlux:
+  luxset: NNPDF40_nnlo_as_01180
+  additional_errors: true # should be set to true only for the last iteration
+  luxseed: 1234567890
+  eps_base: 1e-2

n3fit/src/n3fit/tests/regressions/quickcard_qed_1.json

@@ -0,0 +1,95 @@
+{
+  "preprocessing": [
+    {
+      "fl": "sng",
+      "smallx": 1.1307647228240967,
+      "largex": 2.6348154544830322,
+      "trainable": true
+    },
+    {
+      "fl": "g",
+      "smallx": 1.1853630542755127,
+      "largex": 1.5627975463867188,
+      "trainable": true
+    },
+    {
+      "fl": "v",
+      "smallx": 0.5399999022483826,
+      "largex": 2.004500150680542,
+      "trainable": true
+    },
+    {
+      "fl": "v3",
+      "smallx": 0.3061824142932892,
+      "largex": 2.624323606491089,
+      "trainable": true
+    },
+    {
+      "fl": "v8",
+      "smallx": 0.5774596929550171,
+      "largex": 2.120253801345825,
+      "trainable": true
+    },
+    {
+      "fl": "t3",
+      "smallx": 1.3441987037658691,
+      "largex": 1.7566683292388916,
+      "trainable": true
+    },
+    {
+      "fl": "t8",
+      "smallx": 1.04995858669281,
+      "largex": 1.945939064025879,
+      "trainable": true
+    },
+    {
+      "fl": "cp",
+      "smallx": 0.7400740385055542,
+      "largex": 3.461853504180908,
+      "trainable": true
+    }
+  ],
+  "stop_epoch": 1100,
+  "best_epoch": 1099,
+  "erf_tr": 31.486101150512695,
+  "erf_vl": 28.4218692779541,
+  "chi2": 19.611825942993164,
+  "pos_state": "POS_VETO",
+  "arc_lengths": [
+    1.035533162554303,
+    1.1953713068471692,
+    1.0881025884095246,
+    1.3414978876721764,
+    1.0839843290638607
+  ],
+  "integrability": [
+    0.002653829054906187,
+    0.002653829054905521,
+    0.0002567724250169823,
+    3.2896786332130423,
+    0.003927801561079747
+  ],
+  "timing": {
+    "walltime": {
+      "Total": 58.37841296195984,
+      "start": 0.0,
+      "replica_set": 0.3208029270172119,
+      "replica_fitted": 58.37802076339722,
+      "replica_set_to_replica_fitted": 58.057217836380005
+    },
+    "cputime": {
+      "Total": 61.517351913999995,
+      "start": 0.0,
+      "replica_set": 2.813359167999998,
+      "replica_fitted": 61.516948647,
+      "replica_set_to_replica_fitted": 58.703589479
+    }
+  },
+  "version": {
+    "keras": "2.11.0",
+    "tensorflow": "2.11.0, mkl=False",
+    "numpy": "1.22.4",
+    "nnpdf": "4.0.6.846+g0a926fdf1",
+    "validphys": "4.0.6.846+g0a926fdf1"
+  }
+}

n3fit/src/n3fit/tests/regressions/quickcard_qed_2.json

@@ -0,0 +1,95 @@
+{
+  "preprocessing": [
+    {
+      "fl": "sng",
+      "smallx": 1.1090763807296753,
+      "largex": 2.683891534805298,
+      "trainable": true
+    },
+    {
+      "fl": "g",
+      "smallx": 0.9399998784065247,
+      "largex": 1.6250606775283813,
+      "trainable": true
+    },
+    {
+      "fl": "v",
+      "smallx": 0.7499998211860657,
+      "largex": 1.7267229557037354,
+      "trainable": true
+    },
+    {
+      "fl": "v3",
+      "smallx": 0.21201331913471222,
+      "largex": 1.3532426357269287,
+      "trainable": true
+    },
+    {
+      "fl": "v8",
+      "smallx": 0.7599998712539673,
+      "largex": 2.390087127685547,
+      "trainable": true
+    },
+    {
+      "fl": "t3",
+      "smallx": 1.4388009309768677,
+      "largex": 2.2958621978759766,
+      "trainable": true
+    },
+    {
+      "fl": "t8",
+      "smallx": 1.0386217832565308,
+      "largex": 1.7531665563583374,
+      "trainable": true
+    },
+    {
+      "fl": "cp",
+      "smallx": 0.24638080596923828,
+      "largex": 2.7976953983306885,
+      "trainable": true
+    }
+  ],
+  "stop_epoch": 1100,
+  "best_epoch": 1099,
+  "erf_tr": 3.5809452533721924,
+  "erf_vl": 3.6826603412628174,
+  "chi2": 2.141340732574463,
+  "pos_state": "POS_VETO",
+  "arc_lengths": [
+    1.3194883264768875,
+    1.1995146017416334,
+    1.054266019685804,
+    5.074692492247958,
+    1.0689068380364566
+  ],
+  "integrability": [
+    0.02917606895789332,
+    0.029176068957894374,
+    0.0003924771135637162,
+    12.764093160629272,
+    0.02982003148645135
+  ],
+  "timing": {
+    "walltime": {
+      "Total": 57.77593541145325,
+      "start": 0.0,
+      "replica_set": 0.31868839263916016,
+      "replica_fitted": 57.77560114860535,
+      "replica_set_to_replica_fitted": 57.45691275596619
+    },
+    "cputime": {
+      "Total": 60.771742980000006,
+      "start": 0.0,
+      "replica_set": 2.6913169609999983,
+      "replica_fitted": 60.771408723,
+      "replica_set_to_replica_fitted": 58.080091762
+    }
+  },
+  "version": {
+    "keras": "2.11.0",
+    "tensorflow": "2.11.0, mkl=False",
+    "numpy": "1.22.4",
+    "nnpdf": "4.0.6.846+g0a926fdf1",
+    "validphys": "4.0.6.846+g0a926fdf1"
+  }
+}

n3fit/src/n3fit/tests/test_evolven3fit.py

@@ -161,12 +161,13 @@ def test_eko_utils(tmp_path):
     assert_allclose(list(eko_op.operator_card.raw["mugrid"]), op_card_dict["mugrid"])
 
 
-@pytest.mark.parametrize("fitname", ["Basic_runcard_3replicas_lowprec_399"])
+@pytest.mark.parametrize("fitname", ["Basic_runcard_3replicas_lowprec_399", "Basic_runcard_qed_3replicas_lowprec_398"])
 def test_perform_evolution(tmp_path, fitname):
     """Test that evolven3fit_new is able to utilize the current eko in the respective theory.
     In addition checks that the generated .info files are correct
     """
     fit = API.fit(fit=fitname)
+    _ = API.theoryid(theoryid=fit.as_input()['theory']['theoryid'])
     # Move the fit to a temporary folder
     tmp_fit = tmp_path / fitname
     shutil.copytree(fit.path, tmp_fit)
@@ -185,3 +186,4 @@ def test_perform_evolution(tmp_path, fitname):
     info = check_lhapdf_info(tmp_info)
     for datpath in tmp_nnfit.glob("replica_*/*.dat"):
         check_lhapdf_dat(datpath, info)
+

n3fit/src/n3fit/tests/test_fit.py

@@ -33,6 +33,7 @@
 log = logging.getLogger(__name__)
 REGRESSION_FOLDER = pathlib.Path(__file__).with_name("regressions")
 QUICKNAME = "quickcard"
+QUICKNAME_QED = "quickcard_qed"
 EXE = "n3fit"
 REPLICA = "1"
 EXPECTED_MAX_FITTIME = 130  # seen mac ~ 180  and linux ~ 90
@@ -64,22 +65,22 @@ def test_initialize_seeds():
     assert len({replica_mcseed(rep, 1, True) for rep in same_replicas}) == 1
 
 
-def auxiliary_performfit(tmp_path, replica=1, timing=True, rel_error=2e-3):
+def auxiliary_performfit(tmp_path, runcard=QUICKNAME, replica=1, timing=True, rel_error=2e-3):
     """Fits quickcard and checks the json file to ensure the results have not changed.
     """
-    quickcard = f"{QUICKNAME}.yml"
+    quickcard = f"{runcard}.yml"
     # Prepare the runcard
     quickpath = REGRESSION_FOLDER / quickcard
     weightpath = REGRESSION_FOLDER / f"weights_{replica}.h5"
     # read up the previous json file for the given replica
-    old_json = load_data(REGRESSION_FOLDER / f"{QUICKNAME}_{replica}.json")
+    old_json = load_data(REGRESSION_FOLDER / f"{runcard}_{replica}.json")
     # cp runcard and weights to tmp folder
     shutil.copy(quickpath, tmp_path)
     shutil.copy(weightpath, tmp_path / "weights.h5")
     # run the fit
     sp.run(f"{EXE} {quickcard} {replica}".split(), cwd=tmp_path, check=True)
     # read up json files
-    full_json = tmp_path / f"{QUICKNAME}/nnfit/replica_{replica}/{QUICKNAME}.json"
+    full_json = tmp_path / f"{runcard}/nnfit/replica_{replica}/{runcard}.json"
     new_json = load_data(full_json)
     # Now compare to regression results, taking into account precision won't be 100%
     equal_checks = ["stop_epoch", "pos_state"]
@@ -101,14 +102,16 @@ def auxiliary_performfit(tmp_path, replica=1, timing=True, rel_error=2e-3):
 
 
 @pytest.mark.darwin
-def test_performfit(tmp_path):
-    auxiliary_performfit(tmp_path, replica=2, timing=False, rel_error=1e-1)
+@pytest.mark.parametrize("runcard", [QUICKNAME, QUICKNAME_QED])
+def test_performfit(tmp_path, runcard):
+    auxiliary_performfit(tmp_path, runcard=runcard, replica=2, timing=False, rel_error=1e-1)
 
 
 @pytest.mark.linux
 @pytest.mark.parametrize("replica", [1, 2])
-def test_performfit_and_timing(tmp_path, replica):
-    auxiliary_performfit(tmp_path, replica=replica, timing=True)
+@pytest.mark.parametrize("runcard", [QUICKNAME, QUICKNAME_QED])
+def test_performfit_and_timing(tmp_path, runcard, replica):
+    auxiliary_performfit(tmp_path, runcard=runcard, replica=replica, timing=True)
 
 
 @pytest.mark.skip(reason="Still not implemented in parallel mode")