New CommonData Reader

buildmaster/oldcommondata_porter.py

@@ -94,7 +94,7 @@ def create_uncertainties(df, systype_file, is_default=False, use_multiplicative=
             if info["treatment"] not in ["ADD", "MULT"]:
                 raise ValueError(f"Treatment type: {info['treatment']} not recognized")
             if use_multiplicative:
-                tmp[key] = float(bin_data[2 * n+1]*data[idx-1]/100.0)
+                tmp[key] = float(bin_data[2 * n + 1] * data[idx - 1] / 100.0)
             else:
                 tmp[key] = float(bin_data[2 * n])
         bins.append(tmp)
@@ -340,16 +340,9 @@ def convert_from_old_to_new(dsname, new_info, overwrite=False, dry=False, keep_e
         metadata = safe_load(metadata_path.read_text())
         # Perform sanity checks
         nnpdf_md = metadata["nnpdf_metadata"]
-        try:
-            assert nnpdf_md["experiment"] == plotting_dict["experiment"]
-            assert nnpdf_md["nnpdf31_process"] == plotting_dict["nnpdf31_process"]
-            assert metadata.get("setname") == set_name
-        except AssertionError:
-            print(traceback.format_exc())
-            # If this fails, inspect
-            import ipdb
-
-            ipdb.set_trace()
+        assert nnpdf_md["experiment"] == plotting_dict["experiment"]
+        assert nnpdf_md["nnpdf31_process"] == plotting_dict["nnpdf31_process"]
+        assert metadata.get("setname") == set_name
         # Check whether the observable already exists
         already_implemented = [i["observable_name"] for i in metadata["implemented_observables"]]
         if obs_name in already_implemented:

doc/sphinx/source/tutorials/closuretest.md

@@ -171,13 +171,10 @@ the closure test settings modified as shown
 between running a closure fit in ``n3fit`` and a standard fit is that the user is
 required to run ``vp-setupfit`` on the runcard before running ``n3fit``. This is
 because the filtering of the data is required to generate the pseudodata central
-values. The filtered data should then be rebuilt before the fit, so there is no
-risk of the fit crashing due to multiple replicas rebuilding the data
-simultaneously. The workflow is as follows:
+values. The workflow is as follows:
 
 ```bash
 $ vp-setupfit fitname.yml
-$ vp-rebuild-data fitname
 $ n3fit fitname.yml <replica_number>
 ```
 

extra_tests/regression_fits/central.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -69,12 +69,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/diagonal.yml

@@ -12,9 +12,9 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
 
 ############################################################
 datacuts:
@@ -69,12 +69,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/feature_scaling.yml

@@ -12,9 +12,9 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -68,12 +68,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/flavour.yml

@@ -12,8 +12,8 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
 
 ############################################################
 datacuts:
@@ -67,12 +67,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/no_lagrange.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:

extra_tests/regression_fits/no_msr.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -70,12 +70,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/no_sumrules.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -70,12 +70,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/no_vsr.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -70,12 +70,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/normal_fit.yml

@@ -6,8 +6,8 @@ description: n3fit regression test, normal fit without initial weights
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
 
 ############################################################
 datacuts:
@@ -60,12 +60,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

extra_tests/regression_fits/trainable_prepro.yml

@@ -12,10 +12,10 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
-- { dataset: ATLASTTBARTOT8TEV, frac: 1.0, cfac: ['QCD'] }
+  - {dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy}
+  - {dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy}
+  - {dataset: CMS_Z0_8TEV_PT-Y, frac: 0.5, cfac: [QCD], variant: legacy_10}
+  - {dataset: ATLAS_TTBAR_8TEV_TOT_X-SEC, frac: 1.0, cfac: [QCD], variant: legacy}
 
 ############################################################
 datacuts:
@@ -69,12 +69,12 @@ fitting:
 ############################################################
 positivity:
   posdatasets:
-    - { dataset: POSF2U,   maxlambda: 1e6 }  # Positivity Lagrange Multiplier
-    - { dataset: POSDYS, maxlambda: 1e5 }
+  - {dataset: NNPDF_POS_5GEV_F2U, maxlambda: 1e6}
+  - {dataset: NNPDF_POS_5GEV_DYS, maxlambda: 1e5}
 
 integrability:
   integdatasets:
-    - {dataset: INTEGXT8, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_3GEV_XT8, maxlambda: 1e2}
 
 ############################################################
 debug: true

n3fit/src/n3fit/tests/regressions/hyper-quickcard.yml

@@ -12,9 +12,9 @@ description: n3fit regression test
 # ewk: apply ewk k-factors
 # sys: systematics treatment (see systypes)
 dataset_inputs:
-- { dataset: NMC, frac: 0.5 }
-- { dataset: SLACP, frac: 0.5}
-- { dataset: CMSZDIFF12, frac: 0.5, cfac: ['QCD'], sys: 10 }
+- { dataset: NMC_NC_NOTFIXED_P_EM-SIGMARED, frac: 0.5, variant: legacy }
+- { dataset: SLAC_NC_NOTFIXED_P_EM-F2, frac: 0.5, variant: legacy }
+- { dataset: CMS_WP_7TEV_ELECTRON_ASY, frac: 0.5}
 
 ############################################################
 datacuts:
@@ -74,10 +74,10 @@ kfold:
     threshold: 2.0
     partitions:
         - datasets:
-            - NMC
+            - NMC_NC_NOTFIXED_P_EM-SIGMARED
         - datasets:
-            - SLACP
-            - CMSZDIFF12
+            - SLAC_NC_NOTFIXED_P_EM-F2
+            - CMS_WP_7TEV_ELECTRON_ASY
 
 ############################################################
 genrep: False    # on = generate MC replicas, False = use real data

pyproject.toml

@@ -27,11 +27,14 @@ packages = [
 ]
 # Data files
 include = [
-    # The profile is included together with the validphys package
+    # The default profile is included together with the validphys package
     "validphys2/src/validphys/nnprofile_default.yaml",
-    # While commondata and theory.db are separated into the datafiles folder
+    # Same for commondata and theory.db
     "validphys2/src/validphys/datafiles/commondata/*",
     "validphys2/src/validphys/datafiles/theory.db",
+    # From the new commondata we are only interested on top-level yaml files
+    "validphys2/src/validphys/datafiles/new_commondata/*/*.yaml",
+    "validphys2/src/validphys/datafiles/new_commondata/dataset_names.yml",
     # The version file is ignored by git so it needs to be explicitly included
     "validphys2/src/validphys/_version.py"
 ]
@@ -51,7 +54,6 @@ vp-get = "validphys.scripts.vp_get:main"
 vp-comparefits = "validphys.scripts.vp_comparefits:main"
 vp-fitrename = "validphys.scripts.vp_fitrename:main"
 vp-checktheory = "validphys.scripts.vp_checktheory:main"
-vp-rebuild-data = "validphys.scripts.vp_rebuild_data:main"
 vp-pdfrename = "validphys.scripts.vp_pdfrename:main"
 vp-pdffromreplicas = "validphys.scripts.vp_pdffromreplicas:main"
 vp-list = "validphys.scripts.vp_list:main"