add a template runcard

init work on theory covmat

some other work on pineparser, not working yet

lumi to channels

remove is_polarized and make it working

n3fit/runcards/examples/nnpdfpol20_dis_nlo_mhou.yml

@@ -0,0 +1,119 @@
+############################################################
+# Configuration file for N3FIT Polarized Fits
+############################################################
+description: NNPDFpol2.0 NLO Global fit with ID 20000001 and MHOU (W2min=4)
+
+############################################################
+dataset_inputs:
+# Polarized DIS datasets
+- {dataset: COMPASS15_NC_NOTFIXED_MUP_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: COMPASS15_NC_NOTFIXED_MUD_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: E142_NC_NOTFIXED_EN_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: E143_NC_NOTFIXED_EP_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: E143_NC_NOTFIXED_ED_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: E154_NC_9GEV_EN_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: E155_NC_9GEV_EN_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: E155_NC_9GEV_EP_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: EMC_NC_NOTFIXED_MUP_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: HERMES_NC_7GEV_ED_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: HERMES_NC_7GEV_EP_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: HERMES97_NC_7GEV_EN_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: JLABE06_NC_3GEV_EN_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: JLABEG1B_NC_NOTFIXED_EP_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: JLABEG1B_NC_NOTFIXED_ED_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: JLABE97_NC_NOTFIXED_EN_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: JLABE99_NC_3GEV_EN_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: JLABEG1DVCS_NC_5GEV_ED_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: JLABEG1DVCS_NC_3GEV_EP_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: SMC_NC_NOTFIXED_MUD_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: SMC_NC_NOTFIXED_MUP_G1, frac: 0.60, cfac: [NRM]}
+- {dataset: SMCSX_NC_24GEV_MUD_G1F1RATIO, frac: 0.60, cfac: []}
+- {dataset: SMCSX_NC_17GEV_MUP_G1F1RATIO, frac: 0.60, cfac: []}
+# Polarized DY datasets
+- {dataset: STAR_WMWP_510GEV_WM-AL, frac: 0.60}
+- {dataset: STAR_WMWP_510GEV_WP-AL, frac: 0.60}
+
+############################################################
+datacuts:
+  t0pdfset: NNPDFpol10_100   # PDF set to generate t0 covmat
+  q2min: 1.00                # Q2 minimum
+  w2min: 4.00                # W2 minimum
+
+############################################################
+# Define the unpolarized PDF set to be used as BC for POS
+positivity_bound:
+  unpolarized_bc: NNPDF40_nnlo_pch_as_01180
+  n_std: 1.00 # Standard Deviation to be added as Error
+
+############################################################
+theory:
+  theoryid: 20000001
+
+theorycovmatconfig:
+  point_prescription: '7 point'
+  theoryids:
+    from_: scale_variation_theories
+  pdf: NNPDFpol10_100
+  unpolarized_bc: NNPDF40_nnlo_pch_as_01180
+  use_thcovmat_in_fitting: true
+  use_thcovmat_in_sampling: true
+
+############################################################
+trvlseed: 14613416
+nnseed: 1865683875
+mcseed: 746898422
+save: false
+genrep: true
+
+parameters:
+  nodes_per_layer: [25, 20, 6]
+  activation_per_layer: [tanh, tanh, linear]
+  initializer: glorot_normal
+  optimizer:
+    clipnorm: 6.073e-6
+    learning_rate: 2.621e-3
+    optimizer_name: Nadam
+  epochs: 20000
+  positivity:
+    initial: 1
+    multiplier:
+    threshold: 1e-7
+  integrability:
+    initial: 1
+    multiplier:
+    threshold: 1e-2
+  stopping_patience: 0.1
+  layer_type: dense
+  dropout: 0.0
+  threshold_chi2: 3.5
+
+fitting:
+  fitbasis: POLARIZED_EVOL_CMP
+  sum_rules: TSR
+  basis:
+  - {fl: sng, trainable: false, smallx: [1.094, 1.118], largex: [1.46, 3.003]}
+  - {fl: g, trainable: false, smallx: [0.8189, 1.844], largex: [2.591, 5.697]}
+  - {fl: t3, trainable: false, smallx: [-0.4401, 0.9163], largex: [1.773, 3.333]}
+  - {fl: t8, trainable: false, smallx: [0.5852, 0.8537], largex: [1.533, 3.436]}
+  - {fl: v, trainable: false, smallx: [0.472, 0.9576], largex: [1.571, 3.559]}
+  - {fl: v3, trainable: false, smallx: [0.07483, 0.9501], largex: [1.514, 3.467]}
+
+###########################################################
+positivity:
+  posdatasets:
+  - {dataset: NNPDF_POS_2P24GEV_XGL-POLARIZED, maxlambda: 1e8}
+  - {dataset: NNPDF_POS_2P24GEV_XDQ-POLARIZED, maxlambda: 1e8}
+  - {dataset: NNPDF_POS_2P24GEV_XDB-POLARIZED, maxlambda: 1e8}
+  - {dataset: NNPDF_POS_2P24GEV_XUQ-POLARIZED, maxlambda: 1e8}
+  - {dataset: NNPDF_POS_2P24GEV_XUB-POLARIZED, maxlambda: 1e8}
+  - {dataset: NNPDF_POS_2P24GEV_XSQSUM-POLARIZED, maxlambda: 1e8}
+
+############################################################
+integrability:
+  integdatasets:
+  - {dataset: NNPDF_INTEG_1GEV_XGL-POLARIZED, maxlambda: 1e2}
+  - {dataset: NNPDF_INTEG_1GEV_XSIGMA-POLARIZED, maxlambda: 1e2}
+
+############################################################
+debug: false
+maxcores: 4

n3fit/src/n3fit/layers/observable.py

@@ -111,6 +111,8 @@ def __init__(
 
             # Check if the given Positivity dataset is a Polarized one and if the current FK
             # table is NOT a Polarized FK table.
+            # FIXME:  I have drop is_polarized, but here we are still using it, 
+            # need to replace these with convolution_types
             # `is_polarized` method is deprecated and should be replaced by `convolution_types`
             nopol_fk_pos: bool = self.is_pos_polarized() and not fkdata.is_polarized
             # Check if the current FK table involves `UnpolPDF` when `boundary_condition` is not None

n3fit/src/n3fit/tests/test_layers.py

@@ -32,7 +32,6 @@ class _fake_FKTableData:
     fktable: np.array
     luminosity_mapping: np.array
     xgrid: np.array
-    is_polarized: bool = False
     convolution_types: tuple = ("UnpolPDF",)
 
 
@@ -120,7 +119,6 @@ def generate_had(nfk=1):
                 fk,
                 comb,
                 np.ones((1, XSIZE)),
-                is_polarized=False,
                 convolution_types=("UnpolPDF", "UnpolPDF"),
             )
         )

validphys2/src/validphys/convolution.py

@@ -148,7 +148,7 @@ class PredictionsRequireCutsError(Exception):
     pass
 
 
-def _predictions(dataset, pdf, fkfunc):
+def _predictions(dataset, pdf, fkfunc, unpolarized_bc=None):
     """Combine data on all the FKTables in the database according to the
     reduction operation defined therein. Dispatch the kind of predictions (for
     all replicas, central, etc) according to the provided ``fkfunc``, which
@@ -166,7 +166,9 @@ def _predictions(dataset, pdf, fkfunc):
     all_predictions = []
     for fk in dataset.fkspecs:
         fk_w_cuts = fk.load_with_cuts(cuts)
-        all_predictions.append(fkfunc(fk_w_cuts, pdf))
+        # TODO: here we have to try and except or we 
+        # need to modify all the possible fkfunc...
+        all_predictions.append(fkfunc(fk_w_cuts, pdf, unpolarized_bc))
     # Old fktables repeated values to make DEN and NUM sizes match in RATIO operations
     # pineappl tables instead just contain the one value used
     # The code below works for both situation while keeping `true_div` as the operation
@@ -175,7 +177,7 @@ def _predictions(dataset, pdf, fkfunc):
     return opfunc(*all_predictions)
 
 
-def predictions(dataset, pdf):
+def predictions(dataset, pdf, unpolarized_bc=None):
     """ "Compute theory predictions for a given PDF and dataset. Information
     regading the dataset, on cuts, CFactors and combinations of FKTables is
     taken into account to construct the predictions.
@@ -189,6 +191,8 @@ def predictions(dataset, pdf):
         The dataset containing information on the partonic cross section.
     pdf : validphys.core.PDF
         The PDF set to use for the convolutions.
+    unpolarized_bc : validphys.core.PDF
+        Unpolarized boundary condition PDF for polarized fits.
 
     Returns
     -------
@@ -223,17 +227,17 @@ def predictions(dataset, pdf):
 
 
     """
-    return _predictions(dataset, pdf, fk_predictions)
+    return _predictions(dataset, pdf, fk_predictions, unpolarized_bc)
 
 
-def central_predictions(dataset, pdf):
+def central_predictions(dataset, pdf, unpolarized_bc=None):
     """Same as :py:func:`predictions` but computing the predictions for the
     central member of the PDF set only. For Monte Carlo PDFs, this is a faster
     alternative to computing the central predictions as the average of the
     replica predictions (although a small approximation is involved in the case
     of hadronic predictions).
     """
-    return _predictions(dataset, pdf, central_fk_predictions)
+    return _predictions(dataset, pdf, central_fk_predictions, unpolarized_bc)
 
 
 def linear_predictions(dataset, pdf):
@@ -298,13 +302,28 @@ def fk_predictions(loaded_fk, pdf):
         return dis_predictions(loaded_fk, pdf)
 
 
-def central_fk_predictions(loaded_fk, pdf):
+def central_fk_predictions(loaded_fk, pdf, unpolarized_bc=None):
     """Same as :py:func:`fk_predictions`, but computing predictions for the
     central PDF member only."""
+
+    # if the unpolarized_bc is not None, check the convolution types
+    if unpolarized_bc is not None:
+        map_pdf_to_conv_types = {
+            "UnpolPDF": unpolarized_bc,
+            "PolPDF": pdf,
+            "None": None,
+        }
+        conv_types = loaded_fk.convolution_types
+        pdf_list = [
+            map_pdf_to_conv_types[conv_types[0]],
+            map_pdf_to_conv_types[conv_types[1]]
+        ]
+    else:
+        pdf_list = [pdf, pdf]
     if loaded_fk.hadronic:
-        return central_hadron_predictions(loaded_fk, pdf)
+        return central_hadron_predictions(loaded_fk, pdf_list)
     else:
-        return central_dis_predictions(loaded_fk, pdf)
+        return central_dis_predictions(loaded_fk, pdf_list[0])
 
 
 def linear_fk_predictions(loaded_fk, pdf):
@@ -408,11 +427,12 @@ def hadron_predictions(loaded_fk, pdf):
     return res
 
 
-def central_hadron_predictions(loaded_fk, pdf):
+def central_hadron_predictions(loaded_fk, pdf_list):
     """Implementation of :py:func:`central_fk_predictions` for hadronic
     observables."""
-    gv = functools.partial(evolution.central_grid_values, pdf=pdf)
-    return _gv_hadron_predictions(loaded_fk, gv)
+    gv1 = functools.partial(evolution.central_grid_values, pdf=pdf_list[0])
+    gv2 = functools.partial(evolution.central_grid_values, pdf=pdf_list[1])
+    return _gv_hadron_predictions(loaded_fk, gv1, gv2)
 
 
 def linear_hadron_predictions(loaded_fk, pdf):

validphys2/src/validphys/coredata.py

@@ -22,10 +22,6 @@ class FKTableData:
     """
     Data contained in an FKTable
 
-    TODO: remove `is_polarized` as it is now deprecated and should be replaced by
-    `convolution_particle_`, but to keep backward compatibility with old grids we
-    still have to keep it.
-
     Parameters
     ----------
     hadronic : bool
@@ -67,7 +63,6 @@ class FKTableData:
     ndata: int
     xgrid: np.ndarray
     sigma: pd.DataFrame
-    is_polarized: bool = False
     convolution_types: tuple[str] = ("UnpolPDF",)
     metadata: dict = dataclasses.field(default_factory=dict, repr=False)
     protected: bool = False

validphys2/src/validphys/loader.py

@@ -974,7 +974,7 @@ def download_and_extract(url, local_path, target_name=None):
             folder_dest = tempfile.TemporaryDirectory(dir=local_path, suffix=name)
             dest_path = pathlib.Path(folder_dest.name)
             try:
-                res_tar.extractall(path=dest_path, filter="data")
+                res_tar.extractall(path=dest_path)
             except tarfile.LinkOutsideDestinationError as e:
                 if sys.version_info > (3, 11):
                     raise e

validphys2/src/validphys/pineparser.py

@@ -155,26 +155,16 @@ def pineappl_reader(fkspec):
     # Extract metadata from the first grid
     pine_rep = pines[0]
 
-    # Check if it is a Polarized DIS FK table. The `polarized` key is now deprecated and should be
-    # replaced with the `convolution_type_` that defines the initial states.
-    is_polarized = pine_rep.key_values().get("polarized") == "True"
-
     # Is it hadronic? (at the moment only hadronic and DIS are considered)
-    hadronic = pine_rep.key_values()["initial_state_1"] == pine_rep.key_values()["initial_state_2"]
-
-    # Determine the types of convolutions required by the different initial states. The default when
-    # neither `is_polarized` nor `convolution_type_` is defined is `UnpolPDF`.
-    nb_convolutions = 2 if hadronic else 1
-    if not is_polarized:  # if `polarized` is not in grid but `convolution_type_` is
-        conv_types = tuple(
-            pine_rep.key_values().get(f"convolution_type_{i + 1}", "UnpolPDF")
-            for i in range(nb_convolutions)
-        )
-    else:
-        conv_types = tuple("PolPDF" for _ in range(nb_convolutions))
+    hadronic = pine_rep.key_values()["convolution_particle_1"] == pine_rep.key_values()["convolution_particle_2"]
+
+    # Check if it is a Polarized FK table.
+    convolution_type_1 = pine_rep.key_values().get("convolution_type_1", "UnpolPDF")
+    convolution_type_2 = pine_rep.key_values().get("convolution_type_2", "UnpolPDF")
+    conv_types = (convolution_type_1, convolution_type_2)
 
     # Sanity check (in case at some point we start fitting things that are not protons)
-    if hadronic and pine_rep.key_values()["initial_state_1"] != "2212":
+    if hadronic and pine_rep.key_values()["convolution_particle_1"] != "2212":
         raise ValueError(
             "pineappl_reader is not prepared to read a hadronic fktable with no protons!"
         )
@@ -236,7 +226,7 @@ def pineappl_reader(fkspec):
         # Create the multi-index for the dataframe
         # for optimized pineappls different grids can potentially have different indices
         # so they need to be indexed separately and then concatenated only at the end
-        lumi_columns = _pinelumi_to_columns(p.lumi(), hadronic)
+        lumi_columns = _pinelumi_to_columns(p.channels(), hadronic)
         lf = len(lumi_columns)
         data_idx = np.arange(ndata, ndata + n)
         if hadronic:
@@ -274,7 +264,6 @@ def pineappl_reader(fkspec):
         sigma=sigma,
         ndata=ndata,
         Q0=Q0,
-        is_polarized=is_polarized,
         convolution_types=conv_types,
         metadata=fkspec.metadata,
         hadronic=hadronic,

validphys2/src/validphys/results.py

@@ -135,7 +135,7 @@ def make_label(pdf, dataset):
         return label
 
     @classmethod
-    def from_convolution(cls, pdf, dataset, central_only=False):
+    def from_convolution(cls, pdf, dataset, central_only=False, unpolarized_bc=None):
         # This should work for both single dataset and whole groups
         try:
             datasets = dataset.datasets
@@ -144,9 +144,9 @@ def from_convolution(cls, pdf, dataset, central_only=False):
 
         try:
             if central_only:
-                preds = [central_predictions(d, pdf) for d in datasets]
+                preds = [central_predictions(d, pdf, unpolarized_bc) for d in datasets]
             else:
-                preds = [predictions(d, pdf) for d in datasets]
+                preds = [predictions(d, pdf, unpolarized_bc) for d in datasets]
             th_predictions = pd.concat(preds)
         except PredictionsRequireCutsError as e:
             raise PredictionsRequireCutsError(
@@ -554,11 +554,11 @@ def results(dataset: DataSetSpec, pdf: PDF, covariance_matrix, sqrt_covmat):
     )
 
 
-def results_central(dataset: DataSetSpec, pdf: PDF, covariance_matrix, sqrt_covmat):
+def results_central(dataset: DataSetSpec, pdf: PDF, covariance_matrix, sqrt_covmat, unpolarized_bc = None):
     """Same as :py:func:`results` but only calculates the prediction for replica0."""
     return (
         DataResult(dataset, covariance_matrix, sqrt_covmat),
-        ThPredictionsResult.from_convolution(pdf, dataset, central_only=True),
+        ThPredictionsResult.from_convolution(pdf, dataset, central_only=True, unpolarized_bc=unpolarized_bc),
     )