Merge branch 'new_commondata_collected' into collider_dy_ncd

NNPDF · Nov 17, 2023 · ce5aa48 · ce5aa48
2 parents ac996c0 + e64d092
commit ce5aa48
Show file tree

Hide file tree

Showing 55 changed files with 1,483 additions and 943 deletions.
diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml
@@ -8,7 +8,7 @@ categories:
   - title: Documentation
     labels:
       - documentation
-  - title: Worflow
+  - title: Workflow
     labels:
       - devtools
   - title: Refactoring

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -8,7 +8,7 @@ on:
 
 # some general variables
 env:
-  CONDA_PY: 39
+  CONDA_PY: 310
   IMAGE_NAME: nnpdf
 
 jobs:
@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.9]
+        python-version: ["3.10"]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v1
@@ -27,14 +27,16 @@ jobs:
     - name: Setup conda and install conda-build
       shell: bash -l {0}
       run: |
+        conda install -n base conda-libmamba-solver
+        conda config --set solver libmamba
         conda config --append channels conda-forge
         conda config --prepend channels https://packages.nnpdf.science/public
         conda config --set show_channel_urls true
-        conda install conda-build --yes
+        conda install boa --yes
     - name: Build recipe
       shell: bash -l {0}
       run: |
-        CONDA_PY=$CONDA_PY conda build --no-test -q conda-recipe
+        CONDA_PY=$CONDA_PY conda mambabuild --no-test -q conda-recipe
     # install local build
     - name: Copying conda channel locally
       run: |

diff --git a/buildmaster/dataset_names.yml b/buildmaster/dataset_names.yml
@@ -10,4 +10,4 @@ LHCBWZMU8TEV: LHCB_DY_8TEV_MUON_Y
 LHCBWMU7TEV : LHCB_WPWM_7TEV_MUON_Y       
 LHCBWMU8TEV: 
   dataset: LHCB_WPWM_8TEV_MUON_Y       
-  variant: Null
+  variant: Null
diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml
@@ -50,8 +50,8 @@ requirements:
         - sphinx_rtd_theme >0.5
         - sphinxcontrib-bibtex
         - curio >=1.0
-        - pineappl >=0.5.8
-        - eko >=0.13.5,<0.14
+        - pineappl >=0.6.2
+        - eko >=0.14.0
         - banana-hep >=0.6.8
         - fiatlux
 

diff --git a/doc/sphinx/source/n3fit/hyperopt.rst b/doc/sphinx/source/n3fit/hyperopt.rst
@@ -320,10 +320,10 @@ Changing the hyperoptimization target
 -----------------------------------
 
 Beyond the usual :math:`\chi2`-based optimization figures above, it is possible to utilize other measures as the target for hyperoptimization.
-One possibility is to use a :ref:`future test<futuretests>`-based metric for which the goal is not to get the minimum :math:`\chi2` but to get the same :math:`\chi2` (with PDF errors considered) for different datasets. The idea is that this way we select models of which the prediction is stable upon variations in the dataset. 
+One possibility is to use a :ref:`future test<futuretests>`-based metric for which the goal is not to get the minimum :math:`\chi2` but to get the same :math:`\chi2` (with PDF errors considered) for different datasets. The idea is that this way we select models of which the prediction is stable upon variations in the dataset.
 In order to obtain the PDF errors used in the figure of merit it is necessary to run multiple replicas, luckily ``n3fit`` provides such a possibility also during hyperoptimization.
 
-Take the following modifications to a normal hyperopt runcard 
+Take the following modifications to a normal hyperopt runcard
 (note that for convenience we take the trials directly from a previous run, so we don't have to create a new
 hyperopt configuration dictionary).
 
@@ -345,7 +345,7 @@ hyperopt configuration dictionary).
 
         kfold:
           target: fit_future_tests
-          partitions: 
+          partitions:
           - datasets:
             - HERACOMBCCEP
             - HERACOMBCCEM
@@ -370,3 +370,19 @@ The figure of merit will be the difference between the :math:`\chi2` of the seco
 
 .. math::
    L_{\rm hyperopt} = \chi^{2}_{(1) \rm pdferr} - \chi^{2}_{(2)}
+
+
+Restarting hyperoptimization runs
+---------------------------------
+
+In addition to the ``tries.json`` files, hyperparameter scans also produce ``tries.pkl`` `pickle <https://docs.python.org/3/library/pickle.html>`_ files,
+which are located in the same directory as the corresponding ``tries.json`` file.
+The generated ``tries.pkl`` file stores the complete history of a previous hyperoptimization run, making it possible to resume the process using the ``hyperopt`` framework.
+To achieve this, you can use the ``--restart`` option within the ``n3fit`` command, e.g.,:
+
+.. code-block:: bash
+
+   n3fit runcard.yml 1 -r 10 --hyperopt 20 --restart
+
+The above command example is effective when the number of saved trials in the ``test_run/nnfit/replica_1/tries.pkl`` is
+less than ``20``. If there are ``20`` or more saved trials, ``n3fit`` will simply terminate, displaying the best results.
diff --git a/docker/dockerfile b/docker/dockerfile
@@ -30,7 +30,9 @@ RUN wget "$CONDA_URL"      && \
 ENV PATH /root/miniconda3/bin:$PATH
 
 # Setup conda channels
-RUN conda config --append channels conda-forge                                    && \
+RUN conda install -n base conda-libmamba-solver                                   && \
+    conda config --set solver libmamba                                            && \
+    conda config --append channels conda-forge                                    && \
     conda config --prepend channels https://packages.nnpdf.science/public/        && \
     conda config --set show_channel_urls true                                     && \
     conda init bash

diff --git a/n3fit/runcards/nnpdfenv-tag-4.0.7.yml b/n3fit/runcards/nnpdfenv-tag-4.0.7.yml
@@ -0,0 +1,246 @@
+name: nnpdfenv-tag-4.0.7
+channels:
+  - local
+  - https://packages.nnpdf.science/public
+  - defaults
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - _tflow_select=2.2.0=eigen
+  - abseil-cpp=20211102.0=hd4dd3e8_0
+  - absl-py=1.4.0=py310h06a4308_0
+  - aiohttp=3.8.5=py310h5eee18b_0
+  - aiosignal=1.2.0=pyhd3eb1b0_0
+  - alabaster=0.7.12=pyhd3eb1b0_0
+  - apfel=3.0.6.3=hbdda60e_9
+  - appdirs=1.4.4=pyhd3eb1b0_0
+  - asttokens=2.0.5=pyhd3eb1b0_0
+  - astunparse=1.6.3=py_0
+  - async-timeout=4.0.2=py310h06a4308_0
+  - attrs=23.1.0=py310h06a4308_0
+  - babel=2.11.0=py310h06a4308_0
+  - backcall=0.2.0=pyhd3eb1b0_0
+  - banana-hep=0.6.8=pyhd8ed1ab_0
+  - blas=1.0=openblas
+  - blessings=1.7=py310h06a4308_1002
+  - blinker=1.6.2=py310h06a4308_0
+  - bottleneck=1.3.5=py310ha9d4c09_0
+  - brotli=1.0.9=h5eee18b_7
+  - brotli-bin=1.0.9=h5eee18b_7
+  - brotli-python=1.0.9=py310h6a678d5_7
+  - bzip2=1.0.8=h7b6447c_0
+  - c-ares=1.19.1=h5eee18b_0
+  - ca-certificates=2023.08.22=h06a4308_0
+  - cachetools=4.2.2=pyhd3eb1b0_0
+  - certifi=2023.7.22=py310h06a4308_0
+  - cffi=1.15.1=py310h5eee18b_3
+  - charset-normalizer=2.0.4=pyhd3eb1b0_0
+  - click=8.1.7=py310h06a4308_0
+  - cloudpickle=2.2.1=py310h06a4308_0
+  - colorama=0.4.6=py310h06a4308_0
+  - commonmark=0.9.1=pyhd3eb1b0_0
+  - contourpy=1.0.5=py310hdb19cb5_0
+  - cryptography=41.0.3=py310h130f0dd_0
+  - curio=1.4=pyhd3eb1b0_0
+  - cycler=0.11.0=pyhd3eb1b0_0
+  - cyrus-sasl=2.1.28=h9c0eb46_1
+  - dataclasses=0.8=pyh6d0b6a4_7
+  - dbus=1.13.18=hb2f20db_0
+  - decorator=5.1.1=pyhd3eb1b0_0
+  - docutils=0.16=py310h06a4308_2
+  - eko=0.14.0=pyhd8ed1ab_0
+  - exceptiongroup=1.0.4=py310h06a4308_0
+  - executing=0.8.3=pyhd3eb1b0_0
+  - expat=2.5.0=h6a678d5_0
+  - fiatlux=0.1.2=py310heac68b4_0
+  - flatbuffers=2.0.0=h2531618_0
+  - fontconfig=2.14.1=h52c9d5c_1
+  - fonttools=4.25.0=pyhd3eb1b0_0
+  - freetype=2.12.1=h4a9f257_0
+  - frozenlist=1.4.0=py310h5eee18b_0
+  - future=0.18.3=py310h06a4308_0
+  - gast=0.4.0=pyhd3eb1b0_0
+  - giflib=5.2.1=h5eee18b_3
+  - glib=2.69.1=he621ea3_2
+  - google-auth=2.22.0=py310h06a4308_0
+  - google-auth-oauthlib=0.4.4=pyhd3eb1b0_0
+  - google-pasta=0.2.0=pyhd3eb1b0_0
+  - greenlet=2.0.1=py310h6a678d5_0
+  - grpc-cpp=1.46.1=h33aed49_1
+  - grpcio=1.42.0=py310hce63b2e_0
+  - gsl=2.7.1=h6e86dc7_1
+  - gst-plugins-base=1.14.1=h6a678d5_1
+  - gstreamer=1.14.1=h5eee18b_1
+  - h5py=3.9.0=py310he06866b_0
+  - hdf5=1.12.1=h70be1eb_2
+  - hyperopt=0.2.7=pyhd8ed1ab_0
+  - icu=58.2=he6710b0_3
+  - idna=3.4=py310h06a4308_0
+  - imagesize=1.4.1=py310h06a4308_0
+  - importlib-metadata=6.0.0=py310h06a4308_0
+  - importlib_metadata=6.0.0=hd3eb1b0_0
+  - ipython=8.15.0=py310h06a4308_0
+  - jedi=0.18.1=py310h06a4308_1
+  - jinja2=3.1.2=py310h06a4308_0
+  - jpeg=9e=h5eee18b_1
+  - keras=2.11.0=py310h06a4308_0
+  - keras-preprocessing=1.1.2=pyhd3eb1b0_0
+  - kiwisolver=1.4.4=py310h6a678d5_0
+  - krb5=1.20.1=h568e23c_1
+  - latexcodec=2.0.1=pyh9f0ad1d_0
+  - lcms2=2.12=h3be6417_0
+  - ld_impl_linux-64=2.38=h1181459_1
+  - lerc=3.0=h295c915_0
+  - lhapdf=6.5.0=py310hd41b1e2_1
+  - libarchive=3.4.2=h5de8990_2
+  - libbrotlicommon=1.0.9=h5eee18b_7
+  - libbrotlidec=1.0.9=h5eee18b_7
+  - libbrotlienc=1.0.9=h5eee18b_7
+  - libclang=14.0.6=default_hc6dbbc7_1
+  - libclang13=14.0.6=default_he11475f_1
+  - libcups=2.4.2=ha637b67_0
+  - libcurl=8.2.1=h91b91d3_0
+  - libdeflate=1.17=h5eee18b_1
+  - libedit=3.1.20221030=h5eee18b_0
+  - libev=4.33=h7f8727e_1
+  - libffi=3.4.4=h6a678d5_0
+  - libgcc-ng=13.2.0=h807b86a_3
+  - libgfortran-ng=13.2.0=h69a702a_3
+  - libgfortran5=13.2.0=ha4646dd_3
+  - libgomp=13.2.0=h807b86a_3
+  - libllvm14=14.0.6=hdb19cb5_3
+  - libnghttp2=1.52.0=ha637b67_1
+  - libopenblas=0.3.21=h043d6bf_0
+  - libpng=1.6.39=h5eee18b_0
+  - libpq=12.15=h37d81fd_1
+  - libprotobuf=3.20.3=he621ea3_0
+  - libssh2=1.10.0=h37d81fd_2
+  - libstdcxx-ng=13.2.0=h7e041cc_3
+  - libtiff=4.5.1=h6a678d5_0
+  - libuuid=1.41.5=h5eee18b_0
+  - libwebp=1.3.2=h11a3e52_0
+  - libwebp-base=1.3.2=h5eee18b_0
+  - libxcb=1.15=h7f8727e_0
+  - libxkbcommon=1.0.1=hfa300c1_0
+  - libxml2=2.9.14=h74e7548_0
+  - llvmlite=0.40.1=py310he621ea3_0
+  - lz4=4.3.2=py310h5eee18b_0
+  - lz4-c=1.9.4=h6a678d5_0
+  - markdown=3.4.1=py310h06a4308_0
+  - markupsafe=2.1.1=py310h7f8727e_0
+  - matplotlib=3.7.2=py310h06a4308_0
+  - matplotlib-base=3.7.2=py310h1128e8f_0
+  - matplotlib-inline=0.1.6=py310h06a4308_0
+  - multidict=6.0.2=py310h5eee18b_0
+  - munkres=1.1.4=py_0
+  - mysql=5.7.24=he378463_2
+  - ncurses=6.4=h6a678d5_0
+  - networkx=3.1=py310h06a4308_0
+  - nnpdf=4.0.7.0+g0e6c1479c=py310h6cd0baa_0
+  - numba=0.57.1=py310h0f6aa51_0
+  - numexpr=2.8.7=py310h286c3b5_0
+  - numpy=1.24.3=py310hac523dd_0
+  - numpy-base=1.24.3=py310h375b286_0
+  - oauthlib=3.2.2=py310h06a4308_0
+  - openjpeg=2.4.0=h3ad879b_0
+  - openssl=1.1.1w=h7f8727e_0
+  - opt_einsum=3.3.0=pyhd3eb1b0_1
+  - packaging=23.1=py310h06a4308_0
+  - pandas=1.5.3=py310h1128e8f_0
+  - pandoc=2.12=h06a4308_3
+  - parso=0.8.3=pyhd3eb1b0_0
+  - pcre=8.45=h295c915_0
+  - pendulum=2.1.2=pyhd3eb1b0_1
+  - pexpect=4.8.0=pyhd3eb1b0_3
+  - pickleshare=0.7.5=pyhd3eb1b0_1003
+  - pillow=10.0.1=py310ha6cbd5a_0
+  - pineappl=0.6.2=py310hcb5633a_0
+  - pip=23.3=py310h06a4308_0
+  - pkg-config=0.29.2=h1bed415_8
+  - ply=3.11=py310h06a4308_0
+  - prompt-toolkit=3.0.36=py310h06a4308_0
+  - prompt_toolkit=3.0.36=hd3eb1b0_0
+  - protobuf=3.20.3=py310h6a678d5_0
+  - psutil=5.9.0=py310h5eee18b_0
+  - ptyprocess=0.7.0=pyhd3eb1b0_2
+  - pure_eval=0.2.2=pyhd3eb1b0_0
+  - py4j=0.10.9.7=py310h06a4308_0
+  - pyasn1=0.4.8=pyhd3eb1b0_0
+  - pyasn1-modules=0.2.8=py_0
+  - pybtex=0.24.0=pyhd8ed1ab_2
+  - pybtex-docutils=1.0.3=py310hff52083_1
+  - pycparser=2.21=pyhd3eb1b0_0
+  - pygments=2.15.1=py310h06a4308_1
+  - pyjwt=2.4.0=py310h06a4308_0
+  - pyopenssl=23.2.0=py310h06a4308_0
+  - pyparsing=3.0.9=py310h06a4308_0
+  - pyqt=5.15.10=py310h6a678d5_0
+  - pyqt5-sip=12.13.0=py310h5eee18b_0
+  - pysocks=1.7.1=py310h06a4308_0
+  - python=3.10.13=h7a1cb2a_0
+  - python-dateutil=2.8.2=pyhd3eb1b0_0
+  - python-flatbuffers=2.0=pyhd3eb1b0_0
+  - python_abi=3.10=2_cp310
+  - pytz=2023.3.post1=py310h06a4308_0
+  - pytzdata=2020.1=pyhd3eb1b0_0
+  - pyyaml=6.0.1=py310h5eee18b_0
+  - qt-main=5.15.2=h5b8104b_9
+  - re2=2022.04.01=h295c915_0
+  - readline=8.2=h5eee18b_0
+  - recommonmark=0.6.0=pyhd3eb1b0_0
+  - reportengine=0.30.28=py_0
+  - requests=2.31.0=py310h06a4308_0
+  - requests-oauthlib=1.3.0=py_0
+  - rich=12.5.1=py310h06a4308_0
+  - rsa=4.7.2=pyhd3eb1b0_1
+  - ruamel_yaml=0.15.100=py310h7f8727e_0
+  - scipy=1.11.3=py310heeff2f4_0
+  - seaborn=0.12.2=py310h06a4308_0
+  - setuptools=68.0.0=py310h06a4308_0
+  - sip=6.7.12=py310h6a678d5_0
+  - six=1.16.0=pyhd3eb1b0_1
+  - snappy=1.1.9=h295c915_0
+  - snowballstemmer=2.2.0=pyhd3eb1b0_0
+  - sphinx=5.0.2=py310h06a4308_0
+  - sphinx_rtd_theme=1.1.1=py310h06a4308_0
+  - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0
+  - sphinxcontrib-bibtex=2.6.1=pyhd8ed1ab_1
+  - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0
+  - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0
+  - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0
+  - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0
+  - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0
+  - sqlalchemy=1.4.39=py310h5eee18b_0
+  - sqlite=3.41.2=h5eee18b_0
+  - stack_data=0.2.0=pyhd3eb1b0_0
+  - tensorboard=2.11.0=py310h06a4308_0
+  - tensorboard-data-server=0.6.1=py310h52d8a92_0
+  - tensorboard-plugin-wit=1.8.1=py310h06a4308_0
+  - tensorflow=2.11.0=eigen_py310h0f08fec_0
+  - tensorflow-base=2.11.0=eigen_py310h3323580_0
+  - tensorflow-estimator=2.11.0=py310h06a4308_0
+  - termcolor=2.1.0=py310h06a4308_0
+  - tk=8.6.12=h1ccaba5_0
+  - tomli=2.0.1=py310h06a4308_0
+  - tornado=6.3.3=py310h5eee18b_0
+  - tqdm=4.65.0=py310h2f386ee_0
+  - traitlets=5.7.1=py310h06a4308_0
+  - typing_extensions=4.7.1=py310h06a4308_0
+  - tzdata=2023c=h04d1e81_0
+  - urllib3=1.26.18=py310h06a4308_0
+  - validobj=1.1=pyhd8ed1ab_0
+  - wcwidth=0.2.5=pyhd3eb1b0_0
+  - werkzeug=2.2.3=py310h06a4308_0
+  - wheel=0.41.2=py310h06a4308_0
+  - wrapt=1.14.1=py310h5eee18b_0
+  - xz=5.4.2=h5eee18b_0
+  - yaml=0.2.5=h7b6447c_0
+  - yaml-cpp=0.8.0=h6a678d5_0
+  - yarl=1.8.1=py310h5eee18b_0
+  - zipp=3.11.0=py310h06a4308_0
+  - zlib=1.2.13=h5eee18b_0
+  - zstd=1.5.5=hc292b87_0
+  - pip:
+      - validphys==4.0
+prefix: /root/miniconda3/envs/nnpdfenv-tag-4.0.7