Merge pull request #1914 from NNPDF/develop_merge_20240119, includes: #…

…1871, #1864, #1861 and #1888

WIP Merging

.github/workflows/python_installation.yml

@@ -0,0 +1,44 @@
+name: Test the python installation
+
+on: [push]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.10", "3.11"]
+        include:
+          - os: ubuntu-latest
+            CONDA_OS: linux-64
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    env:
+      NETRC_FILE: ${{ secrets.NETRC_FILE }}
+      NNPDF_SSH_KEY: ${{ secrets.NNPDF_SSH_KEY }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Setup conda and install LHAPDF and pandoc
+      shell: bash -l {0}
+      run: |
+        echo "$NETRC_FILE" | base64 --decode > ~/.netrc
+        conda install -n base conda-libmamba-solver
+        conda config --set solver libmamba
+        conda config --append channels conda-forge
+        conda config --prepend channels https://packages.nnpdf.science/public
+        conda config --set show_channel_urls true
+        conda install lhapdf pandoc
+        conda activate test
+    - name: Install nnpdf with testing and qed extras
+      shell: bash -l {0}
+      run: |
+        conda activate test
+        pip install .[qed,tests]
+    - name: Test n3fit and validphys
+      shell: bash -l {0}
+      run: |
+        conda activate test
+        pytest --pyargs --mpl validphys n3fit

.github/workflows/tests.yml

@@ -4,7 +4,7 @@ name: Tests
 on: [push]
 
 concurrency:
-  group: ${{ github.ref }}
+  group: ${{ github.head_ref || github.ref }}
   cancel-in-progress: true
 
 jobs:

.gitignore

@@ -1,8 +1,13 @@
+# extra files created to hold the versioning information
+validphys2/src/validphys/_version.py
+
 # extra files created by cmake
 libnnpdf/nnprofile.yaml
+libnnpdf/REAMDE.md
 libnnpdf/scripts/nnpdf.pc
 libnnpdf/src/NNPDF/common.h
 libnnpdf/src/NNPDF/config.h
+evolven3fit/evolven3fit.cc
 
 # Created by https://www.gitignore.io/api/macos
 

CMakeLists.txt

@@ -28,21 +28,15 @@ set(VERSION "\"4.0\"")
 set(nnpdfcpp_VERSION 4.0)
 
 # check for dependencies
+find_package(PythonInterp 3 REQUIRED)
 find_package(PkgConfig REQUIRED)
 pkg_search_module(LIBARCHIVE REQUIRED libarchive)
 pkg_search_module(SQLITE3 REQUIRED sqlite3)
 pkg_search_module(GSL REQUIRED gsl)
 pkg_search_module(YAML REQUIRED yaml-cpp)
 
 
-option(ENABLE_OPENMP "Enable openmp parallelism." OFF)
-option(ENABLE_OPENMPI "Enable openMPI parallelism." OFF)
-option(ENABLE_TESTS "Enable unit test" OFF)
-option(ENABLE_ASAN "Enable ASAN" OFF)
-option(ENABLE_DEAD_STRIP "Enable use of flag `-dead_strip-dylibs`" OFF)
-option(VP_DEV "validphys in developer mode" ON)
-option(N3_DEV "n3fit in developer mode" ON)
-option(BURN_TAG "burn down the git tag in vp and n3fit (only local non-dev installation)" ON)
+option(NNPDF_DEV "n3fit and validphys in developer mode" ON)
 set(PROFILE_PREFIX "" CACHE STRING "Where you store the 'data' folder. Default empty uses CMAKE_INSTALL_PREFIX/share/NNPDF.")
 
 if (PROFILE_PREFIX)
@@ -51,19 +45,6 @@ else (PROFILE_PREFIX)
   set(PROFILE_PREFIX "${CMAKE_INSTALL_PREFIX}/share/NNPDF")
 endif()
 
-if (ENABLE_OPENMPI)
-  set(LIBNNPDF_HAVE_MPI "#define OPENMPI")
-endif()
-
-set(LIBNNPDF_HAVE_SSE "#define SSE_CONV")
-
-
-if(ENABLE_OPENMP)
-  find_package(OpenMP REQUIRED)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-endif(ENABLE_OPENMP)
-
 # LHAPDF
 find_program(LHAPDF_CONFIG lhapdf-config REQUIRED)
 if (LHAPDF_CONFIG)
@@ -101,101 +82,28 @@ if (APFEL_CONFIG)
   set(APFEL_LIBRARIES ${APFEL_LIBRARIES} CACHE STRING INTERNAL)
 endif(APFEL_CONFIG)
 
-set(DEFAULT_CXX_OPTIONS "-Wall -Wextra -march=nocona -mtune=haswell -fvisibility-inlines-hidden -fmessage-length=0 -ftree-vectorize -fPIC -fstack-protector-strong -O2 -pipe")
+set(DEFAULT_CXX_OPTIONS "-Wall -Wextra -fvisibility-inlines-hidden -fmessage-length=0 -ftree-vectorize -fPIC -fstack-protector-strong -O2 -pipe")
 
 #strip linker flags to avoid duplication of asan flags
 string(REPLACE "-fsanitize=address" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
 string(REPLACE "-fsanitize=address" "" CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG}")
 
 set(CMAKE_ALL_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_MODULE_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS}")
 
-#dead_strip_dylibs can cause issues, warn if we remove it
-if(ENABLE_DEAD_STRIP)
-  if(NOT ${CMAKE_ALL_LINKER_FLAGS} MATCHES "-Wl,-dead_strip_dylibs")
-    message(WARNING "ENABLE_DEAD_STRIP was set true but there were no occurences of the flag `dead_strip_dylibs` found")
-  endif(NOT ${CMAKE_ALL_LINKER_FLAGS} MATCHES "-Wl,-dead_strip_dylibs")
-else(ENABLE_DEAD_STRIP)
-  if(CMAKE_ALL_LINKER_FLAGS MATCHES "-Wl,-dead_strip_dylibs")
-    message(WARNING "removing all occurence of flag: `dead_strip_dylibs` (default behaviour). If you are sure this flag is required then ENABLE_DEAD_STRIP")
-    string(REPLACE "-Wl,-dead_strip_dylibs" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
-    string(REPLACE "-Wl,-dead_strip_dylibs" "" CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS}")
-    string(REPLACE "-Wl,-dead_strip_dylibs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}")
-    #executable linker flags get set later
-    set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS}" CACHE STRING "Flags used by the linker during the creation of modules during all build types" FORCE)
-    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}" CACHE STRING "Flags used by the linker during the creation of shared libraries during all build types" FORCE)
-  endif(CMAKE_ALL_LINKER_FLAGS MATCHES "-Wl,-dead_strip_dylibs")
-endif(ENABLE_DEAD_STRIP)
-
-# Flags for ASAN
-if(ENABLE_ASAN)
-  set(CMAKE_CXX_FLAGS "-fsanitize=address ${DEFAULT_CXX_OPTIONS} ${LHAPDF_CXX_FLAGS} ${APFEL_CXX_FLAGS} ${YAML_CFLAGS} ${SQLITE3_CFLAGS} ${GSL_CFLAGS} ${LIBARCHIVE_CFLAGS}" CACHE STRING "compile flags" FORCE)
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g" CACHE STRING "debug compile flags" FORCE)
-  set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=address ${CMAKE_EXE_LINKER_FLAGS}" CACHE STRING "linker flags" FORCE)
-  set(CMAKE_EXE_LINKER_FLAGS_DEBUG "-fsanitize=address ${CMAKE_EXE_LINKER_FLAGS_DEBUG}" CACHE STRING "debug linker flags" FORCE)
-else(ENABLE_ASAN)
-  set(CMAKE_CXX_FLAGS "${DEFAULT_CXX_OPTIONS} ${LHAPDF_CXX_FLAGS} ${APFEL_CXX_FLAGS} ${YAML_CFLAGS} ${SQLITE3_CFLAGS} ${GSL_CFLAGS} ${LIBARCHIVE_CFLAGS}" CACHE STRING "compile flags" FORCE)
-  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g" CACHE STRING "debug compile flags" FORCE)
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}" CACHE STRING "linker flags" FORCE)
-  set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG}" CACHE STRING "debug linker flags" FORCE)
-endif(ENABLE_ASAN)
+set(CMAKE_CXX_FLAGS "${DEFAULT_CXX_OPTIONS} ${LHAPDF_CXX_FLAGS} ${APFEL_CXX_FLAGS} ${YAML_CFLAGS} ${SQLITE3_CFLAGS} ${GSL_CFLAGS} ${LIBARCHIVE_CFLAGS}" CACHE STRING "compile flags" FORCE)
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS} -g" CACHE STRING "debug compile flags" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}" CACHE STRING "linker flags" FORCE)
+set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG}" CACHE STRING "debug linker flags" FORCE)
 
 # libnnpdf configuration
 add_subdirectory(libnnpdf)
 
-# nnpdfcpp configuration
-add_subdirectory(nnpdfcpp)
-
 # evolven3fit
-add_subdirectory(n3fit/evolven3fit)
-
-if((BURN_TAG) AND (NOT (VP_DEV) OR NOT (N3_DEV)))
-    # Find out where is the root of the git repository (if available) and finds the correct tag to burn down
-    # in the package. This just sets the variable, the actual burning is done down below by the validphys
-    # or n3fit installation and is found in the version.cmake file as it needs to be done post-installation
-    set(GIT_DIR "--git-dir=${PROJECT_SOURCE_DIR}/.git")
-    # get the current tag (ex: 3.4) and check at the same time whether git is available and the .git folder found
-    execute_process(COMMAND git ${GIT_DIR} describe --abbrev=0 --tags OUTPUT_VARIABLE GIT_TAG ERROR_VARIABLE GIT_ERROR)
-    if(GIT_ERROR STREQUAL "")
-        string(STRIP ${GIT_TAG} GIT_TAG)
-        # get the number of revisions since tag happened
-        execute_process(COMMAND git ${GIT_DIR} rev-list ${GIT_TAG}..HEAD --count OUTPUT_VARIABLE GIT_REVN)
-        string(STRIP ${GIT_REVN} GIT_REVN)
-        # get the shortname for the hash
-        execute_process(COMMAND git ${GIT_DIR} rev-parse --short HEAD OUTPUT_VARIABLE GIT_HASH)
-        string(STRIP ${GIT_HASH} GIT_HASH)
-        # find out whether the repository is in a dirty state
-        # (this returns ${GIT_TAG}-dev so we have to remove the tag)
-        execute_process(COMMAND git ${GIT_DIR} describe --abbrev=0 --tags --dirty=-dev OUTPUT_VARIABLE GIT_DIRTY)
-        string(STRIP ${GIT_DIRTY} GIT_DIRTY)
-        string(REPLACE "${GIT_TAG}" "" GIT_DIRTY ${GIT_DIRTY})
-        # now concatenate everything
-        # with the format ex: 3.4.1880+g00b1741c-dev
-        set(GIT_VERSION "build_version='${GIT_TAG}.${GIT_REVN}+g${GIT_HASH}${GIT_DIRTY}'")
-        # now set the variable at install time
-        install(CODE "set(GIT_VERSION \"${GIT_VERSION}\")")
-    else(GIT_ERROR STREQUAL "")
-        set(BURN_TAG OFF)
-    endif(GIT_ERROR STREQUAL "")
-endif()
-
-# install validphys2
-if(VP_DEV)
-    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install -e ${PROJECT_SOURCE_DIR}/validphys2)")
-else(VP_DEV)
-    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps --ignore-installed ${PROJECT_SOURCE_DIR}/validphys2)")
-    if(BURN_TAG)
-        install(CODE "set(LIBRARY \"validphys\")")
-        install(SCRIPT ${PROJECT_SOURCE_DIR}/version.cmake)
-    endif(BURN_TAG)
-endif(VP_DEV)
-
-# install n3fit
-if(N3_DEV)
-    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install -e ${PROJECT_SOURCE_DIR}/n3fit)")
-else(N3_DEV)
-    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps --ignore-installed ${PROJECT_SOURCE_DIR}/n3fit)")
-    if(BURN_TAG)
-        install(CODE "set(LIBRARY \"n3fit\")")
-        install(SCRIPT ${PROJECT_SOURCE_DIR}/version.cmake)
-    endif(BURN_TAG)
-endif(N3_DEV)
+add_subdirectory(evolven3fit)
+install(FILES ${PROJECT_SOURCE_DIR}/validphys2/src/validphys/datafiles/theory.db DESTINATION ${PROFILE_PREFIX}/)
+
+if(NNPDF_DEV)
+    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps -e ${PROJECT_SOURCE_DIR})")
+else(NNPDF_DEV)
+    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps --ignore-installed ${PROJECT_SOURCE_DIR})")
+endif(NNPDF_DEV)

conda-recipe/build.sh

@@ -2,9 +2,6 @@
 
 mkdir build
 cd build
-echo "build_version=\"${PKG_VERSION}\"" > ../n3fit/src/n3fit/version.py
-echo "build_version=\"${PKG_VERSION}\"" > ../validphys2/src/validphys/version.py
-
-cmake .. -DCMAKE_INSTALL_PREFIX=${PREFIX} -DVP_DEV=OFF -DN3_DEV=OFF -DBURN_TAG=OFF
+cmake .. -DCMAKE_INSTALL_PREFIX=${PREFIX} -DNNPDF_DEV=OFF
 make -j${CPU_COUNT}
 make install

conda-recipe/meta.yaml

@@ -23,6 +23,8 @@ requirements:
         - apfel >=3   # see https://github.com/scarrazza/apfel
         - python
         - numpy
+        - poetry-core >=1.0.0
+        - poetry-dynamic-versioning >=1.1.0
     run:
         - tensorflow >=2.10
         - tensorflow >=2.10 *eigen* # [py < 311]
@@ -50,23 +52,17 @@ requirements:
         - recommonmark
         - sphinx_rtd_theme >0.5
         - sphinxcontrib-bibtex
-        - curio >=1.0
         - pineappl >=0.6.2
         - eko >=0.14.1
         - fiatlux
+        - curio >=1.0 # reportengine uses it but it's not in its dependencies 
 
 test:
     requires:
         - hypothesis
         - pytest
         - coverage
         - pytest-mpl
-        #Build dependencies for catch tests
-        - {{ compiler("cxx") }}
-        - {{ compiler("c") }}
-        - sysroot_linux-64==2.17 # [linux]
-        - swig ==3.0.10
-        - cmake
 
     source_files:
         - "*"

doc/sphinx/source/data/data-config.rst

@@ -4,22 +4,22 @@
 Organisation of data files
 ==========================
 
-The ``nnpdf++`` code needs to be able to handle a great deal of different
+The ``nnpdf`` code needs to be able to handle a great deal of different
 options with regard to the treatment of both experimental data and theoretical
 choices. In the code, every effort has been made to keep experimental and
 theoretical parameters strictly separate.
-In this section we shall specify the layout of the various ``nnpdf++`` data
+In this section we shall specify the layout of the ``nnpdf`` data
 directory. It is in this directory that all of the read-only data to be used in
-the fit are accessed. The data directory is located in the ``nnpdfcpp`` git
-repository, under the path ``/nnpdfcpp/data/``.
+the fit are accessed. The data directory is located in the ``nnpdf`` git
+repository, under the path ``validphys/src/validphys2/datafiles``.
 
 Experimental data storage
 =========================
 
-The central repository for ``CommonData`` in use by ``nnpdf++`` projects is
-located in the ``nnpdfcpp`` git repository at
+The central repository for ``CommonData`` in use by ``nnpdf`` projects is
+located in the ``nnpdf`` git repository at
 
-	``/nnpdfcpp/data/commondata/``
+	``validphys/src/validphys2/datafiles/commondata``
 
 where a separate ``CommonData`` file is stored for each *Dataset* with the
 filename format
@@ -29,7 +29,7 @@ filename format
 Information on the treatment of systematic uncertainties, provided in
 ``SYSTYPE`` files, is located in the subdirectory
 
-	``/nnpdfcpp/data/commondata/systypes``
+	``commondata/systypes``
 
 Here several ``SYSTYPE`` files may be supplied for each *Dataset*. The
 various options are enumerated by suffix to the filename. The filename format
@@ -47,16 +47,16 @@ Theory lookup table
 
 In order to organise the various different theoretical treatments available, a
 lookup table is provided in ``sqlite3`` format. This lookup table can be found
-in the ``nnpdfcpp`` repository data directory at:
+in the ``nnpdf`` repository data directory at:
 
-	``/nnpdfcpp/data/theory.db``
+	``validphys/src/validphys2/datafiles/theory.db``
 
 This file should only be edited in order to add new theory options. It may be
 edited with any appropriate ``sqlite3``-supported software. A script is provided to
 give a brief overview of the various theory options available. It can be found
 at
 
-	``/nnpdfcpp/data/disp_theory.py``
+	``validphys/src/validphys2/datafiles/disp_theory.py``
 
 and should be run without any arguments.