cmdata and make_mat updated (#543)

* updated cmdata to generate h5 histograms by default

* multiple updated for HDF5 format

* updated requirements

* flake8

* updated env

* github action

* gha

* flake

* gth

* again

* updated conda

* conda

* conda

* conda

* conda

* conda

* conda macos

* clean stuff

* miniforge

* again

* again

* escape s+

* removed escaped characters

* codecheck updated

* cmdata regtest

.github/workflows/test.yml

@@ -15,24 +15,19 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/cache@v4
-      with:
-        path: ~/.ccache
-        key: ccache-reset-${{ github.sha }}
-        restore-keys: ccache-reset-
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Add conda to system path
       run: |
         # $CONDA is an environment variable pointing to the root of the miniconda directory
         echo $CONDA/bin >> $GITHUB_PATH
     - name: Install dependencies with conda
       run: |
-        conda install --solver=classic -y python=3.11
-        conda env update --solver=classic --file conda/environment.yml --name base
-        conda install --solver=classic flake8
+        conda update conda
+        conda env update --file conda/environment.yml -n base
+        conda install flake8
     - name: Run flake8 
       run: |
         # Flake8 exit on most issues
@@ -47,11 +42,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.10
+    - name: Set up Python 3.13
       uses: actions/setup-python@v5
       with:
-        python-version: '3.10'
-        cache: 'pip'
+        python-version: '3.13'
     - name: Install requirements
       run: pip install -r requirements.txt
     - name: Run tests
@@ -64,25 +58,18 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/cache@v4
-      with:
-        path: ~/.ccache
-        key: ccache-reset-${{ github.sha }}
-        restore-keys: ccache-reset-
-    - name: Set up Python 3.11
+    - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Add conda to system path
       uses: conda-incubator/setup-miniconda@v3
       with:
-        activate-environment:
-    - name: Install dependencies with conda
-      run: |
-        echo $CONDA/bin >> $GITHUB_PATH
-        conda install -y python=3.11
-        conda env update --file conda/environment.yml --name base
+        environment-file: conda/environment.yml
+        miniforge-version: latest
+        activate-environment: test
     - name: Run tests
+      shell: bash -el {0}
       run: |
         $CONDA/bin/python test/run_tests.py
         cd test; bash run_make_mat.sh; cd ..
@@ -91,11 +78,10 @@ jobs:
     runs-on: macos-latest
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.9
+    - name: Set up Python 3.13
       uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
-        cache: 'pip'
+        python-version: '3.13'
     - name: Install requirements
       run: pip install -r requirements.txt
     - name: Run tests

codecheck.sh

@@ -1,7 +1,7 @@
 # check for black
 pip show black
 if [ $? -eq 0 ]; then
-  python -m black -t py39 -t py310 -t py311 -t py312 --line-length=127 .
+  python -m black -t py310 -t py311 -t py312 -t py313 --line-length=127 .
 else
   echo "black module not found!"
   echo "you can install it as pip install git+https://github.com/psf/black"

conda/environment.yml

@@ -1,13 +1,14 @@
 name: meGO
 channels:
   - conda-forge
-  - defaults
 dependencies:
-  - python=3.11
+  - python=3.12
   - numpy
   - pandas
-  - parmed
-  - gitpython
-  - pyyaml
+  - ParmEd
+  - GitPython
+  - PyYAML
   - scipy
+  - hdf5
   - h5py
+  - pytables

requirements.txt

@@ -1,7 +1,8 @@
-ParmEd>=3.4.3
-numpy>=1.26.2
-pandas>=2.1.3
-pyyaml>=6.0.1
-gitpython>=3.1.43
+GitPython>=3.1.44
 h5py>=3.11.0
-scipy>=1.13.0
+numpy>=2.2.2
+pandas>=2.2.3
+ParmEd>=4.2.2
+PyYAML>=6.0.2
+scipy>=1.15.1
+tables>=3.9.2

src/multiego/io.py

@@ -399,7 +399,7 @@ def read_molecular_contacts(path, ensemble_molecules_idx_sbtype_dictionary, simu
 
     contact_matrix = pd.DataFrame()
     if not h5:
-        contact_matrix = pd.read_csv(path, header=None, sep="\s+", names=col_names, dtype=col_types)
+        contact_matrix = pd.read_csv(path, header=None, sep=r"\s+", names=col_names, dtype=col_types)
         contact_matrix["learned"] = contact_matrix["learned"].fillna(1).astype(bool)
     else:
         contact_matrix = pd.read_hdf(path, key="data", dtype=col_types)
@@ -657,8 +657,8 @@ def print_stats(meGO_LJ):
 \t- Sigma range is: [{intrad_a_s_min_contacts:5.3f}:{intrad_a_s_max_contacts:5.3f}] [{interm_a_s_min_contacts:5.3f}:{interm_a_s_max_contacts:5.3f}] nm
 
 \t- RELEVANT MDP PARAMETERS:
-\t- Suggested rlist value: {1.1*2.5*max(meGO_LJ['sigma'].max(),0.5):4.2f} nm
-\t- Suggested cut-off value: {2.5*max(meGO_LJ['sigma'].max(),0.5):4.2f} nm
+\t- Suggested rlist value: {1.1*2.5*max(meGO_LJ['sigma'].max(), 0.5):4.2f} nm
+\t- Suggested cut-off value: {2.5*max(meGO_LJ['sigma'].max(), 0.5):4.2f} nm
     """
     )
 
@@ -873,7 +873,7 @@ def get_name(parameters):
     if parameters.egos == "mg":
         name = f"{parameters.system}_{parameters.egos}"
     else:
-        name = f"{parameters.system}_{parameters.egos}_epsis_intra{ '-'.join(np.array(parameters.multi_epsilon, dtype=str)) }_{parameters.inter_epsilon}"
+        name = f"{parameters.system}_{parameters.egos}_epsis_intra{'-'.join(np.array(parameters.multi_epsilon, dtype=str))}_{parameters.inter_epsilon}"
     return name
 
 

test/run_cmdata.sh

@@ -2,7 +2,7 @@ set -e
 set -o pipefail
 mkdir test_inputs/cmdata/histo
 cd test_inputs/cmdata/histo
-cmdata -f ../traj.xtc -s ../protein.tpr --mode=intra
+cmdata -f ../traj.xtc -s ../protein.tpr --mode=intra --noh5
 cd ../../../
 tar -zxf test_outputs/cmdata/hh.tgz -C test_outputs/cmdata/
 for i in `ls -1 test_inputs/cmdata/histo`; do

test/run_make_mat.sh

@@ -4,19 +4,12 @@ set -e
 set -o pipefail
 
 tar -zxf test_inputs/make_mat_ttr/hh.tgz -C test_inputs/make_mat_ttr/
-python ../tools/make_mat/make_mat.py --histo test_inputs/make_mat_ttr/hh.tgz --target_top test_inputs/make_mat_ttr/topol_md.top --mego_top test_inputs/make_mat_ttr/topol_ref.top --cutoff 0.75 --mode intra+same --out  test_inputs/make_mat_ttr/ --tar
-#diff <(gzip -dc test_inputs/make_mat_ttr/intramat_1_1.ndx.gz) <(gzip -dc test_outputs/make_mat_ttr/intramat_1_1.ndx.gz)
-#diff <(gzip -dc test_inputs/make_mat_ttr/intermat_1_1.ndx.gz) <(gzip -dc test_outputs/make_mat_ttr/intermat_1_1.ndx.gz)
+python ../tools/make_mat/make_mat.py --histo test_inputs/make_mat_ttr/histo --target_top test_inputs/make_mat_ttr/topol_mego.top --mego_top test_inputs/make_mat_ttr/topol.top --cutoff 0.75 --mode intra+same --out  test_inputs/make_mat_ttr/
+python ../tools/make_mat/HDF52ndx.py --input test_inputs/make_mat_ttr/intramat_1_1.ndx.h5 
+python ../tools/make_mat/HDF52ndx.py --input test_inputs/make_mat_ttr/intermat_1_1.ndx.h5 
+diff test_inputs/make_mat_ttr/intramat_1_1.ndx test_outputs/make_mat_ttr/intramat_1_1.ndx
+diff test_inputs/make_mat_ttr/intermat_1_1.ndx test_outputs/make_mat_ttr/intermat_1_1.ndx
 
 tar -zxf test_inputs/make_mat_popc/hh.tgz -C test_inputs/make_mat_popc/
-python ../tools/make_mat/make_mat.py --histo test_inputs/make_mat_popc/histo --target_top test_inputs/make_mat_popc/topol_md.top --mego_top test_inputs/make_mat_popc/topol_ref.top --cutoff 0.75 --mode intra --out  test_inputs/make_mat_popc/ 
-#diff <(gzip -dc test_inputs/make_mat_popc/intramat_1_1.ndx.gz) <(gzip -dc test_outputs/make_mat_popc/intramat_1_1.ndx.gz)
-
-tar -zxf test_inputs/make_mat_het_trim/hh.tgz -C test_inputs/make_mat_het_trim/
-python ../tools/make_mat/make_mat.py --histo test_inputs/make_mat_het_trim/histo --target_top test_inputs/make_mat_het_trim/topol_mego.top --mego_top test_inputs/make_mat_het_trim/topol.top --cutoff 0.75 --out  test_inputs/make_mat_het_trim/ 
-#diff <(gzip -dc test_inputs/make_mat_het_trim/intramat_1_1.ndx.gz) <(gzip -dc test_outputs/make_mat_het_trim/intramat_1_1.ndx.gz)
-#diff <(gzip -dc test_inputs/make_mat_het_trim/intramat_2_2.ndx.gz) <(gzip -dc test_outputs/make_mat_het_trim/intramat_2_2.ndx.gz)
-#diff <(gzip -dc test_inputs/make_mat_het_trim/intramat_3_3.ndx.gz) <(gzip -dc test_outputs/make_mat_het_trim/intramat_3_3.ndx.gz)
-#diff <(gzip -dc test_inputs/make_mat_het_trim/intermat_1_2.ndx.gz) <(gzip -dc test_outputs/make_mat_het_trim/intermat_1_2.ndx.gz)
-#diff <(gzip -dc test_inputs/make_mat_het_trim/intermat_1_3.ndx.gz) <(gzip -dc test_outputs/make_mat_het_trim/intermat_1_3.ndx.gz)
-#diff <(gzip -dc test_inputs/make_mat_het_trim/intermat_2_3.ndx.gz) <(gzip -dc test_outputs/make_mat_het_trim/intermat_2_3.ndx.gz)
+python ../tools/make_mat/make_mat.py --histo test_inputs/make_mat_popc/histo --target_top test_inputs/make_mat_popc/topol_md.top --mego_top test_inputs/make_mat_popc/topol_ref.top --cutoff 0.75 --mode intra --out  test_inputs/make_mat_popc/ --noh5 
+#diff <(gzip -dc test_inputs/make_mat_popc/intramat_1_1.ndx.h5) <(gzip -dc test_outputs/make_mat_popc/intramat_1_1.ndx.h5)