Merge branch 'dcamarmas-master'

xpn-arcos · Mar 3, 2023 · bb6eda5 · bb6eda5
2 parents 0459f95 + 8b07db8
commit bb6eda5
Show file tree

Hide file tree

Showing 315 changed files with 14,523 additions and 34,780 deletions.
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,6 @@ autom4te.cache
 /install-sh
 /missing
 /stamp-h1
+/Makefile
+Makefile
+
diff --git a/Makefile b/Makefile
@@ -220,29 +220,29 @@ distuninstallcheck_listfiles = find . -type f -print
 am__distuninstallcheck_listfiles = $(distuninstallcheck_listfiles) \
   | sed 's|^\./|$(prefix)/|' | grep -v '$(infodir)/dir$$'
 distcleancheck_listfiles = find . -type f -print
-ACLOCAL = ${SHELL} /home/acaldero/work/xpn/config/missing aclocal-1.16
+ACLOCAL = ${SHELL} /local_test/src/xpn/config/missing aclocal-1.16
 AMTAR = $${TAR-tar}
 AM_DEFAULT_VERBOSITY = 1
 AR = ar
-AUTOCONF = ${SHELL} /home/acaldero/work/xpn/config/missing autoconf
-AUTOHEADER = ${SHELL} /home/acaldero/work/xpn/config/missing autoheader
-AUTOMAKE = ${SHELL} /home/acaldero/work/xpn/config/missing automake-1.16
-AWK = gawk
-CC = /usr/bin/mpicc
+AUTOCONF = ${SHELL} /local_test/src/xpn/config/missing autoconf
+AUTOHEADER = ${SHELL} /local_test/src/xpn/config/missing autoheader
+AUTOMAKE = ${SHELL} /local_test/src/xpn/config/missing automake-1.16
+AWK = mawk
+CC = /opt/software/install-mpich/bin/mpicc
 CCDEPMODE = depmode=gcc3
-CDEFS = -D_REENTRANT -DPOSIX_THREADS -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML -DENABLE_TCP_SERVER -DENABLE_MPI_SERVER
-CFLAGS =     -I../../mxml    -I../../xpn/include    -I../../xpn/include/xpn_client    -I../../xpn/include/base -I../../../mxml -I../../../xpn/include -I../../../xpn/include/xpn_client -I../../../xpn/include/base  -fPIC -O2 -g -g3 -ggdb -Wall -Wextra -std=c11
+CDEFS = -D_REENTRANT -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML -DENABLE_TCP_SERVER -DENABLE_MPI_SERVER
+CFLAGS = -I../../../mxml -I../../../xpn/include -I../../../xpn/include/xpn_client -I../../../xpn/include/base  -fPIC -O2 -g -g3 -Wall -Wextra -std=c11
 CPP = gcc -E
-CPPFLAGS = -D_REENTRANT -DPOSIX_THREADS -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML -DENABLE_TCP_SERVER -DENABLE_MPI_SERVER -I$(top_srcdir)/include -I$(top_srcdir)/include/base -I$(top_srcdir)/include/xpn_client/ -I$(top_srcdir)/include/xpn_client/xpn -I$(top_srcdir)/include/xpn_client/xpn/xpn_simple -I$(top_srcdir)/include/xpn_client/nfi -I$(top_srcdir)/include/xpn_client/nfi/nfi_local -I$(top_srcdir)/include/xpn_client/xpni/ -I$(top_srcdir)/include/xpn_client/xpni/common -I$(top_srcdir)/include/xpn_client/xpni/xpni_null -I$(top_srcdir)/include/xpn_client/xpni/xpni_ft -I$(top_srcdir)/include/xpn_client/xpni/xpni_log -I$(top_srcdir)/include/xpn_client/nfi/nfi_tcp_server -I$(top_srcdir)/include/xpn_client/nfi/nfi_mpi_server  
+CPPFLAGS = -D_REENTRANT -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML -DENABLE_TCP_SERVER -DENABLE_MPI_SERVER -I$(top_srcdir)/include -I$(top_srcdir)/include/base -I$(top_srcdir)/include/xpn_client/ -I$(top_srcdir)/include/xpn_client/xpn -I$(top_srcdir)/include/xpn_client/xpn/xpn_simple -I$(top_srcdir)/include/xpn_client/nfi -I$(top_srcdir)/include/xpn_client/nfi/nfi_local -I$(top_srcdir)/include/xpn_client/xpni/ -I$(top_srcdir)/include/xpn_client/xpni/common -I$(top_srcdir)/include/xpn_client/xpni/xpni_null -I$(top_srcdir)/include/xpn_client/xpni/xpni_ft -I$(top_srcdir)/include/xpn_client/xpni/xpni_log -I$(top_srcdir)/include/xpn_client/nfi/nfi_tcp_server -I$(top_srcdir)/include/xpn_client/nfi/nfi_mpi_server  
 CYGPATH_W = echo
 DEFS = -DHAVE_CONFIG_H
 DEPDIR = .deps
 ECHO_C = 
 ECHO_N = -n
 ECHO_T = 
-EGREP = /usr/bin/grep -E
+EGREP = /bin/grep -E
 EXEEXT = 
-GREP = /usr/bin/grep
+GREP = /bin/grep
 INSTALL = /usr/bin/install -c
 INSTALL_DATA = ${INSTALL} -m 644
 INSTALL_PROGRAM = ${INSTALL}
@@ -254,8 +254,8 @@ LIBOBJS =
 LIBS =     -L../../mxml    -L../../xpn/lib -L../../../mxml -L../../../xpn/lib  -lmxml -ldl -lpthread
 LTLIBOBJS = 
 MAKE = make
-MAKEINFO = ${SHELL} /home/acaldero/work/xpn/config/missing makeinfo
-MKDIR_P = /usr/bin/mkdir -p
+MAKEINFO = ${SHELL} /local_test/src/xpn/config/missing makeinfo
+MKDIR_P = /bin/mkdir -p
 M_HEADERS = 
 NFI_MPI_SERVER = nfi_mpi_server
 NFI_MPI_SERVER_OBJECTS = $(NFI_MPI_SERVER_OBJECTS)
@@ -280,10 +280,10 @@ SET_MAKE =
 SHELL = /bin/bash
 STRIP = 
 VERSION = 2.0
-abs_builddir = /home/acaldero/work/xpn
-abs_srcdir = /home/acaldero/work/xpn
-abs_top_builddir = /home/acaldero/work/xpn
-abs_top_srcdir = /home/acaldero/work/xpn
+abs_builddir = /local_test/src/xpn
+abs_srcdir = /local_test/src/xpn
+abs_top_builddir = /local_test/src/xpn
+abs_top_srcdir = /local_test/src/xpn
 ac_ct_AR = ar
 ac_ct_CC = gcc
 am__include = include
@@ -311,7 +311,7 @@ host_vendor = pc
 htmldir = ${docdir}
 includedir = ${prefix}/include
 infodir = ${datarootdir}/info
-install_sh = ${SHELL} /home/acaldero/work/xpn/config/install-sh
+install_sh = ${SHELL} /local_test/src/xpn/config/install-sh
 libdir = ${exec_prefix}/lib
 libexecdir = ${exec_prefix}/libexec
 localedir = ${datarootdir}/locale
@@ -320,7 +320,7 @@ mandir = ${datarootdir}/man
 mkdir_p = $(MKDIR_P)
 oldincludedir = /usr/include
 pdfdir = ${docdir}
-prefix = /home/acaldero/bin//xpn
+prefix = /local_test/bin//xpn
 program_transform_name = s,x,x,
 psdir = ${docdir}
 runstatedir = ${localstatedir}/run

diff --git a/README.md b/README.md
@@ -1,9 +1,9 @@
-# XPN 2.0
+# XPN 2.1
 
 *Expand Ad-Hoc Parallel File System*
 
 [![License: GPL3](https://img.shields.io/badge/License-GPL3-blue.svg)](https://opensource.org/licenses/GPL-3.0)
-![version](https://img.shields.io/badge/version-2.0-blue)
+![version](https://img.shields.io/badge/version-2.1-blue)
 [![Codacy Badge](https://app.codacy.com/project/badge/Grade/ca0c40db97f64698a2db9992cafdd4ab)](https://www.codacy.com/gh/xpn-arcos/xpn/dashboard?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=xpn-arcos/xpn&amp;utm_campaign=Badge_Grade)
 
 * *Homepage*: <https://xpn-arcos.github.io/xpn-arcos.github.io/>
@@ -41,7 +41,7 @@ You must do both 'git clone' requests in the same directory (e.g.: $HOME/src).
 To build Expand you need to execute:
 ```
 cd $HOME/src
-./xpn/build-me.sh -m <MPICC_PATH> -i <INSTALL_PATH>
+./xpn/build-me -m <MPICC_PATH> -i <INSTALL_PATH>
 ```
 Where:
 * MPICC_PATH is the full path to your mpicc compiler.
@@ -55,58 +55,55 @@ First, you need to get familiar with 4 special files:
 * ```<nameserver file>``` for XPN, it will be a text file (created at runtime) with the list of host names where XPN servers are executing.
 * ```<server file>``` for XPN is a text file with the list of the servers to be stopped (one host name per line).
 
-Then, you need to get familiar with 2 special environment variables for XPN clients:
-* ```XPN_DNS```  with the full path to the nameserver file to be used.
-* ```XPN_CONF``` with the full path to the XPN configuration file to be used.
+Then, you need to get familiar with 5 special environment variables for XPN clients:
+* ```XPN_DNS```      with the full path to the nameserver file to be used (mandatory).
+* ```XPN_CONF```     with the full path to the XPN configuration file to be used (mandatory).
+* ```XPN_THREAD```   with value 0 for without threads, value 1 for thread-on-demand and value 2 for pool-of-threads (optional, default: 0).
+* ```XPN_SESSION```  with value 0 for without session and value 1 for with session (optional, default: 0).
+* ```XPN_LOCALITY``` with value 0 for without locality and value 1 for with locality (optional, default: 0).
+
 
 ### 4.1 Ad-Hoc Expand (based on MPI)
-The typical executions has 4 main steps:
-- First, generate the XPN configuration file:
-
-    ```
-    cd $HOME/src/xpn/bin
-    ./mk_conf.sh --conf ~/tmp/config.xml \
-                 --machinefile ~/tmp/machinefile \
-                 --part_size 512k \
-                 --part_name xpn \
-                 --storage_path /tmp
-    ```
-    Where:
-    * ```--part_size``` is the partition size (by default in bytes but "k" can be used for kilobytes and "m" for megabytes).
-    * ```--part_name``` is the partition name (a string without whitespaces).
-    * ```--storage_path``` is where the data is stogare in the servers (is the same for all servers).
-
-- Then, launch the Expand MPI server (xpn_mpi_server):
-
-    ```
-    mpiexec -np <number of processes> \
-            -hostfile <full path to the hostfile> \
-            -genv LD_LIBRARY_PATH <INSTALL_PATH>/mxml/lib:$LD_LIBRARY_PATH \
-            <INSTALL_PATH>/bin/xpn_mpi_server -ns <nameserver file> -tp &
-    ```
-
-    To use a thread pool to serve the requests add the -tp flag.
+The typical executions has 3 main steps:
+- First, launch the Expand MPI server (xpn_mpi_server):
+
+  ```
+  ./xpn -v -n <number of processes> -l <full path to the hostfile>  start
+  ```
 
 - Then,  launch the program that will use Expand (XPN client):
 
-    ```
-    mpiexec -np <number of processes> \
-            -hostfile <full path to the hostfile> \
-            -genv XPN_DNS  <nameserver file> \
-            -genv XPN_CONF <XPN configuration file> \
-            -genv LD_LIBRARY_PATH <INSTALL_PATH>/mxml/lib:$LD_LIBRARY_PATH \
-            -genv LD_PRELOAD      <INSTALL_PATH>/xpn/lib/xpn_bypass.so:$LD_PRELOAD \
-            <program path>
-    ```
+  ```
+  mpiexec -np <number of processes> \
+          -hostfile <full path to the hostfile> \
+          -genv XPN_DNS  <nameserver file> \
+          -genv XPN_CONF <XPN configuration file> \
+          -genv LD_LIBRARY_PATH <INSTALL_PATH>/mxml/lib:$LD_LIBRARY_PATH \
+          -genv LD_PRELOAD      <INSTALL_PATH>/xpn/lib/xpn_bypass.so:$LD_PRELOAD \
+          <program path>
+  ```
 
 - At the end of your working session, you need to stop the MPI server (xpn_mpi_server):
 
-    ```
-    mpiexec -np 1 \
-            -genv XPN_DNS <nameserver file> \
-            -genv LD_LIBRARY_PATH <INSTALL_PATH>/mxml/lib:$LD_LIBRARY_PATH \
-            <INSTALL_PATH>/bin/xpn_stop_mpi_server -f <server file>
-    ```
+  ```
+  ./xpn -v -l <full path to the hostfile>  stop
+  ```
+
+Summary:
+
+```mermaid
+sequenceDiagram
+    session        ->> xpn_mpi_server: launch the Expand MPI server
+    xpn_mpi_server ->> mk_conf.sh: generate the XPN configuration file
+    mk_conf.sh     ->> xpn.conf: generate the xpn.conf file
+    xpn.conf      -->> xpn_mpi_server: read the XPN configuration file
+    session        ->> XPN client: launch the program that will use Expand
+    xpn.conf      -->> XPN client: read the XPN configuration file
+    XPN client    -->> xpn_mpi_server: write and read data
+    XPN client    -->> session: execution ends
+    session        ->> xpn_mpi_server: stop the MPI server
+```
+
 
 ## 5. XPN Examples
 
@@ -116,6 +113,19 @@ As an build example scenario we will consider the following one:
 * MPI distribution is installed at '/opt/software/install-mpich'
 * Installation directory will be '/opt/xpn'
 
+```mermaid
+stateDiagram-v2
+    direction LR
+    state "1. Install dependencies"                     as step1
+    state "2. Download the source code of XPN and mxml" as step2
+    state "3. Build Expand"                             as step3
+    [*] --> step1
+    step1 --> step2
+    step2 --> step3
+    step3 --> [*]
+```
+
+
 (1) Install dependencies:
   ```
   sudo apt-get install -y autoconf automake gcc g++ make flex libtool doxygen libmpich-dev
@@ -132,9 +142,10 @@ As an build example scenario we will consider the following one:
 (3) To build Expand in this case you need to execute:
    ```
    cd $HOME/src
-   ./xpn/build-me.sh -m /opt/software/install-mpich/bin/mpicc -i /opt/xpn
+   ./xpn/build-me -m /opt/software/install-mpich/bin/mpicc -i /opt/xpn
    ```
 
+
 ### 5.2 Front-end node of a cluster
 
 Imagine you have a cluter with local storage (HDD, SSD, RAM Drive) per-node and shared home.
@@ -143,6 +154,19 @@ As an build example scenario we will consider the following one:
 * MPI distribution is installed at '/opt/software/install-mpich'
 * Installation directory will be $HOME/xpn_bin
 
+```mermaid
+stateDiagram-v2
+    direction LR
+    state "1. Install dependencies"                     as step1
+    state "2. Download the source code of XPN and mxml" as step2
+    state "3. Build Expand"                             as step3
+    [*] --> step1
+    step1 --> step2
+    step2 --> step3
+    step3 --> [*]
+```
+
+
 (1) Install dependencies:
   ```
   module available
@@ -161,6 +185,6 @@ As an build example scenario we will consider the following one:
 (3) To build Expand in this case you need to execute:
   ```
   cd $HOME/src;
-  ./xpn/build-me.sh -m /opt/software/install-mpich/bin/mpicc -i $HOME/xpn_bin
+  ./xpn/build-me -m /opt/software/install-mpich/bin/mpicc -i $HOME/xpn_bin
   ```
 
diff --git a/autom4te.cache/output.1 b/autom4te.cache/output.1
@@ -5549,7 +5549,7 @@ fi
 done
 
 
-for ac_header in fcntl.h string.h strings.h unistd.h sys/ioctl.h sys/time.h netinet/tcp.h netinet/in.h pthread.h windows.h sys/param.h dirent.h rpc/rpc.h rpc/clnt.h rpc/types.h
+for ac_header in fcntl.h string.h strings.h unistd.h sys/ioctl.h sys/time.h netinet/tcp.h netinet/in.h pthread.h windows.h sys/param.h dirent.h rpc/rpc.h rpc/clnt.h rpc/types.h mpi.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
 ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"
@@ -5950,8 +5950,7 @@ done
 #
 # Set initial variables depending on $target_cpu and $target_os...
 #
-IFLAGS="    -I../../mxml    -I../../xpn/include    -I../../xpn/include/xpn_client    -I../../xpn/include/base "
-IFLAGS+="-I../../../mxml -I../../../xpn/include -I../../../xpn/include/xpn_client -I../../../xpn/include/base "
+IFLAGS="-I../../../mxml -I../../../xpn/include -I../../../xpn/include/xpn_client -I../../../xpn/include/base "
 LFLAGS="    -L../../mxml    -L../../xpn/lib "
 LFLAGS+="-L../../../mxml -L../../../xpn/lib "
 
@@ -5961,7 +5960,7 @@ then
   CC="mpifcc"
   MAKE="make"
 
-  CDEFS="-D_REENTRANT -DPOSIX_THREADS -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML"
+  CDEFS="-D_REENTRANT -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML"
   LIBLINK="-lpthread"
 
   CFLAGS=$IFLAGS" -fPIC -O2 -g -std=c11"
@@ -5973,10 +5972,10 @@ then
   CC="mpicc"
   MAKE="make"
 
-  CDEFS="-D_REENTRANT -DPOSIX_THREADS -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML"
+  CDEFS="-D_REENTRANT -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -DENABLE_MXML"
   LIBLINK="-lpthread"
 
-  CFLAGS=$IFLAGS" -fPIC -O2 -g -g3 -ggdb -Wall -Wextra -std=c11"
+  CFLAGS=$IFLAGS" -fPIC -O2 -g -g3 -Wall -Wextra -std=c11"
   LIBS=$LFLAGS" -lmxml -ldl -lpthread"
 
 elif test "$target_os" = "AIX" -o "$target_os" = "aix"
@@ -5997,7 +5996,7 @@ then
   CC="mpicc"
   MAKE="gmake"
 
-  CDEFS="-D_REENTRANT -DPOSIX_THREADS"
+  CDEFS="-D_REENTRANT"
   LIBLINK="-lpthread "
 
   CFLAGS=$IFLAGS" -O2 -Wall -Wextra -fPIC -g -std=c11"
@@ -6008,7 +6007,7 @@ else
   #CC="mpicc"
   MAKE="make"
 
-  CDEFS="-D_REENTRANT -DPOSIX_THREADS -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN"
+  CDEFS="-D_REENTRANT -DHAVE_CONFIG_H -DLINUX -DXPNI_LOWFSI_XPN"
   LIBLINK="-lpthread "
 
   CFLAGS=$IFLAGS" -O2 -Wall -Wextra -fPIC -g -std=c11"