resolve conflict using commandline

.devcontainer/post-create.sh

@@ -38,4 +38,4 @@ make -j && make install
 export LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH"
 export PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH"
 echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh
-echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
+echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh

.devcontainer/post-start.sh

@@ -3,4 +3,4 @@
 nohup /bin/bash /workspaces/pdc/.devcontainer/post-create.sh 2>&1 > /workspaces/pdc_install.out &
 
 echo "Wait for 10 seconds for the building processes to start."
-sleep 10s
+sleep 10s

.docker/base.Dockerfile

@@ -0,0 +1,90 @@
+# Use Ubuntu Jammy (latest LTS) as the base image
+FROM ubuntu:jammy
+
+# Install necessary tools, MPICH, UUID library and developer files
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    git \
+    mpich \
+    libmpich-dev \
+    uuid \
+    uuid-dev \
+    autoconf \
+    libtool \
+    cmake \
+    cmake-curses-gui \
+    wget \
+    axel \
+    curl \
+    vim \
+    nano \
+    gdb \
+    cgdb \
+    curl \
+    valgrind
+
+# Set WORK_SPACE environment variable and create necessary directories
+ENV WORK_SPACE=/home/codespace
+RUN mkdir -p $WORK_SPACE
+
+# Clone the repositories
+WORKDIR $WORK_SPACE/source
+RUN git clone https://github.com/ofiwg/libfabric.git && \
+    git clone https://github.com/mercury-hpc/mercury.git --recursive
+
+COPY ./ ${WORK_SPACE}/source/pdc
+
+ENV LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric
+ENV MERCURY_SRC_DIR=$WORK_SPACE/source/mercury
+ENV PDC_SRC_DIR=$WORK_SPACE/source/pdc
+ENV LIBFABRIC_DIR=$WORK_SPACE/install/libfabric
+ENV MERCURY_DIR=$WORK_SPACE/install/mercury
+ENV PDC_DIR=$WORK_SPACE/install/pdc
+
+RUN mkdir -p $LIBFABRIC_SRC_DIR \
+    mkdir -p $MERCURY_SRC_DIR \
+    mkdir -p $PDC_SRC_DIR \
+    mkdir -p $LIBFABRIC_DIR \
+    mkdir -p $MERCURY_DIR \
+    mkdir -p $PDC_DIR
+
+
+# Save the environment variables to a file
+RUN echo "export LIBFABRIC_SRC_DIR=$WORK_SPACE/source/libfabric" > $WORK_SPACE/pdc_env.sh && \
+    echo "export MERCURY_SRC_DIR=$WORK_SPACE/source/mercury" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export PDC_SRC_DIR=$WORK_SPACE/source/pdc" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export LIBFABRIC_DIR=$WORK_SPACE/install/libfabric" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export MERCURY_DIR=$WORK_SPACE/install/mercury" >> $WORK_SPACE/pdc_env.sh && \
+    echo "export PDC_DIR=$WORK_SPACE/install/pdc" >> $WORK_SPACE/pdc_env.sh
+
+
+# Build and install libfabric
+WORKDIR $LIBFABRIC_SRC_DIR
+RUN git checkout v1.18.0 && \
+    ./autogen.sh && \
+    ./configure --prefix=$LIBFABRIC_DIR CC=mpicc CFLAG="-O2" && \
+    make clean && \
+    make -j && make install && \
+    make check
+
+ENV LD_LIBRARY_PATH="$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH"
+ENV PATH="$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH"
+RUN echo 'export LD_LIBRARY_PATH=$LIBFABRIC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh  && \
+    echo 'export PATH=$LIBFABRIC_DIR/include:$LIBFABRIC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
+
+
+# Build and install Mercury
+WORKDIR $MERCURY_SRC_DIR
+ENV MERCURY_CMAKE_FLAGS="-DCMAKE_INSTALL_PREFIX=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DNA_USE_OFI=ON -DNA_USE_SM=OFF -DNA_OFI_TESTING_PROTOCOL=tcp "
+RUN git checkout v2.2.0 \
+    mkdir -p build 
+WORKDIR ${MERCURY_SRC_DIR}/build
+RUN cmake $MERCURY_CMAKE_FLAGS ../ && \
+    make -j && make install && \
+    ctest
+
+# Set the environment variables
+ENV LD_LIBRARY_PATH="$MERCURY_DIR/lib:$LD_LIBRARY_PATH"
+ENV PATH="$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH"
+RUN echo 'export LD_LIBRARY_PATH=$MERCURY_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh \
+    echo 'export PATH=$MERCURY_DIR/include:$MERCURY_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh

.docker/local.Dockerfile

@@ -0,0 +1,29 @@
+# Note: Run `docker build -f .docker/Dockerfile -t pdc:latest .` from the root directory of the repository to build the docker image.
+
+# Use Ubuntu Jammy (latest LTS) as the base image
+FROM zhangwei217245/pdc_dev_base:latest
+
+# Build and install PDC
+ENV PDC_CMAKE_FLAGS="-DBUILD_MPI_TESTING=ON -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR -DPDC_ENABLE_MPI=ON -DMERCURY_DIR=$MERCURY_DIR -DCMAKE_C_COMPILER=mpicc -DMPI_RUN_CMD=mpiexec "
+
+
+WORKDIR $PDC_SRC_DIR
+RUN rm -rf build && \
+    mkdir -p build
+
+# COPY ../  ${PDC_SRC_DIR}
+# RUN ls -l $PDC_SRC_DIR
+
+WORKDIR ${PDC_SRC_DIR}/build
+RUN cmake $PDC_CMAKE_FLAGS ../ 2>&1 > ./cmake_config.log || echo "ignoring cmake config error and proceed" && \
+    make -j && make install
+
+# Set the environment variables
+ENV LD_LIBRARY_PATH="$PDC_DIR/lib:$LD_LIBRARY_PATH"
+ENV PATH="$PDC_DIR/include:$PDC_DIR/lib:$PATH"
+RUN echo 'export LD_LIBRARY_PATH=$PDC_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh && \
+    echo 'export PATH=$PDC_DIR/include:$PDC_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
+
+
+# WORKDIR $PDC_SRC_DIR/build
+# RUN ctest

.docker/local.Dockerfile.dockerignore

@@ -0,0 +1,2 @@
+# Exclude files and directories from the Docker build context
+!/.git/

src/server/pdc_server.c

@@ -2384,4 +2384,4 @@ server_run(int argc, char *argv[])
     MPI_Finalize();
 #endif
     return 0;
-}
+}

src/server/pdc_server_metadata.c

@@ -1845,7 +1845,12 @@ PDC_Server_query_kvtag_someta(pdc_kvtag_t *in, uint32_t *n_meta, uint64_t **obj_
                             *obj_ids = (void *)realloc(*obj_ids, alloc_size * sizeof(uint64_t));
                         }
                         (*obj_ids)[iter++] = elt->obj_id;
-                        break;
+                        // break; // FIXME: shall we break here? or continue to check other kvtags?
+                    }
+                    else {
+#ifdef PDC_DEBUG_OUTPUT
+                        println("[NOT FOUND]");
+#endif
                     }
                 } // End for each kvtag in list
             }     // End for each metadata from hash table entry

src/tools/CMakeLists.txt

@@ -95,4 +95,4 @@ add_library(cjson cjson/cJSON.c)
 foreach(program ${PROGRAMS})
   add_executable(${program} ${program}.c)
   target_link_libraries(${program} pdc cjson ${TOOLS_EXT_LIB})
-endforeach(program)
+endforeach(program)

tools/CMakeLists.txt

@@ -0,0 +1,136 @@
+cmake_minimum_required (VERSION 2.8.12)
+
+# Setup cmake policies.
+foreach(p
+    CMP0012
+    CMP0013
+    CMP0014
+    CMP0022 # CMake 2.8.12
+    CMP0025 # CMake 3.0
+    CMP0053 # CMake 3.1
+    CMP0054 # CMake 3.1
+    CMP0074 # CMake 3.12
+    CMP0075 # CMake 3.12
+    CMP0083 # CMake 3.14
+    CMP0093 # CMake 3.15
+  )
+  if(POLICY ${p})
+    cmake_policy(SET ${p} NEW)
+  endif()
+endforeach()
+
+project(PDC_VOL C)
+
+include_directories(
+  ${PDC_EXT_INCLUDE_DEPENDENCIES}
+)
+
+find_package(PDC REQUIRED)
+if(PDC_FOUND)
+    #message(STATUS "PDC include directory: ${PDC_INCLUDE_DIR}")
+  set(PDC_EXT_INCLUDE_DEPENDENCIES ${PDC_INCLUDE_DIR}
+    ${PDC_EXT_INCLUDE_DEPENDENCIES}
+    )
+  set(PDC_EXT_LIB_DEPENDENCIES pdc ${PDC_EXT_LIB_DEPENDENCIES})
+endif()
+
+#HDF5
+option(USE_SYSTEM_HDF5 "Use system-installed HDF5." ON)
+  if(USE_SYSTEM_HDF5)
+  find_package(HDF5 NO_MODULE NAMES hdf5 COMPONENTS C shared)
+  if(HDF5_FOUND)
+    set(HDF5_C_SHARED_LIBRARY hdf5-shared)
+#    if(NOT TARGET ${HDF5_C_SHARED_LIBRARY})
+#      message(FATAL_ERROR "Could not find hdf5 shared target, please make "
+#"sure that HDF5 has ben compiled with shared libraries enabled.")
+#    endif()
+    set(PDC_EXT_INCLUDE_DEPENDENCIES
+        ${PDC_EXT_INCLUDE_DEPENDENCIES}
+        ${HDF5_INCLUDE_DIR}
+      )
+    set(PDC_EXT_LIB_DEPENDENCIES
+      ${PDC_EXT_LIB_DEPENDENCIES}
+      ${HDF5_C_SHARED_LIBRARY}
+    )
+  else()
+    # Allow for HDF5 autotools builds
+    find_package(HDF5 MODULE REQUIRED)
+    if(HDF5_FOUND)
+      set(PDC_EXT_INCLUDE_DEPENDENCIES
+        ${PDC_EXT_INCLUDE_DEPENDENCIES}
+        ${HDF5_INCLUDE_DIRS}
+      )
+      set(PDC_EXT_LIB_DEPENDENCIES
+        ${PDC_EXT_LIB_DEPENDENCIES}
+        ${HDF5_LIBRARIES}
+      )
+    else()
+      message(FATAL_ERROR "Could not find HDF5, please check HDF5_DIR.")
+    endif()
+  endif()
+endif()
+
+option(USE_SYSTEM_OPENMP "Use system-installed OpenMP." ON)
+if(USE_SYSTEM_OPENMP)
+  find_package(OpenMP REQUIRED)
+  if(OPENMP_FOUND)
+      add_definitions(-DENABLE_OPENMP=1)
+      set(ENABLE_OPENMP 1)
+      set(OPENMP_LIBRARIES "${OpenMP_C_LIBRARIES}")
+  else()
+      message(FATAL_ERROR "OpenMP not found")
+  endif()
+endif()
+
+
+add_definitions(-DENABLE_MPI=1)
+add_library(cjson cjson/cJSON.c)
+
+
+# set(PROGRAMS
+#   pdc_ls
+#   pdc_import
+#   pdc_export
+#   )
+
+# foreach(program ${PROGRAMS})
+#   add_executable(${program} ${program}.c)
+#   target_link_libraries(${program} ${PDC_EXT_LIB_DEPENDENCIES})
+#   target_link_libraries(${program} pdc)
+#   target_link_libraries(${program} cjson)
+#   target_include_directories(${program} PUBLIC ${PDC_INCLUDE_DIR})
+# endforeach(program)
+
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O3 -fopenmp -DNDEBUG")
+
+# Find LibTIFF
+option(USE_LIB_TIFF "Enable LibTiff." ON)
+if(USE_LIB_TIFF)
+  find_package(TIFF REQUIRED)
+  if(TIFF_FOUND)
+    set(LLSM_LIB_SOURCE 
+      llsm/parallelReadTiff.c
+      llsm/csvReader.c
+      llsm/pdc_list.c
+    )
+    # Add the LibTIFF include directory to the include path
+    include_directories(${TIFF_INCLUDE_DIRS})
+    add_library(llsm_tiff ${LLSM_LIB_SOURCE})
+    target_compile_options(llsm_tiff PRIVATE ${OpenMP_C_FLAGS})
+    target_link_libraries(llsm_tiff PUBLIC ${OpenMP_C_LIBRARIES})
+    target_link_libraries(llsm_tiff PUBLIC ${TIFF_LIBRARIES})
+    target_include_directories(llsm_tiff PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/llsm)
+
+
+    add_executable(llsm_importer llsm_importer.c)
+    target_link_libraries(llsm_importer ${PDC_EXT_LIB_DEPENDENCIES})
+    target_link_libraries(llsm_importer pdc)
+    target_link_libraries(llsm_importer cjson)
+    target_link_libraries(llsm_importer ${TIFF_LIBRARIES})
+    target_link_libraries(llsm_importer llsm_tiff)
+    target_include_directories(llsm_importer PUBLIC ${PDC_INCLUDE_DIR})
+  else()
+    message(WARNING "LibTiff not found, ignore building the executables which requires LibTiff support.")
+  endif()
+endif()

tools/LLSM_IMPORTER.md

@@ -0,0 +1,76 @@
+# LLSM_Importer Tutorial
+
+This is a tutorial for you to run llsm_importer on Perlmutter supercomputer at NERSC. 
+
+## Prerequisite
+
+Before building and installing LLSM_importer tool, you need to make sure you install PDC correctly. Check out the latest update on the `develop` branch of `PDC`. Please refer to [Proactive Data Containers (PDC) Installation Guide](../README.md) 
+
+Once you finish all the steps in the installation guide above, you should have environment variable `$WORK_SPACE` defined.
+
+## Installation
+
+To build and install LLSM_importer, you need to download libtiff 4.4.0 first. 
+
+```bash
+cd $WORK_SPACE/source
+wget https://download.osgeo.org/libtiff/tiff-4.4.0.tar.gz
+tar zxvf tiff-4.4.0.tar.gz
+cd tiff-4.4.0
+./configure --prefix=$WORK_SPACE/install/tiff-4.4.0
+make -j 32 install
+```
+
+Now you should have libtiff 4.4.0 installed and you need to include the path to the library to your environment variables:
+
+```bash
+echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0"
+echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH'
+echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH'
+
+echo "export TIFF_DIR=$WORK_SPACE/install/tiff-4.4.0" >> $WORK_SPACE/pdc_env.sh
+echo 'export LD_LIBRARY_PATH=$TIFF_DIR/lib:$LD_LIBRARY_PATH' >> $WORK_SPACE/pdc_env.sh
+echo 'export PATH=$TIFF_DIR/include:$TIFF_DIR/lib:$PATH' >> $WORK_SPACE/pdc_env.sh
+```
+
+Copy the 3 export commands on your screen and run them, and next time if you need to rebuild the llsm_importer program, you can run `$WORK_SPACE/pdc_env.sh` again in advance!
+
+Now, time to build llsm_importer program.
+
+```bash
+mkdir -p $WORK_SPACE/source/pdc/tools/build
+cd $WORK_SPACE/source/pdc/tools/build
+
+cmake ../ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DPDC_DIR=$PDC_DIR -DUSE_LIB_TIFF=ON -DUSE_SYSTEM_HDF5=ON -DUSE_SYSTEM_OPENMP=ON -DCMAKE_INSTALL_PREFIX=$PDC_DIR/tools/ -DCMAKE_C_COMPILER=cc
+
+make -j 32
+```
+
+After this, you should be able to see `llsm_importer` artifact under your `$WORK_SPACE/source/pdc/tools/build` directory.
+
+## Running LLSM_importer
+
+First, locate the llsm_importer script
+
+```bash
+cd $WORK_SPACE/source/pdc/scripts/llsm_importer
+```
+
+Modify the template script `template.sh`. 
+
+Change `EXECPATH` to where your `pdc_server.exe` is installed
+Change `TOOLPATH` to where your `llsm_importer` artifact is.
+
+Change `LLSM_DATA_PATH` to where your sample dataset is. For exmaple, 
+
+```bash
+LLSM_DATA_PATH=/pscratch/sd/w/wzhang5/data/llsm/20220115_Korra_LLCPK_LFOV_0p1PSAmpKan/run1
+```
+
+Note: you may download the sample dataset from the this [link](https://drive.google.com/file/d/19hH7v58iF_QBJ985ajwLD86MMseBH-YR/view?usp=sharing). It is provided with the courtesy of [Advanced BioImaging Center at UC Berkeley](https://mcb.berkeley.edu/faculty/cdb/upadhyayulas).
+
+Now, run `gen_script.sh` to generate scripts for different settings with various number of servers. 
+
+After this, enter into any directory named with a number, and submit the job with `sbatch` command.
+
+Note: This program is still under development and changes will be made available from time to time. Please always use the develop branch for a stable version of this llsm_importer tool.