Develop

.github/workflows/ci.yml

@@ -0,0 +1,43 @@
+on: [push]
+jobs:
+  test:
+    name: Unit Tests
+    uses: ./.github/workflows/test.yml
+  asan:
+    name: Address Sanitizer
+    needs: test
+    uses: ./.github/workflows/test.yml
+    with:
+      flags: -DTEST_ASAN=OFF
+      #TODO -DTEST_ASAN=ON
+  tsan:
+    name: Thread Sanitizer
+    needs: test
+    uses: ./.github/workflows/test.yml
+    with:
+      flags: -DTEST_TSAN=ON
+  ubsan:
+    name: Undefined Behaviour Sanitizer
+    needs: test
+    uses: ./.github/workflows/test.yml
+    with:
+      flags: -DTEST_UBSAN=ON
+  # TODO check documentation CI so that unnecessary files (.cpp, examples/,...) can be left out
+  # doc:
+  #   name: Documentation
+  #   runs-on: [ubuntu-latest]
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v3
+  #       with:
+  #         submodules: true
+  #     - name: Install Doxygen
+  #       run: sudo apt install doxygen
+  #     - name: Install coverxygen
+  #       run: pip3 install coverxygen
+  #     - name: Generate documentation
+  #       run: cd docs && doxygen Doxyfile
+  #     - name: Generate documentation coverage
+  #       run: cd docs && python3 -m coverxygen --xml-dir xml --src-dir ../src/ --output doc-coverage.info --format json-v3 --scope public,protected --kind enum,enumvalue,friend,typedef,function,class,struct,union,define,file,namespace --prefix */src/
+  #     - name: Documentation report
+  #       run: cd docs && ./show_undocumented.py doc-coverage.info

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+on:
+  workflow_call:
+    inputs:
+      flags:
+        type: string
+        default: ''
+jobs:
+  test:
+    name: Test
+    runs-on: [ubuntu-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Configure
+        run: cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DTEST=ON ${{ inputs.flags }}
+      - name: Build
+        run: cmake --build build --target test
+      - name: Test
+        run: ./build/test/test

.gitignore

@@ -5,4 +5,7 @@ main
 *.so
 build/*
 install/*
-src/defines.hpp
+inst-dir/*
+src/defines.hpp
+.vscode
+

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "data-sources/mt4g"]
 	path = data-sources/mt4g
 	url = https://github.com/caps-tum/mt4g.git
+[submodule "test/ut"]
+	path = test/ut
+	url = git@github.com:boost-ext/ut.git

CMakeLists.txt

@@ -33,7 +33,6 @@ endif()
 
 if(DATA_SOURCES OR DS_HWLOC)
   find_package(HWLOC REQUIRED)
-  include_directories(${HWLOC_INCLUDE_DIRS})
 endif()
 
 
@@ -64,6 +63,11 @@ option(DATA_SOURCES "Build and install all data sources" OFF)
 option(DS_HWLOC "Build and install data source hwloc (Retrieves hwloc topology information)" OFF)
 option(DS_MT4G "Build and install data source mt4g (Compute and memory topology of NVidia GPUs)" OFF)
 option(DS_NUMA "Build and install data source caps-numa-benchmark" OFF)
+option(TEST "Build tests" OFF)
+option(TEST_ASAN "Build tests with enabled address sanitizers" OFF)
+option(TEST_TSAN "Build tests with enabled thread sanitizers" OFF)
+option(TEST_UBSAN "Build tests with enabled undefined behaviour sanitizers" OFF)
+option(TEST_COVERAGE "Build tests with enabled coverage" OFF)
 
 if(DATA_SOURCES)
     set(DS_HWLOC ON)
@@ -75,3 +79,7 @@ endif()
 add_subdirectory(src)
 add_subdirectory(examples)
 add_subdirectory(data-sources)
+
+if(${TEST})
+    add_subdirectory(test)
+endif()

README.md

@@ -46,6 +46,10 @@ cmake ..
 make all install
 ```
 
+## Testing
+
+Please refer to the [Testing Guide](test/README.md) for more information.
+
 ## About
 
 sys-sage has been created by Stepan Vanecek (stepan.vanecek@tum.de) and the [CAPS TUM](https://www.ce.cit.tum.de/en/caps/homepage/). Please contact us in case of questions, bug reporting etc.

data-sources/CMakeLists.txt

@@ -11,7 +11,9 @@ if(DS_HWLOC)
     endif()
 
     add_executable(hwloc-output hwloc-output.cpp)
-    target_link_libraries(hwloc-output hwloc)
+    target_include_directories(hwloc-output PUBLIC ${HWLOC_INCLUDE_DIRS})
+    target_link_directories(hwloc-output PRIVATE ${HWLOC_LIBRARY_DIRS})
+    target_link_libraries(hwloc-output PRIVATE ${HWLOC_LIBRARIES})
     install(TARGETS hwloc-output DESTINATION bin)
 endif()
 

data-sources/hwloc-output.cpp

@@ -13,32 +13,40 @@ using namespace std;
 string get_hwloc_topology_xml_string();
 
 /**
-Binary (entrypoint) for generating hwloc topology XML output (to current directory)
+Binary (entrypoint) for generating hwloc topology XML output (to current directory). Functionality is almost identical to "lstopo --of XML" (with an option to store to a file).
 \n usage: ./hwloc-output [output_filename]
-@param filename of the output file (default: hwloc_topology.xml)
+@param filename of the output file. If no output_filename is selected, print to stdout (also use "-" as output_filename to print to stdout)
 */
-
 int main(int argc, char* argv[])
 {
-
+    bool to_stdout = false;
     string filename;
-    if (argc < 2) {
-        filename = "hwloc_topology.xml";
+    if (argc < 2 || std::strcmp(argv[1], "-") == 0 ) {
+        to_stdout = true;
     }
     else {
        filename = argv[1];
     }
+
     string xml_output = get_hwloc_topology_xml_string();
 
     if (!xml_output.empty()) {
-        ofstream outfile;
-        outfile.open(filename);
-        outfile << xml_output;
-        outfile.close();
-        //cout << "Hwloc XML output exported to " << filename << endl;
+        if(to_stdout)
+        {
+            std::cout << xml_output;
+        }
+        else
+        {
+            ofstream outfile;
+            outfile.open(filename);
+            outfile << xml_output;
+            outfile.close();
+            //cout << "Hwloc XML output exported to " << filename << endl;
+        }
     }
     else {
         cerr << "Failed to generate hwloc topology XML output" << endl;
+        return 1;
     }
 
     return 0;
@@ -66,16 +74,19 @@ string get_hwloc_topology_xml_string() {
         hwloc_topology_destroy(topology);
         return "";
     }
-    //TODO replace with hwloc_topology_export_xmlbuffer?
-    stringstream xml_output_stream;
-    err = hwloc_topology_export_xml(topology, xml_output_stream.str().c_str(), flags);
+    char * xmlbuffer;
+    int buflen;
+    err = hwloc_topology_export_xmlbuffer(topology, &xmlbuffer, &buflen, flags);
     if (err) {
-        cerr << "hwloc: Failed to export xml" << endl;
+        cerr << "hwloc: Failed to export to a temporary buffer" << endl;
+        hwloc_free_xmlbuffer(topology, xmlbuffer);
         hwloc_topology_destroy(topology);
         return "";
     }
+    std::string xml_output(xmlbuffer, buflen);
 
+    hwloc_free_xmlbuffer(topology, xmlbuffer);
     hwloc_topology_destroy(topology);
-    string xml_output = xml_output_stream.str();
+
     return xml_output;
 }

docker-compose.yaml

@@ -0,0 +1,14 @@
+version: '3'
+services:
+  test:
+    build:
+      context: test
+      dockerfile: Dockerfile
+    volumes:
+      - .:/sys-sage
+    entrypoint: [ "/bin/sh", "-c" ]
+    command:
+      - |
+        cmake -S /sys-sage -B /build -DCMAKE_BUILD_TYPE=Debug -DTEST=ON
+        cmake --build build --target test
+        ./build/test/test

docs/.gitignore

@@ -1,2 +1,3 @@
 html/*
 latex/*
+xml/*

docs/Doxyfile

@@ -1849,7 +1849,7 @@ EXTRA_SEARCH_MAPPINGS  =
 # If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
 # The default value is: YES.
 
-GENERATE_LATEX         = YES
+GENERATE_LATEX         = NO
 
 # The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. If a
 # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of
@@ -2134,7 +2134,7 @@ MAN_LINKS              = NO
 # captures the structure of the code including all documentation.
 # The default value is: NO.
 
-GENERATE_XML           = NO
+GENERATE_XML           = YES
 
 # The XML_OUTPUT tag is used to specify where the XML pages will be put. If a
 # relative path is entered the value of OUTPUT_DIRECTORY will be put in front of

docs/show_undocumented.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+# Shows undocumented symbols from the sys-sage project.
+# Requires https://github.com/psycofdj/coverxygen, install via: pip3 install coverxygen
+# Generate coverage file: python3 -m coverxygen --xml-dir xml --src-dir ../src/ --output doc-coverage.info --format json-v3
+
+import sys
+import json
+
+if len(sys.argv) != 2:
+    print("Missing input file argument")
+    exit(-1)
+
+cov = json.load(open(sys.argv[1]))
+missing = 0
+for file in cov['files']:
+    for sym in cov['files'][file]:
+        if not sym['documented'] and sym['kind'] not in ['file', 'namespace']:
+            print("[{}:{}]  \033[0;33m{}\033[0m".format(
+                sym['file'], sym['line'], sym['symbol']
+            ))
+            missing += 1
+
+print(f"Undocumented symbols: {missing}")
+
+exit(missing)

examples/gpu-topo-parser.cpp

@@ -18,7 +18,7 @@ int main(int argc, char *argv[])
         std::string path_prefix(argv[0]);
         std::size_t found = path_prefix.find_last_of("/\\");
         path_prefix=path_prefix.substr(0,found) + "/";
-        gpuTopoPath = path_prefix + "example_data/pascal_gpu_topo.csv";
+        gpuTopoPath = path_prefix + "example_data/ampere_gpu_topo.csv";
     }
     else if(argc == 2){
         gpuTopoPath = argv[1];
@@ -42,9 +42,11 @@ int main(int argc, char *argv[])
 
     string output_name = "sys-sage_gpu_sample_output.xml";
     cout << "-------- Exporting as XML to " << output_name << " --------" << endl;
+
+    //topo->DeleteSubtree();
     exportToXml(topo, output_name);
 
-    delete topo;
-    delete n;
+    topo->Delete(true);
+
     return 0;
 }

examples/sys-sage-benchmarking.cpp

@@ -146,7 +146,9 @@ int main(int argc, char *argv[])
     cout << ", mt4g_dataPaths, "<< mt4g_dataPaths;
     cout << ", all_components, " << allComponentList.size() ;
 
-    cout << ", time_getNumaMaxBw, " << time_getNumaMaxBw; //<< "; bw; " << max_bw << "; ComponentId; " << max_bw_component->GetId() << endl;
+    cout << ", time_getNumaMaxBw, " << time_getNumaMaxBw;
+    if(false)
+        cout << "; bw; " << max_bw << "; ComponentId; " << max_bw_component->GetId() << endl;
     cout << ", time_createNewComponent, " << time_createNewComponent;
 
     cout << ", hwloc_component_size[B], " << hwloc_component_size;

src/DataPath.hpp

@@ -6,23 +6,23 @@
 #include "defines.hpp"
 #include "Topology.hpp"
 
-//void AddDataPath(DataPath* p, int orientation);
-#define SYS_SAGE_DATAPATH_NONE 1
-#define SYS_SAGE_DATAPATH_OUTGOING 2
-#define SYS_SAGE_DATAPATH_INCOMING 4
+//Component pointing to a DataPath 
+#define SYS_SAGE_DATAPATH_NONE 1 /**< TODO */
+#define SYS_SAGE_DATAPATH_OUTGOING 2 /**< This Component is the source DataPath. */
+#define SYS_SAGE_DATAPATH_INCOMING 4 /**< This Component is the taerget DataPath. */
 
 //int oriented
-#define SYS_SAGE_DATAPATH_BIDIRECTIONAL 8
-#define SYS_SAGE_DATAPATH_ORIENTED 16
+#define SYS_SAGE_DATAPATH_BIDIRECTIONAL 8 /**< DataPath has no direction. */
+#define SYS_SAGE_DATAPATH_ORIENTED 16 /**< DataPath is directec from the source to the target. */
 
 //dp_type
-#define SYS_SAGE_DATAPATH_TYPE_NONE 32
-#define SYS_SAGE_DATAPATH_TYPE_LOGICAL 64
-#define SYS_SAGE_DATAPATH_TYPE_PHYSICAL 128
-#define SYS_SAGE_DATAPATH_TYPE_L3CAT 256
-#define SYS_SAGE_DATAPATH_TYPE_MIG 512
-#define SYS_SAGE_DATAPATH_TYPE_DATATRANSFER 1024
-#define SYS_SAGE_DATAPATH_TYPE_C2C 2048
+#define SYS_SAGE_DATAPATH_TYPE_NONE 32 /**< Generic type of DataPath */
+#define SYS_SAGE_DATAPATH_TYPE_LOGICAL 64 /**< DataPath describes a logical connection/relation of two Components. */
+#define SYS_SAGE_DATAPATH_TYPE_PHYSICAL 128 /**< DataPath describes a physical/hardware connection/relation of two Components. */
+#define SYS_SAGE_DATAPATH_TYPE_L3CAT 256 /**< DataPath type describing Cache partitioning settings. */
+#define SYS_SAGE_DATAPATH_TYPE_MIG 512 /**< DataPath type describing GPU partitioning settings. */
+#define SYS_SAGE_DATAPATH_TYPE_DATATRANSFER 1024 /**< DataPath type describing data transfer attributes. */
+#define SYS_SAGE_DATAPATH_TYPE_C2C 2048 /**< DataPath type describing cache-to-cache latencies (cccbench data source). */
 
 using namespace std;
 class Component;
@@ -103,6 +103,9 @@ class DataPath {
     @see dp_type
     */
     int GetDpType();
+    /**
+     * TODO
+    */
     int GetOriented();
 
     /**
@@ -117,16 +120,19 @@ class DataPath {
     */
     void DeleteDataPath();
 
+    /**
+     * TODO
+    */
     map<string,void*> attrib;
 private:
-    Component * source;
-    Component * target;
+    Component * source; /**< TODO */
+    Component * target; /**< TODO */
 
-    const int oriented;
-    const int dp_type; /**< asdasd */
+    const int oriented; /**< TODO */
+    const int dp_type; /**< TODO */
 
-    double bw;
-    double latency;
+    double bw; /**< TODO */
+    double latency; /**< TODO */
 
 };