Merge pull request #31 from caps-tum/develop

Develop

.github/workflows/ci.yml

@@ -3,41 +3,50 @@ jobs:
   test:
     name: Unit Tests
     uses: ./.github/workflows/test.yml
+
   asan:
     name: Address Sanitizer
     needs: test
     uses: ./.github/workflows/test.yml
     with:
       flags: -DTEST_ASAN=OFF
-      #TODO -DTEST_ASAN=ON
+      # TODO: Change to -DTEST_ASAN=ON when ready
+
   tsan:
     name: Thread Sanitizer
     needs: test
     uses: ./.github/workflows/test.yml
     with:
       flags: -DTEST_TSAN=ON
+
   ubsan:
     name: Undefined Behaviour Sanitizer
     needs: test
     uses: ./.github/workflows/test.yml
     with:
       flags: -DTEST_UBSAN=ON
-  # TODO check documentation CI so that unnecessary files (.cpp, examples/,...) can be left out
-  # doc:
-  #   name: Documentation
-  #   runs-on: [ubuntu-latest]
-  #   steps:
-  #     - name: Checkout
-  #       uses: actions/checkout@v3
-  #       with:
-  #         submodules: true
-  #     - name: Install Doxygen
-  #       run: sudo apt install doxygen
-  #     - name: Install coverxygen
-  #       run: pip3 install coverxygen
-  #     - name: Generate documentation
-  #       run: cd docs && doxygen Doxyfile
-  #     - name: Generate documentation coverage
-  #       run: cd docs && python3 -m coverxygen --xml-dir xml --src-dir ../src/ --output doc-coverage.info --format json-v3 --scope public,protected --kind enum,enumvalue,friend,typedef,function,class,struct,union,define,file,namespace --prefix */src/
-  #     - name: Documentation report
-  #       run: cd docs && ./show_undocumented.py doc-coverage.info
+  #  check documentation CI so that unnecessary files (.cpp, examples/,...) can be left out    
+  doc:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Install Doxygen
+        run: sudo apt-get install -y doxygen
+
+      - name: Install coverxygen
+        run: pip3 install coverxygen
+
+      - name: Generate documentation
+        run: cd docs && doxygen Doxyfile
+
+      - name: Documentation coverage
+        # Exclude parsers, external_interfaces, examples, data-sources and .cpp files
+        run: cd docs && python3 -m coverxygen --verbose --xml-dir xml --src-dir ../src/ --output doc-coverage.info --format json-v3 --scope public,protected --kind enum,enumvalue,friend,typedef,function,class,struct,union,define,file,namespace --exclude '.*\/parsers\/.*' --exclude '.*\/external_interfaces\/.*' --exclude '.*\/examples\/.*' --exclude '.*\/data-sources\/.*' --exclude '.*\.cpp$'
+
+      - name: Check documentation completeness
+        run: cd docs && ./check_docs.sh

CMakeLists.txt

@@ -56,9 +56,9 @@ install(FILES ${sys-sage_BINARY_DIR}/pkg/sys-sage.pc
 
 
 ###Options:
-option(CAT_AWARE "Build and install functionality regarding Intel L3 CAT" OFF)
+option(INTEL_PQOS "Build and install functionality regarding Intel L3 CAT" OFF)
 option(NVIDIA_MIG "Build and install functionality regarding NVidia MIG(multi-instance GPU, ampere or newer)" OFF)
-option(CPUINFO "Build and install functionality regarding Linux cpuinfo" OFF)
+option(PROC_CPUINFO "Build and install functionality regarding Linux cpuinfo" OFF)
 option(DATA_SOURCES "Build and install all data sources" OFF)
 option(DS_HWLOC "Build and install data source hwloc (Retrieves hwloc topology information)" OFF)
 option(DS_MT4G "Build and install data source mt4g (Compute and memory topology of NVidia GPUs)" OFF)

README.md

@@ -35,9 +35,9 @@ cd sys-sage
 mkdir build && cd build
 cmake ..
 # build options:
-# -DCAT_AWARE=ON            - builds with Intel CAT functionality. For that, Intel-specific pqos header/library are necessary.
+# -DINTEL_PQOS=ON            - builds with Intel CAT functionality. For that, Intel-specific pqos header/library are necessary.
 # -DNVIDIA_MIG=ON           - Build and install functionality regarding NVidia MIG(multi-instance GPU, ampere or newer).
-# -DCPUINFO=ON              - Build and install functionality regarding Linux cpuinfo (only x86) -- default ON.
+# -DPROC_CPUINFO=ON              - Build and install functionality regarding Linux cpuinfo (only x86) -- default ON.
 # -DDATA_SOURCES=ON         - builds all data sources from folder 'data-sources' listed below. Data sources are used to collecting HW-related information, so it only makes sense to compile that on the system where the topology information is queried.
 # -DDS_HWLOC=ON             - builds the hwloc data source for retrieving the CPU topology
 # -DDS_MT4g=ON              - builds the mt4g data source for retrieving GPU compute and memory topology. If turned on, includes hwloc.

docs/Doxyfile

@@ -2286,7 +2286,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             =
+PREDEFINED             = NVIDIA_MIG PROC_CPUINFO INTEL_PQOS
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The

docs/Installation_Guide.md

@@ -23,7 +23,7 @@ Installation with CMAKE:
 mkdir build && cd build
 cmake ..
 # build options:
-# -DCAT_AWARE=ON            - builds with Intel CAT functionality. For that, Intel-specific pqos header/library are necessary.
+# -DINTEL_PQOS=ON            - builds with Intel CAT functionality. For that, Intel-specific pqos header/library are necessary.
 # -DBUILD_DATA_SOURCES=ON   - builds data sources from folder 'data-sources'. If turned on, includes Linux-specific libraries and hwloc. Data sources are used to collecting HW-related information, so it only makes sense to compile that on the system where the topology information is queried.
 # -DCMAKE_INSTALL_PREFIX=../inst-dir    - to install locally into the git repo folder
 make all install

docs/check_docs.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+./show_undocumented.py doc-coverage.info
+RESULT=$?
+echo "Script exit status: $RESULT"
+if [ $RESULT -ne 0 ]; then
+  echo "Documentation is incomplete. The build will now fail."
+  exit 1
+fi
+echo "Documentation check passed successfully."

docs/show_undocumented.py

@@ -2,7 +2,7 @@
 
 # Shows undocumented symbols from the sys-sage project.
 # Requires https://github.com/psycofdj/coverxygen, install via: pip3 install coverxygen
-# Generate coverage file: python3 -m coverxygen --xml-dir xml --src-dir ../src/ --output doc-coverage.info --format json-v3
+# Generate coverage file: python3 -m coverxygen --verbose --xml-dir xml --src-dir ../src/ --output doc-coverage.info --format json-v3 --scope public,protected --kind enum,enumvalue,friend,typedef,function,class,struct,union,define,file,namespace --exclude '.*\/parsers\/.*' --exclude '.*\/external_interfaces\/.*' --exclude '.*\/examples\/.*' --exclude '.*\/data-sources\/.*' --exclude '.*\.cpp$'
 
 import sys
 import json
@@ -12,6 +12,9 @@
     exit(-1)
 
 cov = json.load(open(sys.argv[1]))
+
+print(f"Loading coverage file: {sys.argv[1]}")
+
 missing = 0
 for file in cov['files']:
     for sym in cov['files'][file]:

examples/CMakeLists.txt

@@ -1,6 +1,8 @@
 
 link_libraries(sys-sage sys-sage)
 include_directories(../src)
+# include_directories(../src/external_interfaces)
+# include_directories(../src/pa)
 
 find_package(OpenMP)
 if (OPENMP_FOUND)
@@ -20,14 +22,14 @@ add_executable(cccbenchplushwloc cccbenchplushwloc.cpp)
 install(TARGETS basic_usage mt4g-parser custom_attributes larger_topo sys-sage-benchmarking use_custom_parser cccbenchplushwloc DESTINATION bin/examples)
 install(DIRECTORY example_data DESTINATION bin/examples)
 
-if(CAT_AWARE)
+if(INTEL_PQOS)
     add_executable(matmul matmul.cpp)
     set_source_files_properties(matmul.cpp PROPERTIES COMPILE_FLAGS -O3)
     target_link_libraries(matmul hwloc pqos )
     install(TARGETS matmul DESTINATION bin/examples)
 endif()
 
-if(CPUINFO)
+if(PROC_CPUINFO)
     add_executable(cpu-frequency cpu-frequency.cpp)
     install(TARGETS cpu-frequency DESTINATION bin/examples)
 endif()

examples/basic_usage.cpp

@@ -13,17 +13,17 @@ void usage(char* argv0)
 
 int main(int argc, char *argv[])
 {
-    string topoPath;
+    string xmlPath;
     string bwPath;
     if(argc < 2){
         std::string path_prefix(argv[0]);
         std::size_t found = path_prefix.find_last_of("/\\");
         path_prefix=path_prefix.substr(0,found) + "/";
-        topoPath = path_prefix + "example_data/skylake_hwloc.xml";
+        xmlPath = path_prefix + "example_data/skylake_hwloc.xml";
         bwPath = path_prefix + "example_data/skylake_caps_numa_benchmark.csv";
     }
     else if(argc == 3){
-        topoPath = argv[1];
+        xmlPath = argv[1];
         bwPath = argv[2];
     }
     else{
@@ -36,14 +36,14 @@ int main(int argc, char *argv[])
     Node* n = new Node(topo, 1);
 
 
-    cout << "-- Parsing Hwloc output from file " << topoPath << endl;
-    if(parseHwlocOutput(n, topoPath) != 0) { //adds topo to a next node
+    cout << "-- Parsing Hwloc output from file " << xmlPath << endl;
+    if(parseHwlocOutput(n, xmlPath) != 0) { //adds topo to a next node
         usage(argv[0]);
         return 1;
     }
     cout << "-- End parseHwlocOutput" << endl;
 
-    cout << "Total num HW threads: " << topo->GetNumThreads() << endl;
+    cout << "Total num HW threads: " << topo->CountAllSubcomponentsByType(SYS_SAGE_COMPONENT_THREAD) << endl;
 
     cout << "---------------- Printing the whole tree ----------------" << endl;
     topo->PrintSubtree();

examples/cccbenchplushwloc.cpp

@@ -12,10 +12,10 @@ void usage(char* argv0)
 
 int main(int argc, char *argv[])
 {
-    string topoPath;
+    string xmlPath;
     const char *cccPath;
     if(argc == 3){
-        topoPath = argv[1];
+        xmlPath = argv[1];
         cccPath = argv[2];
     }
     else{
@@ -27,13 +27,13 @@ int main(int argc, char *argv[])
     Topology* topo = new Topology();
     Node* n = new Node(topo, 1);
 
-    if(parseHwlocOutput(n, topoPath) != 0) { //adds topo to a next node
+    if(parseHwlocOutput(n, xmlPath) != 0) { //adds topo to a next node
         usage(argv[0]);
         return 1;
     }
     cout << "-- End parseHwlocOutput" << endl;
 
-    cout << "Total num HW threads: " << topo->GetNumThreads() << endl;
+    cout << "Total num HW threads: " << topo->CountAllSubcomponentsByType(SYS_SAGE_COMPONENT_THREAD) << endl;
 
     cout << "---------------- Printing the whole tree ----------------" << endl;
     topo->PrintSubtree(2);
@@ -44,7 +44,7 @@ int main(int argc, char *argv[])
     cccparser->applyDataPaths(n);
 
     auto allcores = new vector<Component *>();
-    topo->FindAllSubcomponentsByType(allcores, SYS_SAGE_COMPONENT_CORE);    
+    topo->GetAllSubcomponentsByType(allcores, SYS_SAGE_COMPONENT_CORE);    
     //auto allcores = topo->GetAllChildrenByType(SYS_SAGE_COMPONENT_CORE);
 
     for(auto c0 : *allcores)

examples/cpu-frequency.cpp

@@ -13,19 +13,19 @@ void usage(char* argv0)
 
 int main(int argc, char *argv[])
 {
-    string topoPath;
+    string xmlPath;
     string output_name = "sys-sage_cpu-frequency.xml";
     if(argc < 2){
         std::string path_prefix(argv[0]);
         std::size_t found = path_prefix.find_last_of("/\\");
         path_prefix=path_prefix.substr(0,found) + "/";
-        topoPath = path_prefix + "example_data/skylake_hwloc.xml";
+        xmlPath = path_prefix + "example_data/skylake_hwloc.xml";
     }
     else if(argc == 2){
-        topoPath = argv[1];
+        xmlPath = argv[1];
     }
     else if(argc == 3){
-        topoPath = argv[1];
+        xmlPath = argv[1];
         output_name = argv[2];
     }
     else{
@@ -37,8 +37,8 @@ int main(int argc, char *argv[])
     Topology* topo = new Topology();
     Node* n = new Node(topo, 1);
 
-    cout << "-- Parsing Hwloc output from file " << topoPath << endl;
-    if(parseHwlocOutput(n, topoPath) != 0) { //adds topo to a next node
+    cout << "-- Parsing Hwloc output from file " << xmlPath << endl;
+    if(parseHwlocOutput(n, xmlPath) != 0) { //adds topo to a next node
         usage(argv[0]);
         return 1;
     }

examples/custom_attributes.cpp

@@ -57,17 +57,17 @@ void usage(char* argv0)
 
 int main(int argc, char *argv[])
 {
-    string topoPath;
+    string xmlPath;
     string bwPath;
     if(argc < 2){
         std::string path_prefix(argv[0]);
         std::size_t found = path_prefix.find_last_of("/\\");
         path_prefix=path_prefix.substr(0,found) + "/";
-        topoPath = path_prefix + "example_data/skylake_hwloc.xml";
+        xmlPath = path_prefix + "example_data/skylake_hwloc.xml";
         bwPath = path_prefix + "example_data/skylake_caps_numa_benchmark.csv";
     }
     else if(argc == 3){
-        topoPath = argv[1];
+        xmlPath = argv[1];
         bwPath = argv[2];
     }
     else{
@@ -79,8 +79,8 @@ int main(int argc, char *argv[])
     Topology* topo = new Topology();
     Node* n = new Node(topo, 1);
 
-    cout << "-- Parsing Hwloc output from file " << topoPath << endl;
-    if(parseHwlocOutput(n, topoPath) != 0) { //adds topo to a next node
+    cout << "-- Parsing Hwloc output from file " << xmlPath << endl;
+    if(parseHwlocOutput(n, xmlPath) != 0) { //adds topo to a next node
         usage(argv[0]);
         return 1;
     }
@@ -98,20 +98,20 @@ int main(int argc, char *argv[])
     int r = 15;
     n->attrib["codename"]=(void*)&codename;
     n->attrib["rack_no"]=(void*)&r;
-    n->attrib["unknown_will_not_be_printed"]=(void*)&topoPath;
+    n->attrib["unknown_will_not_be_printed"]=(void*)&xmlPath;
 
     My_core_attributes c1_attrib(38.222, 2000000000);
-    Core* c1 = (Core*)n->FindSubcomponentById(1, SYS_SAGE_COMPONENT_CORE);
+    Core* c1 = (Core*)n->GetSubcomponentById(1, SYS_SAGE_COMPONENT_CORE);
     if(c1 != NULL)
         c1->attrib["my_core_info"]=(void*)&c1_attrib;
 
     My_core_attributes c4_attrib(44.1, 1500000000);
-    Core* c4 = (Core*)n->FindSubcomponentById(4, SYS_SAGE_COMPONENT_CORE);
+    Core* c4 = (Core*)n->GetSubcomponentById(4, SYS_SAGE_COMPONENT_CORE);
     if(c4 != NULL)
         c4->attrib["my_core_info"]=(void*)&c4_attrib;
 
     string benchmark_info="measured with no load on 07.07.";
-    Numa* n2 = (Numa*)n->FindSubcomponentById(2, SYS_SAGE_COMPONENT_NUMA);
+    Numa* n2 = (Numa*)n->GetSubcomponentById(2, SYS_SAGE_COMPONENT_NUMA);
     if(n2 != NULL){
         DataPath * dp = (*(n2->GetDataPaths(SYS_SAGE_DATAPATH_INCOMING)))[0];
         if(dp != NULL)

examples/custom_parser_musa/musa_parser.cpp

@@ -128,10 +128,14 @@ Memory* MusaParser::ParseMemory() {
 	int pos1 = input.find("_");
 	int pos2 = input.find("_", pos1+1);
 	std::string output;
-    if(pos2 == std::string::npos) //no second underscore
+    if(pos2 == std::string::npos) {//no second underscore
         output = input.substr(pos1 + 1);
-    else
-        output = input.substr(pos1 + 1, pos2 - pos1 - 1);
+	}
+    else{
+		output = input.substr(pos1 + 1, pos2 - pos1 - 1);
+	}
+
+
 	long long size = stol(output);
     long long channels = stol(mapping["RAMULATOR"]["channels"]);
     if(channels > 0)

examples/custom_parser_musa/use_custom_parser.cpp

@@ -40,7 +40,7 @@ int main(int argc, char *argv[])
         return 1;
     }
     cout << "-- End parseMusa" << endl;
-    cout << "Total num HW threads: " << topo->GetNumThreads() << endl;
+    cout << "Total num HW threads: " << topo->CountAllSubcomponentsByType(SYS_SAGE_COMPONENT_THREAD) << endl;
 
     cout << "---------------- Printing the whole tree ----------------" << endl;
     topo->PrintSubtree(2);

examples/larger_topo.cpp

@@ -19,16 +19,16 @@ int main(int argc, char *argv[])
     std::string path_prefix(argv[0]);
     std::size_t found = path_prefix.find_last_of("/\\");
     path_prefix=path_prefix.substr(0,found) + "/";
-    string topoPath = "example_data/skylake_hwloc.xml";
+    string xmlPath = "example_data/skylake_hwloc.xml";
     string bwPath = "example_data/skylake_caps_numa_benchmark.csv";
     for(int n_idx=0; n_idx<tot_nodes; n_idx++)
     {
         Node* n = new Node(n_idx);
         n->SetParent((Component*)topo);
         topo->InsertChild((Component*)n);
-        if(parseHwlocOutput(n, path_prefix+topoPath) != 0)
+        if(parseHwlocOutput(n, path_prefix+xmlPath) != 0)
         {
-            cout << "error parsing hwloc in path " << path_prefix+topoPath << endl;
+            cout << "error parsing hwloc in path " << path_prefix+xmlPath << endl;
             return 1;
         }
         if(parseCapsNumaBenchmark((Component*)n, path_prefix+bwPath, ";") != 0)