Merge branch 'devel' into 'master'

MetaCG Release v0.4.0

See merge request tuda-sc/projects/metacg!67

.gitlab-ci.yml

@@ -62,7 +62,8 @@ mcg-container:
   script:
     - podman login ${CI_REGISTRY} -u ${CI_CONTAINER_REG_USER} -p ${CONTAINER_REGISTRY_READ_TOKEN}
     - podman build -t registry.git.rwth-aachen.de/tuda-sc/projects/metacg/metacg:$CI_COMMIT_SHA -f container/metacg .
-    - podman run --rm -t metacg:$CI_COMMIT_SHA /opt/metacg/metacg/build/pgis/test/unit/basictests
+    - podman run --rm -t metacg:$CI_COMMIT_SHA /opt/metacg/metacg/build/pgis/test/unit/pgistests
+    - podman run --rm -t metacg:$CI_COMMIT_SHA /opt/metacg/metacg/build/graph/test/unit/libtests
     - podman image rm metacg:$CI_COMMIT_SHA
 
 build-mcg:
@@ -74,7 +75,7 @@ build-mcg:
     GIT_STRATEGY: none
     GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_SHA
   script:
-    - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=$(dirname $(which cube_info))/../lib -DCUBE_INCLUDE=$(dirname $(which cube_info))/../include/cubelib -DEXTRAP_INCLUDE=./extern/src/extrap/extrap-3.0/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON
+    - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=$(dirname $(which cube_info))/../lib -DCUBE_INCLUDE=$(dirname $(which cube_info))/../include/cubelib -DEXTRAP_INCLUDE=./extern/install/extrap/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON
     - cmake --build build --parallel
 
 test-cgc:
@@ -119,6 +120,16 @@ install-mcg:
     - /tmp/metacg/bin/cgcollector --help
     - rm -r /tmp/metacg
 
+test-graphlib:
+  stage: test
+  tags:
+    - general
+  before_script: *lb-setup
+  variables:
+    GIT_STRATEGY: none
+    GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_SHA
+  script:
+    - cd build/graph/test/unit && ./libtests
 
 test-pgis:
   stage: test
@@ -129,7 +140,7 @@ test-pgis:
     GIT_STRATEGY: none
     GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_COMMIT_SHA
   script:
-    - cd build/pgis/test/unit && ./basictests
+    - cd build/pgis/test/unit && ./pgistests
 
 test-static-pgis:
   stage: integration-test

AUTHORS

@@ -15,3 +15,6 @@ Constantin Völter <Scientific Computing>
 Peter Arzt <Scientific Computing>
 Julian Hindelang <Scientific Computing>
 Jonas Rickert <Scientific Computing>
+Sebastian Kreutzer <Scientific Computing>
+Tim Heldmann <Scientific Computing>
+

CMakeLists.txt

@@ -19,7 +19,7 @@ endif()
 # End of require out-of-source builds
 
 # Project information
-set(METACG_VERSION 0.3.0)
+set(METACG_VERSION 0.4.0)
 project(
   MetaCG
   VERSION ${METACG_VERSION}
@@ -29,7 +29,7 @@ project(
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_VERBOSE_MAKEFILE ON)
 
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -48,6 +48,21 @@ include(CMakePackageConfigHelpers)
 # Making printing / debugging easier
 include(CMakePrintHelpers)
 
+# Obtain ghe git revision of this MetaCG repository
+include(GetGitRevisionDescription)
+
+# Stream the verison info in the project's config.h
+get_git_head_revision(GIT_REFSPEC GIT_SHA1)
+if(NOT
+   DEFINED
+   GIT_SHA1
+)
+  set(GIT_SHA1 "NO_GIT_ROOT")
+endif()
+message(${GIT_SHA1})
+
+configure_file(Config.h.in config.h)
+
 # Component options MetaCG graph library will always be built. The actual graph implementation
 add_subdirectory(graph)
 
@@ -70,3 +85,17 @@ option(
 if(WITH_PGIS)
   add_subdirectory(pgis)
 endif()
+
+# If set to on, CMake looks for installed nlohmann::json
+option(
+  METACG_USE_EXTERNAL_JSON
+  "On or off"
+  OFF
+)
+
+# if set to on, CMake looks for installed spdlog
+option(
+  METACG_USE_EXTERNAL_SPDLOG
+  "On or off"
+  OFF
+)

CONTRIBUTING.md

@@ -6,25 +6,24 @@ In case you want to contribute, please reach out through the issues and we see h
 
 ## Issues
 
-We are aware that metacg has open issues, i.e., a correct handling of function pointers, etc.
+We are aware that MetaCG has open issues, i.e., a correct handling of function pointers, etc.
 If you find a new issue that is not already reported in github, please feel free to open an issue.
 The issue should state clearly what is the problem, and what you did to get there.
 
-If you can, please provide a MWE in the issue, and the tool output.
-Please also share, what you have the tool expected to do.
+If you can, please provide a minimal working example in the issue, and the tool output.
+Please also share, what you expected the tool to do.
 
 ## Workflow
 
-Please contribute only via pull requests and (a) keep your commit history clean, or (b) it will be squashed during merge.
+We currently accept changes via merge requests in our non-public repository.
+All history will be squashed in the merge.
 
-**Internal**: Please branch from `master` for fixes and from `devel` for researchy new stuff.
-`master` is automatically mirrored to github, so, please, only merge to `master`, if the CI builds succeed.
-Preceed a branch either with `fix/` or with `feat/` to indicate it.
+**Internal**: Please always branch from `devel`, and only in very particular bugfix cases (really critical ones) ask in our slack what to do.
+Preceed a branch either with `fix/` or with `feat/` to indicate what it is meant to do, followed by the issue number.
 
 ## Tests
 
 Every contribution must add reasonable tests.
-(We use a Gitlab internally for development, so the github CI / actions are not yet set up.)
 
 ## Commit messages
 

README.md

@@ -51,31 +51,38 @@ Examples are empirically determined performance models, runtime measurements, or
 ## Requirements / Building
 
 MetaCG consists of the graph library, a CG construction tool, and an example analysis tool.
-Currently, all components are built.
+The graph library is always built.
+The CG construction tool `CGCollector` can be disabled via the CMake option `-DWITH_CGCOLLECTOR=OFF`.
+The example analysis tool `PGIS` can be disabled via the CMake option `-DWITH_PGIS=OFF`.
 
-**Build Requirements**
+**Build Requirements (for full build)**
 - Clang/LLVM version 10.
-- nlohmann/json library [github](https://github.com/nlohmann/json)
 - Cube 4.5 [scalasca.org](https://www.scalasca.org/software/cube-4.x/download.html)
+- Extra-P 3.0 [.tar.gz](http://apps.fz-juelich.de/scalasca/releases/extra-p/extrap-3.0.tar.gz)
+- nlohmann/json library [github](https://github.com/nlohmann/json)
 - spdlog [github](https://github.com/gabime/spdlog)
 - cxxopts [github](https://github.com/jarro2783/cxxopts)
-- Extra-P 3.0 [.tar.gz](http://apps.fz-juelich.de/scalasca/releases/extra-p/extrap-3.0.tar.gz)
 - PyQt5
 - matplotlib
 
-### Building the package
+### Building
 
-Currently, one first needs to build the dependencies for PGIS.
-This can be done by running the provided convenience script, which we use in a Ubuntu 20.04 container and on a RedHat 8.5 (with certain LMod modules).
-This will build and install the required software into `./deps/src` and `./deps/install`, respectively.
-No guarantees that the script works on your machine.
+Clang/LLVM (v 10) and the cube library are assumed to be available on the system.
+Extra-P can be built using the `build_submodules.sh` script provided in the repository, though it is not tested outside of our CI system.
+It builds and installs Extra-P into `./deps/src` and `./deps/install`, respectively.
+
+The `nlohmann-json`, `spdlog` and `cxxopts` library are downloaded at configure time, unless `-DMETACG_USE_EXTERNAL_JSON=ON` or `-DMETACG_USE_EXTERNAL_SPDLOG=ON` is given.
+This will search for the respective libraries in the common CMake locations.
+While CMake looks for `nlohmann-json` version 3.10., MetaCG should work starting from version 3.9.2.
+For spdlog, we rely on version 1.8.2 -- other versions *may* work.
 
 ```{.sh}
-?> cd pgis
 ?> ./build_submodules.sh
 ```
 
 Thereafter, the package can be configured and built from the top-level CMake.
+Change the `CMAKE_INSTALL_PREFIX` to where you want your MetaCG installation to live.
+Providing `SPDLOG_SHARED=ON` is necessary to build the shared object version of spdlog and prevent linker errors.
 
 ```{.sh}
 ?> cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INSTALL_PREFIX=/tmp/metacg -DCUBE_LIB=$(dirname $(which cube_info))/../lib -DCUBE_INCLUDE=$(dirname $(which cube_info))/../include/cubelib -DEXTRAP_INCLUDE=./extern/src/extrap/extrap-3.0/include -DEXTRAP_LIB=./extern/install/extrap/lib -DSPDLOG_BUILD_SHARED=ON -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON
@@ -84,6 +91,17 @@ Thereafter, the package can be configured and built from the top-level CMake.
 ?> cmake --install build
 ```
 
+### CMake Options
+
+- Path `CUBE_LIB`: Path to the libcube library directory
+- Path `CUBE_INCLUDE`: Path to the libcube include directory
+- Path `EXTRAP_LIB`: Path to the Extra-P library directory
+- Path `EXTRAP_INCLUDE`: Path to the Extra-P include directory
+- Bool `WITH_CGCOLLECTOR`: Whether to build call-graph construction tool
+- Bool `WITH_PGIS`: Whether to build demo-analysis tool
+- Bool `METACG_USE_EXTERNAL_JSON`: Search for installed version of nlohmann-json
+- Bool `METACG_USE_EXTERNAL_SPDLOG`: Search for installed version of spdlog
+
 ## Usage
 
 ### Graph Library

build_submodules.sh

@@ -1,8 +1,8 @@
 #! /usr/bin/env bash
 #"""
 # File: build_submodules.sh
-# License: Part of the PIRA project. Licensed under BSD 3 clause license. See LICENSE.txt file at https://github.com/jplehr/pira/LICENSE.txt
-# Description: Helper script to build the git submodules useed in PIRA.
+# License: Part of the MetaCG project. Licensed under BSD 3 clause license. See LICENSE.txt file at https://github.com/tudasc/MetaCG/LICENSE.txt
+# Description: Helper script to build the git submodules useed in MetaCG.
 #"""
 
 scriptdir="$(
@@ -20,12 +20,12 @@ parallel_jobs="$1"
 add_flags="$2"
 
 # Extra-P (https://www.scalasca.org/software/extra-p/download.html)
-echo "[PIRA] Building Extra-P (for PIRA II modeling)"
-echo "[PIRA] Getting prerequisites ..."
+echo "[MetaCG] Building Extra-P (for PIRA II modeling)"
+echo "[MetaCG] Getting prerequisites ..."
 pip3 install --user PyQt5 2>&1 >/dev/null
 pip3 install --user matplotlib 2>&1 >/dev/null
 if [ $? -ne 0 ]; then
-  echo "[PIRA] Installting Extra-P dependencies failed."
+  echo "[MetaCG] Installting Extra-P dependencies failed."
   exit 1
 fi
 
@@ -34,7 +34,7 @@ cd $extsourcedir/extrap
 
 # TODO check if extra-p is already there, if so, no download / no build?
 if [ ! -f "extrap-3.0.tar.gz" ]; then
-  echo "[PIRA] Downloading and building Extra-P"
+  echo "[MetaCG] Downloading and building Extra-P"
   wget http://apps.fz-juelich.de/scalasca/releases/extra-p/extrap-3.0.tar.gz
 fi
 tar xzf extrap-3.0.tar.gz
@@ -46,35 +46,27 @@ mkdir build && cd build
 # but at least with python 3.9 on ubuntu it is a bit buggy and some distributions don't support it at all
 pythonheader=$(python3 -c "from sysconfig import get_paths; print(get_paths()[\"include\"])")
 if [ -z $pythonheader ]; then
-  echo "[PIRA] Python header not found."
+  echo "[MetaCG] Python header not found."
   exit 1
 fi
-echo "[PIRA] Found Python.h at " $pythonheader
+echo "[MetaCG] Found Python.h at " $pythonheader
 ../configure --prefix=$extinstalldir/extrap CPPFLAGS=-I$pythonheader 2>&1 >/dev/null
 if [ $? -ne 0 ]; then
-  echo "[PIRA] Configuring Extra-P failed."
+  echo "[MetaCG] Configuring Extra-P failed."
   exit 1
 fi
+
 make -j $parallel_jobs 2>&1 >/dev/null
 if [ $? -ne 0 ]; then
-  echo "[PIRA] Building Extra-P failed."
+  echo "[MetaCG] Building Extra-P failed."
   exit 1
 fi
-make install 2>&1 >/dev/null
 
-# CXX Opts
-echo "[PIRA] Getting cxxopts library"
-cd $extsourcedir
-if [ ! -d "$extsourcedir/cxxopts" ]; then
-  git clone --branch 2_1 --depth 1 https://github.com/jarro2783/cxxopts cxxopts 2>&1 >/dev/null
+make install 2>&1 >/dev/null
+if [ $? -ne 0 ]; then
+  echo "[MetaCG] Installing Extra-P failed."
+  exit 1
 fi
-#cd cxxopts
-#echo "[PIRA] Select release branch 2_1 for cxxopts."
-#git checkout 2_1 2>&1 >/dev/null
 
-# JSON library
-echo "[PIRA] Getting json library"
-cd $extsourcedir
-if [ ! -d "$extsourcedir/json" ]; then
-  git clone --depth 1 --branch v3.9.1 https://github.com/nlohmann/json json 2>&1 >/dev/null
-fi
+mkdir $extinstalldir/extrap/include
+cp $extsourcedir/extrap/extrap-3.0/include/* $extinstalldir/extrap/include

cgcollector/CMakeLists.txt

@@ -1,10 +1,6 @@
 set(PROJECT_NAME CGCollector)
 set(TARGETS_EXPORT_NAME ${PROJECT_NAME}-target)
 
-function(add_config_include target)
-  target_include_directories(${target} PUBLIC "${PROJECT_BINARY_DIR}")
-endfunction()
-
 # include directories
 function(add_collector_include target)
   target_include_directories(
@@ -49,19 +45,6 @@ function(default_compile_options target)
   endif()
 endfunction()
 
-include(GetGitRevisionDescription)
-
-get_git_head_revision(GIT_REFSPEC GIT_SHA1)
-if(NOT
-   DEFINED
-   GIT_SHA1
-)
-  set(GIT_SHA1 "NO_GIT_ROOT")
-endif()
-message(${GIT_SHA1})
-
-configure_file(Config.h.in config.h)
-
 add_subdirectory(lib)
 add_subdirectory(tools)
 add_subdirectory(test)

cgcollector/tools/CGMerge.cpp

@@ -47,7 +47,7 @@ nlohmann::json mergeFileFormatTwo(std::string wholeCGFilename, std::vector<std::
   }
 
   // {
-  // "_Meta": {...}, <-- Gets attached by "attachFormatTwoHeader"
+  // "_Meta": {...}, <-- Gets attached by "attachMCGFormatHeader"
   // "_CG": { /** Gets constructed in the for loop **/ }
   // }
   attachFormatTwoHeader(wholeCGFinal);
@@ -270,8 +270,26 @@ int main(int argc, char **argv) {
   }
 
   nlohmann::json j;
-  readIPCG(inputFiles.front(), j);
-  if (j.contains("_CG")) {
+  bool foundValidFile{false};
+  bool useFileFormatTwo{false};
+
+  for (auto &inFile : inputFiles) {
+    readIPCG(inFile, j);
+    if (!j.is_null()) {
+      foundValidFile = true;
+      if (j.contains("_CG")) {
+        useFileFormatTwo = true;
+      }
+      break;
+    }
+  }
+
+  if (!foundValidFile) {
+    std::cerr << "[Error] All input files are NULL" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+
+  if (useFileFormatTwo) {
     auto wholeCG = mergeFileFormatTwo(argv[1], inputFiles);
     writeIPCG(argv[1], wholeCG);
   } else {