common: a complete stack analysis toolchain

A comprehensive analysis of stack sizes requires several steps. To facilitate this process, these steps have been aggregated into a single script run_call_stacks_analysis.sh. In addition, it is possible to perform this operation based on the set of functions used in the DAOS project. The cflow program, a key component of stack analysis, is not available by default on Linux. The install_cflow.sh script solves this problem by downloading the sources, compiling and then installing the cflow program on the system. Both scripts has been integrated into their respective GHA workflows to avoid code redundancy. Scan GHA workflow shall be run every PR to detect an issue before it is integrated into main branch of PMDK source code. Signed-off-by: Tomasz Gromadzki <[email protected]>
pmem · Mar 10, 2025 · aa52446 · aa52446
1 parent 2da26b7
commit aa52446
Show file tree

Hide file tree

Showing 8 changed files with 121 additions and 25 deletions.
diff --git a/.github/workflows/scan_stack_usage.yml b/.github/workflows/scan_stack_usage.yml
@@ -26,6 +26,11 @@ jobs:
         run: |
           sudo apt-get install libdaxctl-dev libndctl-dev
 
+      - name: Install cflow
+        working-directory: ${{ env.CALL_STACKS_TOOLS_PATH }}
+        run: |
+          ./install_cflow.sh
+
       - name: Build PMDK
         env:
           # just to speed up the job
@@ -35,32 +40,10 @@ jobs:
           cd pmdk/src
           make -j$(nproc)
 
-      - name: Install cflow
-        run: |
-          wget https://mirror.easyname.at/gnu/cflow/cflow-latest.tar.gz
-          tar -xzf cflow-latest.tar.gz
-          rm cflow-latest.tar.gz
-          echo cflow-*
-          cd cflow-*
-          ./configure
-          make
-          sudo make install
-
       - name: Make call stacks
         working-directory: ${{ env.CALL_STACKS_TOOLS_PATH }}
         run: |
-          ./make_stack_usage.sh
-          ./make_api.sh
-          ./make_extra.py
-          ./make_cflow.sh
-          # The lower limit comes up from the DAOS memory requirements.
-          # 16kB - 4kB - 720B = 11568B
-          # 16kB = Stack allocated for a single Argobot's ULT
-          #  4kB = a maximum DAOS' stack usage up to calling a PMDK API calls
-          #  720B = safety margin
-          #    ~ = Some OSes, e.g. Ubuntu 22.04, generate call stacks of size
-          # a little bit over the exact limit which is not deemed a problem at the moment.
-          ./make_call_stacks.py --filter-api-file examples/api_filter.txt --filter-lower-limit 11568
+          ./run_call_stacks_analysis.sh
 
       - name: List per-function stack usage for all call stacks of interest
         working-directory: ${{ env.CALL_STACKS_TOOLS_PATH }}

diff --git a/.github/workflows/scans.yml b/.github/workflows/scans.yml
@@ -6,6 +6,7 @@ on:
   schedule:
     # run this job at 00:00 UTC every day
     - cron:  '0 0 * * *'
+  pull_request:
 
 permissions: {}
 

diff --git a/utils/call_stacks_analysis/README.md b/utils/call_stacks_analysis/README.md
@@ -17,6 +17,11 @@
 ./make_call_stacks.py --filter-api-file examples/api_filter.txt \
     --filter-lower-limit 11568 --dump-all-stacks
 ```
+or
+
+```sh
+./run_call_stack_analysis.py
+```
 
 If succesfull, it produces:
 

diff --git a/utils/call_stacks_analysis/examples/api_filter.txt b/utils/call_stacks_analysis/examples/api_filter.txt
@@ -8,6 +8,7 @@ pmemobj_ctl_get
 pmemobj_ctl_set
 pmemobj_defer_free
 pmemobj_direct
+pmemobj_errormsg
 pmemobj_flush
 pmemobj_free
 pmemobj_memcpy_persist
@@ -16,6 +17,7 @@ pmemobj_reserve
 pmemobj_root
 pmemobj_tx_abort
 pmemobj_tx_add_range
+pmemobj_tx_add_range_direct
 pmemobj_tx_begin
 pmemobj_tx_commit
 pmemobj_tx_end

diff --git a/utils/call_stacks_analysis/install_cflow.sh b/utils/call_stacks_analysis/install_cflow.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2025, Hewlett Packard Enterprise Development LP
+#
+#
+# Install cflow tool
+#
+
+function main() {
+	if  which "cflow" >/dev/null 2>&1; then
+		echo "cflow already installed"
+		#return 0
+	fi
+	wget "https://mirror.easyname.at/gnu/cflow/cflow-latest.tar.gz" && \
+		tar -xvzf cflow-latest.tar.gz && \
+		rm cflow-latest.tar.gz
+	echo $?
+	if [ $? != 0 ]; then
+		echo "Can not download cflow source code"
+		return 1
+	fi
+	pushd cflow-*
+	./configure && make && sudo make install
+	if [ $? != 0 ]; then
+		popd
+		echo "Can not install cflow"
+		return 1
+	fi
+	popd
+	rm -rf cflow-*
+	return 0
+}
+
+main
+
+exit $?
diff --git a/utils/call_stacks_analysis/make_cflow.sh b/utils/call_stacks_analysis/make_cflow.sh
@@ -12,6 +12,12 @@ UNSAFE=$1 # '-f' to omit security checks
 WD=$(realpath $(dirname "$0"))
 SRC=$(realpath $WD/../../src)
 
+if ! which "cflow" >/dev/null 2>&1; then
+	echo "cflow is not found in the PATH, please check if it is available."
+	echo "If not, please install it using the install_cflow.sh script."
+	exit 1
+fi
+
 API=$WD/api.txt
 if [ ! -f "$API" ]; then
 	echo "$API is missing"

diff --git a/utils/call_stacks_analysis/make_stack_usage.sh b/utils/call_stacks_analysis/make_stack_usage.sh
@@ -12,12 +12,12 @@ WD=$(realpath $(dirname $0))
 TOP=$(realpath $WD/../..)
 SRC=src/$BUILD
 
-cd $TOP
+pushd $TOP
 
 SU_FILES="$SRC/core/*.su $SRC/common/*.su $SRC/libpmem/*.su $SRC/libpmemobj/*.su"
 
 grep -v ^$ $SU_FILES | \
 	gawk -F "[:\t]" '{print $6 " " $5 " : " $1 ":" $2 " " $7}' | \
 	sort -n -r > $WD/stack_usage.txt
 
-cd - > /dev/null
+popd > /dev/null
diff --git a/utils/call_stacks_analysis/run_call_stacks_analysis.sh b/utils/call_stacks_analysis/run_call_stacks_analysis.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright 2025, Hewlett Packard Enterprise Development LP
+#
+#
+# Execute complete call stack analysis based on API subset defined in examples/api_filter.txt.
+#
+# Update the examples/api_filter.txt file based on the actual DAOS source code. (Optional)
+#
+
+function help() {
+        echo "usage:"
+        echo "       $0 [-h|--help] [-d|--daos]"
+        echo
+        echo "       -h/--help    print help"
+        echo "       -d/--daos    use the actual pmemobj API calls from the DAOS source code"
+        echo
+}
+
+function main() {
+        local update=$1
+        local filter_name="examples/api_filter.txt"
+
+        if [ $update == 1 ]; then
+                wget -q https://github.com/daos-stack/daos/archive/refs/heads/master.zip -O daos.zip
+                unzip -q -o daos.zip
+                rm daos.zip
+                grep -r -E -h -o 'pmemobj_[^(]*\(' 'daos-master/src' | sed 's/(.*$//' | \
+                        sort | uniq > $filter_name
+                rm -rf daos-master
+        fi
+
+        # The lower limit comes up from the DAOS memory requirements.
+        # 16kB - 4kB - 720B = 11568B
+        # 16kB = Stack allocated for a single Argobot's ULT
+        #  4kB = a maximum DAOS' stack usage up to calling a PMDK API calls
+        #  720B = safety margin
+        #    ~ = Some OSes, e.g. Ubuntu 22.04, generate call stacks of size
+        # a little bit over the exact limit which is not deemed a problem at the moment.
+
+        ./make_stack_usage.sh && \
+                ./make_api.sh && \
+                ./make_extra.py && \
+                ./make_cflow.sh && \
+                ./make_call_stacks.py --filter-api-file $filter_name --filter-lower-limit 11568 \
+                        --dump-all-stacks
+        return $?
+}
+
+if [[ "$1" == "-h" || "$1" == "--help" ]]; then
+        help
+        exit 1
+fi
+
+if [[ "$1" == "-d" || "$1" == "--daos" ]]; then
+        daos=1
+else
+        daos=0
+fi
+
+main $daos
+
+exit $?