pmdabcc: add USDT JVM threads module

Add a USDT module collecting metrics from JVM thread starts/stops.
This module does not require any specific Java parameters to work.
Could be used as an example / reference or for collecting metrics.

Wrt the return value in the BPF program even in the unlikely case
that lookup fails, iovisor/bcc#139 has
explanation on BPF program values and it should be OK for BCC PMDA
modules to use 0 (by default, an admin could of course change it).

qa/1151

@@ -0,0 +1,78 @@
+#!/bin/sh
+# PCP QA Test No. 1151
+# Exercise the BCC PMDA USDT JVM threads module - install, remove and values.
+#
+# Copyright (c) 2018 Red Hat.
+#
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+. ./common.bcc
+
+_pmdabcc_check
+
+status=1       # failure is the default!
+signal=$PCP_BINADM_DIR/pmsignal
+$sudo rm -rf $tmp.* $seq.full
+
+jvm_path=""
+if [ -f /etc/alternatives/jre/lib/server/libjvm.so ]
+then
+    jvm_path=/etc/alternatives/jre/lib/server/libjvm.so
+fi
+if [ -z "$jvm_path" ]
+then
+    java_major=$(java -version 2>&1 | sed -e 's,openjdk version "1.,,' -e 's,\..*,,')
+    jvm_path=/usr/lib/jvm/java-${java_major}-openjdk-amd64/jre/lib/amd64/server/libjvm.so
+fi
+[ -n "$jvm_path" ] || _notrun "libjvm.so not found"
+
+_log_filter()
+{
+    sed -e 's,Using PID.*,Found a Java process.,g'
+}
+
+_value_filter()
+{
+    awk '/value [1-9]\d*/ {print "OK"; exit}'
+}
+
+_prepare_pmda bcc
+trap "_pmdabcc_cleanup; exit \$status" 0 1 2 3 15
+_stop_auto_restart pmcd
+
+# Launch a Java test program
+cd java
+java USDTJVMTest &
+javapid=$!
+disown
+cd ..
+
+# real QA test starts here
+cat <<EOF | _pmdabcc_install | _log_filter
+[pmda]
+modules = usdt_jvm_threads
+prefix = bcc.
+[usdt_jvm_threads]
+module = usdt_jvm_threads
+cluster = 110
+jvm_path = $jvm_path
+process = java
+EOF
+_pmdabcc_wait_for_metric
+
+# Wait for values to appear
+pmsleep 1
+
+echo "=== report metric values ==="
+pminfo -dfmtT bcc.usdt.jvm.threads.start 2>&1 | tee -a $here/$seq.full \
+| _value_filter
+pminfo -dfmtT bcc.usdt.jvm.threads.stop 2>&1 | tee -a $here/$seq.full \
+| _value_filter
+
+kill -9 $javapid > /dev/null 2>&1
+_pmdabcc_remove 2>&1
+
+status=0
+exit

qa/1151.out

@@ -0,0 +1,56 @@
+QA output created by 1151
+
+=== bcc agent installation ===
+Info: Initializing, currently in 'notready' state.
+Info: Enabled modules:
+Info: ['usdt_jvm_threads']
+Info: Configuring modules:
+Info: usdt_jvm_threads
+Info: Modules configured.
+Info: Initializing modules:
+Info: usdt_jvm_threads
+Info: usdt_jvm_threads: Found a Java process.
+Info: usdt_jvm_threads: Initialized.
+Info: Modules initialized.
+Info: Registering metrics:
+Info: usdt_jvm_threads
+Info: Metrics registered.
+Info: Registering helpers:
+Info: Helpers registered.
+Info: Ready to process requests.
+Info: Initializing, currently in 'notready' state.
+Info: Enabled modules:
+Info: ['usdt_jvm_threads']
+Info: Configuring modules:
+Info: usdt_jvm_threads
+Info: Modules configured.
+Info: Initializing modules:
+Info: usdt_jvm_threads
+Info: usdt_jvm_threads: Found a Java process.
+Info: usdt_jvm_threads: Initialized.
+Info: Modules initialized.
+Info: Registering metrics:
+Info: usdt_jvm_threads
+Info: Metrics registered.
+Info: Registering helpers:
+Info: Helpers registered.
+Info: Ready to process requests.
+Updating the Performance Metrics Name Space (PMNS) ...
+Terminate PMDA if already installed ...
+[...install files, make output...]
+Updating the PMCD control file, and notifying PMCD ...
+Check bcc metrics have appeared ... 2 metrics and 2 values
+
+=== report metric values ===
+OK
+OK
+
+=== remove bcc agent ===
+Culling the Performance Metrics Name Space ...
+bcc ... done
+Updating the PMCD control file, and notifying PMCD ...
+[...removing files...]
+Check bcc metrics have gone away ... OK
+Waiting for pmcd to terminate ...
+Starting pmcd ... 
+Starting pmlogger ... 

qa/group

@@ -1477,6 +1477,7 @@ BAD
 1141 selinux local
 1144 pmda.proc local
 1150 pmda.bcc local
+1151 pmda.bcc local
 1159 pmlogger local pmdumplog
 1164 pmda.linux local
 1166 libpcp local

qa/java/USDTJVMTest.java

@@ -0,0 +1,39 @@
+/*
+ * PCP BCC PMDA USDT JVM test helper
+ */
+
+import java.util.concurrent.ThreadLocalRandom;
+
+class TestUnit implements Runnable {
+    private int lifetime;
+
+    public TestUnit(int lifetime) {
+        this.lifetime = lifetime * 100;
+    }
+
+    public void run() {
+        int i = 0;
+        while (lifetime-- > 0) {
+            i++;
+            try {
+                Thread.sleep(10);
+            } catch (Exception ex) { ex.printStackTrace(); }
+        }
+    }
+}
+
+public class USDTJVMTest {
+    public static void main(String[] args) {
+        int i = 0;
+        while (true) {
+            i++;
+            try {
+                Thread.sleep(100);
+            } catch (Exception ex) { ex.printStackTrace(); }
+
+            int lifetime = ThreadLocalRandom.current().nextInt(1, 21);
+            Thread t = new Thread(new TestUnit(lifetime));
+            t.start();
+        }
+    }
+}

src/pmdas/bcc/bcc.conf

@@ -16,6 +16,7 @@
 modules = biolatency,sysfork,tcplife,runqlat,ext4dist
 #modules = biotop
 #modules = tracepoint_hits,usdt_hits,uprobe_hits
+#modules = usdt_jvm_threads
 prefix = bcc.
 
 
@@ -75,3 +76,10 @@ cluster = 102
 #process = 1
 uprobes = c:malloc,c:strlen
 #uprobes = bcc-uprobe.conf
+
+[usdt_jvm_threads]
+module = usdt_jvm_threads
+cluster = 110
+jvm_path = /etc/alternatives/jre/lib/server/libjvm.so
+# Mandatory for now - https://github.com/iovisor/bcc/issues/1774
+process = java

src/pmdas/bcc/modules/usdt_jvm_threads.bpf

@@ -0,0 +1,21 @@
+// Copyright (C) 2018 Marko Myllynen <[email protected]>
+// Licensed under the Apache License, Version 2.0 (the "License")
+
+#include <uapi/linux/ptrace.h>
+
+BPF_ARRAY(stats, u64, 2);
+
+static void inc(int key) {
+    u64 *val = stats.lookup(&key);
+    if (val) (*val)++;
+}
+
+int trace_jvm_thread_start(struct pt_regs *ctx) {
+    inc(0);
+    return 0;
+}
+
+int trace_jvm_thread_stop(struct pt_regs *ctx) {
+    inc(1);
+    return 0;
+}

src/pmdas/bcc/modules/usdt_jvm_threads.python

@@ -0,0 +1,115 @@
+#
+# Copyright (C) 2018 Marko Myllynen <[email protected]>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+""" PCP BCC PMDA USDT JVM threads module """
+
+# Configuration options
+# Name - type - default
+#
+# jvm_path - string -  [1]  : path to libjvm.so
+# process  - string - unset : list of names/pids or regex of processes to monitor
+#
+# 1) Default for jvm_path: /etc/alternatives/jre/lib/server/libjvm.so
+
+# pylint: disable=invalid-name
+
+from ctypes import c_int
+from bcc import BPF, USDT
+
+from pcp.pmapi import pmUnits
+from cpmapi import PM_TYPE_U64, PM_SEM_COUNTER, PM_COUNT_ONE
+from cpmapi import PM_ERR_AGAIN
+from modules.pcpbcc import PCPBCCBase
+
+#
+# BPF program
+#
+bpf_src = "modules/usdt_jvm_threads.bpf"
+
+#
+# PCP BCC PMDA constants
+#
+MODULE = 'usdt_jvm_threads'
+BASENS = 'usdt.jvm.threads.'
+units_count = pmUnits(0, 0, 1, 0, 0, PM_COUNT_ONE)
+
+#
+# PCP BCC Module
+#
+class PCPBCCModule(PCPBCCBase):
+    """ PCP BCC JVM threads module """
+    def __init__(self, config, log, err):
+        """ Constructor """
+        PCPBCCBase.__init__(self, MODULE, config, log, err)
+
+        self.stats = [0, 0]
+        self.pid = None
+        self.jvm_path = "/etc/alternatives/jre/lib/server/libjvm.so"
+
+        for opt in self.config.options(MODULE):
+            if opt == 'jvm_path':
+                self.jvm_path = self.config.get(MODULE, opt)
+            if opt == 'process':
+                procs = self.get_proc_info(self.config.get(MODULE, opt))
+                if len(procs) > 1:
+                    self.log("Several PIDs found, using the first one.")
+                if procs:
+                    self.log("Using PID %s: %s %s" % (str(procs[0][0]), procs[0][1], procs[0][2]))
+                    self.pid = procs[0][0]
+
+        if self.pid is None:
+            # https://github.com/iovisor/bcc/issues/1774
+            self.err("No process to attach found/set, will not activate!")
+
+        self.log("Initialized.")
+
+    def metrics(self):
+        """ Get metric definitions """
+        name = BASENS
+        self.items = (
+            # Name - reserved - type - semantics - units - help
+            (name + 'start', None, PM_TYPE_U64, PM_SEM_COUNTER, units_count, 'thread starts'),
+            (name + 'stop' , None, PM_TYPE_U64, PM_SEM_COUNTER, units_count, 'thread stops'),
+        )
+        return False, self.items
+
+    def compile(self):
+        """ Compile BPF """
+        try:
+            if self.pid is None:
+                # https://github.com/iovisor/bcc/issues/1774
+                raise RuntimeError("No PID found/set.")
+            usdt = USDT(pid=self.pid)
+            usdt.enable_probe("thread__start", "trace_jvm_thread_start")
+            usdt.enable_probe("thread__stop", "trace_jvm_thread_stop")
+            self.bpf = BPF(src_file=bpf_src, usdt_contexts=[usdt])
+            self.log("Compiled.")
+        except Exception as error: # pylint: disable=broad-except
+            self.err(str(error))
+            self.err("Module NOT active!")
+            self.bpf = None
+
+    def refresh(self):
+        """ Refresh BPF data """
+        if self.bpf is None:
+            return None
+
+        self.stats[0] = self.bpf["stats"][c_int(0)].value
+        self.stats[1] = self.bpf["stats"][c_int(1)].value
+
+    def bpfdata(self, item, inst):
+        """ Return BPF data as PCP metric value """
+        try:
+            return [self.stats[item], 1]
+        except Exception: # pylint: disable=broad-except
+            return [PM_ERR_AGAIN, 0]