performance monitoring with pcp

[stefwalter]

It would be desirable to have some support for monitoring performance. The framework called Performance Co-Pilot (pcp), which could serve as a basis for this.

​http://oss.sgi.com/projects/pcp/index.html

[stefwalter]

Example pcp monitor from mvollmer: http://pastebin.com/CPKcmuzr

[stefwalter]

From mvollmer: What we get from pcp is:

• API guarantee, no danger that /proc/net/dev changes format without us noticing.
• Tons more metrics so adding new ones is very easy.
• Background data logging so that we have historical data without needing to keep rhscd running.

Initially, I think we should concentrate on the presentation side (nice graphs, nice layout), and pcp doesn't help much with that. Let's stick with our current code for a while.

[mvollmer]

Code from pastebin, in case it goes away.

    /* gcc -Wall -g -o monitor monitor.c -lpcp */

    /*
     * Copyright (c) 2012 Red Hat.
     *
     * This program is free software; you can redistribute it and/or modify it
     * under the terms of the GNU General Public License as published by the
     * Free Software Foundation; either version 2 of the License, or (at your
     * option) any later version.
     *
     * This program is distributed in the hope that it will be useful, but
     * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
     * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
     * for more details.
     */

    #include <pcp/pmapi.h>

    static char * metricnames[] = {
    #define MEM 0
        "swap.used",            
        "mem.util.free",        
        "mem.util.used",        
        "mem.util.bufmem",      
        "mem.util.cached",      

    #define NET 5
        "network.interface.in.bytes",              
        "network.interface.out.bytes",            

    #define IOPS 7
        "disk.all.total",    

    #define IO 8
        "disk.all.read_bytes",
        "disk.all.write_bytes",    

    #define CPU 10
        "kernel.all.cpu.nice",  
        "kernel.all.cpu.user",  
        "kernel.all.cpu.intr",
        "kernel.all.cpu.sys",
        "kernel.all.cpu.idle",
        "kernel.all.cpu.wait.total",
        "kernel.all.cpu.steal",
    };

    #define nummetrics (sizeof(metricnames)/sizeof(metricnames[0]))

    static int      context;
    static pmID     pmids[nummetrics];
    static pmDesc   pmdesc[nummetrics];

    static pmResult * pmresult[2];
    static int      flip;

    static float    period;
    static struct timeval sleeptime = { 1, 0 };
    extern char     * pmProgname;

    static void
    timeval_sleep(struct timeval interval)
    {
        struct timespec delay, left;
        delay.tv_sec = interval.tv_sec;
        delay.tv_nsec = interval.tv_usec * 1000;

        for (;;) {          /* loop to catch early wakeup by nanosleep */
            int sts = nanosleep(&delay, &left);
            if (sts == 0 || (sts < 0 && errno != EINTR))
                break;
            delay = left;
        }
    }

    static int
    new_context(int type, char * host, int quiet)
    {
        int i, sts;

        if ((sts = pmNewContext(type, host)) < 0 ) {
            if (!quiet)
                fprintf(stderr, "%s: Cannot create %s context to query: %s\n",
                        pmProgname, host, pmErrStr(sts));
            return sts;
        } else {
            context = sts;
            pmresult[0] = pmresult[1] = NULL;
            flip = 0;

            /* convert external metric names to internal metric identifiers (PMIDs) */
            if ((sts = pmLookupName(nummetrics, metricnames, pmids)) != nummetrics) {
                if (sts < 0) {
                    fprintf(stderr, "%s: pmLookupName: %s\n", pmProgname, pmErrStr(sts));
                    return sts;
                }
                for (i = sts = 0; i < nummetrics; i++)
                    if (pmids[i] == PM_ID_NULL)
                        fprintf(stderr, "%s: %s: no such metric \"%s\"\n",
                                pmProgname, host, metricnames[i]);
                return -ENODATA;        /* one or more was missing, wimp out */
            }

            /* get the descriptor (metadata) for each of the metrics */
            for (i = 0; i < nummetrics; i++) {
                if ((sts = pmLookupDesc(pmids[i], &pmdesc[i])) < 0) {
                    fprintf(stderr,
                            "%s: %s: Warning: cannot retrieve description for "
                            "metric \"%s\" (PMID: %s)\nReason: %s\n",
                            pmProgname, host, metricnames[i], pmIDStr(pmids[i]),
                            pmErrStr(sts));
                    pmdesc[i].indom = PM_INDOM_NULL;
                    pmdesc[i].pmid = PM_ID_NULL;
                }
            }

            /* exclusions: for network interfaces, no loopback thanks */
            i = pmLookupInDom(pmdesc[NET].indom, "lo");
            if (i != PM_ERR_INST) {
                pmAddProfile(pmdesc[NET].indom, 0, NULL);   /* all on */
                pmDelProfile(pmdesc[NET].indom, 1, &i);     /* lo off */
            }
        }

        return 0;
    }

    long long
    difference(pmDesc * d, pmValueSet * now, pmValueSet * was)
    {
        long long diff = 0;
        pmAtomValue a, b;

        pmExtractValue(was->valfmt,  &was->vlist[0], d->type, &a, d->type);
        pmExtractValue(now->valfmt,  &now->vlist[0], d->type, &b, d->type);

        switch (d->type) {
        case PM_TYPE_32:
            diff = b.l - a.l;
            break;
        case PM_TYPE_U32:
            diff = b.ul - a.ul;
            break;
        case PM_TYPE_64:
            diff = b.ll - a.ll;
            break;
        case PM_TYPE_U64:
            diff = b.ull - a.ull;
            break;
        default:    /* eh? */
            abort();
        }
        return diff;
    }

    static void
    scale_and_print_bytes(long long value)
    {
        if (value < 10000) {
            printf (" %4lld", value);
        } else {
            value /= 1000;        /* '000s */
            if (value < 1000)
                printf(" %3lldK", value);
            else {
                value /= 1000;        /* '000,000s */
                printf(" %3lldM", value);
            }
        }
    }

    static void
    scale_and_print(unsigned long value)
    {
        if (value < 1000000) {
            printf(" %6lu", value);
        } else {
            value /= 1024;        /* PM_SPACE_MBYTE now */
            if (value < 100000)
                printf(" %5lum", value);
            else {
                value /= 1024;      /* PM_SPACE_GBYTE now */
                printf(" %5lug", value);
            }
        }
    }

    static int
    fetch_values(int ctxtype)
    {
        int sts;

        if ((sts = pmFetch(nummetrics, pmids, pmresult + flip)) < 0) {
            if (ctxtype == PM_CONTEXT_HOST &&
                (sts == PM_ERR_IPC || sts == PM_ERR_TIMEOUT)) {
                puts(" Fetch failed. Reconnecting ...");
                if (pmresult[1-flip] != NULL ) {
                    pmFreeResult(pmresult[1-flip]);
                    pmresult[1-flip] = NULL;
                }
                pmReconnectContext(context);
                return -EAGAIN;
            } else {
                printf(" pmFetch: %s\n", pmErrStr(sts));
                exit(1);
            }
        }
        return 0;
    }

    static void
    report_values(void)
    {
        int i, j;
        unsigned long long dtot = 0;
        unsigned long long diffs[7];
        pmAtomValue la;
        pmResult * cur = pmresult[flip];
        pmResult * prev = pmresult[1 - flip];

        /* Memory state */
        for (i = 0; i < 5; i++) {
            if (cur->vset[MEM+i]->numval != 1) {
                printf(" %6.6s", "?");
            } else {
                pmUnits kb = { .dimSpace = 1, .scaleSpace = PM_SPACE_KBYTE };
                pmExtractValue(cur->vset[MEM+i]->valfmt,
                               &cur->vset[MEM+i]->vlist[0],
                               pmdesc[MEM+i].type,
                               &la, PM_TYPE_U32);
                pmConvScale(pmdesc[MEM+i].type, &la,
                               &pmdesc[MEM+i].units, &la, &kb);
                scale_and_print(la.ul);
            }
        }

        /* Network in/out throughput */
        for (i = 0; i < 2; i++) {
            if (prev == NULL || prev->vset[NET+i]->numval < 1 ||
                prev->vset[NET+i]->numval != cur->vset[NET+i]->numval) {
                printf(" %4.4s", "?");
            } else {
                for (la.ll = 0, j = 0; j < cur->vset[NET+i]->numval; j++)
                    la.ll += difference(&pmdesc[NET+i], cur->vset[NET+i], prev->vset[NET+i]);
                scale_and_print_bytes(la.ll / period);
            }
        }

        /* Aggregate IOPS */
        if (prev == NULL ||
            prev->vset[IOPS]->numval != 1 || cur->vset[IOPS]->numval != 1)
            printf(" %6.6s", "?");
        else {
            la.ll = difference(&pmdesc[IOPS], cur->vset[IOPS], prev->vset[IOPS]);
            scale_and_print(la.ll / period);
        }

        /* Disk in/out throughput */
        for (i = 0; i < 2; i++) {
            if (prev == NULL ||
                prev->vset[IO+i]->numval != 1 ||
                cur->vset[IO+i]->numval != 1) {
                printf(" %4.4s", "?");
            } else {
                la.ll = difference(&pmdesc[IO+i], cur->vset[IO+i], prev->vset[IO+i]);
                scale_and_print_bytes(la.ll / period);
            }
        }

        /* CPU utilization - report percentage */
        for (i = 0; i < 7; i++) {
            if (prev == NULL ||
                cur->vset[CPU+i]->numval != 1 ||
                prev->vset[CPU+i]->numval != 1) {
                break;
            } else {
                diffs[i] = difference(&pmdesc[CPU+i], cur->vset[CPU+i], prev->vset[CPU+i]);
                dtot += diffs[i];
            }
        }

        if (i != 7 || dtot == 0) {
                printf(" %3.3s %3.3s %3.3s %3.3s %3.3s",
                        "?", "?", "?", "?", "?");
        } else {
                unsigned long long fill = dtot/2;
                printf(" %3u %3u %3u %3u %3u",
                   (unsigned int)((100*(diffs[0]+diffs[1])+fill)/dtot),
                   (unsigned int)((100*(diffs[2]+diffs[3])+fill)/dtot),
                   (unsigned int)((100*diffs[4]+fill)/dtot),
                   (unsigned int)((100*diffs[5]+fill)/dtot),
                   (unsigned int)((100*diffs[6]+fill)/dtot));
        }

        putchar('\n');
    }

    static void
    print_header(pmResult * r)
    {
        char timebuf[26];
        time_t now;

        if (r != NULL)
            now = (time_t)(r->timestamp.tv_sec + 0.5 + r->timestamp.tv_usec/ 1.0e6);
        else        /* fetch failed, but live mode only so just report local time */
            now = time(NULL);
        printf("@ %s", pmCtime(&now, timebuf));
        printf("%35s%10s%17s%20s\n",
                   "memory","net","disk","cpu");
        printf(" %6s %6s %6s %6s %6s %4s %4s %6s %4s %4s %3s %3s %3s %3s %3s\n",
                    "swpd","free","used","buff","cache", "in","out","iops","in","out",
                    "us","sy","id","wa","st");
    }

    static void
    usage()
    {
        fprintf(stderr,
            "Usage: %s [options]\n\n"
            "Options:\n"
            "  -h hostname    read metrics from pmcd on named host\n"
            "  -L             metrics source is local context, no pmcd\n"
            "  -s samples     terminate after this many iterations\n"
            "  -t interval    sample interval [default %d sec]\n",
                    pmProgname, (int)sleeptime.tv_sec);
    }

    int
    main(int argc, char *argv[])
    {
        int ctxtype = 0;
        int errflag = 0;
        int samples = 0;
        int count;
        int sts;
        int c;
        char * endnum;
        char * hostname = NULL;

        pmProgname = basename(argv[0]);

        while ((c = getopt(argc, argv, "h:Ls:t:?")) != EOF) {
            switch (c) {
            case 'h':
                if (ctxtype) {
                    fprintf(stderr, "%s: -h and -L are mutually exclusive\n",
                             pmProgname);
                    errflag++;
                } else {
                    ctxtype = PM_CONTEXT_HOST;
                    hostname = optarg;
                }
                break;

            case 'L':        /* local PMDA connection, no PMCD */
                if (ctxtype) {
                    fprintf(stderr, "%s: -h and -L are mutually exclusive\n",
                            pmProgname);
                    errflag++;
                } else {
                    ctxtype = PM_CONTEXT_LOCAL;
                }
                break;

            case 's':        /* sample count */
                    samples = (int)strtol(optarg, &endnum, 10);
                    if (*endnum != '\0' || samples < 0) {
                        fprintf(stderr,
                                "%s: -s requires numeric argument\n", pmProgname);
                        errflag++;
                    }
                break;

            case 't':        /* update interval */
                if (pmParseInterval(optarg, &sleeptime, &endnum) < 0) {
                    fprintf(stderr,
                            "%s: -t argument not in pmParseInterval(3) format:\n",
                            pmProgname);
                    fprintf(stderr, "%s\n", endnum);
                    free(endnum);
                    errflag++;
                }
                break;

            case '?':
            default:
                errflag++;
                break;
            }
        }

        if (argc != optind) {
            fprintf (stderr, "%s: too many options\n", pmProgname);
            errflag++;
        }

        if (errflag) {
            usage();
            exit(1);
        }

        if (!ctxtype)
            ctxtype = PM_CONTEXT_HOST;

        period = (sleeptime.tv_sec * 1.0e6 + sleeptime.tv_usec) / 1e6;

        if (hostname) {
            if (new_context(ctxtype, hostname, 0) < 0)
               exit(1);
        } else {
            /* Read metrics from the local host.  If we fail to talk to pmcd we
             * fallback to local context mode automagically.
             */
            char local[MAXHOSTNAMELEN];
            gethostname(local, MAXHOSTNAMELEN);
            local[MAXHOSTNAMELEN-1] = '\0';

            if (new_context(ctxtype, local, 1) < 0) {
                ctxtype = PM_CONTEXT_LOCAL;
                if (new_context(ctxtype, local, 0) < 0)
                    exit (1);
            }
        }

        /* Do first fetch, priming one of the results structures */
        if ((sts = pmFetch(nummetrics, pmids, pmresult + flip)) < 0)
            pmresult[flip] = NULL;
        else
            flip = 1 - flip;

        for (count = 0; (samples==0) || (count < samples); count++ ) {
            if (count % 10 == 0)        /* dump header after ten iterations */
                print_header(pmresult[1-flip]);
            timeval_sleep(sleeptime);

            if (fetch_values(ctxtype) < 0)
                continue;
            report_values();

            flip = 1 - flip;
            if (pmresult[flip])
                pmFreeResult(pmresult[flip]);
            pmresult[flip] = NULL;
        }

        exit(EXIT_SUCCESS);
    }

[fche]

An incomplete draft of a pcp backend for the netdev monitor only, is at fpaste [1, no expiry]. However, I'm not happy with it [2], so please look at it only as an exercise. A future version of PCP should make this sort of code much smaller.

[1] http://fpaste.org/136593/
[2] http://oss.sgi.com/pipermail/pcp/2014-September/005684.html

[stefwalter]

http://fpaste.org/136593/

[stefwalter]

pminfo -f cgroup

[stefwalter]

@mvollmer there's a pmConvScale with CPU percentage in these examples.

[stefwalter]

All done yay. Picking up the pieces :)