Fetches control group resource information

docs/setup/settings.asciidoc

@@ -69,6 +69,8 @@ information and all requests.
 The minimum value is 100.
 `status.allowAnonymous`:: *Default: false* If authentication is enabled, setting this to `true` allows
 unauthenticated users to access the Kibana server status API and status page.
+`cpu.cgroup.path.override`:: Override for cgroup cpu path when mounted in manner that is inconsistent with `/proc/self/cgroup`
+`cpuacct.cgroup.path.override`:: Override for cgroup cpuacct path when mounted in manner that is inconsistent with `/proc/self/cgroup`
 `console.enabled`:: *Default: true* Set to false to disable Console.  Toggling this will cause the server to regenerate assets on the next startup, which may cause a delay before pages start being served.
 
 `elasticsearch.tribe.url:`:: Optional URL of the Elasticsearch tribe instance to use for all your

package.json

@@ -249,6 +249,7 @@
     "makelogs": "3.2.3",
     "marked-text-renderer": "0.1.0",
     "mocha": "2.5.3",
+    "mock-fs": "4.0.0",
     "murmurhash3js": "3.0.1",
     "ncp": "2.0.0",
     "nock": "8.0.0",

src/server/config/schema.js

@@ -27,6 +27,21 @@ module.exports = () => Joi.object({
     exclusive: Joi.boolean().default(false)
   }).default(),
 
+  cpu: Joi.object({
+    cgroup: Joi.object({
+      path: Joi.object({
+        override: Joi.string().default()
+      })
+    })
+  }),
+
+  cpuacct: Joi.object({
+    cgroup: Joi.object({
+      path: Joi.object({
+        override: Joi.string().default()
+      })
+    })
+  }),
 
   server: Joi.object({
     uuid: Joi.string().guid().default(),

src/server/status/__tests__/cgroup.js

@@ -0,0 +1,170 @@
+import expect from 'expect.js';
+import mockFs from 'mock-fs';
+import { cGroups as cGroupsFsStub } from './fs_stubs';
+import { getAllStats, readControlGroups, readCPUStat } from '../cgroup';
+
+describe('Control Group', function () {
+  const fsStub = cGroupsFsStub();
+
+  afterEach(() => {
+    mockFs.restore();
+  });
+
+  describe('readControlGroups', () => {
+    it('parses the file', async () => {
+      mockFs({ '/proc/self/cgroup': fsStub.cGroupContents });
+      const cGroup = await readControlGroups();
+
+      expect(cGroup).to.eql({
+        freezer: '/',
+        net_cls: '/',
+        net_prio: '/',
+        pids: '/',
+        blkio: '/',
+        memory: '/',
+        devices: '/user.slice',
+        hugetlb: '/',
+        perf_event: '/',
+        cpu: `/${fsStub.hierarchy}`,
+        cpuacct: `/${fsStub.hierarchy}`,
+        cpuset: `/${fsStub.hierarchy}`,
+        'name=systemd': '/user.slice/user-1000.slice/session-2359.scope'
+      });
+    });
+  });
+
+  describe('readCPUStat', () => {
+    it('parses the file', async () => {
+      mockFs({ '/sys/fs/cgroup/cpu/fakeGroup/cpu.stat': fsStub.cpuStatContents });
+      const cpuStat = await readCPUStat('fakeGroup');
+
+      expect(cpuStat).to.eql({
+        number_of_elapsed_periods: 0,
+        number_of_times_throttled: 10,
+        time_throttled_nanos:  20
+      });
+    });
+
+    it('returns default stats for missing file', async () => {
+      mockFs();
+      const cpuStat = await readCPUStat('fakeGroup');
+
+      expect(cpuStat).to.eql({
+        number_of_elapsed_periods: -1,
+        number_of_times_throttled: -1,
+        time_throttled_nanos:  -1
+      });
+    });
+  });
+
+  describe('getAllStats', () => {
+    it('can override the cpu group path', async () => {
+      mockFs({
+        '/proc/self/cgroup': fsStub.cGroupContents,
+        [`${fsStub.cpuAcctDir}/cpuacct.usage`]: '357753491408',
+        '/sys/fs/cgroup/cpu/docker/cpu.cfs_period_us': '100000',
+        '/sys/fs/cgroup/cpu/docker/cpu.cfs_quota_us': '5000',
+        '/sys/fs/cgroup/cpu/docker/cpu.stat': fsStub.cpuStatContents,
+      });
+
+      console.log('fsStub.cpuAcctDir', fsStub.cpuAcctDir);
+      const stats = await getAllStats({ cpuPath: '/docker' });
+
+      expect(stats).to.eql({
+        cpuacct: {
+          control_group: `/${fsStub.hierarchy}`,
+          usage_nanos: 357753491408,
+        },
+        cpu: {
+          control_group: '/docker',
+          cfs_period_micros: 100000,
+          cfs_quota_micros: 5000,
+          stat: {
+            number_of_elapsed_periods: 0,
+            number_of_times_throttled: 10,
+            time_throttled_nanos: 20
+          }
+        }
+      });
+    });
+
+    it('can override the cpuacct group path', async () => {
+      mockFs({
+        '/proc/self/cgroup': fsStub.cGroupContents,
+        '/sys/fs/cgroup/cpuacct/docker/cpuacct.usage': '357753491408',
+        [`${fsStub.cpuDir}/cpu.cfs_period_us`]: '100000',
+        [`${fsStub.cpuDir}/cpu.cfs_quota_us`]: '5000',
+        [`${fsStub.cpuDir}/cpu.stat`]: fsStub.cpuStatContents,
+      });
+
+      const stats = await getAllStats({ cpuAcctPath: '/docker' });
+
+      expect(stats).to.eql({
+        cpuacct: {
+          control_group: '/docker',
+          usage_nanos: 357753491408,
+        },
+        cpu: {
+          control_group: `/${fsStub.hierarchy}`,
+          cfs_period_micros: 100000,
+          cfs_quota_micros: 5000,
+          stat: {
+            number_of_elapsed_periods: 0,
+            number_of_times_throttled: 10,
+            time_throttled_nanos: 20
+          }
+        }
+      });
+    });
+
+    it('extracts control group stats', async () => {
+      mockFs(fsStub.files);
+      const stats = await getAllStats();
+
+      expect(stats).to.eql({
+        cpuacct: {
+          control_group: `/${fsStub.hierarchy}`,
+          usage_nanos: 357753491408,
+        },
+        cpu: {
+          control_group: `/${fsStub.hierarchy}`,
+          cfs_period_micros: 100000,
+          cfs_quota_micros: 5000,
+          stat: {
+            number_of_elapsed_periods: 0,
+            number_of_times_throttled: 10,
+            time_throttled_nanos: 20
+          }
+        }
+      });
+    });
+
+    it('returns null when all files are missing', async () => {
+      mockFs({});
+      const stats = await getAllStats();
+      expect(stats).to.be.null;
+    });
+
+    it('returns null if CPU accounting files are missing', async () => {
+      mockFs({
+        '/proc/self/cgroup': fsStub.cGroupContents,
+        [`${fsStub.cpuDir}/cpu.stat`]: fsStub.cpuStatContents
+      });
+      const stats = await getAllStats();
+
+      expect(stats).to.be.null;
+    });
+
+    it('returns null if cpuStat file is missing', async () => {
+      mockFs({
+        '/proc/self/cgroup': fsStub.cGroupContents,
+        [`${fsStub.cpuAcctDir}/cpuacct.usage`]: '357753491408',
+        [`${fsStub.cpuDir}/cpu.cfs_period_us`]: '100000',
+        [`${fsStub.cpuDir}/cpu.cfs_quota_us`]: '5000'
+      });
+      const stats = await getAllStats();
+
+      expect(stats).to.be.null;
+    });
+  });
+});

src/server/status/__tests__/fs_stubs.js

@@ -0,0 +1,42 @@
+export function cGroups(hierarchy) {
+  if (!hierarchy) {
+    hierarchy = Math.random().toString(36).substring(7);
+  }
+
+  const cpuAcctDir = `/sys/fs/cgroup/cpuacct/${hierarchy}`;
+  const cpuDir = `/sys/fs/cgroup/cpu/${hierarchy}`;
+
+  const cGroupContents = [
+    '10:freezer:/',
+    '9:net_cls,net_prio:/',
+    '8:pids:/',
+    '7:blkio:/',
+    '6:memory:/',
+    '5:devices:/user.slice',
+    '4:hugetlb:/',
+    '3:perf_event:/',
+    '2:cpu,cpuacct,cpuset:/' + hierarchy,
+    '1:name=systemd:/user.slice/user-1000.slice/session-2359.scope'
+  ].join('\n');
+
+  const cpuStatContents = [
+    'nr_periods 0',
+    'nr_throttled 10',
+    'throttled_time 20'
+  ].join('\n');
+
+  return {
+    hierarchy,
+    cGroupContents,
+    cpuStatContents,
+    cpuAcctDir,
+    cpuDir,
+    files: {
+      '/proc/self/cgroup': cGroupContents,
+      [`${cpuAcctDir}/cpuacct.usage`]: '357753491408',
+      [`${cpuDir}/cpu.cfs_period_us`]: '100000',
+      [`${cpuDir}/cpu.cfs_quota_us`]: '5000',
+      [`${cpuDir}/cpu.stat`]: cpuStatContents,
+    }
+  };
+}

src/server/status/__tests__/metrics.js

@@ -1,5 +1,8 @@
 import _ from 'lodash';
 import expect from 'expect.js';
+import sinon from 'sinon';
+import mockFs from 'mock-fs';
+import { cGroups as cGroupsFsStub } from './fs_stubs';
 
 import { getMetrics } from '../metrics';
 
@@ -20,7 +23,8 @@ describe('Metrics', function () {
     'psdelay': 1.6091690063476562,
     'host': '123'
   };
-  const config = {
+
+  const sampleConfig = {
     ops: {
       interval: 5000
     },
@@ -29,28 +33,104 @@ describe('Metrics', function () {
     }
   };
 
-  let metrics;
-  beforeEach(() => {
-    metrics = getMetrics({
-      event: _.cloneDeep(mockOps),
-      config: {
-        get: path => _.get(config, path)
-      }
+  describe('with cgroups', () => {
+    it('should provide cgroups', async () => {
+      const fsStub = cGroupsFsStub();
+      const event = _.cloneDeep(mockOps);
+      const config = { get: path => _.get(sampleConfig, path) };
+      const kbnServer = { log: sinon.mock() };
+
+      mockFs(fsStub.files);
+      const metrics = await getMetrics(event, config, kbnServer);
+      mockFs.restore();
+
+      expect(_.get(metrics, 'os.cgroup')).to.eql({
+        cpuacct: {
+          control_group: `/${fsStub.hierarchy}`,
+          usage_nanos: 357753491408,
+        },
+        cpu: {
+          control_group: `/${fsStub.hierarchy}`,
+          cfs_period_micros: 100000,
+          cfs_quota_micros: 5000,
+          stat: {
+            number_of_elapsed_periods: 0,
+            number_of_times_throttled: 10,
+            time_throttled_nanos: 20
+          }
+        }
+      });
     });
-  });
 
-  it('should snake case the request object', () => {
-    expect(metrics.requests.status_codes).not.to.be(undefined);
-    expect(metrics.requests.statusCodes).to.be(undefined);
-  });
+    it('can override cgroup path', async () => {
+      const fsStub = cGroupsFsStub('foo');
+      const event = _.cloneDeep(mockOps);
+      const configOverride = Object.assign(sampleConfig, {
+        cpu: {
+          cgroup: {
+            path: {
+              override: '/foo'
+            }
+          }
+        },
+
+        cpuacct: {
+          cgroup: {
+            path: {
+              override: '/foo'
+            }
+          }
+        },
+      });
+      const config = { get: path => _.get(configOverride, path) };
+      const kbnServer = { log: sinon.mock() };
+
+      mockFs(fsStub.files);
+      const metrics = await getMetrics(event, config, kbnServer);
+      mockFs.restore();
 
-  it('should provide defined metrics', () => {
-    (function checkMetrics(currentMetric) {
-      _.forOwn(currentMetric, value => {
-        if (typeof value === 'object') return checkMetrics(value);
-        expect(currentMetric).not.to.be(undefined);
+      expect(_.get(metrics, 'os.cgroup')).to.eql({
+        cpuacct: {
+          control_group: `/foo`,
+          usage_nanos: 357753491408,
+        },
+        cpu: {
+          control_group: `/foo`,
+          cfs_period_micros: 100000,
+          cfs_quota_micros: 5000,
+          stat: {
+            number_of_elapsed_periods: 0,
+            number_of_times_throttled: 10,
+            time_throttled_nanos: 20
+          }
+        }
       });
+    });
+  });
+
+  describe('without cgroups', () => {
+    let metrics;
+    beforeEach(async () => {
+      const event = _.cloneDeep(mockOps);
+      const config = { get: path => _.get(sampleConfig, path) };
+      const kbnServer = { log: sinon.mock() };
+
+      metrics = await getMetrics(event, config, kbnServer);
+    });
 
-    }(metrics));
+    it('should snake case the request object', () => {
+      expect(metrics.requests.status_codes).not.to.be(undefined);
+      expect(metrics.requests.statusCodes).to.be(undefined);
+    });
+
+    it('should provide defined metrics', () => {
+      (function checkMetrics(currentMetric) {
+        _.forOwn(currentMetric, value => {
+          if (typeof value === 'object') return checkMetrics(value);
+          expect(currentMetric).not.to.be(undefined);
+        });
+
+      }(metrics));
+    });
   });
 });