From eb1401c91338f2c68495eb39bd0bcc4d3a979401 Mon Sep 17 00:00:00 2001 From: Drew Perttula Date: Sun, 8 May 2016 04:06:39 -0700 Subject: [PATCH 1/6] Add cyclone support (using the same code as tornado) --- src/greplin/scales/cyclonehandler.py | 32 ++++++++++++++++++++++ src/greplin/scales/tornadohandler.py | 40 ++-------------------------- src/greplin/scales/tornadolike.py | 39 +++++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 38 deletions(-) create mode 100644 src/greplin/scales/cyclonehandler.py create mode 100644 src/greplin/scales/tornadolike.py diff --git a/src/greplin/scales/cyclonehandler.py b/src/greplin/scales/cyclonehandler.py new file mode 100644 index 0000000..505f019 --- /dev/null +++ b/src/greplin/scales/cyclonehandler.py @@ -0,0 +1,32 @@ +# Copyright 2011 The scales Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines a Cyclone request handler for status reporting. + +Install like: + handlers=[ + ... + (r'/stats/(.*)', StatsHandler, {'serverName': 'my_app_name'}), + ] + +""" + + +from greplin.scales import tornadolike + +import cyclone.web + + +class StatsHandler(tornadolike.Handler, cyclone.web.RequestHandler): + """Cyclone request handler for a status page.""" diff --git a/src/greplin/scales/tornadohandler.py b/src/greplin/scales/tornadohandler.py index b2cb78d..78b17bc 100644 --- a/src/greplin/scales/tornadohandler.py +++ b/src/greplin/scales/tornadohandler.py @@ -15,46 +15,10 @@ """Defines a Tornado request handler for status reporting.""" -from greplin import scales -from greplin.scales import formats, util +from greplin.scales import tornadolike import tornado.web - -class StatsHandler(tornado.web.RequestHandler): +class StatsHandler(tornadolike.Handler, tornado.web.RequestHandler): """Tornado request handler for a status page.""" - - serverName = None - - - def initialize(self, serverName): # pylint: disable=W0221 - """Initializes the handler.""" - self.serverName = serverName - - - def get(self, path): # pylint: disable=W0221 - """Renders a GET request, by showing this nodes stats and children.""" - path = path or '' - path = path.lstrip('/') - parts = path.split('/') - if not parts[0]: - parts = parts[1:] - statDict = util.lookup(scales.getStats(), parts) - - if statDict is None: - self.set_status(404) - self.finish('Path not found.') - return - - outputFormat = self.get_argument('format', default='html') - query = self.get_argument('query', default=None) - if outputFormat == 'json': - formats.jsonFormat(self, statDict, query) - elif outputFormat == 'prettyjson': - formats.jsonFormat(self, statDict, query, pretty=True) - else: - formats.htmlHeader(self, '/' + path, self.serverName, query) - formats.htmlFormat(self, tuple(parts), statDict, query) - - return None diff --git a/src/greplin/scales/tornadolike.py b/src/greplin/scales/tornadolike.py new file mode 100644 index 0000000..7576111 --- /dev/null +++ b/src/greplin/scales/tornadolike.py @@ -0,0 +1,39 @@ +from greplin import scales +from greplin.scales import formats, util + + +class Handler(object): + + serverName = None + + + def initialize(self, serverName): # pylint: disable=W0221 + """Initializes the handler.""" + self.serverName = serverName + + + def get(self, path): # pylint: disable=W0221 + """Renders a GET request, by showing this nodes stats and children.""" + path = path or '' + path = path.lstrip('/') + parts = path.split('/') + if not parts[0]: + parts = parts[1:] + statDict = util.lookup(scales.getStats(), parts) + + if statDict is None: + self.set_status(404) + self.finish('Path not found.') + return + + outputFormat = self.get_argument('format', default='html') + query = self.get_argument('query', default=None) + if outputFormat == 'json': + formats.jsonFormat(self, statDict, query) + elif outputFormat == 'prettyjson': + formats.jsonFormat(self, statDict, query, pretty=True) + else: + formats.htmlHeader(self, '/' + path, self.serverName, query) + formats.htmlFormat(self, tuple(parts), statDict, query) + + return None From 23e3a43cb57661a204476c8b8b437bfc0caa87a5 Mon Sep 17 00:00:00 2001 From: Drew Perttula Date: Sun, 8 May 2016 04:06:48 -0700 Subject: [PATCH 2/6] Remove stale comment --- src/greplin/scales/bottlehandler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/greplin/scales/bottlehandler.py b/src/greplin/scales/bottlehandler.py index d6f568d..652723a 100644 --- a/src/greplin/scales/bottlehandler.py +++ b/src/greplin/scales/bottlehandler.py @@ -37,8 +37,7 @@ def bottlestats(server_name, path=''): def register_stats_handler(app, server_name, prefix='/status/'): """Register the stats handler with a Flask app, serving routes - with a given prefix. The prefix defaults to '/_stats/', which is - generally what you want.""" + with a given prefix.""" if not prefix.endswith('/'): prefix += '/' handler = functools.partial(bottlestats, server_name) From 448d59fb491b7631877528e7695a93553bfaaa93 Mon Sep 17 00:00:00 2001 From: Drew Perttula Date: Sun, 8 May 2016 04:07:57 -0700 Subject: [PATCH 3/6] PmfStat.time() can now be used as a decorator too --- src/greplin/scales/__init__.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/greplin/scales/__init__.py b/src/greplin/scales/__init__.py index ade0692..aef38b8 100644 --- a/src/greplin/scales/__init__.py +++ b/src/greplin/scales/__init__.py @@ -15,6 +15,7 @@ """Classes for tracking system statistics.""" import collections +import functools import inspect import itertools import gc @@ -465,7 +466,7 @@ class PmfStatDict(UserDict): """Ugly hack defaultdict-like thing.""" class TimeManager(object): - """Context manager for timing.""" + """Context manager for timing. Also works as a function decorator.""" def __init__(self, container): self.container = container @@ -501,6 +502,13 @@ def discard(self): """Discard this sample.""" self.__discard = True + def __call__(self, func): + """Decorator mode""" + def newFunc(*args, **kw): + with self: + return func(*args, **kw) + functools.update_wrapper(newFunc, func) + return newFunc def __init__(self, sample = None): UserDict.__init__(self) @@ -542,7 +550,15 @@ def addValue(self, value): def time(self): - """Measure the time this section of code takes. For use in with statements.""" + """Measure the time this section of code takes. For use in with- + statements or as a function decorator. + + Decorator example: + + @STATS.foo.time() + def foo(): + ... + """ return self.TimeManager(self) From 3f2609c3398dd80d8fc79aade80cff0252596b21 Mon Sep 17 00:00:00 2001 From: Drew Perttula Date: Wed, 29 May 2019 18:49:58 -0700 Subject: [PATCH 4/6] new RecentFpsStat with rate() decorator that reports the rate of a call --- src/greplin/scales/__init__.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/greplin/scales/__init__.py b/src/greplin/scales/__init__.py index aef38b8..075e571 100644 --- a/src/greplin/scales/__init__.py +++ b/src/greplin/scales/__init__.py @@ -607,7 +607,37 @@ class NamedPmfDictStat(Stat): def _getDefault(self, _): return NamedPmfDict() +class RecentFps(object): + def __init__(self, window=20): + self.window = window + self.recentTimes = [] + def mark(self): + now = time.time() + self.recentTimes.append(now) + self.recentTimes = self.recentTimes[-self.window:] + + def rate(self): + def dec(innerFunc): + def f(*a, **kw): + self.mark() + return innerFunc(*a, **kw) + return f + return dec + + def __call__(self): + if len(self.recentTimes) < 2: + return {} + recents = sorted(round(1 / (b - a), 3) + for a, b in zip(self.recentTimes[:-1], + self.recentTimes[1:])) + avg = (len(self.recentTimes) - 1) / ( + self.recentTimes[-1] - self.recentTimes[0]) + return {'average': round(avg, 5), 'recents': recents} + +class RecentFpsStat(Stat): + def _getDefault(self, _): + return RecentFps() class StateTimeStatDict(UserDict): """Special dict that tracks time spent in current state.""" From 4b011434f7469a442c3fc1d7e81685c0bfa56eeb Mon Sep 17 00:00:00 2001 From: Drew Perttula Date: Sun, 2 Jun 2019 14:09:06 -0700 Subject: [PATCH 5/6] PmfStat creator can override the 20-second recalc period I have a dashboard display that updates at 1Hz, and the PmfStats would stall because of the deferred recalcs. --- src/greplin/scales/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/greplin/scales/__init__.py b/src/greplin/scales/__init__.py index 075e571..6fc46de 100644 --- a/src/greplin/scales/__init__.py +++ b/src/greplin/scales/__init__.py @@ -510,13 +510,14 @@ def newFunc(*args, **kw): functools.update_wrapper(newFunc, func) return newFunc - def __init__(self, sample = None): + def __init__(self, sample = None, recalcPeriod = 20): UserDict.__init__(self) if sample: self.__sample = sample else: self.__sample = ExponentiallyDecayingReservoir() self.__timestamp = 0 + self.__recalcPeriod = recalcPeriod self.percentile99 = None self['count'] = 0 @@ -532,7 +533,7 @@ def addValue(self, value): """Updates the dictionary.""" self['count'] += 1 self.__sample.update(value) - if time.time() > self.__timestamp + 20 and len(self.__sample) > 1: + if time.time() > self.__timestamp + self.__recalcPeriod and len(self.__sample) > 1: self.__timestamp = time.time() self['min'] = self.__sample.min self['max'] = self.__sample.max @@ -569,12 +570,13 @@ class PmfStat(Stat): bit expensive, so its child values are only updated once every twenty seconds.""" - def __init__(self, name, _=None): + def __init__(self, name, _=None, recalcPeriod=20): Stat.__init__(self, name, None) + self.__recalcPeriod = recalcPeriod def _getDefault(self, _): - return PmfStatDict() + return PmfStatDict(recalcPeriod=self.__recalcPeriod) def __set__(self, instance, value): From 550bcba42c5e96152ff5c5bd753fbb33ffdfe460 Mon Sep 17 00:00:00 2001 From: Drew Perttula Date: Sun, 2 Feb 2020 14:23:30 -0800 Subject: [PATCH 6/6] RecentFps also returns period (1/average) --- src/greplin/scales/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/greplin/scales/__init__.py b/src/greplin/scales/__init__.py index 6fc46de..ffe2be6 100644 --- a/src/greplin/scales/__init__.py +++ b/src/greplin/scales/__init__.py @@ -635,7 +635,9 @@ def __call__(self): self.recentTimes[1:])) avg = (len(self.recentTimes) - 1) / ( self.recentTimes[-1] - self.recentTimes[0]) - return {'average': round(avg, 5), 'recents': recents} + return {'average': round(avg, 5), + 'recents': recents, + 'period': round(1 / avg, 1) if avg > 0 else None} class RecentFpsStat(Stat): def _getDefault(self, _):