From 235c7bde3c685404db29e6d069b50c677779b075 Mon Sep 17 00:00:00 2001 From: Dave Shawley Date: Mon, 12 Apr 2021 08:08:07 -0400 Subject: [PATCH] Add support for disabling tornado connector. --- CHANGELOG.rst | 10 ++-- docs/conf.py | 6 ++ docs/index.rst | 22 +++++-- setup.cfg | 1 + sprockets_statsd/statsd.py | 114 +++++++++++++++++++++--------------- sprockets_statsd/tornado.py | 47 +++++++++++---- tests/test_tornado.py | 57 ++++++++++++------ 7 files changed, 173 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d364bc2..bcb20c5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,5 +1,7 @@ -`0.0.1`_ (08-Apr-2021) ----------------------- -- support for sending counters & timers to statsd over a TCP or UDP socket +:tag:`Next release <0.0.1...main>` +---------------------------------- +- Added :envvar:`STATSD_ENABLED` environment variable to disable the Tornado integration -.. _0.0.1: https://github.com/sprockets/sprockets-statsd/compare/832f8af7...0.0.1 +:tag:`0.0.1 <832f8af7...0.0.1>` (08-Apr-2021) +--------------------------------------------- +- Simple support for sending counters & timers to statsd over a TCP or UDP socket diff --git a/docs/conf.py b/docs/conf.py index 916de7d..7f1cdb5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,5 +16,11 @@ intersphinx_mapping = { 'tornado': ('https://www.tornadoweb.org/en/branch6.0/', None), } +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +extensions.append('sphinx.ext.extlinks') +extlinks = { + 'tag': ("https://github.com/sprockets/sprockets-statsd/compare/%s", "%s"), +} + # https://pypi.org/project/sphinx-autodoc-typehints/ extensions.append('sphinx_autodoc_typehints') diff --git a/docs/index.rst b/docs/index.rst index 7777077..96657cc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,10 +3,10 @@ sprockets-statsd .. include:: ../README.rst -Configuration -============= -The statsd connection is configured by the ``statsd`` application settings key. The default values can be set by -the following environment variables. +Tornado configuration +===================== +The Tornado statsd connection is configured by the ``statsd`` application settings key. The default values can be set +by the following environment variables. .. envvar:: STATSD_HOST @@ -26,6 +26,17 @@ the following environment variables. The IP protocol to use when connecting to the StatsD server. You can specify either "tcp" or "udp". The default is "tcp" if it not not configured. +.. envvar:: STATSD_ENABLED + + Define this variable and set it to a *falsy* value to **disable** the Tornado integration. If you omit + this variable, then the connector is enabled. The following values are considered *truthy*: + + - non-zero integer + - case-insensitive match of ``yes``, ``true``, ``t``, or ``on`` + + All other values are considered *falsy*. You only want to define this environment variables when you + want to explicitly disable an otherwise installed and configured connection. + If you are using the Tornado helper clases, then you can fine tune the metric payloads and the connector by setting additional values in the ``statsd`` key of :attr:`tornado.web.Application.settings`. See the :class:`sprockets_statsd.tornado.Application` class documentation for a description of the supported settings. @@ -46,6 +57,9 @@ Tornado helpers Internals --------- +.. autoclass:: sprockets_statsd.statsd.AbstractConnector + :members: + .. autoclass:: sprockets_statsd.statsd.Processor :members: diff --git a/setup.cfg b/setup.cfg index a9c956c..5218f4c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,6 +63,7 @@ warning_is_error = 1 [coverage:report] exclude_lines = + pass pragma: no cover raise NotImplementedError fail_under = 100 diff --git a/sprockets_statsd/statsd.py b/sprockets_statsd/statsd.py index b35e605..d8439af 100644 --- a/sprockets_statsd/statsd.py +++ b/sprockets_statsd/statsd.py @@ -4,7 +4,71 @@ import socket import typing -class Connector: +class AbstractConnector: + """StatsD connector that does not send metrics or connect. + + Use this connector when you want to maintain the application + interface without doing any real work. + + """ + async def start(self) -> None: + pass + + async def stop(self) -> None: + pass + + def inject_metric(self, path: str, value: str, type_code: str) -> None: + pass + + def incr(self, path: str, value: int = 1) -> None: + """Increment a counter metric. + + :param path: counter to increment + :param value: amount to increment the counter by + + """ + self.inject_metric(f'counters.{path}', str(value), 'c') + + def decr(self, path: str, value: int = 1) -> None: + """Decrement a counter metric. + + :param path: counter to decrement + :param value: amount to decrement the counter by + + This is equivalent to ``self.incr(path, -value)``. + + """ + self.inject_metric(f'counters.{path}', str(-value), 'c') + + def gauge(self, path: str, value: int, delta: bool = False) -> None: + """Manipulate a gauge metric. + + :param path: gauge to adjust + :param value: value to send + :param delta: is this an adjustment of the gauge? + + If the `delta` parameter is ``False`` (or omitted), then + `value` is the new value to set the gauge to. Otherwise, + `value` is an adjustment for the current gauge. + + """ + if delta: + payload = f'{value:+d}' + else: + payload = str(value) + self.inject_metric(f'gauges.{path}', payload, 'g') + + def timing(self, path: str, seconds: float) -> None: + """Send a timer metric. + + :param path: timer to append a value to + :param seconds: number of **seconds** to record + + """ + self.inject_metric(f'timers.{path}', str(seconds * 1000.0), 'ms') + + +class Connector(AbstractConnector): """Sends metrics to a statsd server. :param host: statsd server to send metrics to @@ -69,6 +133,7 @@ class Connector: *, prefix: str = '', **kwargs: typing.Any) -> None: + super().__init__() self.logger = logging.getLogger(__package__).getChild('Connector') self.prefix = f'{prefix}.' if prefix else prefix self.processor = Processor(host=host, port=port, **kwargs) @@ -95,53 +160,6 @@ class Connector: """ await self.processor.stop() - def incr(self, path: str, value: int = 1) -> None: - """Increment a counter metric. - - :param path: counter to increment - :param value: amount to increment the counter by - - """ - self.inject_metric(f'counters.{path}', str(value), 'c') - - def decr(self, path: str, value: int = 1) -> None: - """Decrement a counter metric. - - :param path: counter to decrement - :param value: amount to decrement the counter by - - This is equivalent to ``self.incr(path, -value)``. - - """ - self.inject_metric(f'counters.{path}', str(-value), 'c') - - def gauge(self, path: str, value: int, delta: bool = False) -> None: - """Manipulate a gauge metric. - - :param path: gauge to adjust - :param value: value to send - :param delta: is this an adjustment of the gauge? - - If the `delta` parameter is ``False`` (or omitted), then - `value` is the new value to set the gauge to. Otherwise, - `value` is an adjustment for the current gauge. - - """ - if delta: - payload = f'{value:+d}' - else: - payload = str(value) - self.inject_metric(f'gauges.{path}', payload, 'g') - - def timing(self, path: str, seconds: float) -> None: - """Send a timer metric. - - :param path: timer to append a value to - :param seconds: number of **seconds** to record - - """ - self.inject_metric(f'timers.{path}', str(seconds * 1000.0), 'ms') - def inject_metric(self, path: str, value: str, type_code: str) -> None: """Send a metric to the statsd server. diff --git a/sprockets_statsd/tornado.py b/sprockets_statsd/tornado.py index 5e368c6..22e5586 100644 --- a/sprockets_statsd/tornado.py +++ b/sprockets_statsd/tornado.py @@ -13,7 +13,7 @@ class Application(web.Application): """Mix this into your application to add a statsd connection. .. attribute:: statsd_connector - :type: sprockets_statsd.statsd.Connector + :type: sprockets_statsd.statsd.AbstractConnector Connection to the StatsD server that is set between calls to :meth:`.start_statsd` and :meth:`.stop_statsd`. @@ -21,6 +21,8 @@ class Application(web.Application): This mix-in is configured by the ``statsd`` settings key. The value is a dictionary with the following keys. + +-------------------+---------------------------------------------+ + | enabled | should the statsd connector be enabled? | +-------------------+---------------------------------------------+ | host | the statsd host to send metrics to | +-------------------+---------------------------------------------+ @@ -38,6 +40,13 @@ class Application(web.Application): | | connection | +-------------------+---------------------------------------------+ + **enabled** defaults to the :envvar:`STATSD_ENABLED` environment + variable coerced to a :class:`bool`. If this variable is not set, + then the statsd connector *WILL BE* enabled. Set this to a *falsy* + value to disable the connector. The following values are considered + *truthy*: a non-zero integer or a case-insensitive match of "on", + "t", "true", or "yes". All other values are considered *falsy*. + **host** defaults to the :envvar:`STATSD_HOST` environment variable. If this value is not set, then the statsd connector *WILL NOT* be enabled. @@ -72,10 +81,12 @@ class Application(web.Application): processor quickly responds to connection faults. """ - statsd_connector: typing.Optional[statsd.Connector] + statsd_connector: typing.Optional[statsd.AbstractConnector] def __init__(self, *args: typing.Any, **settings: typing.Any): statsd_settings = settings.setdefault('statsd', {}) + statsd_settings.setdefault('enabled', + os.environ.get('STATSD_ENABLED', 'yes')) statsd_settings.setdefault('host', os.environ.get('STATSD_HOST')) statsd_settings.setdefault('port', os.environ.get('STATSD_PORT', '8125')) @@ -98,6 +109,8 @@ class Application(web.Application): super().__init__(*args, **settings) + self.settings['statsd']['enabled'] = _parse_bool( + self.settings['statsd']['enabled']) self.settings['statsd']['port'] = int(self.settings['statsd']['port']) self.statsd_connector = None @@ -111,17 +124,22 @@ class Application(web.Application): """ if self.statsd_connector is None: - kwargs = self.settings['statsd'].copy() - protocol = kwargs.pop('protocol', None) - if protocol == 'tcp': - kwargs['ip_protocol'] = socket.IPPROTO_TCP - elif protocol == 'udp': - kwargs['ip_protocol'] = socket.IPPROTO_UDP + if not self.settings['statsd']['enabled']: + self.statsd_connector = statsd.AbstractConnector() else: - raise RuntimeError(f'statsd configuration error: {protocol} ' - f'is not a valid protocol') + kwargs = self.settings['statsd'].copy() + kwargs.pop('enabled', None) # consume this one + protocol = kwargs.pop('protocol', None) + if protocol == 'tcp': + kwargs['ip_protocol'] = socket.IPPROTO_TCP + elif protocol == 'udp': + kwargs['ip_protocol'] = socket.IPPROTO_UDP + else: + raise RuntimeError( + f'statsd configuration error: {protocol} is not ' + f'a valid protocol') + self.statsd_connector = statsd.Connector(**kwargs) - self.statsd_connector = statsd.Connector(**kwargs) await self.statsd_connector.start() async def stop_statsd(self) -> None: @@ -195,3 +213,10 @@ class RequestHandler(web.RequestHandler): self.record_timing(self.request.request_time(), self.__class__.__name__, self.request.method, self.get_status()) + + +def _parse_bool(value: str) -> bool: + try: + return int(value) != 0 + except ValueError: + return value.lower() in {'true', 't', 'yes', 'on'} diff --git a/tests/test_tornado.py b/tests/test_tornado.py index 77559b8..14ba997 100644 --- a/tests/test_tornado.py +++ b/tests/test_tornado.py @@ -6,6 +6,7 @@ import typing from tornado import testing, web +import sprockets_statsd.statsd import sprockets_statsd.tornado from tests import helpers @@ -49,21 +50,22 @@ class ApplicationTests(AsyncTestCaseWithTimeout): else: os.environ.pop(name, None) - def setenv(self, name, value): - self._environ.setdefault(name, os.environ.pop(name, None)) - os.environ[name] = value + def setenv(self, **variables): + for name, value in variables.items(): + self._environ.setdefault(name, os.environ.pop(name, None)) + os.environ[name] = value - def unsetenv(self, name): - self._environ.setdefault(name, os.environ.pop(name, None)) + def unsetenv(self, *names): + for name in names: + self._environ.setdefault(name, os.environ.pop(name, None)) def test_statsd_setting_defaults(self): - self.unsetenv('STATSD_HOST') - self.unsetenv('STATSD_PORT') - self.unsetenv('STATSD_PREFIX') - self.unsetenv('STATSD_PROTOCOL') + self.unsetenv('STATSD_ENABLED', 'STATSD_HOST', 'STATSD_PORT', + 'STATSD_PREFIX', 'STATSD_PROTOCOL') app = sprockets_statsd.tornado.Application(statsd={'prefix': ''}) self.assertIn('statsd', app.settings) + self.assertTrue(app.settings['statsd']['enabled']) self.assertIsNone(app.settings['statsd']['host'], 'default host value should be None') self.assertEqual(8125, app.settings['statsd']['port']) @@ -71,13 +73,15 @@ class ApplicationTests(AsyncTestCaseWithTimeout): self.assertEqual('tcp', app.settings['statsd']['protocol']) def test_that_statsd_settings_read_from_environment(self): - self.setenv('STATSD_HOST', 'statsd') - self.setenv('STATSD_PORT', '5218') - self.setenv('STATSD_PREFIX', 'my-service') - self.setenv('STATSD_PROTOCOL', 'udp') + self.setenv(STATSD_ENABLED='no', + STATSD_HOST='statsd', + STATSD_PORT='5218', + STATSD_PREFIX='my-service', + STATSD_PROTOCOL='udp') app = sprockets_statsd.tornado.Application() self.assertIn('statsd', app.settings) + self.assertFalse(app.settings['statsd']['enabled']) self.assertEqual('statsd', app.settings['statsd']['host']) self.assertEqual(5218, app.settings['statsd']['port']) self.assertEqual('my-service', app.settings['statsd']['prefix']) @@ -99,18 +103,22 @@ class ApplicationTests(AsyncTestCaseWithTimeout): app.settings['statsd']['prefix']) def test_overridden_settings(self): - self.setenv('STATSD_HOST', 'statsd') - self.setenv('STATSD_PORT', '9999') - self.setenv('STATSD_PREFIX', 'service') - self.setenv('STATSD_PROTOCOL', 'tcp') + self.setenv(STATSD_ENABLED='0', + STATSD_HOST='statsd', + STATSD_PORT='9999', + STATSD_PREFIX='service', + STATSD_PROTOCOL='tcp') + app = sprockets_statsd.tornado.Application( statsd={ + 'enabled': 'true', 'host': 'statsd.example.com', 'port': 5218, 'prefix': 'myapp', 'protocol': 'udp', }) self.assertEqual('statsd.example.com', app.settings['statsd']['host']) + self.assertTrue(app.settings['statsd']['enabled']) self.assertEqual(5218, app.settings['statsd']['port']) self.assertEqual('myapp', app.settings['statsd']['prefix']) self.assertEqual('udp', app.settings['statsd']['protocol']) @@ -231,6 +239,21 @@ class ApplicationTests(AsyncTestCaseWithTimeout): self.assertEqual(app.statsd_connector.prefix, '') self.run_coroutine(app.stop_statsd()) + def test_disabling_statsd_connector(self): + app = sprockets_statsd.tornado.Application( + environment='development', + service='my-service', + version='1.0.0', + statsd={ + 'enabled': 'no', + 'host': 'localhost', + }, + ) + self.run_coroutine(app.start_statsd()) + self.assertIsNotNone(app.statsd_connector) + self.assertIsInstance(app.statsd_connector, + sprockets_statsd.statsd.AbstractConnector) + class RequestHandlerTests(AsyncTestCaseWithTimeout, testing.AsyncHTTPTestCase): def setUp(self):