Add Application & RequestHandler mixins.

CHANGELOG.rst

@@ -1,2 +1,3 @@
-Next Release
-------------
+Initial release
+---------------
+- support for sending counters & timers to statsd over a TCP socket

MANIFEST.in

@@ -1,4 +1,6 @@
 graft docs
 graft tests
-include LICENSE
+
 include CHANGELOG.rst
+include example.py
+include LICENSE

README.rst

@@ -1,5 +1,64 @@
-Report metrics from your tornado_ web application to a StatsD_ instance.
+Report metrics from your tornado_ web application to a statsd_ instance.
 
-.. _StatsD: https://github.com/statsd/statsd/
+.. code-block:: python
+
+   import asyncio
+   import logging
+   
+   from tornado import ioloop, web
+   
+   import sprockets_statsd.mixins
+   
+   
+   class MyHandler(sprockets_statsd.mixins.RequestHandler,
+                   web.RequestHandler):
+       async def get(self):
+           with self.execution_timer('some-operation'):
+               await self.do_something()
+           self.set_status(204)
+   
+       async def do_something(self):
+           await asyncio.sleep(1)
+   
+   
+   class Application(sprockets_statsd.mixins.Application, web.Application):
+       def __init__(self, **settings):
+           super().__init__([web.url('/', MyHandler)], **settings)
+   
+       async def on_start(self):
+           await self.start_statsd()
+   
+       async def on_stop(self):
+           await self.stop_statsd()
+   
+   
+   if __name__ == '__main__':
+       logging.basicConfig(level=logging.DEBUG)
+       app = Application()
+       app.listen(8888)
+       iol = ioloop.IOLoop.current()
+       try:
+           iol.add_callback(app.on_start)
+           iol.start()
+       except KeyboardInterrupt:
+           iol.add_future(asyncio.ensure_future(app.on_stop()),
+                          lambda f: iol.stop())
+           iol.start()
+
+This application will emit two timing metrics each time that the endpoint is invoked::
+
+   applications.timers.some-operation:1001.3449192047119|ms
+   applications.timers.MyHandler.GET.204:1002.4960041046143|ms
+
+You will need to set the ``$STATSD_HOST`` environment variable to enable the statsd processing inside of the
+application.  The ``RequestHandler`` class exposes methods that send counter and timing metrics to a statsd server.
+The connection is managed by the ``Application`` provided that you call the ``start_statsd`` method during application
+startup.
+
+Metrics are sent by a ``asyncio.Task`` that is started by ``start_statsd``.  The request handler methods insert the
+metric data onto a ``asyncio.Queue`` that the task reads from.  Metric data remains on the queue when the task is
+not connected to the server and will be sent in the order received when the task establishes the server connection.
+
+.. _statsd: https://github.com/statsd/statsd/
 .. _tornado: https://tornadoweb.org/
 

docs/index.rst

@@ -4,16 +4,39 @@ sprockets-statsd
 
 .. include:: ../README.rst
 
+Configuration
+=============
+The statsd connection is configured by the ``statsd`` application settings key.  The default values can be set by
+the following environment variables.
+
+.. envvar:: STATSD_HOST
+
+   The host or IP address of the StatsD server to send metrics to.
+
+.. envvar:: STATSD_PORT
+
+   The TCP port number that the StatsD server is listening on.  This defaults to 8125 if it is not configured.
+
+You can fine tune the metric payloads and the connector by setting additional values in the ``stats`` key of
+:attr:`tornado.web.Application.settings`.  See the :class:`sprockets_statsd.mixins.Application` class
+documentation for a description of the supported settings.
+
 Reference
 =========
 
-Connector
+Mixin classes
+-------------
+.. autoclass:: sprockets_statsd.mixins.Application
+   :members:
+
+.. autoclass:: sprockets_statsd.mixins.RequestHandler
+   :members:
+
+Internals
 ---------
 .. autoclass:: sprockets_statsd.statsd.Connector
    :members:
 
-Processor internals
--------------------
 .. autoclass:: sprockets_statsd.statsd.Processor
    :members:
 

example.py

@@ -0,0 +1,42 @@
+import asyncio
+import logging
+
+from tornado import ioloop, web
+
+import sprockets_statsd.mixins
+
+
+class MyHandler(sprockets_statsd.mixins.RequestHandler,
+                web.RequestHandler):
+    async def get(self):
+        with self.execution_timer('some-operation'):
+            await self.do_something()
+        self.set_status(204)
+
+    async def do_something(self):
+        await asyncio.sleep(1)
+
+
+class Application(sprockets_statsd.mixins.Application, web.Application):
+    def __init__(self, **settings):
+        super().__init__([web.url('/', MyHandler)], **settings)
+
+    async def on_start(self):
+        await self.start_statsd()
+
+    async def on_stop(self):
+        await self.stop_statsd()
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.DEBUG)
+    app = Application()
+    app.listen(8888)
+    iol = ioloop.IOLoop.current()
+    try:
+        iol.add_callback(app.on_start)
+        iol.start()
+    except KeyboardInterrupt:
+        iol.add_future(asyncio.ensure_future(app.on_stop()),
+                       lambda f: iol.stop())
+        iol.start()

setup.cfg

@@ -59,7 +59,7 @@ branch = 1
 source = sprockets_statsd
 
 [flake8]
-application_import_names = statsd
+application_import_names = sprockets_statsd,tests
 exclude = build,env,dist
 import_order_style = pycharm
 

sprockets_statsd/mixins.py

@@ -0,0 +1,144 @@
+import contextlib
+import os
+import time
+
+from tornado import web
+
+from sprockets_statsd import statsd
+
+
+class Application(web.Application):
+    """Mix this into your application to add a statsd connection.
+
+    This mix-in is configured by the ``statsd`` settings key.  The
+    value should be a dictionary with the following keys.
+
+    +--------+---------------------------------------------+
+    | host   | the statsd host to send metrics to          |
+    +--------+---------------------------------------------+
+    | port   | TCP port number that statsd is listening on |
+    +--------+---------------------------------------------+
+    | prefix | segment to prefix to metrics                |
+    +--------+---------------------------------------------+
+
+    *host* defaults to the :envvar:`STATSD_HOST` environment variable.
+    If this value is not set, then the statsd connector **WILL NOT**
+    be enabled.
+
+    *port* defaults to the :envvar:`STATSD_PORT` environment variable
+    with a back up default of 8125 if the environment variable is not
+    set.
+
+    *prefix* defaults to ``applications.<service>.<environment>`` where
+    *<service>* and *<environment>* are replaced with the keys from
+    `settings` if they are present.
+
+    """
+    def __init__(self, *args, **settings):
+        statsd_settings = settings.setdefault('statsd', {})
+        statsd_settings.setdefault('host', os.environ.get('STATSD_HOST'))
+        statsd_settings.setdefault('port',
+                                   os.environ.get('STATSD_PORT', '8125'))
+
+        prefix = ['applications']
+        if 'service' in settings:
+            prefix.append(settings['service'])
+        if 'environment' in settings:
+            prefix.append(settings['environment'])
+        statsd_settings.setdefault('prefix', '.'.join(prefix))
+
+        super().__init__(*args, **settings)
+
+        self.settings['statsd']['port'] = int(self.settings['statsd']['port'])
+        self.__statsd_connector = None
+
+    async def start_statsd(self):
+        """Start the connector during startup.
+
+        Call this method during application startup to enable the statsd
+        connection.  A new :class:`~sprockets_statsd.statsd.Connector`
+        instance will be created and started.  This method does not return
+        until the connector is running.
+
+        """
+        statsd_settings = self.settings['statsd']
+        if statsd_settings.get('_connector') is None:
+            connector = statsd.Connector(host=statsd_settings['host'],
+                                         port=statsd_settings['port'])
+            await connector.start()
+            self.settings['statsd']['_connector'] = connector
+
+    async def stop_statsd(self):
+        """Stop the connector during shutdown.
+
+        If the connector was started, then this method will gracefully
+        terminate it.  The method does not return until after the
+        connector is stopped.
+
+        """
+        connector = self.settings['statsd'].pop('_connector', None)
+        if connector is not None:
+            await connector.stop()
+
+
+class RequestHandler(web.RequestHandler):
+    """Mix this into your handler to send metrics to a statsd server."""
+    __connector: statsd.Connector
+
+    def initialize(self, **kwargs):
+        super().initialize(**kwargs)
+        self.__connector = self.settings.get('statsd', {}).get('_connector')
+
+    def __build_path(self, *path):
+        full_path = '.'.join(str(c) for c in path)
+        if self.settings.get('statsd', {}).get('prefix', ''):
+            return f'{self.settings["statsd"]["prefix"]}.{full_path}'
+        return full_path
+
+    def record_timing(self, secs: float, *path):
+        """Record the duration.
+
+        :param secs: number of seconds to record
+        :param path: path to record the duration under
+
+        """
+        if self.__connector is not None:
+            self.__connector.inject_metric(self.__build_path('timers', *path),
+                                           secs * 1000.0, 'ms')
+
+    def increase_counter(self, *path, amount: int = 1):
+        """Adjust a counter.
+
+        :param path: path of the counter to adjust
+        :param amount: amount to adjust the counter by.  Defaults to
+            1 and can be negative
+
+        """
+        if self.__connector is not None:
+            self.__connector.inject_metric(
+                self.__build_path('counters', *path), amount, 'c')
+
+    @contextlib.contextmanager
+    def execution_timer(self, *path):
+        """Record the execution duration of a block of code.
+
+        :param path: path to record the duration as
+
+        """
+        start = time.time()
+        try:
+            yield
+        finally:
+            self.record_timing(time.time() - start, *path)
+
+    def on_finish(self):
+        """Extended to record the request time as a duration.
+
+        This method extends :meth:`tornado.web.RequestHandler.on_finish`
+        to record ``self.request.request_time`` as a timing metric.
+
+        """
+        super().on_finish()
+        self.record_timing(self.request.request_time(),
+                           self.__class__.__name__, self.request.method,
+                           self.get_status())

sprockets_statsd/statsd.py

@@ -34,8 +34,14 @@ class Connector:
         self._processor_task = None
 
     async def start(self):
-        """Start the processor in the background."""
+        """Start the processor in the background.
+
+        This is a *blocking* method and does not return until the
+        processor task is actually running.
+
+        """
         self._processor_task = asyncio.create_task(self.processor.run())
+        await self.processor.running.wait()
 
     async def stop(self):
         """Stop the background processor.
@@ -105,6 +111,13 @@ class Processor(asyncio.Protocol):
 
        Is the TCP connection currently connected?
 
+    .. attribute:: running
+       :type: asyncio.Event
+
+       Is the background task currently running?  This is the event that
+       :meth:`.run` sets when it starts and it remains set until the task
+       exits.
+
     .. attribute:: stopped
        :type: asyncio.Event
 
@@ -115,9 +128,15 @@ class Processor(asyncio.Protocol):
     """
     def __init__(self, *, host, port: int = 8125):
         super().__init__()
+        if not host:
+            raise RuntimeError('host must be set')
+        if not port or port < 1:
+            raise RuntimeError('port must be a positive integer')
+
         self.host = host
         self.port = port
 
+        self.running = asyncio.Event()
         self.stopped = asyncio.Event()
         self.stopped.set()
         self.connected = asyncio.Event()
@@ -130,6 +149,7 @@ class Processor(asyncio.Protocol):
 
     async def run(self):
         """Maintains the connection and processes metric payloads."""
+        self.running.set()
         self.stopped.clear()
         self.should_terminate = False
         while not self.should_terminate:
@@ -156,6 +176,7 @@ class Processor(asyncio.Protocol):
             await asyncio.sleep(0.1)
 
         self.logger.info('processor is exiting')
+        self.running.clear()
         self.stopped.set()
 
     async def stop(self):

tests/helpers.py

@@ -1,8 +1,11 @@
 import asyncio
 import io
+import typing
 
 
 class StatsdServer(asyncio.Protocol):
+    metrics: typing.List[bytes]
+
     def __init__(self):
         self.service = None
         self.host = '127.0.0.1'

tests/test_mixins.py

@@ -0,0 +1,213 @@
+import asyncio
+import os
+import time
+import typing
+
+from tornado import testing, web
+
+import sprockets_statsd.mixins
+from tests import helpers
+
+ParsedMetric = typing.Tuple[str, float, str]
+
+
+class Handler(sprockets_statsd.mixins.RequestHandler, web.RequestHandler):
+    async def get(self):
+        with self.execution_timer('execution-timer'):
+            await asyncio.sleep(0.1)
+        self.increase_counter('request-count')
+        self.write('true')
+
+
+class Application(sprockets_statsd.mixins.Application, web.Application):
+    def __init__(self, **settings):
+        super().__init__([web.url('/', Handler)], **settings)
+
+
+class ApplicationTests(testing.AsyncTestCase):
+    def setUp(self):
+        super().setUp()
+        self._environ = {}
+
+    def setenv(self, name, value):
+        self._environ.setdefault(name, os.environ.pop(name, None))
+        os.environ[name] = value
+
+    def unsetenv(self, name):
+        self._environ.setdefault(name, os.environ.pop(name, None))
+
+    def test_statsd_setting_defaults(self):
+        self.unsetenv('STATSD_HOST')
+        self.unsetenv('STATSD_PORT')
+
+        app = sprockets_statsd.mixins.Application()
+        self.assertIn('statsd', app.settings)
+        self.assertIsNone(app.settings['statsd']['host'],
+                          'default host value should be None')
+        self.assertEqual(8125, app.settings['statsd']['port'])
+        self.assertEqual('applications', app.settings['statsd']['prefix'])
+
+    def test_that_statsd_settings_read_from_environment(self):
+        self.setenv('STATSD_HOST', 'statsd')
+        self.setenv('STATSD_PORT', '5218')
+
+        app = sprockets_statsd.mixins.Application()
+        self.assertIn('statsd', app.settings)
+        self.assertEqual('statsd', app.settings['statsd']['host'])
+        self.assertEqual(5218, app.settings['statsd']['port'])
+
+    def test_that_service_included_in_prefix_if_set(self):
+        app = sprockets_statsd.mixins.Application(service='blah')
+        self.assertIn('statsd', app.settings)
+        self.assertEqual('applications.blah', app.settings['statsd']['prefix'])
+
+    def test_that_environment_included_in_prefix_if_set(self):
+        app = sprockets_statsd.mixins.Application(environment='whatever')
+        self.assertIn('statsd', app.settings)
+        self.assertEqual('applications.whatever',
+                         app.settings['statsd']['prefix'])
+
+    def test_fully_specified_prefix(self):
+        app = sprockets_statsd.mixins.Application(environment='whatever',
+                                                  service='blah')
+        self.assertIn('statsd', app.settings)
+        self.assertEqual('applications.blah.whatever',
+                         app.settings['statsd']['prefix'])
+
+    def test_overridden_settings(self):
+        self.setenv('STATSD_HOST', 'statsd')
+        self.setenv('STATSD_PORT', '9999')
+        app = sprockets_statsd.mixins.Application(statsd={
+            'host': 'statsd.example.com',
+            'port': 5218,
+            'prefix': 'myapp',
+        })
+        self.assertEqual('statsd.example.com', app.settings['statsd']['host'])
+        self.assertEqual(5218, app.settings['statsd']['port'])
+        self.assertEqual('myapp', app.settings['statsd']['prefix'])
+
+    def test_that_starting_without_configuration_fails(self):
+        self.unsetenv('STATSD_HOST')
+        app = sprockets_statsd.mixins.Application()
+        with self.assertRaises(RuntimeError):
+            self.io_loop.run_sync(app.start_statsd)
+
+    def test_starting_twice(self):
+        app = sprockets_statsd.mixins.Application(statsd={
+            'host': 'localhost',
+            'port': '8125',
+        })
+        try:
+            self.io_loop.run_sync(app.start_statsd)
+            connector = app.settings['statsd']['_connector']
+            self.assertIsNotNone(connector, 'statsd.Connector not created')
+
+            self.io_loop.run_sync(app.start_statsd)
+            self.assertIs(app.settings['statsd']['_connector'], connector,
+                          'statsd.Connector should not be recreated')
+        finally:
+            self.io_loop.run_sync(app.stop_statsd)
+
+    def test_stopping_without_starting(self):
+        app = sprockets_statsd.mixins.Application(statsd={
+            'host': 'localhost',
+            'port': '8125',
+        })
+        self.io_loop.run_sync(app.stop_statsd)
+
+
+class RequestHandlerTests(testing.AsyncHTTPTestCase):
+    def setUp(self):
+        super().setUp()
+        self.statsd_server = helpers.StatsdServer()
+        self.io_loop.spawn_callback(self.statsd_server.run)
+        self.io_loop.run_sync(self.statsd_server.wait_running)
+
+        self.app.settings['statsd']['host'] = self.statsd_server.host
+        self.app.settings['statsd']['port'] = self.statsd_server.port
+        self.io_loop.run_sync(self.app.start_statsd)
+
+    def tearDown(self):
+        self.io_loop.run_sync(self.app.stop_statsd)
+        self.statsd_server.close()
+        self.io_loop.run_sync(self.statsd_server.wait_closed)
+        super().tearDown()
+
+    def get_app(self):
+        self.app = Application()
+        return self.app
+
+    def wait_for_metrics(self, metric_count=3):
+        timeout_remaining = testing.get_async_test_timeout()
+        for _ in range(metric_count):
+            start = time.time()
+            self.io_loop.run_sync(self.statsd_server.message_received.acquire,
+                                  timeout=timeout_remaining)
+            timeout_remaining -= (time.time() - start)
+
+    def parse_metric(self, metric_line: bytes) -> ParsedMetric:
+        metric_line = metric_line.decode()
+        path, _, rest = metric_line.partition(':')
+        value, _, type_code = rest.partition('|')
+        try:
+            value = float(value)
+        except ValueError:
+            self.fail(f'value of {path} is not a number: value={value!r}')
+        return path, value, type_code
+
+    def find_metric(self, needle: str) -> ParsedMetric:
+        needle = needle.encode()
+        for line in self.statsd_server.metrics:
+            if needle in line:
+                return self.parse_metric(line)
+        self.fail(f'failed to find metric containing {needle!r}')
+
+    def test_the_request_metric_is_sent_last(self):
+        rsp = self.fetch('/')
+        self.assertEqual(200, rsp.code)
+        self.wait_for_metrics()
+
+        path, _, type_code = self.find_metric('Handler.GET.200')
+        self.assertEqual(path, 'applications.timers.Handler.GET.200')
+        self.assertEqual('ms', type_code)
+
+    def test_execution_timer(self):
+        rsp = self.fetch('/')
+        self.assertEqual(200, rsp.code)
+        self.wait_for_metrics()
+
+        path, _, type_code = self.find_metric('execution-timer')
+        self.assertEqual('applications.timers.execution-timer', path)
+        self.assertEqual('ms', type_code)
+
+    def test_counter(self):
+        rsp = self.fetch('/')
+        self.assertEqual(200, rsp.code)
+        self.wait_for_metrics()
+
+        path, value, type_code = self.find_metric('request-count')
+        self.assertEqual('applications.counters.request-count', path)
+        self.assertEqual(1.0, value)
+        self.assertEqual('c', type_code)
+
+    def test_handling_request_without_statsd_configured(self):
+        self.io_loop.run_sync(self.app.stop_statsd)
+
+        rsp = self.fetch('/')
+        self.assertEqual(200, rsp.code)
+
+    def test_handling_request_without_prefix(self):
+        self.app.settings['statsd']['prefix'] = ''
+
+        rsp = self.fetch('/')
+        self.assertEqual(200, rsp.code)
+        self.wait_for_metrics()
+
+        path, _, _ = self.find_metric('Handler.GET.200')
+        self.assertEqual('timers.Handler.GET.200', path)
+
+        path, _, _ = self.find_metric('execution-timer')
+        self.assertEqual('timers.execution-timer', path)
+
+        path, _, _ = self.find_metric('request-count')
+        self.assertEqual('counters.request-count', path)

tests/test_processor.py

@@ -3,7 +3,6 @@ import time
 import unittest
 
 from sprockets_statsd import statsd
-
 from tests import helpers
 
 
@@ -118,6 +117,24 @@ class ProcessorTests(ProcessorTestCase):
                                      port=self.statsd_server.port)
         await self.wait_for(processor.stop())
 
+    def test_that_processor_fails_when_host_is_none(self):
+        with self.assertRaises(RuntimeError) as context:
+            statsd.Processor(host=None, port=12345)
+        self.assertIn('host', str(context.exception))
+
+    def test_that_processor_fails_when_port_is_invalid(self):
+        with self.assertRaises(RuntimeError) as context:
+            statsd.Processor(host='localhost', port=None)
+        self.assertIn('port', str(context.exception))
+
+        with self.assertRaises(RuntimeError) as context:
+            statsd.Processor(host='localhost', port=0)
+        self.assertIn('port', str(context.exception))
+
+        with self.assertRaises(RuntimeError) as context:
+            statsd.Processor(host='localhost', port=-1)
+        self.assertIn('port', str(context.exception))
+
 
 class ConnectorTests(ProcessorTestCase):
     async def asyncSetUp(self):