From 3af2d18b7e564b0d62dc0255a027fdc90c428f05 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 17:24:08 -0500 Subject: [PATCH 01/13] Drop support for Python < 3.7 --- .travis.yml | 11 +++-------- setup.py | 8 +------- tox.ini | 2 +- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a01dbe..cc3e6c1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,7 @@ language: python +dist: xenial python: -- 2.7 -- 3.4 -- 3.5 -- 3.6 -- 3.7-dev -- pypy -- pypy3 +- 3.7 before_install: - pip install nose coverage codecov - pip install -r requires/testing.txt @@ -25,4 +20,4 @@ deploy: tags: true distributions: sdist bdist_wheel all_branches: true - python: 3.6 + python: 3.7 diff --git a/setup.py b/setup.py index 3a60e97..f4bf020 100755 --- a/setup.py +++ b/setup.py @@ -45,18 +45,12 @@ setuptools.setup( 'License :: OSI Approved :: BSD License', 'Natural Language :: English', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules'], test_suite='nose.collector', + python_requires='>=3.7', zip_safe=True, ) diff --git a/tox.ini b/tox.ini index f90ef7c..50d52a0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py34,py35,py36,py37,pypy2,pypy3 +envlist = py37 indexserver = default = https://pypi.python.org/simple toxworkdir = build/tox From 567b408c2952f1dab9dbb2524e4b6b2e5a0f5c0e Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 17:26:35 -0500 Subject: [PATCH 02/13] Drop InfluxDB - use sprockets-influxdb --- README.rst | 73 +----- docs/api.rst | 21 -- docs/examples.rst | 19 -- examples/influxdb.py | 86 ------- examples/statsd.py | 10 - requires/testing.txt | 1 - sprockets/mixins/metrics/influxdb.py | 362 --------------------------- sprockets/mixins/metrics/statsd.py | 9 - sprockets/mixins/metrics/testing.py | 88 +------ tests.py | 180 +------------ 10 files changed, 4 insertions(+), 845 deletions(-) delete mode 100644 examples/influxdb.py delete mode 100644 sprockets/mixins/metrics/influxdb.py diff --git a/README.rst b/README.rst index 9e7bffe..c62cf6f 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ sprockets.mixins.metrics |Version| |Status| |Coverage| |License| -Adjust counter and timer metrics in `InfluxDB`_ or `StatsD`_ using the same API. +Adjust counter and timer metrics in `StatsD`_ using the same API. The mix-in is configured through the ``tornado.web.Application`` settings property using a key defined by the specific mix-in. @@ -63,76 +63,6 @@ Settings :prepend_metric_type: Optional flag to prepend bucket path with the StatsD metric type -InfluxDB Mixin --------------- - -The following snippet configures the InfluxDB mix-in from common environment -variables: - -.. code-block:: python - - import os - - from sprockets.mixins.metrics import influxdb - from sprockets.mixins import postgresql - from tornado import gen, web - - def make_app(**settings): - settings[influxdb.SETTINGS_KEY] = { - 'measurement': 'rollup', - } - - application = web.Application( - [ - web.url(r'/', MyHandler), - ], **settings) - - influxdb.install({'url': 'http://localhost:8086', - 'database': 'tornado-app'}) - return application - - - class MyHandler(influxdb.InfluxDBMixin, - postgresql.HandlerMixin, - web.RequestHandler): - - @gen.coroutine - def get(self, obj_id): - with self.execution_timer('dbquery', 'get'): - result = yield self.postgresql_session.query( - 'SELECT * FROM foo WHERE id=%s', obj_id) - self.send_response(result) - -If your application handles signal handling for shutdowns, the -:meth:`~sprockets.mixins.influxdb.shutdown` method will try to cleanly ensure -that any buffered metrics in the InfluxDB collector are written prior to -shutting down. The method returns a :class:`~tornado.concurrent.TracebackFuture` -that should be waited on prior to shutting down. - -For environment variable based configuration, use the ``INFLUX_SCHEME``, -``INFLUX_HOST``, and ``INFLUX_PORT`` environment variables. The defaults are -``https``, ``localhost``, and ``8086`` respectively. - -To use authentication with InfluxDB, set the ``INFLUX_USER`` and the -``INFLUX_PASSWORD`` environment variables. Once installed, the -``INFLUX_PASSWORD`` value will be masked in the Python process. - -Settings -^^^^^^^^ - -:url: The InfluxDB API URL -:database: the database to write measurements into -:submission_interval: How often to submit metric batches in - milliseconds. Default: ``5000`` -:max_batch_size: The number of measurements to be submitted in a - single HTTP request. Default: ``1000`` -:tags: Default tags that are to be submitted with each metric. The tag - ``hostname`` is added by default along with ``environment`` and ``service`` - if the corresponding ``ENVIRONMENT`` or ``SERVICE`` environment variables - are set. -:auth_username: A username to use for InfluxDB authentication, if desired. -:auth_password: A password to use for InfluxDB authentication, if desired. - Development Quickstart ---------------------- .. code-block:: bash @@ -164,7 +94,6 @@ Development Quickstart (env)$ open build/sphinx/html/index.html .. _StatsD: https://github.com/etsy/statsd -.. _InfluxDB: https://influxdata.com .. |Version| image:: https://img.shields.io/pypi/v/sprockets_mixins_metrics.svg diff --git a/docs/api.rst b/docs/api.rst index 618040a..c68183d 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -43,30 +43,12 @@ implements the same interface: with self.execution_timer('db', 'query', 'foo'): rows = yield self.session.query('SELECT * FROM foo') - .. method:: set_metric_tag(tag, value) - :noindex: - - :param str tag: the tag to set - :param str value: the value to assign to the tag - - This method stores a tag and value pair to be reported with - metrics. It is only implemented on back-ends that support - tagging metrics (e.g., :class:`sprockets.mixins.metrics.InfluxDBMixin`) - Statsd Implementation --------------------- .. autoclass:: sprockets.mixins.metrics.statsd.StatsdMixin :members: -InfluxDB Implementation ------------------------ -.. autoclass:: sprockets.mixins.metrics.influxdb.InfluxDBMixin - :members: - -.. autoclass:: sprockets.mixins.metrics.influxdb.InfluxDBCollector - :members: - Testing Helpers --------------- *So who actually tests that their metrics are emitted as they expect?* @@ -76,6 +58,3 @@ contains some helper that make testing a little easier. .. autoclass:: sprockets.mixins.metrics.testing.FakeStatsdServer :members: - -.. autoclass:: sprockets.mixins.metrics.testing.FakeInfluxHandler - :members: diff --git a/docs/examples.rst b/docs/examples.rst index 3c2e649..42c8992 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -16,22 +16,3 @@ as a base class. .. literalinclude:: ../examples/statsd.py :pyobject: SimpleHandler - -Sending measurements to InfluxDB --------------------------------- -This simple application sends per-request measurements to an InfluxDB -server listening on ``localhost``. The mix-in is configured by passing -a ``sprockets.mixins.metrics.influxdb`` key into the application settings -as shown below. - -.. literalinclude:: ../examples/influxdb.py - :pyobject: make_application - -The InfluxDB database and measurement name are also configured in the -application settings object. The request handler is responsible for -providing the tag and value portions of the measurement. The standard -:class:`Metric Mixin API` is used to set -tagged values. - -.. literalinclude:: ../examples/influxdb.py - :pyobject: SimpleHandler diff --git a/examples/influxdb.py b/examples/influxdb.py deleted file mode 100644 index b794002..0000000 --- a/examples/influxdb.py +++ /dev/null @@ -1,86 +0,0 @@ -import signal - -from sprockets.mixins.metrics import influxdb -from tornado import concurrent, gen, ioloop, web - - -class SimpleHandler(influxdb.InfluxDBMixin, web.RequestHandler): - """ - Simply emits a few metrics around the GET method. - - The ``InfluxDBMixin`` sends all of the metrics gathered during - the processing of a request as a single measurement when the - request is finished. Each request of this sample will result - in a single measurement using the service name as the key. - - The following tag keys are defined by default: - - handler="examples.influxdb.SimpleHandler" - host="$HOSTNAME" - method="GET" - - and the following values are written: - - duration=0.2573668956756592 - sleepytime=0.255108118057251 - slept=42 - status_code=204 - - The duration and status_code values are handled by the mix-in - and the slept and sleepytime values are added in the method. - - """ - - def initialize(self): - super(SimpleHandler, self).initialize() - self.set_metric_tag('environment', 'testing') - - @gen.coroutine - def prepare(self): - maybe_future = super(SimpleHandler, self).prepare() - if concurrent.is_future(maybe_future): - yield maybe_future - - if 'Correlation-ID' in self.request.headers: - self.set_metric_tag('correlation_id', - self.request.headers['Correlation-ID']) - - @gen.coroutine - def get(self): - with self.execution_timer('sleepytime'): - yield gen.sleep(0.25) - self.increase_counter('slept', amount=42) - self.set_status(204) - self.finish() - - -def _sig_handler(*args_): - iol = ioloop.IOLoop.instance() - iol.add_callback_from_signal(iol.stop) - - -def make_application(): - """ - Create a application configured to send metrics. - - Measurements will be sent to the ``testing`` database on the - configured InfluxDB instance. The measurement name is set - by the ``service`` setting. - - """ - settings = { - influxdb.SETTINGS_KEY: { - 'measurement': 'example', - } - } - application = web.Application([web.url('/', SimpleHandler)], **settings) - influxdb.install(application, **{'database': 'testing'}) - return application - - -if __name__ == '__main__': - app = make_application() - app.listen(8000) - signal.signal(signal.SIGINT, _sig_handler) - signal.signal(signal.SIGTERM, _sig_handler) - ioloop.IOLoop.instance().start() diff --git a/examples/statsd.py b/examples/statsd.py index 0278984..27cca9a 100644 --- a/examples/statsd.py +++ b/examples/statsd.py @@ -14,16 +14,6 @@ class SimpleHandler(statsd.StatsdMixin, web.RequestHandler): """ - @gen.coroutine - def prepare(self): - maybe_future = super(SimpleHandler, self).prepare() - if concurrent.is_future(maybe_future): - yield maybe_future - - if 'Correlation-ID' in self.request.headers: - self.set_metric_tag('correlation_id', - self.request.headers['Correlation-ID']) - @gen.coroutine def get(self): yield gen.sleep(0.25) diff --git a/requires/testing.txt b/requires/testing.txt index 0ec0feb..fd00fa2 100644 --- a/requires/testing.txt +++ b/requires/testing.txt @@ -1,3 +1,2 @@ -mock>=2,<3 nose>=1.3,<2 tornado>=4.2,<4.3 diff --git a/sprockets/mixins/metrics/influxdb.py b/sprockets/mixins/metrics/influxdb.py deleted file mode 100644 index 57fba7a..0000000 --- a/sprockets/mixins/metrics/influxdb.py +++ /dev/null @@ -1,362 +0,0 @@ -import contextlib -import logging -import os -import socket -import time - -from tornado import concurrent, httpclient, ioloop - -from sprockets.mixins.metrics import __version__ - -LOGGER = logging.getLogger(__name__) - -SETTINGS_KEY = 'sprockets.mixins.metrics.influxdb' -"""``self.settings`` key that configures this mix-in.""" - -_USER_AGENT = 'sprockets.mixins.metrics/v{}'.format(__version__) - - -class InfluxDBMixin(object): - """Mix this class in to record measurements to a InfluxDB server.""" - - def __init__(self, application, request, **kwargs): - self.__metrics = [] - self.__tags = { - 'handler': '{}.{}'.format(self.__module__, - self.__class__.__name__), - 'method': request.method, - } - - # Call to super().__init__() needs to be *AFTER* we create our - # properties since it calls initialize() which may want to call - # methods like ``set_metric_tag`` - super(InfluxDBMixin, self).__init__(application, request, **kwargs) - - def set_metric_tag(self, tag, value): - """ - Add a tag to the measurement key. - - :param str tag: name of the tag to set - :param str value: value to assign - - This will overwrite the current value assigned to a tag - if one exists. - - """ - self.__tags[tag] = value - - def record_timing(self, duration, *path): - """ - Record a timing. - - :param float duration: timing to record in seconds - :param path: elements of the metric path to record - - A timing is a named duration value. - - """ - self.__metrics.append('{}={}'.format( - self.application.influxdb.escape_str('.'.join(path)), duration)) - - def increase_counter(self, *path, **kwargs): - """ - Increase a counter. - - :param path: elements of the path to record - :keyword int amount: value to record. If omitted, the counter - value is one. - - Counters are simply values that are summed in a query. - - """ - self.__metrics.append('{}={}'.format( - self.application.influxdb.escape_str('.'.join(path)), - kwargs.get('amount', 1))) - - @contextlib.contextmanager - def execution_timer(self, *path): - """ - Record the time it takes to run an arbitrary code block. - - :param path: elements of the metric path to record - - This method returns a context manager that records the amount - of time spent inside of the context and records a value - named `path` using (:meth:`record_timing`). - - """ - start = time.time() - try: - yield - finally: - self.record_timing(max(time.time(), start) - start, *path) - - def on_finish(self): - super(InfluxDBMixin, self).on_finish() - self.set_metric_tag('status_code', self.get_status()) - self.record_timing(self.request.request_time(), 'duration') - self.application.influxdb.submit( - self.settings[SETTINGS_KEY]['measurement'], - self.__tags, - self.__metrics) - - -class InfluxDBCollector(object): - """Collects and submits stats to InfluxDB on a periodic callback. - - :param str url: The InfluxDB API URL - :param str database: the database to write measurements into - :param tornado.ioloop.IOLoop: the IOLoop to spawn callbacks on. - If this parameter is :data:`None`, then the active IOLoop, - as determined by :meth:`tornado.ioloop.IOLoop.instance`, - is used. - :param int submission_interval: How often to submit metric batches in - milliseconds. Default: ``5000`` - :param max_batch_size: The number of measurements to be submitted in a - single HTTP request. Default: ``1000`` - :param dict tags: Default tags that are to be submitted with each metric. - :param str auth_username: Optional username for authenticated requests. - :param str auth_password: Optional password for authenticated requests. - - This class should be constructed using the - :meth:`~sprockets.mixins.influxdb.install` method. When installed, it is - attached to the :class:`~tornado.web.Application` instance for your web - application and schedules a periodic callback to submit metrics to InfluxDB - in batches. - - """ - SUBMISSION_INTERVAL = 5000 - MAX_BATCH_SIZE = 1000 - WARN_THRESHOLD = 25000 - - def __init__(self, url='http://localhost:8086', database='sprockets', - io_loop=None, submission_interval=SUBMISSION_INTERVAL, - max_batch_size=MAX_BATCH_SIZE, tags=None, - auth_username=None, auth_password=None): - self._buffer = list() - self._database = database - self._influxdb_url = '{}?db={}'.format(url, database) - self._interval = submission_interval or self.SUBMISSION_INTERVAL - self._io_loop = io_loop or ioloop.IOLoop.current() - self._max_batch_size = max_batch_size or self.MAX_BATCH_SIZE - self._pending = 0 - self._tags = tags or {} - - # Configure the default - defaults = {'user_agent': _USER_AGENT} - if auth_username and auth_password: - LOGGER.debug('Adding authentication info to defaults (%s)', - auth_username) - defaults['auth_username'] = auth_username - defaults['auth_password'] = auth_password - - self._client = httpclient.AsyncHTTPClient(force_instance=True, - defaults=defaults, - io_loop=self._io_loop) - - # Add the periodic callback for submitting metrics - LOGGER.info('Starting PeriodicCallback for writing InfluxDB metrics') - self._callback = ioloop.PeriodicCallback(self._write_metrics, - self._interval) - self._callback.start() - - @staticmethod - def escape_str(value): - """Escape the value with InfluxDB's wonderful escaping logic: - - "Measurement names, tag keys, and tag values must escape any spaces or - commas using a backslash (\). For example: \ and \,. All tag values are - stored as strings and should not be surrounded in quotes." - - :param str value: The value to be escaped - :rtype: str - - """ - return str(value).replace(' ', '\ ').replace(',', '\,') - - @property - def database(self): - """Return the configured database name. - - :rtype: str - - """ - return self._database - - def shutdown(self): - """Invoke on shutdown of your application to stop the periodic - callbacks and flush any remaining metrics. - - Returns a future that is complete when all pending metrics have been - submitted. - - :rtype: :class:`~tornado.concurrent.TracebackFuture()` - - """ - future = concurrent.TracebackFuture() - self._callback.stop() - self._write_metrics() - self._shutdown_wait(future) - return future - - def submit(self, measurement, tags, values): - """Add a measurement to the buffer that will be submitted to InfluxDB - on the next periodic callback for writing metrics. - - :param str measurement: The measurement name - :param dict tags: The measurement tags - :param list values: The recorded measurements - - """ - self._buffer.append('{},{} {} {:d}'.format( - self.escape_str(measurement), - self._get_tag_string(tags), - ','.join(values), - int(time.time() * 1000000000))) - if len(self._buffer) > self.WARN_THRESHOLD: - LOGGER.warning('InfluxDB metric buffer is > %i (%i)', - self.WARN_THRESHOLD, len(self._buffer)) - - def _get_tag_string(self, tags): - """Return the tags to be submitted with a measurement combining the - default tags that were passed in when constructing the class along - with any measurement specific tags passed into the - :meth:`~InfluxDBConnection.submit` method. Tags will be properly - escaped and formatted for submission. - - :param dict tags: Measurement specific tags - :rtype: str - - """ - values = dict(self._tags) - values.update(tags) - return ','.join(['{}={}'.format(self.escape_str(k), self.escape_str(v)) - for k, v in values.items()]) - - def _on_write_response(self, response): - """This is invoked by the Tornado IOLoop when the HTTP request to - InfluxDB has returned with a result. - - :param response: The response from InfluxDB - :type response: :class:`~tornado.httpclient.HTTPResponse` - - """ - self._pending -= 1 - LOGGER.debug('InfluxDB batch response: %s', response.code) - if response.error: - LOGGER.error('InfluxDB batch submission error: %s', response.error) - - def _shutdown_wait(self, future): - """Pause briefly allowing any pending metric writes to complete before - shutting down. - - :param future tornado.concurrent.TracebackFuture: The future to resulve - when the shutdown is complete. - - """ - if not self._pending: - future.set_result(True) - return - LOGGER.debug('Waiting for pending metric writes') - self._io_loop.add_timeout(self._io_loop.time() + 0.1, - self._shutdown_wait, - (future,)) - - def _write_metrics(self): - """Submit the metrics in the buffer to InfluxDB. This is invoked - by the periodic callback scheduled when the class is created. - - It will submit batches until the buffer is empty. - - """ - if not self._buffer: - return - LOGGER.debug('InfluxDB buffer has %i items', len(self._buffer)) - while self._buffer: - body = '\n'.join(self._buffer[:self._max_batch_size]) - self._buffer = self._buffer[self._max_batch_size:] - self._pending += 1 - self._client.fetch(self._influxdb_url, method='POST', - body=body.encode('utf-8'), - raise_error=False, - callback=self._on_write_response) - LOGGER.debug('Submitted all InfluxDB metrics for writing') - - -def install(application, **kwargs): - """Call this to install the InfluxDB collector into a Tornado application. - - :param tornado.web.Application application: the application to - install the collector into. - :param kwargs: keyword parameters to pass to the - :class:`InfluxDBCollector` initializer. - :returns: :data:`True` if the client was installed by this call - and :data:`False` otherwise. - - - Optional configuration values: - - - **url** The InfluxDB API URL. If URL is not specified, the - ``INFLUX_HOST`` and ``INFLUX_PORT`` environment variables will be used - to construct the URL to pass into the :class:`InfluxDBCollector`. - - **database** the database to write measurements into. - The default is ``sprockets``. - - **io_loop** A :class:`~tornado.ioloop.IOLoop` to use - - **submission_interval** How often to submit metric batches in - milliseconds. Default: ``5000`` - - **max_batch_size** The number of measurements to be submitted in a - single HTTP request. Default: ``1000`` - - **tags** Default tags that are to be submitted with each metric. - - **auth_username** A username to use for InfluxDB authentication - - **auth_password** A password to use for InfluxDB authentication - - If ``auth_password`` is specified as an environment variable, it will be - masked in the Python process. - - """ - if getattr(application, 'influxdb', None) is not None: - LOGGER.warning('InfluxDBCollector is already installed') - return False - - # Get config values - url = '{}://{}:{}/write'.format(os.environ.get('INFLUX_SCHEME', 'http'), - os.environ.get('INFLUX_HOST', 'localhost'), - os.environ.get('INFLUX_PORT', 8086)) - kwargs.setdefault('url', url) - - # Build the full tag dict and replace what was passed in - tags = {'hostname': socket.gethostname()} - if os.environ.get('ENVIRONMENT'): - tags['environment'] = os.environ.get('ENVIRONMENT') - if os.environ.get('SERVICE'): - tags['service'] = os.environ.get('SERVICE') - tags.update(kwargs.get('tags', {})) - kwargs['tags'] = tags - - # Check if auth variables are set as env vars and set them if so - if os.environ.get('INFLUX_USER'): - kwargs.setdefault('auth_username', os.environ.get('INFLUX_USER')) - kwargs.setdefault('auth_password', - os.environ.get('INFLUX_PASSWORD', '')) - - # Don't leave the environment variable out there with the password - if os.environ.get('INFLUX_PASSWORD'): - os.environ['INFLUX_PASSWORD'] = 'X' * len(kwargs['auth_password']) - - # Create and start the collector - setattr(application, 'influxdb', InfluxDBCollector(**kwargs)) - return True - - -def shutdown(application): - """Invoke to shutdown the InfluxDB collector, writing any pending - measurements to InfluxDB before stopping. - - :param tornado.web.Application application: the application to - install the collector into. - :rtype: tornado.concurrent.TracebackFuture or None - - """ - collector = getattr(application, 'influxdb', None) - if collector: - return collector.shutdown() diff --git a/sprockets/mixins/metrics/statsd.py b/sprockets/mixins/metrics/statsd.py index f0e3817..7cc5477 100644 --- a/sprockets/mixins/metrics/statsd.py +++ b/sprockets/mixins/metrics/statsd.py @@ -18,15 +18,6 @@ class StatsdMixin(object): def initialize(self): super(StatsdMixin, self).initialize() - def set_metric_tag(self, tag, value): - """Ignored for statsd since it does not support tagging. - - :param str tag: name of the tag to set - :param str value: value to assign - - """ - pass - def record_timing(self, duration, *path): """Record a timing. diff --git a/sprockets/mixins/metrics/testing.py b/sprockets/mixins/metrics/testing.py index f50acdb..2b486af 100644 --- a/sprockets/mixins/metrics/testing.py +++ b/sprockets/mixins/metrics/testing.py @@ -2,7 +2,7 @@ import logging import re import socket -from tornado import gen, iostream, locks, tcpserver, testing, web +from tornado import gen, iostream, locks, tcpserver, testing LOGGER = logging.getLogger(__name__) @@ -119,89 +119,3 @@ class FakeStatsdServer(tcpserver.TCPServer): raise AssertionError( 'Expected metric starting with "{}" in {!r}'.format( prefix, self.datagrams)) - - -class FakeInfluxHandler(web.RequestHandler): - """ - Request handler that mimics the InfluxDB write endpoint. - - Install this handler into your testing application and configure - the metrics plugin to write to it. After running a test, you can - examine the received measurements by iterating over the - ``influx_db`` list in the application object. - - .. code-block:: python - - class TestThatMyStuffWorks(testing.AsyncHTTPTestCase): - - def get_app(self): - self.app = web.Application([ - web.url('/', HandlerUnderTest), - web.url('/write', metrics.testing.FakeInfluxHandler), - ]) - return self.app - - def setUp(self): - super(TestThatMyStuffWorks, self).setUp() - self.app.settings[metrics.InfluxDBMixin.SETTINGS_KEY] = { - 'measurement': 'stuff', - 'write_url': self.get_url('/write'), - 'database': 'requests', - } - - def test_that_measurements_are_emitted(self): - self.fetch('/') # invokes handler under test - measurements = metrics.testing.FakeInfluxHandler( - self.app, 'requests', self) - for key, fields, timestamp in measurements: - # inspect measurements - - """ - def initialize(self): - super(FakeInfluxHandler, self).initialize() - self.logger = LOGGER.getChild(__name__) - if not hasattr(self.application, 'influx_db'): - self.application.influx_db = {} - if self.application.influxdb.database not in self.application.influx_db: - self.application.influx_db[self.application.influxdb.database] = [] - - def post(self): - db = self.get_query_argument('db') - payload = self.request.body.decode('utf-8') - for line in payload.splitlines(): - self.logger.debug('received "%s"', line) - key, fields, timestamp = line.split() - self.application.influx_db[db].append((key, fields, timestamp, - self.request.headers)) - self.set_status(204) - - @staticmethod - def get_messages(application, test_case): - """ - Wait for measurements to show up and return them. - - :param tornado.web.Application application: application that - :class:`.FakeInfluxHandler` is writing to - :param str database: database to retrieve - :param tornado.testing.AsyncTestCase test_case: test case - that is being executed - :return: measurements received as a :class:`list` of - (key, fields, timestamp) tuples - - Since measurements are sent asynchronously from within the - ``on_finish`` handler they are usually not sent by the time - that the test case has stopped the IOloop. This method accounts - for this by running the IOloop until measurements have been - received. It will raise an assertion error if measurements - are not received in a reasonable number of runs. - - """ - for _ in range(0, 15): - if hasattr(application, 'influx_db'): - if application.influx_db.get(application.influxdb.database): - return application.influx_db[application.influxdb.database] - test_case.io_loop.add_future(gen.sleep(0.1), - lambda _: test_case.stop()) - test_case.wait() - else: - test_case.fail('Message not published to InfluxDB before timeout') diff --git a/tests.py b/tests.py index 2161d9e..6c890fa 100644 --- a/tests.py +++ b/tests.py @@ -1,20 +1,13 @@ -import base64 import itertools -import logging -import os import socket -import time import unittest -import uuid from tornado import gen, iostream, testing, web import mock from mock import patch -from sprockets.mixins.metrics import influxdb, statsd -from sprockets.mixins.metrics.testing import ( - FakeInfluxHandler, FakeStatsdServer) -import examples.influxdb +from sprockets.mixins.metrics import statsd +from sprockets.mixins.metrics.testing import FakeStatsdServer import examples.statsd @@ -389,172 +382,3 @@ class StatsdInstallationTests(unittest.TestCase): statsd.install(self.application, **{'namespace': 'testing'}) self.assertEqual(self.application.statsd._host, '127.0.0.1') self.assertEqual(self.application.statsd._port, 8125) - - -class InfluxDbTests(testing.AsyncHTTPTestCase): - - def get_app(self): - self.application = web.Application([ - web.url(r'/', examples.influxdb.SimpleHandler), - web.url(r'/write', FakeInfluxHandler), - ]) - influxdb.install(self.application, **{'database': 'requests', - 'submission_interval': 1, - 'url': self.get_url('/write')}) - self.application.influx_db = {} - return self.application - - def setUp(self): - self.application = None - super(InfluxDbTests, self).setUp() - self.application.settings[influxdb.SETTINGS_KEY] = { - 'measurement': 'my-service' - } - logging.getLogger(FakeInfluxHandler.__module__).setLevel(logging.DEBUG) - - @gen.coroutine - def tearDown(self): - yield influxdb.shutdown(self.application) - super(InfluxDbTests, self).tearDown() - - @property - def influx_messages(self): - return FakeInfluxHandler.get_messages(self.application, self) - - def test_that_http_method_call_details_are_recorded(self): - start = int(time.time()) - response = self.fetch('/') - self.assertEqual(response.code, 204) - - for key, fields, timestamp, _headers in self.influx_messages: - if key.startswith('my-service,'): - tag_dict = dict(a.split('=') for a in key.split(',')[1:]) - self.assertEqual(tag_dict['handler'], - 'examples.influxdb.SimpleHandler') - self.assertEqual(tag_dict['method'], 'GET') - self.assertEqual(tag_dict['hostname'], socket.gethostname()) - self.assertEqual(tag_dict['status_code'], '204') - - value_dict = dict(a.split('=') for a in fields.split(',')) - self.assertIn('duration', value_dict) - self.assertTrue(float(value_dict['duration']) > 0) - - nanos_since_epoch = int(timestamp) - then = nanos_since_epoch / 1000000000 - assert_between(start, then, time.time()) - break - else: - self.fail('Expected to find "request" metric in {!r}'.format( - list(self.application.influx_db['requests']))) - - def test_that_execution_timer_is_tracked(self): - response = self.fetch('/') - self.assertEqual(response.code, 204) - - for key, fields, timestamp, _headers in self.influx_messages: - if key.startswith('my-service,'): - value_dict = dict(a.split('=') for a in fields.split(',')) - assert_between(0.25, float(value_dict['sleepytime']), 0.3) - break - else: - self.fail('Expected to find "request" metric in {!r}'.format( - list(self.application.influx_db['requests']))) - - def test_that_counter_is_tracked(self): - response = self.fetch('/') - self.assertEqual(response.code, 204) - - for key, fields, timestamp, _headers in self.influx_messages: - if key.startswith('my-service,'): - value_dict = dict(a.split('=') for a in fields.split(',')) - self.assertEqual(int(value_dict['slept']), 42) - break - else: - self.fail('Expected to find "request" metric in {!r}'.format( - list(self.application.influx_db['requests']))) - - def test_that_cached_db_connection_is_used(self): - cfg = self.application.settings[influxdb.SETTINGS_KEY] - conn = mock.Mock() - cfg['db_connection'] = conn - response = self.fetch('/') - self.assertEqual(response.code, 204) - self.assertIs(cfg['db_connection'], conn) - - def test_that_metric_tag_is_tracked(self): - cid = str(uuid.uuid4()) - response = self.fetch('/', headers={'Correlation-ID': cid}) - self.assertEqual(response.code, 204) - - for key, fields, timestamp, _headers in self.influx_messages: - if key.startswith('my-service,'): - tag_dict = dict(a.split('=') for a in key.split(',')[1:]) - self.assertEqual(tag_dict['correlation_id'], cid) - break - else: - self.fail('Expected to find "request" metric in {!r}'.format( - list(self.application.influx_db['requests']))) - - def test_metrics_with_buffer_not_flush(self): - self.application.settings[influxdb] = { - 'measurement': 'my-service' - } - - # 2 requests - response = self.fetch('/') - self.assertEqual(response.code, 204) - response = self.fetch('/') - self.assertEqual(response.code, 204) - with self.assertRaises(AssertionError): - self.assertEqual(0, len(self.influx_messages)) - - -class InfluxDbAuthTests(testing.AsyncHTTPTestCase): - - def setUp(self): - self.application = None - self.username, self.password = str(uuid.uuid4()), str(uuid.uuid4()) - os.environ['INFLUX_USER'] = self.username - os.environ['INFLUX_PASSWORD'] = self.password - super(InfluxDbAuthTests, self).setUp() - self.application.settings[influxdb.SETTINGS_KEY] = { - 'measurement': 'my-service' - } - logging.getLogger(FakeInfluxHandler.__module__).setLevel(logging.DEBUG) - - @gen.coroutine - def tearDown(self): - yield influxdb.shutdown(self.application) - super(InfluxDbAuthTests, self).tearDown() - - @property - def influx_messages(self): - return FakeInfluxHandler.get_messages(self.application, self) - - def get_app(self): - self.application = web.Application([ - web.url(r'/', examples.influxdb.SimpleHandler), - web.url(r'/write', FakeInfluxHandler), - ]) - influxdb.install(self.application, **{'database': 'requests', - 'submission_interval': 1, - 'url': self.get_url('/write')}) - self.application.influx_db = {} - return self.application - - def test_that_authentication_header_was_sent(self): - print(os.environ) - response = self.fetch('/') - self.assertEqual(response.code, 204) - - for _key, _fields, _timestamp, headers in self.influx_messages: - self.assertIn('Authorization', headers) - scheme, value = headers['Authorization'].split(' ') - self.assertEqual(scheme, 'Basic') - temp = base64.b64decode(value.encode('utf-8')) - values = temp.decode('utf-8').split(':') - self.assertEqual(values[0], self.username) - self.assertEqual(values[1], self.password) - break - else: - self.fail('Did not have an Authorization header') From 694c5276d009b9a9c00cbfbc1c050e6d5b34169f Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 17:29:09 -0500 Subject: [PATCH 03/13] Add support for Tornado >=5, drop Tornado <5 --- requires/installation.txt | 2 +- requires/testing.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/requires/installation.txt b/requires/installation.txt index ef1e120..293b5b2 100644 --- a/requires/installation.txt +++ b/requires/installation.txt @@ -1 +1 @@ -tornado>=4.0,<4.5 +tornado>=5 diff --git a/requires/testing.txt b/requires/testing.txt index fd00fa2..1ead251 100644 --- a/requires/testing.txt +++ b/requires/testing.txt @@ -1,2 +1 @@ nose>=1.3,<2 -tornado>=4.2,<4.3 From 8cc566acfd318169dec565335691f5e44d800ff1 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 17:30:42 -0500 Subject: [PATCH 04/13] Reorganize and update reqs --- requires/development.txt | 4 ++-- requires/docs.txt | 1 + requires/testing.txt | 3 ++- sprockets/mixins/metrics/statsd.py | 8 ++++---- sprockets/mixins/metrics/testing.py | 2 +- tests.py | 27 +++++++++++++-------------- 6 files changed, 23 insertions(+), 22 deletions(-) create mode 100644 requires/docs.txt diff --git a/requires/development.txt b/requires/development.txt index 5be4469..35ac586 100644 --- a/requires/development.txt +++ b/requires/development.txt @@ -1,3 +1,3 @@ +-e . +-r docs.txt -r testing.txt -coverage>=4.5,<5 -Sphinx diff --git a/requires/docs.txt b/requires/docs.txt new file mode 100644 index 0000000..57c7a26 --- /dev/null +++ b/requires/docs.txt @@ -0,0 +1 @@ +sphinx==1.8.2 diff --git a/requires/testing.txt b/requires/testing.txt index 1ead251..fb9480e 100644 --- a/requires/testing.txt +++ b/requires/testing.txt @@ -1 +1,2 @@ -nose>=1.3,<2 +coverage==4.5.2 +nose==1.3.7 diff --git a/sprockets/mixins/metrics/statsd.py b/sprockets/mixins/metrics/statsd.py index 7cc5477..96ed44a 100644 --- a/sprockets/mixins/metrics/statsd.py +++ b/sprockets/mixins/metrics/statsd.py @@ -12,11 +12,11 @@ SETTINGS_KEY = 'sprockets.mixins.metrics.statsd' """``self.settings`` key that configures this mix-in.""" -class StatsdMixin(object): +class StatsdMixin: """Mix this class in to record metrics to a Statsd server.""" def initialize(self): - super(StatsdMixin, self).initialize() + super().initialize() def record_timing(self, duration, *path): """Record a timing. @@ -79,13 +79,13 @@ class StatsdMixin(object): to send the metric, so the configured namespace is used as well. """ - super(StatsdMixin, self).on_finish() + super().on_finish() self.record_timing(self.request.request_time(), self.__class__.__name__, self.request.method, self.get_status()) -class StatsDCollector(object): +class StatsDCollector: """Collects and submits stats to StatsD. This class should be constructed using the diff --git a/sprockets/mixins/metrics/testing.py b/sprockets/mixins/metrics/testing.py index 2b486af..465738b 100644 --- a/sprockets/mixins/metrics/testing.py +++ b/sprockets/mixins/metrics/testing.py @@ -44,7 +44,7 @@ class FakeStatsdServer(tcpserver.TCPServer): def tcp_server(self): self.event = locks.Event() - super(FakeStatsdServer, self).__init__() + super().__init__() sock, port = testing.bind_unused_port() self.add_socket(sock) diff --git a/tests.py b/tests.py index 6c890fa..7cfef5b 100644 --- a/tests.py +++ b/tests.py @@ -1,10 +1,9 @@ import itertools import socket import unittest +from unittest import mock from tornado import gen, iostream, testing, web -import mock -from mock import patch from sprockets.mixins.metrics import statsd from sprockets.mixins.metrics.testing import FakeStatsdServer @@ -67,7 +66,7 @@ class TCPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): self.application = None self.namespace = 'testing' - super(TCPStatsdMetricCollectionTests, self).setUp() + super().setUp() self.statsd = FakeStatsdServer(self.io_loop, protocol='tcp') statsd.install(self.application, **{'namespace': self.namespace, @@ -80,7 +79,7 @@ class TCPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): path_sleep = 'tornado.gen.sleep' path_statsd = self.application.statsd with mock.patch(path_sleep) as gen_sleep, \ - patch.object(path_statsd, '_tcp_socket') as mock_tcp_socket: + mock.patch.object(path_statsd, '_tcp_socket') as mock_tcp_socket: f = web.Future() f.set_result(None) gen_sleep.return_value = f @@ -88,13 +87,13 @@ class TCPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): self.application.statsd._tcp_on_closed() mock_tcp_socket.assert_called_once_with() - @patch.object(iostream.IOStream, 'write') + @mock.patch.object(iostream.IOStream, 'write') def test_write_not_executed_when_connection_is_closed(self, mock_write): self.application.statsd._sock.close() self.application.statsd.send('foo', 500, 'c') mock_write.assert_not_called() - @patch.object(iostream.IOStream, 'write') + @mock.patch.object(iostream.IOStream, 'write') def test_expected_counters_data_written(self, mock_sock): path = ('foo', 'bar') value = 500 @@ -107,7 +106,7 @@ class TCPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): self.application.statsd.send(path, value, metric_type) mock_sock.assert_called_once_with(expected.encode()) - @patch.object(iostream.IOStream, 'write') + @mock.patch.object(iostream.IOStream, 'write') def test_expected_timers_data_written(self, mock_sock): path = ('foo', 'bar') value = 500 @@ -183,7 +182,7 @@ class TCPStatsdConfigurationTests(testing.AsyncHTTPTestCase): self.application = None self.namespace = 'testing' - super(TCPStatsdConfigurationTests, self).setUp() + super().setUp() self.statsd = FakeStatsdServer(self.io_loop, protocol='tcp') statsd.install(self.application, **{'namespace': self.namespace, @@ -223,7 +222,7 @@ class UDPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): self.application = None self.namespace = 'testing' - super(UDPStatsdMetricCollectionTests, self).setUp() + super().setUp() self.statsd = FakeStatsdServer(self.io_loop, protocol='udp') statsd.install(self.application, **{'namespace': self.namespace, @@ -234,9 +233,9 @@ class UDPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): def tearDown(self): self.statsd.close() - super(UDPStatsdMetricCollectionTests, self).tearDown() + super().tearDown() - @patch.object(socket.socket, 'sendto') + @mock.patch.object(socket.socket, 'sendto') def test_expected_counters_data_written(self, mock_sock): path = ('foo', 'bar') value = 500 @@ -251,7 +250,7 @@ class UDPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): expected.encode(), (self.statsd.sockaddr[0], self.statsd.sockaddr[1])) - @patch.object(socket.socket, 'sendto') + @mock.patch.object(socket.socket, 'sendto') def test_expected_timers_data_written(self, mock_sock): path = ('foo', 'bar') value = 500 @@ -329,7 +328,7 @@ class UDPStatsdConfigurationTests(testing.AsyncHTTPTestCase): self.application = None self.namespace = 'testing' - super(UDPStatsdConfigurationTests, self).setUp() + super().setUp() self.statsd = FakeStatsdServer(self.io_loop, protocol='udp') statsd.install(self.application, **{'namespace': self.namespace, @@ -340,7 +339,7 @@ class UDPStatsdConfigurationTests(testing.AsyncHTTPTestCase): def tearDown(self): self.statsd.close() - super(UDPStatsdConfigurationTests, self).tearDown() + super().tearDown() def test_that_http_method_call_is_recorded(self): response = self.fetch('/') From ad19d649116074cad2550b0c198fe5d69636959f Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 17:38:19 -0500 Subject: [PATCH 05/13] Use native async Also remove a test that isn't that useful --- README.rst | 7 +++---- examples/statsd.py | 8 ++++---- sprockets/mixins/metrics/statsd.py | 8 ++++---- sprockets/mixins/metrics/testing.py | 7 +++---- tests.py | 20 ++++---------------- 5 files changed, 18 insertions(+), 32 deletions(-) diff --git a/README.rst b/README.rst index c62cf6f..f8a2a46 100644 --- a/README.rst +++ b/README.rst @@ -26,7 +26,7 @@ call to the ``get`` method as well as a separate metric for the database query. from sprockets.mixins import mediatype from sprockets.mixins.metrics import statsd - from tornado import gen, web + from tornado import web import queries def make_application(): @@ -47,10 +47,9 @@ call to the ``get`` method as well as a separate metric for the database query. super(MyHandler, self).initialize() self.db = queries.TornadoSession(os.environ['MY_PGSQL_DSN']) - @gen.coroutine - def get(self, obj_id): + async def get(self, obj_id): with self.execution_timer('dbquery', 'get'): - result = yield self.db.query('SELECT * FROM foo WHERE id=%s', + result = await self.db.query('SELECT * FROM foo WHERE id=%s', obj_id) self.send_response(result) diff --git a/examples/statsd.py b/examples/statsd.py index 27cca9a..6e0413f 100644 --- a/examples/statsd.py +++ b/examples/statsd.py @@ -1,7 +1,8 @@ +import asyncio import signal from sprockets.mixins.metrics import statsd -from tornado import concurrent, gen, ioloop, web +from tornado import ioloop, web class SimpleHandler(statsd.StatsdMixin, web.RequestHandler): @@ -14,9 +15,8 @@ class SimpleHandler(statsd.StatsdMixin, web.RequestHandler): """ - @gen.coroutine - def get(self): - yield gen.sleep(0.25) + async def get(self): + await asyncio.sleep(0.25) self.set_status(204) self.finish() diff --git a/sprockets/mixins/metrics/statsd.py b/sprockets/mixins/metrics/statsd.py index 96ed44a..78308bf 100644 --- a/sprockets/mixins/metrics/statsd.py +++ b/sprockets/mixins/metrics/statsd.py @@ -1,10 +1,11 @@ +import asyncio import contextlib import logging import os import socket import time -from tornado import gen, iostream +from tornado import iostream LOGGER = logging.getLogger(__name__) @@ -134,12 +135,11 @@ class StatsDCollector: sock.set_close_callback(self._tcp_on_closed) return sock - @gen.engine - def _tcp_on_closed(self): + async def _tcp_on_closed(self): """Invoked when the socket is closed.""" LOGGER.warning('Not connected to statsd, connecting in %s seconds', self._tcp_reconnect_sleep) - yield gen.sleep(self._tcp_reconnect_sleep) + await asyncio.sleep(self._tcp_reconnect_sleep) self._sock = self._tcp_socket() def _tcp_on_connected(self): diff --git a/sprockets/mixins/metrics/testing.py b/sprockets/mixins/metrics/testing.py index 465738b..3155fe8 100644 --- a/sprockets/mixins/metrics/testing.py +++ b/sprockets/mixins/metrics/testing.py @@ -2,7 +2,7 @@ import logging import re import socket -from tornado import gen, iostream, locks, tcpserver, testing +from tornado import iostream, locks, tcpserver, testing LOGGER = logging.getLogger(__name__) @@ -67,11 +67,10 @@ class FakeStatsdServer(tcpserver.TCPServer): self.socket.close() self.socket = None - @gen.coroutine - def handle_stream(self, stream, address): + async def handle_stream(self, stream, address): while True: try: - result = yield stream.read_until_regex(self.TCP_PATTERN) + result = await stream.read_until_regex(self.TCP_PATTERN) except iostream.StreamClosedError: break else: diff --git a/tests.py b/tests.py index 7cfef5b..2f90665 100644 --- a/tests.py +++ b/tests.py @@ -1,9 +1,10 @@ +import asyncio import itertools import socket import unittest from unittest import mock -from tornado import gen, iostream, testing, web +from tornado import iostream, testing, web from sprockets.mixins.metrics import statsd from sprockets.mixins.metrics.testing import FakeStatsdServer @@ -12,10 +13,9 @@ import examples.statsd class CounterBumper(statsd.StatsdMixin, web.RequestHandler): - @gen.coroutine - def get(self, counter, value): + async def get(self, counter, value): with self.execution_timer(*counter.split('.')): - yield gen.sleep(float(value)) + await asyncio.sleep(float(value)) self.set_status(204) self.finish() @@ -75,18 +75,6 @@ class TCPStatsdMetricCollectionTests(testing.AsyncHTTPTestCase): 'protocol': 'tcp', 'prepend_metric_type': True}) - def test_tcp_reconnect_on_stream_close(self): - path_sleep = 'tornado.gen.sleep' - path_statsd = self.application.statsd - with mock.patch(path_sleep) as gen_sleep, \ - mock.patch.object(path_statsd, '_tcp_socket') as mock_tcp_socket: - f = web.Future() - f.set_result(None) - gen_sleep.return_value = f - - self.application.statsd._tcp_on_closed() - mock_tcp_socket.assert_called_once_with() - @mock.patch.object(iostream.IOStream, 'write') def test_write_not_executed_when_connection_is_closed(self, mock_write): self.application.statsd._sock.close() From e64a672050921770207bf7274e5bb0e80421b57b Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:03:01 -0500 Subject: [PATCH 06/13] Update testing --- .travis.yml | 11 ++++++----- tox.ini | 6 ++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index cc3e6c1..103f6f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,13 @@ language: python dist: xenial python: - 3.7 -before_install: -- pip install nose coverage codecov -- pip install -r requires/testing.txt install: -- pip install -e . -script: nosetests --with-coverage +- pip install -r requires/development.txt +script: +- nosetests --with-coverage +- python setup.py build_sphinx +- python setup.py check +- flake8 after_success: - codecov sudo: false diff --git a/tox.ini b/tox.ini index 50d52a0..14e942f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,8 @@ [tox] envlist = py37 -indexserver = - default = https://pypi.python.org/simple toxworkdir = build/tox skip_missing_interpreters = True [testenv] -deps = -rrequires/testing.txt -commands = nosetests [] +deps = -r requires/testing.txt +commands = nosetests From c008e5d866d509520fd74ab722cfdc190a91b35b Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:06:14 -0500 Subject: [PATCH 07/13] Cleanup setup.py --- setup.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index f4bf020..b6ab2cf 100755 --- a/setup.py +++ b/setup.py @@ -1,28 +1,11 @@ -#!/usr/bin/env python -# - -import os.path +import pathlib import setuptools from sprockets.mixins import metrics -def read_requirements(name): - requirements = [] - try: - with open(os.path.join('requires', name)) as req_file: - for line in req_file: - if '#' in line: - line = line[:line.index('#')] - line = line.strip() - if line.startswith('-r'): - requirements.extend(read_requirements(line[2:].strip())) - elif line and not line.startswith('-'): - requirements.append(line) - except IOError: - pass - return requirements +REPO = pathlib.Path(__file__).parent setuptools.setup( @@ -34,8 +17,8 @@ setuptools.setup( author_email='api@aweber.com', license='BSD', url='https://github.com/sprockets/sprockets.mixins.metrics', - install_requires=read_requirements('installation.txt'), - tests_require=read_requirements('testing.txt'), + install_requires=REPO.joinpath('requires/installation.txt').read_text(), + tests_require=REPO.joinpath('requires/testing.txt').read_text(), packages=setuptools.find_packages(exclude=['examples.']), namespace_packages=['sprockets', 'sprockets.mixins'], classifiers=[ From b6e04b08a6467f6ded4504146f6cb0237da81da4 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:07:01 -0500 Subject: [PATCH 08/13] Remove unused --- sprockets/mixins/metrics/statsd.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sprockets/mixins/metrics/statsd.py b/sprockets/mixins/metrics/statsd.py index 78308bf..de737de 100644 --- a/sprockets/mixins/metrics/statsd.py +++ b/sprockets/mixins/metrics/statsd.py @@ -16,9 +16,6 @@ SETTINGS_KEY = 'sprockets.mixins.metrics.statsd' class StatsdMixin: """Mix this class in to record metrics to a Statsd server.""" - def initialize(self): - super().initialize() - def record_timing(self, duration, *path): """Record a timing. From 6e3ae0fbaf3c5b016084fc9dc7479c0be5412899 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:07:42 -0500 Subject: [PATCH 09/13] Strict setup.py check --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index cbab8e8..327f043 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,9 @@ [bdist_wheel] universal = 1 +[check] +strict = 1 + [nosetests] cover-package = sprockets.mixins.metrics cover-branches = 1 From 0e8b29f034ae37a1a313d4cd862009a5ba955975 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:09:05 -0500 Subject: [PATCH 10/13] Cleanup and fix docs --- docs/conf.py | 9 --------- docs/history.rst | 3 ++- setup.cfg | 4 ++++ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 041e6de..38c9fff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import alabaster from sprockets.mixins import metrics @@ -7,23 +6,15 @@ copyright = 'AWeber Communications, Inc.' version = metrics.__version__ release = '.'.join(str(v) for v in metrics.version_info[0:2]) -needs_sphinx = '1.0' extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode', ] -templates_path = [] -source_suffix = '.rst' -source_encoding = 'utf-8-sig' master_doc = 'index' -exclude_patterns = [] -pygments_style = 'sphinx' html_style = 'custom.css' html_static_path = ['_static'] -html_theme = 'alabaster' -html_theme_path = [alabaster.get_path()] html_sidebars = { '**': ['about.html', 'navigation.html'], } diff --git a/docs/history.rst b/docs/history.rst index f0902da..faa865f 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -79,7 +79,8 @@ Release History - Add :class:`sprockets.mixins.metrics.InfluxDBMixin` - Add :class:`sprockets.mixins.metrics.influxdb.InfluxDBConnection` -.. _Next Release: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.1.0...master +.. _Next Release: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.1.1...master +.. _3.1.1: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.1.0...3.1.1 .. _3.1.0: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.0.4...3.1.0 .. _3.0.4: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.0.3...3.0.4 .. _3.0.3: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.0.2...3.0.3 diff --git a/setup.cfg b/setup.cfg index 327f043..f8f9546 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,10 @@ [bdist_wheel] universal = 1 +[build_sphinx] +fresh-env = 1 +warning-is-error = 1 + [check] strict = 1 From fda8a86074995523a2d6ebd6a72bc51b1eaed38e Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:11:39 -0500 Subject: [PATCH 11/13] Update history --- docs/history.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/history.rst b/docs/history.rst index faa865f..ea28b00 100644 --- a/docs/history.rst +++ b/docs/history.rst @@ -3,6 +3,15 @@ Release History =============== +`Next`_ +------- +- Add support for Tornado 5 +- Remove support for Tornado < 5 +- Remove support for Python < 3.7 +- Remove InfluxDB support (use `sprockets-influxdb`_) + +.. _sprockets-influxdb: https://github.com/sprockets/sprockets-influxdb + `3.1.1`_ (07-Aug-2018) ---------------------- - Fixed bad formatted TCP StatsD messages by appending a newline @@ -79,7 +88,7 @@ Release History - Add :class:`sprockets.mixins.metrics.InfluxDBMixin` - Add :class:`sprockets.mixins.metrics.influxdb.InfluxDBConnection` -.. _Next Release: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.1.1...master +.. _Next: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.1.1...master .. _3.1.1: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.1.0...3.1.1 .. _3.1.0: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.0.4...3.1.0 .. _3.0.4: https://github.com/sprockets/sprockets.mixins.metrics/compare/3.0.3...3.0.4 From df0d35282c41bb6a80caa6813e0c1317f049e9d2 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Thu, 13 Dec 2018 19:30:25 -0500 Subject: [PATCH 12/13] Add flake8 --- requires/testing.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requires/testing.txt b/requires/testing.txt index fb9480e..266cc8b 100644 --- a/requires/testing.txt +++ b/requires/testing.txt @@ -1,2 +1,3 @@ coverage==4.5.2 +flake8==3.6.0 nose==1.3.7 From bb48684aceb2f3376420643a6df0913e30c19981 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Fri, 14 Dec 2018 11:04:39 -0500 Subject: [PATCH 13/13] Fix flake8 violation --- docs/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 38c9fff..b84ad8d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -import alabaster from sprockets.mixins import metrics project = 'sprockets.mixins.metrics'