Merge pull request #1 from sprockets/initial-version

Initial version

.travis.yml

@@ -0,0 +1,24 @@
+sudo: false
+language: python
+python:
+- 2.7
+- pypy
+- 3.4
+- 3.5
+- 3.6
+install:
+- pip install -r requires/testing.txt -r docs/requirements.txt
+script:
+- nosetests
+after_success:
+- codecov
+- ./setup.py build_sphinx
+deploy:
+  provider: pypi
+  user: sprockets
+  password:
+    secure: "HncR2JUjQicR1czZfM2fHtlFc7rbc5DNdiBsu6uLvNVQPmdBC/fM+FAkuhXhCPrxgp8aRAKXX3mJebZK0R2NVJCui3Y20h4cmHAqTmkNchqytQVZEeDU9pxFYKZ6dhXb6dy6kq/nqxT4ZIDP6IeDaz7n03ZMlsNzMbt2Hj0b5E8BIghiaOLOIZVI3s4meOv/MGxZ5IL378ssKh/+X+mHa5JTVNoZ0gmlIM0Cq6enxQquPgPiL6c6R+EfEk7hbuIKlRwKLpves9qlAM9NCgOwGoo0lv0ashxmIGSbKQApyYtrLbGXmv1TnWAgM64uFTmR6KGKvmLnDD/avx6BCiI+0Fe1V+qDzdDaZf9A2ibPNIdSBR/bwJNueKgDm2yUvf67R0KoucsPnqxJjTObh7Uo/WShie9QjmBNh+FVeNMCPS7SumLeWK5utJx6IYlffuGaVM7VEaTuCahgLFLoYxN/Iig0REhAaG+bxDT+9jcRxgxNcUzTgrPC7TK/AwxfMIP+Ux+0j3M7bAByipek40VBFNI3VZFqfZYsgUl9/sm/ihRzENan6GeNevHx2IpvxnS9CNP0wuYBo4n8Rgvfl7ZMrUZxgl5gkXCnlAwt4Mow0/SNiuPFNqCJARt66w0KFKB5d2eLNgvowV3Mhsh+s3wWm3jxaY5KqBEtqGojQiBDYuk="
+  on:
+    python: 3.5
+    tags: true
+    repo: sprockets/sprockets.mixins.http

README.rst

@@ -35,8 +35,6 @@ This examples demonstrates the most basic usage of ``sprockets.mixins.http``
 
 .. code:: python
 
-   import json
-
    from tornado import gen, web
    from sprockets.mixins import amqp
 

docs/api.rst

@@ -0,0 +1,4 @@
+API
+---
+.. automodule:: sprockets.mixins.http
+   :members:

docs/conf.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+import sphinx_rtd_theme
+
+needs_sphinx = '1.0'
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
+templates_path = []
+source_suffix = '.rst'
+master_doc = 'index'
+project = 'sprockets.mixins.http'
+author = 'AWeber Communications'
+copyright = '2017, AWeber Communications'
+version = '1.0.0'
+release = '1.0'
+exclude_patterns = []
+pygments_style = 'sphinx'
+html_theme = 'sphinx_rtd_theme'
+html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/', None),
+    'tornado': ('http://www.tornadoweb.org/en/stable/', None),
+}

docs/history.rst

@@ -0,0 +1,9 @@
+Version History
+===============
+
+`1.0.0`_ Apr 26, 2017
+---------------------
+- Initial Version
+
+.. _Next Release: https://github.com/sprockets/sprockets.amqp/compare/1.0.0...HEAD
+.. _0.1.0: https://github.com/sprockets/sprockets.amqp/compare/2fc5bad...1.0.0

docs/index.rst

@@ -0,0 +1,11 @@
+.. include:: ../README.rst
+
+Issues
+------
+Please report any issues to the Github project at `https://github.com/sprockets/sprockets.mixins.http/issues <https://github.com/sprockets/sprockets.mixins.http/issues>`_
+
+.. toctree::
+   :hidden:
+
+   api
+   history

docs/requirements.txt

@@ -0,0 +1,2 @@
+sphinx-rtd-theme
+sphinxcontrib-httpdomain

requires/installation.txt

@@ -0,0 +1,3 @@
+ietfparse
+tornado>=4.2.0,<5.0.0
+u-msgpack-python==2.1

requires/testing.txt

@@ -0,0 +1,8 @@
+coverage>3,<5
+codecov
+flake8
+mock
+pylint
+nose>=1.3.1,<2.0.0
+wheel
+-r installation.txt

setup.cfg

@@ -0,0 +1,16 @@
+[bdist_wheel]
+universal = 1
+
+[nosetests]
+cover-branches=1
+cover-erase=1
+cover-html=1
+cover-html-dir=build/coverage
+cover-package=sprockets.mixins.http
+cover-tests=1
+with-coverage=1
+logging-level=DEBUG
+verbosity=2
+
+[upload_docs]
+upload-dir = build/sphinx/html

setup.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+import os.path
+
+import setuptools
+
+from sprockets.mixins.http import __version__
+
+
+def read_requirements(name):
+    requirements = []
+    try:
+        with open(os.path.join('requires', name)) as req_file:
+            for line in req_file:
+                if '#' in line:
+                    line = line[:line.index('#')]
+                line = line.strip()
+                if line.startswith('-r'):
+                    requirements.extend(read_requirements(line[2:].strip()))
+                elif line and not line.startswith('-'):
+                    requirements.append(line)
+    except IOError:
+        pass
+    return requirements
+
+
+setuptools.setup(
+    name='sprockets.mixins.http',
+    version=__version__,
+    description='HTTP Client Mixin for Tornado RequestHandlers',
+    long_description=open('README.rst').read(),
+    url='https://github.com/sprockets/sprockets.mixins.http',
+    author='AWeber Communications, Inc.',
+    author_email='api@aweber.com',
+    license='BSD',
+    classifiers=[
+        'Development Status :: 3 - Alpha', 'Intended Audience :: Developers',
+        'License :: OSI Approved :: BSD License', 'Natural Language :: English',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
+        'Topic :: Software Development :: Libraries',
+        'Topic :: Software Development :: Libraries :: Python Modules'
+    ],
+    packages=setuptools.find_packages(),
+    namespace_packages=['sprockets', 'sprockets.mixins'],
+    install_requires=read_requirements('installation.txt'),
+    zip_safe=True)

sprockets/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

sprockets/mixins/__init__.py

@@ -0,0 +1 @@
+__import__('pkg_resources').declare_namespace(__name__)

sprockets/mixins/http/__init__.py

@@ -0,0 +1,309 @@
+"""
+HTTP Client Mixin
+=================
+A Tornado Request Handler Mixin that provides functions for making HTTP
+requests.
+
+"""
+import collections
+import json
+import logging
+import socket
+import time
+
+try:
+    from ietfparse import algorithms, datastructures, errors, headers
+except ImportError:  # pragma: nocover
+    logging.getLogger().error('Could not load ietfparse modules')
+    algorithms, datastructures, errors = None, None, None
+
+    class headers(object):
+        """Dummy class for installation"""
+        @staticmethod
+        def parse_content_type(value):
+            """Dummy method"""
+            return value
+
+try:
+    from tornado import gen, httpclient
+except ImportError:  # pragma: nocover
+    logging.getLogger().error('Could not load tornado modules')
+    gen, httpclient = None, None
+
+try:
+    import umsgpack
+except ImportError:  # pragma: nocover
+    logging.getLogger().error('Could not load umsgpack module')
+    umsgpack = None
+
+__version__ = '0.1.0'
+
+LOGGER = logging.getLogger(__name__)
+
+CONTENT_TYPE_JSON = headers.parse_content_type('application/json')
+CONTENT_TYPE_MSGPACK = headers.parse_content_type('application/msgpack')
+DEFAULT_USER_AGENT = 'sprockets.mixins.http/{}'.format(__version__)
+
+HTTPResponse = collections.namedtuple(
+    'HTTPResponse',
+    ['ok', 'code', 'headers', 'body', 'raw', 'attempts', 'duration'])
+"""Response returned from :meth:`sprockets.mixins.http.HTTPClientMixin.fetch`
+that provides a slightly higher level of functionality than Tornado's
+:cls:`tornado.httpclient.HTTPResponse` class.
+
+Attributes
+----------
+ok : bool
+    The response status code was between 200 and 308
+code : int
+    The HTTP response status code
+headers : dict
+    The HTTP response headers
+body : mixed
+    The deserialized HTTP response body if available/supported
+raw : tornado.httpclient.HTTPResponse
+    The original Tornado HTTP response object for the request
+attempts : int
+    The number of HTTP request attempts made
+duration : float
+    The total duration of time spent making the request(s)
+
+"""
+
+
+class HTTPClientMixin(object):
+    """Mixin for making http requests. Requests using the asynchronous
+    :meth:`HTTPClientMixin.http_fetch` method """
+
+    AVAILABLE_CONTENT_TYPES = [CONTENT_TYPE_JSON, CONTENT_TYPE_MSGPACK]
+
+    DEFAULT_CONNECT_TIMEOUT = 10
+    DEFAULT_REQUEST_TIMEOUT = 60
+
+    MAX_HTTP_RETRIES = 3
+
+    @gen.coroutine
+    def http_fetch(self, url,
+                   method='GET',
+                   request_headers=None,
+                   body=None,
+                   content_type=CONTENT_TYPE_MSGPACK,
+                   follow_redirects=False,
+                   connect_timeout=DEFAULT_CONNECT_TIMEOUT,
+                   request_timeout=DEFAULT_REQUEST_TIMEOUT,
+                   auth_username=None,
+                   auth_password=None):
+        """Perform a HTTP request
+
+        Will retry up to ``self.MAX_HTTP_RETRIES`` times.
+
+        :param str url: The URL for the request
+        :param str method: The HTTP request method, defaults to ``GET``
+        :param dict request_headers: Headers to include in the HTTP request
+        :param mixed body: The HTTP request body to send with the request
+        :param content_type: The mime type to use for requests & responses.
+            Defaults to ``application/msgpack``
+        :type content_type: :cls:`ietfparse.datastructures.ContentType` or str
+        :param bool follow_redirects: Follow HTTP redirects when received
+        :param float connect_timeout: Timeout for initial connection in
+            seconds, default 20 seconds
+        :param float request_timeout:  Timeout for entire request in seconds,
+            default 20 seconds
+        :param str auth_username: Username for HTTP authentication
+        :param str auth_password: Password for HTTP authentication
+        :rtype: HTTPResponse
+
+        """
+        request_headers = self._http_req_apply_default_headers(
+            request_headers, content_type, body)
+        if body:
+            body = self._http_req_body_serialize(
+                body, request_headers['Content-Type'])
+        client = httpclient.AsyncHTTPClient()
+        response, start_time = None, time.time()
+        for attempt in range(0, self.MAX_HTTP_RETRIES):
+            LOGGER.debug('%s %s (Attempt %i of %i) %r',
+                         method, url, attempt, self.MAX_HTTP_RETRIES,
+                         request_headers)
+            try:
+                response = yield client.fetch(
+                    url,
+                    method=method,
+                    headers=request_headers,
+                    body=body,
+                    auth_username=auth_username,
+                    auth_password=auth_password,
+                    connect_timeout=connect_timeout,
+                    request_timeout=request_timeout,
+                    user_agent=self._http_req_user_agent(),
+                    follow_redirects=follow_redirects,
+                    raise_error=False)
+            except (OSError, socket.gaierror) as error:
+                LOGGER.debug('HTTP Request Error for %s to %s'
+                             'attempt %i of %i: %s',
+                             method, url, attempt + 1,
+                             self.MAX_HTTP_RETRIES, error)
+                continue
+            if 200 <= response.code < 400:
+                raise gen.Return(
+                    HTTPResponse(
+                        True, response.code, dict(response.headers),
+                        self._http_resp_deserialize(response),
+                        response, attempt + 1, time.time() - start_time))
+            elif response.code in {423, 429}:
+                yield self._http_resp_rate_limited(response)
+            elif 400 <= response.code < 500:
+                error = self._http_resp_error_message(response)
+                LOGGER.debug('HTTP Response Error for %s to %s'
+                             'attempt %i of %i (%s): %s',
+                             method, url, response.code, attempt + 1,
+                             self.MAX_HTTP_RETRIES, error)
+                raise gen.Return(
+                    HTTPResponse(
+                        False, response.code, dict(response.headers),
+                        error, response, attempt + 1,
+                        time.time() - start_time))
+            elif response.code >= 500:
+                LOGGER.error('HTTP Response Error for %s to %s, '
+                             'attempt %i of %i (%s): %s',
+                             method, url, attempt + 1, self.MAX_HTTP_RETRIES,
+                             response.code,
+                             self._http_resp_error_message(response))
+
+        LOGGER.warning('HTTP Get %s failed after %i attempts', url,
+                       self.MAX_HTTP_RETRIES)
+        if response:
+            raise gen.Return(
+                HTTPResponse(
+                    False, response.code, dict(response.headers),
+                    self._http_resp_error_message(response) or response.body,
+                    response, self.MAX_HTTP_RETRIES,
+                    time.time() - start_time))
+        raise gen.Return(
+            HTTPResponse(
+                False, 599, None, None, None, self.MAX_HTTP_RETRIES,
+                time.time() - start_time))
+
+    def _http_req_apply_default_headers(self, request_headers,
+                                        content_type, body):
+        """Set default values for common HTTP request headers
+
+        :param dict request_headers: The HTTP request headers
+        :param content_type: The mime-type used in the request/response
+        :type content_type: :cls:`ietfparse.datastructures.ContentType` or str
+        :param mixed body: The request body
+        :rtype: dict
+
+        """
+        if not request_headers:
+            request_headers = {}
+        request_headers.setdefault(
+            'Accept', str(content_type) or str(CONTENT_TYPE_MSGPACK))
+        if body:
+            request_headers.setdefault(
+                'Content-Type', str(content_type) or str(CONTENT_TYPE_MSGPACK))
+        if hasattr(self, 'request'):
+            if self.request.headers.get('Correlation-Id'):
+                request_headers.setdefault(
+                    'Correlation-Id', self.request.headers['Correlation-Id'])
+        return request_headers
+
+    @staticmethod
+    def _http_req_body_serialize(body, content_type):
+        """Conditionally serialize the request body value if mime_type is set
+        and it's serializable.
+
+        :param mixed body: The request body
+        :param str content_type: The content type for the request body
+        :raises: ValueError
+
+        """
+        if not body or not isinstance(body, (dict, list)):
+            return body
+
+        content_type = headers.parse_content_type(content_type)
+        if content_type == CONTENT_TYPE_JSON:
+            return json.dumps(body)
+        elif content_type == CONTENT_TYPE_MSGPACK:
+            return umsgpack.packb(body)
+        raise ValueError('Unsupported Content Type')
+
+    def _http_req_user_agent(self):
+        """Return the User-Agent value to specify in HTTP requests, defaulting
+        to ``service/version`` if configured in the application settings,
+        otherwise defaulting to ``sprockets.mixins.http/[VERSION]``.
+
+        :rtype: str
+
+        """
+        if hasattr(self, 'application'):
+            if self.application.settings.get('service') and \
+                    self.application.settings.get('version'):
+                return '{}/{}'.format(
+                    self.application.settings['service'],
+                    self.application.settings['version'])
+        return DEFAULT_USER_AGENT
+
+    def _http_resp_decode(self, value):
+        """Decode bytes to UTF-8 strings as a singe value, list, or dict.
+
+        :param mixed value:
+        :rtype: mixed
+        """
+        if isinstance(value, list):
+            return [self._http_resp_decode(v) for v in value]
+        elif isinstance(value, dict):
+            return dict([(self._http_resp_decode(k),
+                          self._http_resp_decode(v))
+                         for k, v in value.items()])
+        elif isinstance(value, bytes):
+            return value.decode('utf-8')
+        return value
+
+    def _http_resp_deserialize(self, response):
+        """Try and deserialize a response body based upon the specified
+        content type.
+
+        :param tornado.httpclient.HTTPResponse: The HTTP response to decode
+        :rtype: mixed
+
+        """
+        if not response.body:
+            return None
+        try:
+            content_type = algorithms.select_content_type(
+                [headers.parse_content_type(response.headers['Content-Type'])],
+                self.AVAILABLE_CONTENT_TYPES)
+        except errors.NoMatch:
+            return response.body
+
+        if content_type[0] == CONTENT_TYPE_JSON:
+            return self._http_resp_decode(
+                json.loads(self._http_resp_decode(response.body)))
+        elif content_type[0] == CONTENT_TYPE_MSGPACK:
+            return self._http_resp_decode(umsgpack.unpackb(response.body))
+
+    def _http_resp_error_message(self, response):
+        """Try and extract the error message from a HTTP error response.
+
+        :param tornado.httpclient.HTTPResponse response: The response
+        :rtype: str
+
+        """
+        response_body = self._http_resp_deserialize(response)
+        if isinstance(response_body, dict) and 'message' in response_body:
+            return response_body['message']
+        return response_body
+
+    @staticmethod
+    def _http_resp_rate_limited(response):
+        """Extract the ``Retry-After`` header value if the request was rate
+        limited and return a future to sleep for the specified duration.
+
+        :param tornado.httpclient.HTTPResponse response: The response
+        :rtype: tornado.concurrent.Future
+
+        """
+        duration = int(response.headers.get('Retry-After', 3))
+        LOGGER.warning('Rate Limited by, retrying in %i seconds', duration)
+        return gen.sleep(duration)

tests.py

@@ -0,0 +1,359 @@
+import json
+import logging
+import uuid
+
+from tornado import httputil, testing, web
+import mock
+import umsgpack
+
+from sprockets.mixins import http
+
+LOGGER = logging.getLogger(__name__)
+
+
+def decode(value):
+    """Decode bytes to UTF-8 strings as a singe value, list, or dict.
+
+    :param mixed value:
+    :rtype: mixed
+
+    """
+    if isinstance(value, list):
+        return [decode(v) for v in value]
+    elif isinstance(value, dict):
+        return dict([(decode(k), decode(v)) for k, v in value.items()])
+    elif isinstance(value, bytes):
+        return value.decode('utf-8')
+    return value
+
+
+class TestHandler(web.RequestHandler):
+
+    def prepare(self):
+        status_code = self.status_code()
+        if status_code == 429:
+            self.add_header('Retry-After', '1')
+            self.set_status(429, 'Rate Limited')
+            self.finish()
+        elif status_code in {502, 504}:
+            self.set_status(status_code)
+            self.finish()
+
+    def delete(self, *args, **kwargs):
+        self.respond()
+
+    def get(self, *args, **kwargs):
+        self.respond()
+
+    def head(self, *args, **kwargs):
+        status_code = self.status_code() or 204
+        self.set_status(status_code)
+
+    def patch(self, *args, **kwargs):
+        self.respond()
+
+    def post(self, *args, **kwargs):
+        self.respond()
+
+    def put(self, *args, **kwargs):
+        self.respond()
+
+    def get_request_body(self):
+        if 'Content-Type' in self.request.headers:
+            if self.request.headers['Content-Type'] == 'application/json':
+                return json.loads(self.request.body.decode('utf-8'))
+            elif self.request.headers['Content-Type'] == 'application/msgpack':
+                return umsgpack.unpackb(self.request.body)
+        if self.request.body_arguments:
+            return self.request.body_arguments
+        return self.request.body
+
+    def respond(self):
+        status_code = self.status_code() or 200
+        self.set_status(status_code)
+        if status_code >= 400:
+            self.send_response({
+                'message': self.get_argument('message',
+                                             'Error Message Text'),
+                'type': self.get_argument('message', 'Error Type Text'),
+                'traceback': None})
+        else:
+            body = self.get_request_body()
+            if isinstance(body, dict) and 'response' in body:
+                return self.send_response(body['response'])
+            self.send_response({'headers': dict(self.request.headers),
+                                'path': self.request.path,
+                                'args': self.request.arguments,
+                                'body': self.get_request_body()})
+
+    def send_response(self, payload):
+        if isinstance(payload, (dict, list)):
+            if self.request.headers.get('Accept') == 'application/json':
+                self.set_header('Content-Type', 'application/json')
+                return self.write(decode(payload))
+            elif self.request.headers.get('Accept') == 'application/msgpack':
+                self.set_header('Content-Type', 'application/msgpack')
+                return self.write(umsgpack.packb(decode(payload)))
+        LOGGER.debug('Bypassed serialization')
+        content_type = self.get_argument('content_type', None)
+        if content_type:
+            LOGGER.debug('Setting response content-type: %r', content_type)
+            self.set_header('Content-Type', content_type)
+        return self.write(decode(payload))
+
+    def status_code(self):
+        value = self.get_argument('status_code', None)
+        return int(value) if value is not None else None
+
+
+class MixinTestCase(testing.AsyncHTTPTestCase):
+
+    def setUp(self):
+        super(MixinTestCase, self).setUp()
+        self.correlation_id = str(uuid.uuid4())
+        self.mixin = self.create_mixin()
+
+    def get_app(self):
+        return web.Application([(r'/(.*)', TestHandler)],
+                               **{'service': 'test', 'version': '0.1.0'})
+
+    def create_mixin(self, add_correlation=True):
+        mixin = http.HTTPClientMixin()
+        mixin.application = self._app
+        mixin.request = httputil.HTTPServerRequest(
+            'GET', 'http://test:9999/test',
+            headers=httputil.HTTPHeaders(
+                {'Correlation-ID': self.correlation_id} if
+                add_correlation else {}))
+        return mixin
+
+    @testing.gen_test()
+    def test_default_user_agent(self):
+        mixin = http.HTTPClientMixin()
+        response = yield mixin.http_fetch(
+            self.get_url('/test?foo=bar&status_code=200'))
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'),
+            'sprockets.mixins.http/{}'.format(http.__version__))
+
+    @testing.gen_test()
+    def test_default_user_agent_with_partial_config(self):
+        del self._app.settings['version']
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test?foo=bar&status_code=200'))
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'),
+            'sprockets.mixins.http/{}'.format(http.__version__))
+
+    @testing.gen_test()
+    def test_socket_errors(self):
+        with mock.patch(
+                'tornado.httpclient.AsyncHTTPClient.fetch') as fetch:
+            fetch.side_effect = OSError
+            response = yield self.mixin.http_fetch(self.get_url('/test'))
+        self.assertFalse(response.ok)
+        self.assertEqual(response.code, 599)
+        self.assertEqual(response.attempts, 3)
+
+    @testing.gen_test()
+    def test_without_correlation_id_behavior(self):
+        mixin = self.create_mixin(False)
+        response = yield mixin.http_fetch(
+            self.get_url('/error?status_code=502'))
+        self.assertFalse(response.ok)
+        self.assertEqual(response.code, 502)
+        self.assertEqual(response.attempts, 3)
+
+    @testing.gen_test()
+    def test_get(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test?foo=bar&status_code=200'))
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['args'],
+                             {'foo': ['bar'], 'status_code': ['200']})
+
+    @testing.gen_test()
+    def test_post(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test'),
+            method='POST',
+            body={'foo': 'bar', 'status_code': 200})
+
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['body'],
+                             {'foo': 'bar', 'status_code': 200})
+
+    @testing.gen_test()
+    def test_get_json(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test?foo=bar&status_code=200'),
+            request_headers={'Accept': 'application/json'})
+
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['args'],
+                             {'foo': ['bar'], 'status_code': ['200']})
+
+    @testing.gen_test()
+    def test_post_html(self):
+        expectation = '<html>foo</html>'
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test'),
+            method='POST',
+            body=expectation,
+            request_headers={'Accept': 'text/html',
+                             'Content-Type': 'text/html'})
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertEqual(response.body['body'], expectation)
+
+    @testing.gen_test()
+    def test_post_json(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test'),
+            method='POST',
+            body={'foo': 'bar', 'status_code': 200},
+            request_headers={'Accept': 'application/json',
+                             'Content-Type': 'application/json'})
+
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['body'],
+                             {'foo': 'bar', 'status_code': 200})
+
+    @testing.gen_test()
+    def test_post_msgpack(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test'),
+            method='POST',
+            body={'foo': 'bar', 'status_code': 200},
+            request_headers={'Accept': 'application/msgpack',
+                             'Content-Type': 'application/msgpack'})
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['body'],
+                             {'foo': 'bar', 'status_code': 200})
+
+    @testing.gen_test()
+    def test_post_pre_serialized_json(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test'),
+            method='POST',
+            body=json.dumps({'foo': 'bar', 'status_code': 200}),
+            request_headers={'Accept': 'application/json',
+                             'Content-Type': 'application/json'})
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['body'],
+                             {'foo': 'bar', 'status_code': 200})
+
+    @testing.gen_test()
+    def test_post_pre_serialized_msgpack(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/test'),
+            method='POST',
+            body=umsgpack.packb({'foo': 'bar', 'status_code': 200}),
+            request_headers={'Accept': 'application/msgpack',
+                             'Content-Type': 'application/msgpack'})
+        self.assertTrue(response.ok)
+        self.assertEqual(response.code, 200)
+        self.assertEqual(response.body['headers'].get('Correlation-Id'),
+                         self.correlation_id)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(
+            response.body['headers'].get('User-Agent'), 'test/0.1.0')
+        self.assertDictEqual(response.body['body'],
+                             {'foo': 'bar', 'status_code': 200})
+
+    @testing.gen_test()
+    def test_rate_limiting_behavior(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/error?status_code=429'))
+        self.assertFalse(response.ok)
+        self.assertEqual(response.code, 429)
+        self.assertEqual(response.attempts, 3)
+
+    @testing.gen_test()
+    def test_error_response(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/error?status_code=400&message=Test%20Error'))
+        self.assertFalse(response.ok)
+        self.assertEqual(response.code, 400)
+        self.assertEqual(response.attempts, 1)
+        self.assertEqual(response.body, 'Test Error')
+
+    @testing.gen_test()
+    def test_error_retry(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/error?status_code=502'))
+        self.assertFalse(response.ok)
+        self.assertEqual(response.code, 502)
+        self.assertEqual(response.attempts, 3)
+        self.assertEqual(response.body, b'')
+
+    @testing.gen_test()
+    def test_unsupported_content_type(self):
+        with self.assertRaises(ValueError):
+            yield self.mixin.http_fetch(
+                self.get_url('/test'),
+                method='POST',
+                body=['foo', 'bar'],
+                request_headers={'Content-Type': 'text/html'})
+
+    @testing.gen_test()
+    def test_unsupported_accept(self):
+        expectation = '<html>foo</html>'
+        response = yield self.mixin.http_fetch(
+                self.get_url('/test?content_type=text/html'),
+                method='POST',
+                body={'response': expectation},
+                request_headers={'Accept': 'text/html',
+                                 'Content-Type': 'application/json'})
+        self.assertTrue(response.ok)
+        self.assertEqual(response.headers['Content-Type'], 'text/html')
+        self.assertEqual(response.body.decode('utf-8'), expectation)
+
+