Merge pull request #8 from sprockets/add-smart-content-handling

Add pluggable transcoders

README.rst

@@ -12,8 +12,8 @@ This mix-in adds two methods to a ``tornado.web.RequestHandler`` instance:
 - ``send_response(object)``: serializes the response into the content type
   requested by the ``Accept`` header.
 
-Support for a content types is enabled by calling either the
-``add_binary_content_type`` or ``add_text_content_type`` function with the
+Support for a content types is enabled by calling ``add_binary_content_type``,
+``add_text_content_type`` or the ``add_transcoder`` functions with the
 ``tornado.web.Application`` instance, the content type, encoding and decoding
 functions as parameters:
 
@@ -37,6 +37,28 @@ functions as parameters:
 
 The *add content type* functions will add a attribute to the ``Application``
 instance that the mix-in uses to manipulate the request and response bodies.
+The *add_transcoder* function is similar except that it takes an object
+that implements transcoding methods instead of simple functions.  The
+``transcoders`` module includes ready-to-use transcoders for a few content
+types:
+
+.. code-block:: python
+
+   from sprockets.mixins.mediatype import content, transcoders
+   from tornado import web
+
+   def make_application():
+       application = web.Application([
+           # insert your handlers here
+       ])
+
+       content.add_transcoder(application, transcoders.JSONTranscoder())
+
+       return application
+
+In either case, the ``ContentMixin`` uses the registered content type
+information to provide transparent content type negotiation for your
+request handlers.
 
 .. code-block:: python
 

docs/api.rst

@@ -15,5 +15,20 @@ Content Type Registration
 
 .. autofunction:: add_text_content_type
 
+.. autofunction:: add_transcoder
+
 .. autoclass:: ContentSettings
    :members:
+
+Bundled Transcoders
+-------------------
+.. currentmodule:: sprockets.mixins.mediatype.transcoders
+
+.. autoclass:: JSONTranscoder
+   :members:
+
+.. autoclass:: MsgPackTranscoder
+   :members:
+
+.. autoclass:: BinaryWrapper
+   :members:

docs/conf.py

@@ -31,7 +31,7 @@ html_theme_options = {
 }
 
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/', None),
+    'python': ('https://docs.python.org/3', None),
     'requests': ('https://requests.readthedocs.org/en/latest/', None),
     'sprockets': ('https://sprockets.readthedocs.org/en/latest/', None),
     'tornado': ('http://tornadoweb.org/en/latest/', None),

docs/history.rst

@@ -5,6 +5,11 @@ Version History
 ---------------
 - Repackage from a module into a package.  Distributing raw modules inside
   of a namespace package is unreliable and questionably correct.
+- Add :func:`sprockets.mixins.mediatype.content.add_transcoder`.
+- Add :class:`sprockets.mixins.mediatype.transcoders.JSONTranscoder`
+- Add :class:`sprockets.mixins.mediatype.transcoders.MsgPackTranscoder`
+- Add :class:`sprockets.mixins.mediatype.transcoders.BinaryWrapper`
+- Normalize registered MIME types.
 
 `1.0.4`_ (14 Sep 2015)
 ----------------------

examples.py

@@ -1,10 +1,8 @@
-import json
 import logging
 import signal
 
-from sprockets.mixins.mediatype import content
+from sprockets.mixins.mediatype import content, transcoders
 from tornado import ioloop, web
-import msgpack
 
 
 class SimpleHandler(content.ContentMixin, web.RequestHandler):
@@ -20,10 +18,8 @@ def make_application(**settings):
     application = web.Application([web.url(r'/', SimpleHandler)], **settings)
     content.set_default_content_type(application, 'application/json',
                                      encoding='utf-8')
-    content.add_binary_content_type(application, 'application/msgpack',
-                                    msgpack.packb, msgpack.unpackb)
-    content.add_text_content_type(application, 'application/json', 'utf-8',
-                                  json.dumps, json.loads)
+    content.add_transcoder(application, transcoders.MsgPackTranscoder())
+    content.add_transcoder(application, transcoders.JSONTranscoder())
     return application
 
 

requires/testing.txt

@@ -1,4 +1,4 @@
 coverage>=3.7,<3.99  # prevent installing 4.0b on ALL pip versions
 mock>=1.3,<2
-msgpack-python>=0.4,<0.5
+u-msgpack-python>=2,<3
 nose>=1.3,<2

sprockets/mixins/mediatype/content.py

@@ -7,6 +7,8 @@ Content handling for Tornado.
   content type
 - :func:`.add_text_content_type` register transcoders for a textual
   content type
+- :func:`.add_transcoder` register a custom transcoder instance
+  for a content type
 - :class:`.ContentSettings` an instance of this is attached to
   :class:`tornado.web.Application` to hold the content mapping
   information for the application
@@ -81,15 +83,18 @@ class ContentSettings(object):
         self.default_encoding = None
 
     def __getitem__(self, content_type):
-        return self._handlers[content_type]
+        parsed = headers.parse_content_type(content_type)
+        return self._handlers[str(parsed)]
 
     def __setitem__(self, content_type, handler):
+        parsed = headers.parse_content_type(content_type)
+        content_type = str(parsed)
         if content_type in self._handlers:
             logger.warning('handler for %s already set to %r',
-                           content_type, self._handers[content_type])
+                           content_type, self._handlers[content_type])
             return
 
-        self._available_types.append(headers.parse_content_type(content_type))
+        self._available_types.append(parsed)
         self._handlers[content_type] = handler
 
     def get(self, content_type, default=None):
@@ -126,9 +131,8 @@ def add_binary_content_type(application, content_type, pack, unpack):
         dictionary.  ``unpack(bytes) -> dict``
 
     """
-    settings = ContentSettings.from_application(application)
-    settings[content_type] = handlers.BinaryContentHandler(
-        content_type, pack, unpack)
+    add_transcoder(application,
+                   handlers.BinaryContentHandler(content_type, pack, unpack))
 
 
 def add_text_content_type(application, content_type, default_encoding,
@@ -144,10 +148,52 @@ def add_text_content_type(application, content_type, default_encoding,
     :param loads: function that loads a dictionary from a string.
         ``loads(str, encoding:str) -> dict``
 
+    Note that the ``charset`` parameter is stripped from `content_type`
+    if it is present.
+
+    """
+    parsed = headers.parse_content_type(content_type)
+    parsed.parameters.pop('charset', None)
+    normalized = str(parsed)
+    add_transcoder(application,
+                   handlers.TextContentHandler(normalized, dumps, loads,
+                                               default_encoding))
+
+
+def add_transcoder(application, transcoder, content_type=None):
+    """
+    Register a transcoder for a specific content type.
+
+    :param tornado.web.Application application: the application to modify
+    :param transcoder: object that translates between :class:`bytes` and
+        :class:`object` instances
+    :param str content_type: the content type to add.  If this is
+        unspecified or :data:`None`, then the transcoder's ``content_type``
+        attribute is used.
+
+    The `transcoder` instance is required to implement the following
+    simple protocol:
+
+    .. attribute:: transcoder.content_type
+
+       :class:`str` that identifies the MIME type that the transcoder
+       implements.
+
+    .. method:: transcoder.to_bytes(inst_data, encoding=None) -> bytes
+
+       :param object inst_data: the object to encode
+       :param str encoding: character encoding to apply or :data:`None`
+       :returns: the encoded :class:`bytes` instance
+
+    .. method:: transcoder.from_bytes(data_bytes, encoding=None) -> object
+
+       :param bytes data_bytes: the :class:`bytes` instance to decode
+       :param str encoding: character encoding to use or :data:`None`
+       :returns: the decoded :class:`object` instance
+
     """
     settings = ContentSettings.from_application(application)
-    settings[content_type] = handlers.TextContentHandler(
-        content_type, dumps, loads, default_encoding)
+    settings[content_type or transcoder.content_type] = transcoder
 
 
 def set_default_content_type(application, content_type, encoding=None):

sprockets/mixins/mediatype/transcoders.py

@@ -0,0 +1,303 @@
+"""
+Bundled media type transcoders.
+
+- :class:`.JSONTranscoder` implements JSON encoding/decoding
+- :class:`.MsgPackTranscoder` implements msgpack encoding/decoding
+
+"""
+import base64
+import json
+import sys
+import uuid
+
+import collections
+
+try:
+    import umsgpack
+except ImportError:
+    umsgpack = None
+
+from sprockets.mixins.mediatype import handlers
+
+
+class BinaryWrapper(bytes):
+    """
+    Ensures that a Python 2 ``str`` is treated as binary.
+
+    Since :class:`bytes` is a synonym for :class:`str` in Python 2,
+    you cannot distinguish between something that should be binary
+    and something that should be encoded as a string.  This is a
+    problem in formats `such as msgpack`_ where binary data and
+    strings are encoded differently.  The :class:`MsgPackTranscoder`
+    accomodates this by trying to UTF-8 encode a :class:`str` instance
+    and falling back to binary encoding if the transcode fails.
+
+    You can avoid this by wrapping binary content in an instance of
+    this class.  The transcoder will then treat it as a binary payload
+    instead of trying to detect whether it is a string or not.
+
+    .. _such as msgpack: http://msgpack.org
+
+    """
+    pass
+
+
+class JSONTranscoder(handlers.TextContentHandler):
+    """
+    JSON transcoder instance.
+
+    :param str content_type: the content type that this encoder instance
+        implements. If omitted, ``application/json`` is used. This is
+        passed directly to the ``TextContentHandler`` initializer.
+    :param str default_encoding: the encoding to use if none is specified.
+        If omitted, this defaults to ``utf-8``. This is passed directly to
+        the ``TextContentHandler`` initializer.
+
+    This JSON encoder uses :func:`json.loads` and :func:`json.dumps` to
+    implement JSON encoding/decoding.  The :meth:`dump_object` method is
+    configured to handle types that the standard JSON module does not
+    support.
+
+    .. attribute:: dump_options
+
+       Keyword parameters that are passed to :func:`json.dumps` when
+       :meth:`.dumps` is called.  By default, the :meth:`dump_object`
+       method is enabled as the default object hook.
+
+    .. attribute:: load_options
+
+       Keyword parameters that are passed to :func:`json.loads` when
+       :meth:`.loads` is called.
+
+    """
+
+    def __init__(self, content_type='application/json',
+                 default_encoding='utf-8'):
+        super(JSONTranscoder, self).__init__(content_type, self.dumps,
+                                             self.loads, default_encoding)
+        self.dump_options = {
+            'default': self.dump_object,
+            'separators': (',', ':'),
+        }
+        self.load_options = {}
+
+    def dumps(self, obj):
+        """
+        Dump a :class:`object` instance into a JSON :class:`str`
+
+        :param object obj: the object to dump
+        :return: the JSON representation of :class:`object`
+
+        """
+        return json.dumps(obj, **self.dump_options)
+
+    def loads(self, str_repr):
+        """
+        Transform :class:`str` into an :class:`object` instance.
+
+        :param str str_repr: the UNICODE representation of an object
+        :return: the decoded :class:`object` representation
+
+        """
+        return json.loads(str_repr, **self.load_options)
+
+    def dump_object(self, obj):
+        """
+        Called to encode unrecognized object.
+
+        :param object obj: the object to encode
+        :return: the encoded object
+        :raises TypeError: when `obj` cannot be encoded
+
+        This method is passed as the ``default`` keyword parameter
+        to :func:`json.dumps`.  It provides default representations for
+        a number of Python language/standard library types.
+
+        +----------------------------+---------------------------------------+
+        | Python Type                | String Format                         |
+        +----------------------------+---------------------------------------+
+        | :class:`bytes`,            | Base64 encoded string.                |
+        | :class:`bytearray`,        |                                       |
+        | :class:`memoryview`        |                                       |
+        +----------------------------+---------------------------------------+
+        | :class:`datetime.datetime` | ISO8601 formatted timestamp in the    |
+        |                            | extended format including separators, |
+        |                            | milliseconds, and the timezone        |
+        |                            | designator.                           |
+        +----------------------------+---------------------------------------+
+        | :class:`uuid.UUID`         | Same as ``str(value)``                |
+        +----------------------------+---------------------------------------+
+
+        .. warning::
+
+           :class:`bytes` instances are treated as character strings by the
+           standard JSON module in Python 2.7 so the *default* object hook
+           is never called.  In other words, :class:`bytes` values will not
+           be serialized as Base64 strings in Python 2.7.
+
+        """
+        if isinstance(obj, uuid.UUID):
+            return str(obj)
+        if hasattr(obj, 'isoformat'):
+            return obj.isoformat()
+        if isinstance(obj, (bytes, bytearray, memoryview)):
+            return base64.b64encode(obj).decode('ASCII')
+        raise TypeError('{!r} is not JSON serializable'.format(obj))
+
+
+class MsgPackTranscoder(handlers.BinaryContentHandler):
+    """
+    Msgpack Transcoder instance.
+
+    :param str content_type: the content type that this encoder instance
+        implements. If omitted, ``application/msgpack`` is used. This
+        is passed directly to the ``BinaryContentHandler`` initializer.
+
+    This transcoder uses the `umsgpack`_ library to encode and decode
+    objects according to the `msgpack format`_.
+
+    .. _umsgpack: https://github.com/vsergeev/u-msgpack-python
+    .. _msgpack format: http://msgpack.org/index.html
+
+    """
+    if sys.version_info[0] < 3:
+        PACKABLE_TYPES = (bool, int, float, long)
+    else:
+        PACKABLE_TYPES = (bool, int, float)
+
+    def __init__(self, content_type='application/msgpack'):
+        if umsgpack is None:
+            raise RuntimeError('Cannot import MsgPackTranscoder, '
+                               'umsgpack is not available')
+
+        super(MsgPackTranscoder, self).__init__(content_type, self.packb,
+                                                self.unpackb)
+
+    def packb(self, data):
+        """Pack `data` into a :class:`bytes` instance."""
+        return umsgpack.packb(self.normalize_datum(data))
+
+    def unpackb(self, data):
+        """Unpack a :class:`object` from a :class:`bytes` instance."""
+        return umsgpack.unpackb(data)
+
+    def normalize_datum(self, datum):
+        """
+        Convert `datum` into something that umsgpack likes.
+
+        :param datum: something that we want to process with umsgpack
+        :return: a packable version of `datum`
+        :raises TypeError: if `datum` cannot be packed
+
+        This message is called by :meth:`.packb` to recursively normalize
+        an input value before passing it to :func:`umsgpack.packb`.  Values
+        are normalized according to the following table.
+
+        +-------------------------------+-------------------------------+
+        | **Value**                     | **MsgPack Family**            |
+        +-------------------------------+-------------------------------+
+        | :data:`None`                  | `nil byte`_ (0xC0)            |
+        +-------------------------------+-------------------------------+
+        | :data:`True`                  | `true byte`_ (0xC3)           |
+        +-------------------------------+-------------------------------+
+        | :data:`False`                 | `false byte`_ (0xC2)          |
+        +-------------------------------+-------------------------------+
+        | :class:`int`                  | `integer family`_             |
+        +-------------------------------+-------------------------------+
+        | :class:`float`                | `float family`_               |
+        +-------------------------------+-------------------------------+
+        | String (see note)             | `str family`_                 |
+        +-------------------------------+-------------------------------+
+        | :class:`bytes`                | `bin family`_                 |
+        +-------------------------------+-------------------------------+
+        | :class:`bytearray`            | `bin family`_                 |
+        +-------------------------------+-------------------------------+
+        | :class:`memoryview`           | `bin family`_                 |
+        +-------------------------------+-------------------------------+
+        | :class:`.BinaryWrapper`       | `bin family`_                 |
+        +-------------------------------+-------------------------------+
+        | :class:`collections.Sequence` | `array family`_               |
+        +-------------------------------+-------------------------------+
+        | :class:`collections.Set`      | `array family`_               |
+        +-------------------------------+-------------------------------+
+        | :class:`collections.Mapping`  | `map family`_                 |
+        +-------------------------------+-------------------------------+
+        | :class:`uuid.UUID`            | Converted to String           |
+        +-------------------------------+-------------------------------+
+
+        .. note::
+
+           :class:`str` and :class:`bytes` are the same before Python 3.
+           If you want a value to be treated as a binary value, then you
+           should wrap it in :class:`.BinaryWrapper` if there is any
+           chance of running under Python 2.7.
+
+           The processing of :class:`str` in Python 2.x attempts to
+           encode the string as a UTF-8 stream.  If the ``encode`` succeeds,
+           then the string is encoded according to the `str family`_.
+           If ``encode`` fails, then the string is encoded according to
+           the `bin family`_ .
+
+        .. _nil byte: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#formats-nil
+        .. _true byte: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#bool-format-family
+        .. _false byte: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#bool-format-family
+        .. _integer family: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#int-format-family
+        .. _float family: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#float-format-family
+        .. _str family: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#str-format-family
+        .. _array family: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#array-format-family
+        .. _map family: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md
+           #mapping-format-family
+        .. _bin family: https://github.com/msgpack/msgpack/blob/
+           0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md#bin-format-family
+
+        """
+        if datum is None:
+            return datum
+
+        if isinstance(datum, self.PACKABLE_TYPES):
+            return datum
+
+        if isinstance(datum, uuid.UUID):
+            datum = str(datum)
+
+        if isinstance(datum, bytearray):
+            datum = bytes(datum)
+
+        if isinstance(datum, memoryview):
+            datum = datum.tobytes()
+
+        if hasattr(datum, 'isoformat'):
+            datum = datum.isoformat()
+
+        if sys.version_info[0] < 3 and isinstance(datum, (str, unicode)):
+            if isinstance(datum, str) and not isinstance(datum, BinaryWrapper):
+                # try to decode this into a string to make the common
+                # case work.  If we fail, then send along the bytes.
+                try:
+                    datum = datum.decode('utf-8')
+                except UnicodeDecodeError:
+                    pass
+            return datum
+
+        if isinstance(datum, (bytes, str)):
+            return datum
+
+        if isinstance(datum, (collections.Sequence, collections.Set)):
+            return [self.normalize_datum(item) for item in datum]
+
+        if isinstance(datum, collections.Mapping):
+            out = {}
+            for k, v in datum.items():
+                out[k] = self.normalize_datum(v)
+            return out
+
+        raise TypeError(
+            '{} is not msgpackable'.format(datum.__class__.__name__))

tests.py

@@ -1,11 +1,65 @@
+import base64
+import datetime
 import json
+import os
+import pickle
+import struct
+import sys
+import unittest
+import uuid
 
 from tornado import testing
-import msgpack
+import umsgpack
 
+from sprockets.mixins.mediatype import content, handlers, transcoders
 import examples
 
 
+class UTC(datetime.tzinfo):
+    ZERO = datetime.timedelta(0)
+
+    def utcoffset(self, dt):
+        return self.ZERO
+
+    def dst(self, dt):
+        return self.ZERO
+
+    def tzname(self, dt):
+        return 'UTC'
+
+
+class Context(object):
+    """Super simple class to call setattr on"""
+    pass
+
+
+def pack_string(obj):
+    """Optimally pack a string according to msgpack format"""
+    payload = str(obj).encode('ASCII')
+    l = len(payload)
+    if l < (2 ** 5):
+        prefix = struct.pack('B', 0b10100000 | l)
+    elif l < (2 ** 8):
+        prefix = struct.pack('BB', 0xD9, l)
+    elif l < (2 ** 16):
+        prefix = struct.pack('>BH', 0xDA, l)
+    else:
+        prefix = struct.pack('>BI', 0xDB, l)
+    return prefix + payload
+
+
+def pack_bytes(payload):
+    """Optimally pack a byte string according to msgpack format"""
+    l = len(payload)
+    if l < (2 ** 8):
+        prefix = struct.pack('BB', 0xC4, l)
+    elif l < (2 ** 16):
+        prefix = struct.pack('>BH', 0xC5, l)
+    else:
+        prefix = struct.pack('>BI', 0xC6, l)
+    return prefix + payload
+
+
 class SendResponseTests(testing.AsyncHTTPTestCase):
 
     def get_app(self):
@@ -35,7 +89,7 @@ class SendResponseTests(testing.AsyncHTTPTestCase):
                          'application/msgpack')
 
     def test_that_default_content_type_is_set_on_response(self):
-        response = self.fetch('/', method='POST', body=msgpack.packb({}),
+        response = self.fetch('/', method='POST', body=umsgpack.packb({}),
                               headers={'Content-Type': 'application/msgpack'})
         self.assertEqual(response.code, 200)
         self.assertEqual(response.headers['Content-Type'],
@@ -62,7 +116,234 @@ class GetRequestBodyTests(testing.AsyncHTTPTestCase):
                 'utf8': u'\u2731'
             }
         }
-        response = self.fetch('/', method='POST', body=msgpack.packb(body),
+        response = self.fetch('/', method='POST', body=umsgpack.packb(body),
                               headers={'Content-Type': 'application/msgpack'})
         self.assertEqual(response.code, 200)
         self.assertEqual(json.loads(response.body.decode('utf-8')), body)
+
+
+class JSONTranscoderTests(unittest.TestCase):
+
+    def setUp(self):
+        super(JSONTranscoderTests, self).setUp()
+        self.transcoder = transcoders.JSONTranscoder()
+
+    def test_that_uuids_are_dumped_as_strings(self):
+        obj = {'id': uuid.uuid4()}
+        dumped = self.transcoder.dumps(obj)
+        self.assertEqual(dumped.replace(' ', ''), '{"id":"%s"}' % obj['id'])
+
+    def test_that_datetimes_are_dumped_in_isoformat(self):
+        obj = {'now': datetime.datetime.now()}
+        dumped = self.transcoder.dumps(obj)
+        self.assertEqual(dumped.replace(' ', ''),
+                         '{"now":"%s"}' % obj['now'].isoformat())
+
+    def test_that_tzaware_datetimes_include_tzoffset(self):
+        obj = {'now': datetime.datetime.now().replace(tzinfo=UTC())}
+        self.assertTrue(obj['now'].isoformat().endswith('+00:00'))
+        dumped = self.transcoder.dumps(obj)
+        self.assertEqual(dumped.replace(' ', ''),
+                         '{"now":"%s"}' % obj['now'].isoformat())
+
+    @unittest.skipIf(sys.version_info[0] == 2, 'bytes unsupported on python 2')
+    def test_that_bytes_are_base64_encoded(self):
+        bin = bytes(os.urandom(127))
+        dumped = self.transcoder.dumps({'bin': bin})
+        self.assertEqual(
+            dumped, '{"bin":"%s"}' % base64.b64encode(bin).decode('ASCII'))
+
+    def test_that_bytearrays_are_base64_encoded(self):
+        bin = bytearray(os.urandom(127))
+        dumped = self.transcoder.dumps({'bin': bin})
+        self.assertEqual(
+            dumped, '{"bin":"%s"}' % base64.b64encode(bin).decode('ASCII'))
+
+    def test_that_memoryviews_are_base64_encoded(self):
+        bin = memoryview(os.urandom(127))
+        dumped = self.transcoder.dumps({'bin': bin})
+        self.assertEqual(
+            dumped, '{"bin":"%s"}' % base64.b64encode(bin).decode('ASCII'))
+
+    def test_that_unhandled_objects_raise_type_error(self):
+        with self.assertRaises(TypeError):
+            self.transcoder.dumps(object())
+
+
+class ContentSettingsTests(unittest.TestCase):
+
+    def test_that_from_application_creates_instance(self):
+
+        context = Context()
+        settings = content.ContentSettings.from_application(context)
+        self.assertIs(content.ContentSettings.from_application(context),
+                      settings)
+
+    def test_that_handler_listed_in_available_content_types(self):
+        settings = content.ContentSettings()
+        settings['application/json'] = object()
+        self.assertEqual(len(settings.available_content_types), 1)
+        self.assertEqual(settings.available_content_types[0].content_type,
+                         'application')
+        self.assertEqual(settings.available_content_types[0].content_subtype,
+                         'json')
+
+    def test_that_handler_is_not_overwritten(self):
+        settings = content.ContentSettings()
+        settings['application/json'] = handler = object()
+        settings['application/json'] = object()
+        self.assertIs(settings.get('application/json'), handler)
+
+    def test_that_registered_content_types_are_normalized(self):
+        settings = content.ContentSettings()
+        handler = object()
+        settings['application/json; VerSion=foo; type=WhatEver'] = handler
+        self.assertIs(settings['application/json; type=whatever; version=foo'],
+                      handler)
+        self.assertIn('application/json; type=whatever; version=foo',
+                      (str(c) for c in settings.available_content_types))
+
+    def test_that_normalized_content_types_do_not_overwrite(self):
+        settings = content.ContentSettings()
+        settings['application/json; charset=UTF-8'] = handler = object()
+        settings['application/json; charset=utf-8'] = object()
+        self.assertEqual(len(settings.available_content_types), 1)
+        self.assertEqual(settings.available_content_types[0].content_type,
+                         'application')
+        self.assertEqual(settings.available_content_types[0].content_subtype,
+                         'json')
+        self.assertEqual(settings['application/json; charset=utf-8'], handler)
+
+
+class ContentFunctionTests(unittest.TestCase):
+
+    def setUp(self):
+        super(ContentFunctionTests, self).setUp()
+        self.context = Context()
+
+    def test_that_add_binary_content_type_creates_binary_handler(self):
+        content.add_binary_content_type(self.context,
+                                        'application/vnd.python.pickle',
+                                        pickle.dumps, pickle.loads)
+        settings = content.ContentSettings.from_application(self.context)
+        transcoder = settings['application/vnd.python.pickle']
+        self.assertIsInstance(transcoder, handlers.BinaryContentHandler)
+        self.assertIs(transcoder._pack, pickle.dumps)
+        self.assertIs(transcoder._unpack, pickle.loads)
+
+    def test_that_add_text_content_type_creates_text_handler(self):
+        content.add_text_content_type(self.context, 'application/json', 'utf8',
+                                      json.dumps, json.loads)
+        settings = content.ContentSettings.from_application(self.context)
+        transcoder = settings['application/json']
+        self.assertIsInstance(transcoder, handlers.TextContentHandler)
+        self.assertIs(transcoder._dumps, json.dumps)
+        self.assertIs(transcoder._loads, json.loads)
+
+    def test_that_add_text_content_type_discards_charset_parameter(self):
+        content.add_text_content_type(self.context,
+                                      'application/json;charset=UTF-8', 'utf8',
+                                      json.dumps, json.loads)
+        settings = content.ContentSettings.from_application(self.context)
+        transcoder = settings['application/json']
+        self.assertIsInstance(transcoder, handlers.TextContentHandler)
+
+
+class MsgPackTranscoderTests(unittest.TestCase):
+
+    def setUp(self):
+        super(MsgPackTranscoderTests, self).setUp()
+        self.transcoder = transcoders.MsgPackTranscoder()
+
+    def test_that_strings_are_dumped_as_strings(self):
+        dumped = self.transcoder.packb(u'foo')
+        self.assertEqual(self.transcoder.unpackb(dumped), 'foo')
+        self.assertEqual(dumped, pack_string('foo'))
+
+    def test_that_none_is_packed_as_nil_byte(self):
+        self.assertEqual(self.transcoder.packb(None), b'\xC0')
+
+    def test_that_bools_are_dumped_appropriately(self):
+        self.assertEqual(self.transcoder.packb(False), b'\xC2')
+        self.assertEqual(self.transcoder.packb(True), b'\xC3')
+
+    def test_that_ints_are_packed_appropriately(self):
+        self.assertEqual(self.transcoder.packb((2 ** 7) - 1), b'\x7F')
+        self.assertEqual(self.transcoder.packb(2 ** 7), b'\xCC\x80')
+        self.assertEqual(self.transcoder.packb(2 ** 8), b'\xCD\x01\x00')
+        self.assertEqual(self.transcoder.packb(2 ** 16),
+                         b'\xCE\x00\x01\x00\x00')
+        self.assertEqual(self.transcoder.packb(2 ** 32),
+                         b'\xCF\x00\x00\x00\x01\x00\x00\x00\x00')
+
+    def test_that_negative_ints_are_packed_accordingly(self):
+        self.assertEqual(self.transcoder.packb(-(2 ** 0)), b'\xFF')
+        self.assertEqual(self.transcoder.packb(-(2 ** 5)), b'\xE0')
+        self.assertEqual(self.transcoder.packb(-(2 ** 7)), b'\xD0\x80')
+        self.assertEqual(self.transcoder.packb(-(2 ** 15)), b'\xD1\x80\x00')
+        self.assertEqual(self.transcoder.packb(-(2 ** 31)),
+                         b'\xD2\x80\x00\x00\x00')
+        self.assertEqual(self.transcoder.packb(-(2 ** 63)),
+                         b'\xD3\x80\x00\x00\x00\x00\x00\x00\x00')
+
+    def test_that_lists_are_treated_as_arrays(self):
+        dumped = self.transcoder.packb(list())
+        self.assertEqual(self.transcoder.unpackb(dumped), [])
+        self.assertEqual(dumped, b'\x90')
+
+    def test_that_tuples_are_treated_as_arrays(self):
+        dumped = self.transcoder.packb(tuple())
+        self.assertEqual(self.transcoder.unpackb(dumped), [])
+        self.assertEqual(dumped, b'\x90')
+
+    def test_that_sets_are_treated_as_arrays(self):
+        dumped = self.transcoder.packb(set())
+        self.assertEqual(self.transcoder.unpackb(dumped), [])
+        self.assertEqual(dumped, b'\x90')
+
+    def test_that_unhandled_objects_raise_type_error(self):
+        with self.assertRaises(TypeError):
+            self.transcoder.packb(object())
+
+    def test_that_uuids_are_dumped_as_strings(self):
+        uid = uuid.uuid4()
+        dumped = self.transcoder.packb(uid)
+        self.assertEqual(self.transcoder.unpackb(dumped), str(uid))
+        self.assertEqual(dumped, pack_string(uid))
+
+    def test_that_datetimes_are_dumped_in_isoformat(self):
+        now = datetime.datetime.now()
+        dumped = self.transcoder.packb(now)
+        self.assertEqual(self.transcoder.unpackb(dumped), now.isoformat())
+        self.assertEqual(dumped, pack_string(now.isoformat()))
+
+    def test_that_tzaware_datetimes_include_tzoffset(self):
+        now = datetime.datetime.now().replace(tzinfo=UTC())
+        self.assertTrue(now.isoformat().endswith('+00:00'))
+        dumped = self.transcoder.packb(now)
+        self.assertEqual(self.transcoder.unpackb(dumped), now.isoformat())
+        self.assertEqual(dumped, pack_string(now.isoformat()))
+
+    def test_that_bytes_are_sent_as_bytes(self):
+        data = bytes(os.urandom(127))
+        dumped = self.transcoder.packb(data)
+        self.assertEqual(self.transcoder.unpackb(dumped), data)
+        self.assertEqual(dumped, pack_bytes(data))
+
+    def test_that_bytearrays_are_sent_as_bytes(self):
+        data = bytearray(os.urandom(127))
+        dumped = self.transcoder.packb(data)
+        self.assertEqual(self.transcoder.unpackb(dumped), data)
+        self.assertEqual(dumped, pack_bytes(data))
+
+    def test_that_memoryviews_are_sent_as_bytes(self):
+        data = memoryview(os.urandom(127))
+        dumped = self.transcoder.packb(data)
+        self.assertEqual(self.transcoder.unpackb(dumped), data)
+        self.assertEqual(dumped, pack_bytes(data.tobytes()))
+
+    def test_that_utf8_values_can_be_forced_to_bytes(self):
+        data = b'a ascii value'
+        dumped = self.transcoder.packb(transcoders.BinaryWrapper(data))
+        self.assertEqual(self.transcoder.unpackb(dumped), data)
+        self.assertEqual(dumped, pack_bytes(data))

tox.ini

@@ -3,7 +3,7 @@ envlist = py27,py34,py35,pypy
 indexserver =
 	default = https://pypi.python.org/simple
 toxworkdir = build/tox
-skip_unknown_interpreters = true
+skip_missing_interpreters = true
 
 [testenv]
 deps =