From 9f4e81b3a833d3bdcd66e4087ad90019cdd79f61 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Tue, 6 Sep 2016 11:28:40 -0400 Subject: [PATCH] Marshalling and Unmarshalling Changes - Remove the maybe_convert strings to UUIDs - Fix the binary data behavior --- sprockets/clients/dynamodb/__init__.py | 2 +- sprockets/clients/dynamodb/utils.py | 41 ++++++++++---------------- tests/api_tests.py | 2 +- tests/utils_tests.py | 15 ++++++---- 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/sprockets/clients/dynamodb/__init__.py b/sprockets/clients/dynamodb/__init__.py index c0cfcea..54f35a6 100644 --- a/sprockets/clients/dynamodb/__init__.py +++ b/sprockets/clients/dynamodb/__init__.py @@ -4,7 +4,7 @@ except ImportError as error: def DynamoDB(*args, **kwargs): raise error -version_info = (0, 2, 3) +version_info = (0, 3, 0) __version__ = '.'.join(str(v) for v in version_info) # Response constants diff --git a/sprockets/clients/dynamodb/utils.py b/sprockets/clients/dynamodb/utils.py index 91fdb26..50848ec 100644 --- a/sprockets/clients/dynamodb/utils.py +++ b/sprockets/clients/dynamodb/utils.py @@ -14,6 +14,7 @@ the vast majority of types that we use. APIReference/API_AttributeValue.html """ +import base64 import datetime import uuid import sys @@ -58,12 +59,12 @@ def _marshall_value(value): """ if PYTHON3 and isinstance(value, bytes): - return {'B': value} + return {'B': base64.b64encode(value).decode('ascii')} elif PYTHON3 and isinstance(value, str): return {'S': value} elif not PYTHON3 and isinstance(value, str): if _is_binary(value): - return {'B': value} + return {'B': base64.b64encode(value).decode('ascii')} return {'S': value} elif isinstance(value, dict): return {'M': marshall(value)} @@ -81,14 +82,14 @@ def _marshall_value(value): return {'L': [_marshall_value(v) for v in value]} elif isinstance(value, set): if PYTHON3 and all([isinstance(v, bytes) for v in value]): - return {'BS': sorted(list(value))} + return {'BS': _encode_binary_set(value)} elif PYTHON3 and all([isinstance(v, str) for v in value]): return {'SS': sorted(list(value))} elif all([isinstance(v, (int, float)) for v in value]): return {'NS': sorted([str(v) for v in value])} elif not PYTHON3 and all([isinstance(v, str) for v in value]) and \ all([_is_binary(v) for v in value]): - return {'BS': sorted(list(value))} + return {'BS': _encode_binary_set(value)} elif not PYTHON3 and all([isinstance(v, str) for v in value]) and \ all([_is_binary(v) is False for v in value]): return {'SS': sorted(list(value))} @@ -99,6 +100,11 @@ def _marshall_value(value): raise ValueError('Unsupported type: %s' % type(value)) +def _encode_binary_set(value): + return sorted([base64.b64encode(v).decode('ascii') for v in value]) + + + def unmarshall(values): """ Transform a response payload from DynamoDB to a native dict @@ -125,9 +131,10 @@ def _unmarshall_dict(value): """ key = list(value.keys()).pop() if key == 'B': - return bytes(value[key]) + return base64.b64decode(value[key].encode('ascii')) elif key == 'BS': - return set([bytes(v) for v in value[key]]) + return set([base64.b64decode(v.encode('ascii')) + for v in value[key]]) elif key == 'BOOL': return value[key] elif key == 'L': @@ -141,9 +148,9 @@ def _unmarshall_dict(value): elif key == 'NS': return set([_to_number(v) for v in value[key]]) elif key == 'S': - return _maybe_convert(value[key]) + return value[key] elif key == 'SS': - return set([_maybe_convert(v) for v in value[key]]) + return set([v for v in value[key]]) raise ValueError('Unsupported value type: %s' % key) @@ -158,24 +165,6 @@ def _to_number(value): return float(value) if '.' in value else int(value) -def _maybe_convert(value): - """ - Try to convert a string into something useful. - - :param str value: The value to convert - :rtype: uuid.UUID|datetime.datetime|str - - Possibly convert the value to a :py:class:`uuid.UUID` or - :py:class:`datetime.datetime` if possible, otherwise just return - the value. - - """ - try: - return uuid.UUID(value) - except ValueError: - return value - - def _is_binary(value): """ Check to see if a string contains binary data in Python2 diff --git a/tests/api_tests.py b/tests/api_tests.py index 541d426..41b2810 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -247,4 +247,4 @@ class PutGetDeleteTests(AsyncTestCase): response = yield self.client.get_item(definition['TableName'], {'id': row_id}) - self.assertEqual(response['id'], row_id) + self.assertEqual(response['id'], str(row_id)) diff --git a/tests/utils_tests.py b/tests/utils_tests.py index d7edd0e..91325d8 100644 --- a/tests/utils_tests.py +++ b/tests/utils_tests.py @@ -1,3 +1,4 @@ +import base64 import datetime import unittest import uuid @@ -60,8 +61,9 @@ class MarshallTests(unittest.TestCase): 'key7': {'NS': ['1', '2', '3', '4']}, 'key8': {'S': arrow_value.isoformat()}, 'key9': {'S': str(uuid_value)}, - 'key10': {'B': b'\0x01\0x02\0x03'}, - 'key11': {'BS': [b'\0x01\0x02\0x03', b'\0x04\0x05\0x06']}, + 'key10': {'B': base64.b64encode(b'\0x01\0x02\0x03').decode('ascii')}, + 'key11': {'BS': [base64.b64encode(b'\0x01\0x02\0x03').decode('ascii'), + base64.b64encode(b'\0x04\0x05\0x06').decode('ascii')]}, 'key12': {'S': dt_value.isoformat()} } self.assertDictEqual(expectation, utils.marshall(value)) @@ -77,7 +79,7 @@ class UnmarshallTests(unittest.TestCase): maxDiff = None def test_complex_document(self): - uuid_value = uuid.uuid4() + uuid_value = str(uuid.uuid4()) dt_value = arrow.utcnow() value = { 'key1': {'S': 'str'}, @@ -95,9 +97,10 @@ class UnmarshallTests(unittest.TestCase): 'key6': {'SS': ['a', 'b', 'c']}, 'key7': {'NS': ['1', '2', '3', '4']}, 'key8': {'S': dt_value.isoformat()}, - 'key9': {'S': str(uuid_value)}, - 'key10': {'B': b'\0x01\0x02\0x03'}, - 'key11': {'BS': [b'\0x01\0x02\0x03', b'\0x04\0x05\0x06']} + 'key9': {'S': uuid_value}, + 'key10': {'B': base64.b64encode(b'\0x01\0x02\0x03').decode('ascii')}, + 'key11': {'BS': [base64.b64encode(b'\0x01\0x02\0x03').decode('ascii'), + base64.b64encode(b'\0x04\0x05\0x06').decode('ascii')]} } expectation = { 'key1': 'str',