Merge pull request #2 from sprockets/merge-tornado-dynamodb

Merge tornado dynamodb
This commit is contained in:
dave-shawley 2016-03-04 10:00:43 -05:00
commit 5a08f214f7
12 changed files with 1793 additions and 33 deletions

1
.gitignore vendored
View file

@ -5,3 +5,4 @@ dist
env env
*.egg-info *.egg-info
.coverage .coverage
.idea

View file

@ -1,14 +1,33 @@
sudo: required
services:
- docker
language: python language: python
python: python:
- 2.7 - 2.7
- 3.4 - 3.4
- 3.5 - 3.5
before_install: before_install:
- pip install nose coverage codecov - docker pull tray/dynamodb-local
- docker run -d -p 7777:7777 tray/dynamodb-local -inMemory -port 7777
- mkdir /home/travis/.aws
- printf "[default]\nregion=us-east-1\noutput=json\n" > /home/travis/.aws/config
- printf "[default]\naws_access_key_id = FAKE0000000000000000\naws_secret_access_key = FAKE000000000000000000000000000000000000\n" > /home/travis/.aws/credentials
- pip install -r requires/testing.txt - pip install -r requires/testing.txt
install: install:
- pip install -e . - pip install -e .
env:
DYNAMODB_ENDPOINT: http://localhost:7777
script: nosetests --with-coverage script: nosetests --with-coverage
after_success: after_success:
- codecov - codecov
sudo: false sudo: false
deploy:
distributions: sdist bdist_wheel
provider: pypi
user: sprockets
on:
python: 3.5
tags: true
all_branches: true
password:
secure: "pCvF0ROHU/p+mDgZT40yoRdNUmpov5B1jUh7mJ6bAUlsMNEaugX/cL+cUGNLgIhrcwBF93B7kdfuhGjO/2uF+k8aPhPocewwJ9qPTTyNMLGjpIclWp56KH9KLNISGmeTPguw06bpV0xOUw40AvSfTw4nmf4jaZsx1Ai2DUuoji7m1OvXwLL5+zXclngmxF7zVvPTnKmPDbJWUsF3n4DEJml8GBr7NW92yIo0Zu1LG3AiNrZWBebWa58Uv/DKOHQXYgyK0j3EixzTPkptoQgAByA6OVPPh6UOE2GUXuV83vDKeciyr/AExLQnlIaONa2FS4utOFdu2zoLsUJy+jeCJxVZ5D+jfYXSx1LyeQKjOZikUKNhI3O3XH7IYwd2YqhlRAE6SvFGQB1nYn6mXklSwdyOEaQ0ufUY4aCH9PRvswOUDJKIJw4xsiEUF46enrGWHVCnW3l0fPbhPx1GbB/PfzcJS3WSEgOKHbZ2u7PHrIkElxAOrI6Vabmrr0g5GD1T2DqBls600lQ/+HkRQ9cXVjegiUach3xj3IKL/gZJUuqiwl2xMPdIfi33GsZp5OItSt1fNmBZo4gz5zBEXYShpgeqx0hP0XEfQWnDLNoaHNhzaW9d1PYs7JHIsiLRw9HcJNdRzm4u08442m42WyEP5i3XnpmylFu6U+2a1mR4VEg="

View file

@ -1,5 +1,5 @@
include LICENSE include LICENSE
include tests.py
graft docs graft docs
graft examples graft examples
graft requires graft requires
graft tests

80
bootstrap Executable file
View file

@ -0,0 +1,80 @@
#!/bin/sh
#
# NAME
# bootstrap -- initialize/update docker environment
#
# SYNOPSIS
# bootstrap
# bootstrap shellinit
#
# DESCRIPTION
# Execute this script without parameters to build the local docker
# environment. Once bootstrapped, dependent services are running
# via docker-compose and the environment variables are written to
# *build/test-environment* for future use.
#
# Running this script with the _shellinit_ command line parameter
# causes it to simply interrogate the running docker environment,
# update *build/test-environment*, and print the environment to
# the standard output stream in a shell executable manner. This
# makes the following pattern for setting environment variables
# in the current shell work.
#
# prompt% $(./bootstrap shellinit)
#
# vim: set ts=2 sts=2 sw=2 et:
PROJECT=sprockets
if test -e /var/run/docker.sock
then
DOCKER_IP=127.0.0.1
else
docker-machine status ${PROJECT} >/dev/null 2>/dev/null
RESULT=$?
if [ ${RESULT} -ne 0 ]
then
docker-machine create --driver virtualbox ${PROJECT}
fi
eval $(docker-machine env ${PROJECT} 2>/dev/null) || {
echo "Failed to initialize docker environment"
exit 2
}
DOCKER_IP=$(docker-machine ip ${PROJECT})
fi
COMPOSE_ARGS=
if test -n "${DOCKER_COMPOSE_PREFIX}"
then
COMPOSE_ARGS="-p ${DOCKER_COMPOSE_PREFIX}"
fi
get_exposed_port() {
docker-compose ${COMPOSE_ARGS} port $1 $2 | cut -d: -f2
}
build_env_file() {
DYNAMODB_PORT=$(get_exposed_port dynamodb 7777)
(echo "export DOCKER_COMPOSE_PREFIX=${DOCKER_COMPOSE_PREFIX}"
echo "export DOCKER_TLS_VERIFY=${DOCKER_TLS_VERIFY}"
echo "export DOCKER_HOST=${DOCKER_HOST}"
echo "export DOCKER_CERT_PATH=${DOCKER_CERT_PATH}"
echo "export DOCKER_MACHINE_NAME=${DOCKER_MACHINE_NAME}"
echo "export DYNAMODB_ENDPOINT=http://${DOCKER_IP}:${DYNAMODB_PORT}"
) > $1
}
set -e
mkdir -p build
if test "$1" = 'shellinit'
then
# just build the environment file from docker containers
build_env_file build/test-environment
else
docker-compose ${COMPOSE_ARGS} stop
docker-compose ${COMPOSE_ARGS} rm --force
docker-compose ${COMPOSE_ARGS} up -d
build_env_file build/test-environment
fi
cat build/test-environment

7
docker-compose.yml Normal file
View file

@ -0,0 +1,7 @@
%YAML 1.2
---
dynamodb:
image: tray/dynamodb-local
command: -inMemory -port 7777
ports:
- 7777

View file

@ -1,2 +1,5 @@
nose>=1.3.7,<2 nose>=1.3.7,<2
coverage>=3.7,<4 coverage>=3.7,<4
arrow>=0.7.0,<1
mock>=1.3.0,<2
codecov>=1.6.3,<2

View file

@ -6,4 +6,11 @@ except ImportError as error:
version_info = (0, 0, 0) version_info = (0, 0, 0)
__version__ = '.'.join(str(v) for v in version_info) __version__ = '.'.join(str(v) for v in version_info)
__all__ = ['DynamoDB', 'version_info', '__version__']
# Response constants
TABLE_ACTIVE = 'ACTIVE'
TABLE_CREATING = 'CREATING'
TABLE_DELETING = 'DELETING'
TABLE_DISABLED = 'DISABLED'
TABLE_UPDATING = 'UPDATING'

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,194 @@
"""
DynamoDB Exceptions
===================
"""
class DynamoDBException(Exception):
"""Base exception that is extended by all exceptions raised by
tornado_dynamodb.
:ivar msg: The error message
"""
def __init__(self, *args, **kwargs):
super(DynamoDBException, self).__init__(*args, **kwargs)
class ConditionalCheckFailedException(DynamoDBException):
"""A condition specified in the operation could not be evaluated."""
pass
class ConfigNotFound(DynamoDBException):
"""The configuration file could not be parsed."""
pass
class ConfigParserError(DynamoDBException):
"""Error raised when parsing a configuration file with
:class:`~configparser.RawConfigParser`
"""
pass
class InternalFailure(DynamoDBException):
"""The request processing has failed because of an unknown error, exception
or failure.
"""
pass
class ItemCollectionSizeLimitExceeded(DynamoDBException):
"""An item collection is too large. This exception is only returned for
tables that have one or more local secondary indexes.
"""
pass
class InvalidAction(DynamoDBException):
"""The action or operation requested is invalid. Verify that the action is
typed correctly.
"""
pass
class InvalidParameterCombination(DynamoDBException):
"""Parameters that must not be used together were used together."""
pass
class InvalidParameterValue(DynamoDBException):
"""An invalid or out-of-range value was supplied for the input parameter."""
pass
class InvalidQueryParameter(DynamoDBException):
"""The AWS query string is malformed or does not adhere to AWS standards."""
pass
class LimitExceeded(DynamoDBException):
"""The number of concurrent table requests (cumulative number of tables in
the ``CREATING``, ``DELETING`` or ``UPDATING`` state) exceeds the maximum
allowed of ``10``.
Also, for tables with secondary indexes, only one of those tables can be in
the ``CREATING`` state at any point in time. Do not attempt to create more
than one such table simultaneously.
The total limit of tables in the ``ACTIVE`` state is ``250``.
"""
pass
class MalformedQueryString(DynamoDBException):
"""The query string contains a syntax error."""
pass
class MissingParameter(DynamoDBException):
"""A required parameter for the specified action is not supplied."""
pass
class NoCredentialsError(DynamoDBException):
"""Raised when the credentials could not be located."""
pass
class NoProfileError(DynamoDBException):
"""Raised when the specified profile could not be located."""
pass
class OptInRequired(DynamoDBException):
"""The AWS access key ID needs a subscription for the service."""
pass
class ThroughputExceeded(DynamoDBException):
"""Your request rate is too high. The AWS SDKs for DynamoDB automatically
retry requests that receive this exception. Your request is eventually
successful, unless your retry queue is too large to finish. Reduce the
frequency of requests and use exponential backoff. For more information, go
to `Error Retries and Exponential Backoff <http://docs.aws.amazon.com/
amazondynamodb/latest/developerguide/ErrorHandling.html#APIRetries>`_ in
the Amazon DynamoDB Developer Guide.
"""
pass
class RequestException(DynamoDBException):
"""A generic HTTP request exception has occurred when communicating with
DynamoDB.
"""
pass
class RequestExpired(DynamoDBException):
"""The request reached the service more than 15 minutes after the date
stamp on the request or more than 15 minutes after the request expiration
date (such as for pre-signed URLs), or the date stamp on the request is
more than 15 minutes in the future.
"""
pass
class ResourceInUse(DynamoDBException):
"""he operation conflicts with the resource's availability. For example,
you attempted to recreate an existing table, or tried to delete a table
currently in the ``CREATING`` state.
"""
pass
class ResourceNotFound(DynamoDBException):
"""The operation tried to access a nonexistent table or index. The resource
might not be specified correctly, or its status might not be ``ACTIVE``.
"""
pass
class ServiceUnavailable(DynamoDBException):
"""The request has failed due to a temporary failure of the server."""
pass
class ThrottlingException(DynamoDBException):
"""The request was denied due to request throttling."""
pass
class TimeoutException(DynamoDBException):
"""The request to DynamoDB timed out."""
pass
class ValidationException(DynamoDBException):
"""The input fails to satisfy the constraints specified by an AWS service.
"""
pass
MAP = {
'com.amazonaws.dynamodb.v20120810#InternalFailure': InternalFailure,
'com.amazonaws.dynamodb.v20120810#ProvisionedThroughputExceeded':
ThroughputExceeded,
'com.amazonaws.dynamodb.v20120810#ResourceInUseException': ResourceInUse,
'com.amazonaws.dynamodb.v20120810#ResourceNotFoundException':
ResourceNotFound,
'com.amazon.coral.validate#ValidationException': ValidationException
}

View file

216
tests/api_tests.py Normal file
View file

@ -0,0 +1,216 @@
import datetime
import os
import uuid
import mock
from tornado import concurrent
from tornado import httpclient
from tornado import testing
from tornado_aws import exceptions as aws_exceptions
from sprockets.clients import dynamodb
from sprockets.clients.dynamodb import exceptions
class AsyncTestCase(testing.AsyncTestCase):
def setUp(self):
super(AsyncTestCase, self).setUp()
self.client = self.get_client()
@property
def endpoint(self):
return os.getenv('DYNAMODB_ENDPOINT')
@staticmethod
def generic_table_definition():
return {
'TableName': str(uuid.uuid4()),
'AttributeDefinitions': [{'AttributeName': 'id',
'AttributeType': 'S'}],
'KeySchema': [{'AttributeName': 'id', 'KeyType': 'HASH'}],
'ProvisionedThroughput': {
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
}
}
def get_client(self):
return dynamodb.DynamoDB(endpoint=self.endpoint)
class AWSClientTests(AsyncTestCase):
@testing.gen_test
def test_raises_config_not_found_exception(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
fetch.side_effect = aws_exceptions.ConfigNotFound(path='/test')
with self.assertRaises(exceptions.ConfigNotFound):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_raises_config_parser_error(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
fetch.side_effect = aws_exceptions.ConfigParserError(path='/test')
with self.assertRaises(exceptions.ConfigParserError):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_raises_no_credentials_error(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
fetch.side_effect = aws_exceptions.NoCredentialsError()
with self.assertRaises(exceptions.NoCredentialsError):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_raises_no_profile_error(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
fetch.side_effect = aws_exceptions.NoProfileError(profile='test-1',
path='/test')
with self.assertRaises(exceptions.NoProfileError):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_raises_request_exception(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
fetch.side_effect = httpclient.HTTPError(500, 'uh-oh')
with self.assertRaises(exceptions.RequestException):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_raises_timeout_exception(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
fetch.side_effect = httpclient.HTTPError(599)
with self.assertRaises(exceptions.TimeoutException):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_fetch_future_exception(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
future = concurrent.Future()
fetch.return_value = future
future.set_exception(exceptions.DynamoDBException())
with self.assertRaises(exceptions.DynamoDBException):
yield self.client.create_table(self.generic_table_definition())
@testing.gen_test
def test_empty_fetch_response_raises_dynamodb_exception(self):
with mock.patch('tornado_aws.client.AsyncAWSClient.fetch') as fetch:
future = concurrent.Future()
fetch.return_value = future
future.set_result(None)
with self.assertRaises(exceptions.DynamoDBException):
yield self.client.create_table(self.generic_table_definition())
class CreateTableTests(AsyncTestCase):
@testing.gen_test
def test_simple_table(self):
definition = self.generic_table_definition()
response = yield self.client.create_table(definition)
self.assertEqual(response['TableName'], definition['TableName'])
self.assertIn(response['TableStatus'],
[dynamodb.TABLE_ACTIVE,
dynamodb.TABLE_CREATING])
@testing.gen_test
def test_invalid_request(self):
definition = {
'TableName': str(uuid.uuid4()),
'AttributeDefinitions': [{'AttributeName': 'id'}],
'KeySchema': [],
'ProvisionedThroughput': {
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
}
}
with self.assertRaises(exceptions.ValidationException):
yield self.client.create_table(definition)
@testing.gen_test
def test_double_create(self):
definition = self.generic_table_definition()
response = yield self.client.create_table(definition)
self.assertEqual(response['TableName'], definition['TableName'])
self.assertIn(response['TableStatus'],
[dynamodb.TABLE_ACTIVE,
dynamodb.TABLE_CREATING])
with self.assertRaises(exceptions.ResourceInUse):
response = yield self.client.create_table(definition)
class DeleteTableTests(AsyncTestCase):
@testing.gen_test
def test_delete_table(self):
definition = self.generic_table_definition()
response = yield self.client.create_table(definition)
self.assertEqual(response['TableName'], definition['TableName'])
yield self.client.delete_table(definition['TableName'])
with self.assertRaises(exceptions.ResourceNotFound):
yield self.client.describe_table(definition['TableName'])
@testing.gen_test
def test_table_not_found(self):
table = str(uuid.uuid4())
with self.assertRaises(exceptions.ResourceNotFound):
yield self.client.delete_table(table)
class DescribeTableTests(AsyncTestCase):
@testing.gen_test
def test_describe_table(self):
# Create the table first
definition = self.generic_table_definition()
response = yield self.client.create_table(definition)
self.assertEqual(response['TableName'], definition['TableName'])
# Describe the table
response = yield self.client.describe_table(definition['TableName'])
self.assertEqual(response['TableName'], definition['TableName'])
self.assertEqual(response['TableStatus'],
dynamodb.TABLE_ACTIVE)
@testing.gen_test
def test_table_not_found(self):
table = str(uuid.uuid4())
with self.assertRaises(exceptions.ResourceNotFound):
yield self.client.describe_table(table)
class ListTableTests(AsyncTestCase):
@testing.gen_test
def test_list_tables(self):
# Create the table first
definition = self.generic_table_definition()
response = yield self.client.create_table(definition)
self.assertEqual(response['TableName'], definition['TableName'])
# Describe the table
response = yield self.client.list_tables(limit=100)
self.assertIn(definition['TableName'], response['TableNames'])
class PutGetDeleteTests(AsyncTestCase):
@testing.gen_test
def test_put_item(self):
# Create the table first
definition = self.generic_table_definition()
response = yield self.client.create_table(definition)
self.assertEqual(response['TableName'], definition['TableName'])
row_id = uuid.uuid4()
# Describe the table
yield self.client.put_item(
definition['TableName'],
{'id': row_id, 'created_at': datetime.datetime.utcnow()})
response = yield self.client.get_item(definition['TableName'],
{'id': row_id})
self.assertEqual(response['id'], row_id)

122
tests/utils_tests.py Normal file
View file

@ -0,0 +1,122 @@
import datetime
import unittest
import uuid
import arrow
from sprockets.clients.dynamodb import utils
class UTC(datetime.tzinfo):
def utcoffset(self, dt):
return datetime.timedelta(0)
def tzname(self, dt):
return 'UTC'
def dst(self, dt):
return datetime.timedelta(0)
class MarshallTests(unittest.TestCase):
maxDiff = None
def test_complex_document(self):
uuid_value = uuid.uuid4()
arrow_value = arrow.utcnow()
dt_value = datetime.datetime.utcnow().replace(tzinfo=UTC())
value = {
'key1': 'str',
'key2': 10,
'key3': {
'sub-key1': 20,
'sub-key2': True,
'sub-key3': 'value'
},
'key4': None,
'key5': ['one', 'two', 'three', 4, None, True],
'key6': set(['a', 'b', 'c']),
'key7': {1, 2, 3, 4},
'key8': arrow_value,
'key9': uuid_value,
'key10': b'\0x01\0x02\0x03',
'key11': {b'\0x01\0x02\0x03', b'\0x04\0x05\0x06'},
'key12': dt_value
}
expectation = {
'key1': {'S': 'str'},
'key2': {'N': '10'},
'key3': {'M':
{
'sub-key1': {'N': '20'},
'sub-key2': {'BOOL': True},
'sub-key3': {'S': 'value'}
}
},
'key4': {'NULL': True},
'key5': {'L': [{'S': 'one'}, {'S': 'two'}, {'S': 'three'},
{'N': '4'}, {'NULL': True}, {'BOOL': True}]},
'key6': {'SS': ['a', 'b', 'c']},
'key7': {'NS': ['1', '2', '3', '4']},
'key8': {'S': arrow_value.isoformat()},
'key9': {'S': str(uuid_value)},
'key10': {'B': b'\0x01\0x02\0x03'},
'key11': {'BS': [b'\0x01\0x02\0x03', b'\0x04\0x05\0x06']},
'key12': {'S': dt_value.isoformat()}
}
self.assertDictEqual(expectation, utils.marshall(value))
def test_value_error_raised_on_unsupported_type(self):
self.assertRaises(ValueError, utils.marshall, {'key': self})
def test_value_error_raised_on_mixed_set(self):
self.assertRaises(ValueError, utils.marshall, {'key': {1, 'two', 3}})
class UnmarshallTests(unittest.TestCase):
maxDiff = None
def test_complex_document(self):
uuid_value = uuid.uuid4()
dt_value = arrow.utcnow()
value = {
'key1': {'S': 'str'},
'key2': {'N': '10'},
'key3': {'M':
{
'sub-key1': {'N': '20'},
'sub-key2': {'BOOL': True},
'sub-key3': {'S': 'value'}
}
},
'key4': {'NULL': True},
'key5': {'L': [{'S': 'one'}, {'S': 'two'}, {'S': 'three'},
{'N': '4'}, {'NULL': True}, {'BOOL': True}]},
'key6': {'SS': ['a', 'b', 'c']},
'key7': {'NS': ['1', '2', '3', '4']},
'key8': {'S': dt_value.isoformat()},
'key9': {'S': str(uuid_value)},
'key10': {'B': b'\0x01\0x02\0x03'},
'key11': {'BS': [b'\0x01\0x02\0x03', b'\0x04\0x05\0x06']}
}
expectation = {
'key1': 'str',
'key2': 10,
'key3': {
'sub-key1': 20,
'sub-key2': True,
'sub-key3': 'value'
},
'key4': None,
'key5': ['one', 'two', 'three', 4, None, True],
'key6': {'a', 'b', 'c'},
'key7': {1, 2, 3, 4},
'key8': dt_value.isoformat(),
'key9': uuid_value,
'key10': b'\0x01\0x02\0x03',
'key11': {b'\0x01\0x02\0x03', b'\0x04\0x05\0x06'}
}
self.assertDictEqual(expectation, utils.unmarshall(value))
def test_value_error_raised_on_unsupported_type(self):
self.assertRaises(ValueError, utils.unmarshall, {'key': {'T': 1}})