From ca362e2bc5788a25b9d522e8bcbcb27d20ddb4a4 Mon Sep 17 00:00:00 2001 From: "Gavin M. Roy" Date: Fri, 8 Jan 2021 14:55:25 -0500 Subject: [PATCH] Allow for settings based configuration --- .github/workflows/testing.yaml | 2 +- README.rst | 63 ++++++++++++++------------ VERSION | 2 +- docs/configuration.rst | 81 ++++++++++++++++++++++------------ setup.cfg | 2 +- sprockets_postgres.py | 52 +++++++++++++--------- tests.py | 5 +++ 7 files changed, 129 insertions(+), 78 deletions(-) diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index ec0276d..5e08fd7 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -26,7 +26,7 @@ jobs: POSTGRES_HOST_AUTH_METHOD: trust strategy: matrix: - python: [3.7, 3.8] + python: [3.7, 3.8, 3.9] container: image: python:${{ matrix.python }}-alpine steps: diff --git a/README.rst b/README.rst index 4ff1a94..8494aec 100644 --- a/README.rst +++ b/README.rst @@ -19,39 +19,48 @@ Documentation is available at `sprockets-postgres.readthedocs.io `_ +dictionary. The `sprockets_postgres.ApplicationMixin `_ +will use configuration as applied to the settings dictionary, falling back to the +environment variable if the value is not set in the dictionary. Keys in the +settings dictionary are lowercase, and if provided as environment variables, +are uppercase. -+---------------------------------+--------------------------------------------------+-----------+ -| Variable | Definition | Default | -+=================================+==================================================+===========+ -| ``POSTGRES_URL`` | The PostgreSQL URL to connect to | | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_MAX_POOL_SIZE`` | Maximum connection count to Postgres per backend | ``10`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_MIN_POOL_SIZE`` | Minimum or starting pool size. | ``1`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_CONNECTION_TIMEOUT`` | The maximum time in seconds to spend attempting | ``10`` | -| | to create a new connection. | | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_CONNECTION_TTL`` | Time-to-life in seconds for a pooled connection. | ``300`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_QUERY_TIMEOUT`` | Maximum execution time for a query in seconds. | ``60`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_HSTORE`` | Enable HSTORE support in the client. | ``FALSE`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_JSON`` | Enable JSON support in the client. | ``FALSE`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_UUID`` | Enable UUID support in the client. | ``TRUE`` | -+---------------------------------+--------------------------------------------------+-----------+ +The following table details the available configuration options: -If ``POSTGRES_URL`` uses a scheme of ``postgresql+srv``, a SRV DNS lookup will be ++---------------------------------+--------------------------------------------------+------+-----------+ +| Variable | Definition | Type | Default | ++=================================+==================================================+======+===========+ +| ``postgres_url`` | The PostgreSQL URL to connect to | str | | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_max_pool_size`` | Maximum connection count to Postgres per backend | int | ``10`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_min_pool_size`` | Minimum or starting pool size. | int | ``1`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_connection_timeout`` | The maximum time in seconds to spend attempting | int | ``10`` | +| | to create a new connection. | | | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_connection_ttl`` | Time-to-life in seconds for a pooled connection. | int | ``300`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_query_timeout`` | Maximum execution time for a query in seconds. | int | ``60`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_hstore`` | Enable HSTORE support in the client. | bool | ``FALSE`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_json`` | Enable JSON support in the client. | bool | ``FALSE`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_uuid`` | Enable UUID support in the client. | bool | ``TRUE`` | ++---------------------------------+--------------------------------------------------+------+-----------+ + +If ``postgres_url`` uses a scheme of ``postgresql+srv``, a SRV DNS lookup will be performed and the lowest priority record with the highest weight will be selected for connecting to Postgres. AWS's ECS service discovery does not follow the SRV standard, but creates SRV -records. If ``POSTGRES_URL`` uses a scheme of ``aws+srv``, a SRV DNS lookup will be -performed using the correct format for ECS service discovery. The lowest priority -record with the highest weight will be selected for connecting to Postgres. +records. If ``postgres_url`` uses a scheme of ``aws+srv``, a SRV DNS lookup will be +performed and the URL will be constructed containing all host and port combinations +in priority and weighted order, utilizing `libpq's supoprt `_ +for multiple hosts in a URL. Requirements ------------ diff --git a/VERSION b/VERSION index bc80560..dc1e644 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.0 +1.6.0 diff --git a/docs/configuration.rst b/docs/configuration.rst index 4d9ef97..db0e10b 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -1,36 +1,61 @@ Configuration ============= -:py:mod:`sprockets-postgres ` is configured via environment variables. The following table -details the configuration options and their defaults. +Configuration of :py:mod:`sprockets-postgres ` is done by +using of environment variables or :py:attr:`tornado.web.Application.settings` +dictionary. The :py:class:`sprockets_postgres.ApplicationMixin` will use configuration +as applied to the settings dictionary, falling back to the environment variable +if the value is not set in the dictionary. Keys in the settings dictionary are +lowercase, and if provided as environment variables, are uppercase. For example +to set the Postgres URL in a :py:attr:`tornado.web.Application.settings`, +you'd do the following: -+---------------------------------+--------------------------------------------------+-----------+ -| Variable | Definition | Default | -+=================================+==================================================+===========+ -| ``POSTGRES_URL`` | The PostgreSQL URL to connect to | | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_MAX_POOL_SIZE`` | Maximum connection count to Postgres per backend | ``10`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_MIN_POOL_SIZE`` | Minimum or starting pool size. | ``1`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_CONNECTION_TIMEOUT`` | The maximum time in seconds to spend attempting | ``10`` | -| | to create a new connection. | | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_CONNECTION_TTL`` | Time-to-life in seconds for a pooled connection. | ``300`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_QUERY_TIMEOUT`` | Maximum execution time for a query in seconds. | ``60`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_HSTORE`` | Enable HSTORE support in the client. | ``FALSE`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_JSON`` | Enable JSON support in the client. | ``FALSE`` | -+---------------------------------+--------------------------------------------------+-----------+ -| ``POSTGRES_UUID`` | Enable UUID support in the client. | ``TRUE`` | -+---------------------------------+--------------------------------------------------+-----------+ +.. code-block:: -If ``POSTGRES_URL`` uses a scheme of ``postgresql+srv``, a SRV DNS lookup will be + settings = {'postgres_url': 'postgresql://postgres@localhost:5432/postgres'} + app = web.Application([routes], settings=settings) + +and as an environment variable: + +.. code-block:: + + POSTGRES_URL=postgresql://postgres@localhost:5432/postgres + + +Available Settings +------------------ + +The following table details the available configuration options: + ++---------------------------------+--------------------------------------------------+------+-----------+ +| Variable | Definition | Type | Default | ++=================================+==================================================+======+===========+ +| ``postgres_url`` | The PostgreSQL URL to connect to | str | | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_max_pool_size`` | Maximum connection count to Postgres per backend | int | ``10`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_min_pool_size`` | Minimum or starting pool size. | int | ``1`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_connection_timeout`` | The maximum time in seconds to spend attempting | int | ``10`` | +| | to create a new connection. | | | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_connection_ttl`` | Time-to-life in seconds for a pooled connection. | int | ``300`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_query_timeout`` | Maximum execution time for a query in seconds. | int | ``60`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_hstore`` | Enable HSTORE support in the client. | bool | ``FALSE`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_json`` | Enable JSON support in the client. | bool | ``FALSE`` | ++---------------------------------+--------------------------------------------------+------+-----------+ +| ``postgres_uuid`` | Enable UUID support in the client. | bool | ``TRUE`` | ++---------------------------------+--------------------------------------------------+------+-----------+ + +If ``postgres_url`` uses a scheme of ``postgresql+srv``, a SRV DNS lookup will be +performed and the lowest priority record with the highest weight will be selected +for connecting to Postgres. + +AWS's ECS service discovery does not follow the SRV standard, but creates SRV +records. If ``postgres_url`` uses a scheme of ``aws+srv``, a SRV DNS lookup will be performed and the URL will be constructed containing all host and port combinations in priority and weighted order, utilizing `libpq's supoprt `_ for multiple hosts in a URL. -AWS's ECS service discovery does not follow the SRV standard, but creates SRV -records. If ``POSTGRES_URL`` uses a scheme of ``aws+srv``, a SRV DNS lookup will be -performed using the correct format for ECS service discovery. diff --git a/setup.cfg b/setup.cfg index 20f299b..c4d2fd9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,7 @@ project_urls = Documentation = https://sprockets-postgres.readthedocs.io Source Code = https://github.com/sprockets/sprockets-postgres/ classifiers = - Development Status :: 3 - Alpha + Development Status :: 4 - Beta Intended Audience :: Developers License :: OSI Approved :: BSD License Natural Language :: English diff --git a/sprockets_postgres.py b/sprockets_postgres.py index 0c253ec..905a749 100644 --- a/sprockets_postgres.py +++ b/sprockets_postgres.py @@ -459,31 +459,43 @@ class ApplicationMixin: try: self._postgres_pool = await pool.Pool.from_pool_fill( url, - maxsize=int( - os.environ.get( + maxsize=self.settings.get( + 'postgres_max_pool_size', + int(os.environ.get( 'POSTGRES_MAX_POOL_SIZE', - DEFAULT_POSTGRES_MAX_POOL_SIZE)), - minsize=int( - os.environ.get( + DEFAULT_POSTGRES_MAX_POOL_SIZE))), + minsize=self.settings.get( + 'postgres_min_pool_size', + int(os.environ.get( 'POSTGRES_MIN_POOL_SIZE', - DEFAULT_POSTGRES_MIN_POOL_SIZE)), - timeout=int( - os.environ.get( - 'POSTGRES_CONNECT_TIMEOUT', - DEFAULT_POSTGRES_CONNECTION_TIMEOUT)), - enable_hstore=util.strtobool( - os.environ.get( - 'POSTGRES_HSTORE', DEFAULT_POSTGRES_HSTORE)), - enable_json=util.strtobool( - os.environ.get('POSTGRES_JSON', DEFAULT_POSTGRES_JSON)), - enable_uuid=util.strtobool( - os.environ.get('POSTGRES_UUID', DEFAULT_POSTGRES_UUID)), + DEFAULT_POSTGRES_MIN_POOL_SIZE))), + timeout=self.settings.get( + 'postgres_connect_timeout', + int(os.environ.get( + 'POSTGRES_CONNECT_TIMEOUT', + DEFAULT_POSTGRES_CONNECTION_TIMEOUT))), + enable_hstore=self.settings.get( + 'postgres_hstore', + util.strtobool( + os.environ.get( + 'POSTGRES_HSTORE', DEFAULT_POSTGRES_HSTORE))), + enable_json=self.settings.get( + 'enable_json', + util.strtobool( + os.environ.get( + 'POSTGRES_JSON', DEFAULT_POSTGRES_JSON))), + enable_uuid=self.settings.get( + 'postgres_uuid', + util.strtobool( + os.environ.get( + 'POSTGRES_UUID', DEFAULT_POSTGRES_UUID))), echo=False, on_connect=None, - pool_recycle=int( - os.environ.get( + pool_recycle=self.settings.get( + 'postgres_connection_ttl', + int(os.environ.get( 'POSTGRES_CONNECTION_TTL', - DEFAULT_POSTGRES_CONNECTION_TTL))) + DEFAULT_POSTGRES_CONNECTION_TTL)))) except (psycopg2.OperationalError, psycopg2.Error) as error: # pragma: nocover LOGGER.warning( diff --git a/tests.py b/tests.py index 56b6152..b3e5041 100644 --- a/tests.py +++ b/tests.py @@ -319,6 +319,11 @@ class TestCase(testing.SprocketsHttpTestCase): name, _, value = line.strip().partition('=') os.environ[name] = value + def tearDown(self): + if self.app._postgres_pool is not None: + self.app._postgres_pool.terminate() + super().tearDown() + def get_app(self): self.app = Application(handlers=[ web.url('/callproc', CallprocRequestHandler),