Don't obey rogue Retry-After headers.

docs/history.rst

@@ -1,6 +1,12 @@
 Version History
 ===============
 
+Next Release
+------------
+- Address `#27`_ by using the shortest appropriate timeout
+
+.. _#27: https://github.com/sprockets/sprockets.mixins.http/issues/27
+
 `2.3.0`_ Dec 9, 2019
 --------------------
 - Added an option to control response body transformation for errors, i.e. HTTP

sprockets/mixins/http/__init__.py

@@ -371,7 +371,8 @@ class HTTPClientMixin:
             elif resp.code in dont_retry:
                 break
             elif resp.code in {423, 429}:
-                await self._http_resp_rate_limited(resp)
+                await self._http_resp_rate_limited(
+                    resp, min(connect_timeout, request_timeout))
             elif resp.code < 500:
                 LOGGER.debug('HTTP Response Error for %s to %s'
                              'attempt %i of %i (%s): %s',
@@ -464,11 +465,13 @@ class HTTPClientMixin:
         return DEFAULT_USER_AGENT
 
     @staticmethod
-    def _http_resp_rate_limited(response):
+    def _http_resp_rate_limited(response, timeout=3.0):
         """Extract the ``Retry-After`` header value if the request was rate
         limited and return a future to sleep for the specified duration.
 
         :param tornado.httpclient.HTTPResponse response: The response
+        :param float timeout: Maximum number of seconds to wait regardless
+            of ``Retry-After`` header
         :rtype: tornado.concurrent.Future
 
         """
@@ -476,4 +479,4 @@ class HTTPClientMixin:
         duration = int(response.headers.get('Retry-After', 3))
         LOGGER.warning('Rate Limited by %s, retrying in %i seconds',
                        parsed.netloc, duration)
-        return asyncio.sleep(duration)
+        return asyncio.sleep(min(duration, timeout))

tests.py

@@ -32,7 +32,8 @@ class TestHandler(web.RequestHandler):
     def prepare(self):
         status_code = self.status_code()
         if status_code == 429:
-            self.add_header('Retry-After', '1')
+            self.add_header('Retry-After',
+                            self.get_argument('retry_after', '1'))
             self.set_status(429, 'Rate Limited')
             self.finish()
         elif status_code in {502, 504}:
@@ -602,3 +603,15 @@ class MixinTestCase(testing.AsyncHTTPTestCase):
         self.assertTrue(response.ok)
         self.assertEqual(response.code, 200)
         self.assertEqual(response.attempts, 1)
+
+    @testing.gen_test
+    def test_with_obscene_retry_after(self):
+        response = yield self.mixin.http_fetch(
+            self.get_url('/error?status_code=429&retry_after=86400'),
+            max_http_attempts=2,
+            request_timeout=0.25,
+        )
+        self.assertFalse(response.ok)
+        self.assertAlmostEqual(response.duration,
+                               response.attempts * 0.25,
+                               places=1)