T-2221 Report a server 5xx after the retries run out, like a connection error

logtail/flusher.py

@@ -89,13 +89,13 @@ def step(self) -> None:
                 if delay is not None:
                     time.sleep(delay)
 
-            # A network error surfaces as a Fake500 that carries an `exception`
-            # attribute; a genuine server HTTP 500 is a real `requests.Response`
-            # that has no such attribute. Use a default so the latter doesn't
-            # raise AttributeError here and kill the flush thread (dropping all
-            # subsequent logs for the life of the process).
-            if response.status_code == 500 and getattr(response, "exception", None) is not None:
-                print('Failed to send logs to Better Stack after {} retries: {}'.format(len(RETRY_SCHEDULE), response.exception))
+            # The batch is dropped once the retries run out. A network error
+            # surfaces as a Fake500 that carries the `exception`; a 5xx answer
+            # from the server is a real `requests.Response` without one, so it
+            # is reported by its status code instead.
+            if _should_retry(response.status_code):
+                reason = getattr(response, 'exception', 'HTTP {}'.format(response.status_code))
+                print('Failed to send logs to Better Stack after {} retries: {}'.format(len(RETRY_SCHEDULE), reason))
 
         self._clean = True
         if shutdown and self.pipe.empty():

tests/test_flusher.py

@@ -143,9 +143,44 @@ def uploader(frame):
         for log in first_frame:
             pipe.put(log, block=False)
 
-        fw.step()  # must not raise AttributeError
+        with patch('builtins.print') as mock_print:
+            fw.step()  # must not raise AttributeError
 
         self.assertTrue(fw._clean)
+        # The batch is dropped here, so say why, as the network-error path does.
+        mock_print.assert_called_once_with('Failed to send logs to Better Stack after 3 retries: HTTP 500')
+
+    @patch('logtail.flusher.time.sleep')
+    def test_step_logs_any_server_5xx_after_retries(self, _mock_sleep):
+        def uploader(frame):
+            resp = requests.models.Response()
+            resp.status_code = 503
+            return resp
+
+        pipe, _, fw = self._setup_worker(uploader)
+        pipe.put('log', block=False)
+
+        with patch('builtins.print') as mock_print:
+            fw.step()
+
+        mock_print.assert_called_once_with('Failed to send logs to Better Stack after 3 retries: HTTP 503')
+
+    @patch('logtail.flusher.time.sleep')
+    def test_step_does_not_log_when_a_retry_succeeds(self, _mock_sleep):
+        status_codes = iter([500, 503, 202])
+
+        def uploader(frame):
+            resp = requests.models.Response()
+            resp.status_code = next(status_codes)
+            return resp
+
+        pipe, _, fw = self._setup_worker(uploader)
+        pipe.put('log', block=False)
+
+        with patch('builtins.print') as mock_print:
+            fw.step()
+
+        mock_print.assert_not_called()
 
     @patch('logtail.flusher.time.sleep')
     def test_step_still_logs_network_error_fake500(self, _mock_sleep):
@@ -164,7 +199,7 @@ def uploader(frame):
         with patch('builtins.print') as mock_print:
             fw.step()
 
-        mock_print.assert_called_once()
+        mock_print.assert_called_once_with('Failed to send logs to Better Stack after 3 retries: network is down')
 
     def test_shutdown_condition_empties_queue_and_shuts_down(self):
         self.buffer_capacity = 10