feat: honor Retry-After on HTTP 429 with jittered exponential backoff (#27)

api_request now retries HTTP 429 responses instead of failing immediately:

- Parse and honor a valid Retry-After header (seconds or HTTP-date).
- Fall back to full-jitter capped exponential backoff when the header is
  missing or invalid.
- Bound retries by max_retries and delay by max_delay (env-overridable via
  GITEA_MAX_RETRIES / GITEA_RETRY_BASE_DELAY / GITEA_RETRY_MAX_DELAY) — no
  infinite loops.
- Non-429 errors and successful responses are unchanged.

Sleep, randomness, and clock are injectable so retry timing is tested
deterministically. Adds tests/test_retry_backoff.py (23 cases).

Closes #27

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-01 21:28:51 -04:00
parent 989856a007
commit 1b3c961ff2
2 changed files with 311 additions and 9 deletions
+113 -9
View File
@@ -7,10 +7,14 @@ antivirus alerts (e.g. Bitdefender).
import os
import glob
import json
import time
import base64
import random
import datetime
import subprocess
import urllib.request
import urllib.error
from email.utils import parsedate_to_datetime
from dotenv import dotenv_values, load_dotenv
# Load standard .env if present
@@ -148,23 +152,123 @@ def add_remote_args(parser):
parser.add_argument("--repo", help="Override the repository.")
def api_request(method, url, auth_header, payload=None):
def _env_int(name, default):
"""Read a non-negative int from the environment, falling back to *default*."""
try:
value = int(os.environ[name])
except (KeyError, ValueError, TypeError):
return default
return value if value >= 0 else default
def _env_float(name, default):
"""Read a non-negative float from the environment, falling back to *default*."""
try:
value = float(os.environ[name])
except (KeyError, ValueError, TypeError):
return default
return value if value >= 0 else default
# Retry/backoff configuration for HTTP 429 (rate-limit) responses.
# Overridable via environment; safe defaults otherwise.
DEFAULT_MAX_RETRIES = _env_int("GITEA_MAX_RETRIES", 3)
DEFAULT_BASE_DELAY = _env_float("GITEA_RETRY_BASE_DELAY", 1.0) # seconds
DEFAULT_MAX_DELAY = _env_float("GITEA_RETRY_MAX_DELAY", 60.0) # seconds
def parse_retry_after(value, now=None):
"""Parse a ``Retry-After`` header into a non-negative delay in seconds.
Supports both forms defined by RFC 7231:
- a non-negative integer number of seconds (e.g. ``"120"``)
- an HTTP-date (e.g. ``"Wed, 21 Oct 2015 07:28:00 GMT"``)
Returns ``None`` when *value* is missing, blank, or unparseable, so the
caller can fall back to computed backoff. Past dates clamp to ``0``.
"""
if value is None:
return None
value = value.strip()
if not value:
return None
# Seconds form (integer). Reject non-integer numerics like "1.5".
try:
seconds = int(value)
return max(0, seconds)
except ValueError:
pass
# HTTP-date form.
try:
when = parsedate_to_datetime(value)
except (TypeError, ValueError):
return None
if when is None:
return None
if when.tzinfo is None:
# RFC dates without a zone are UTC.
when = when.replace(tzinfo=datetime.timezone.utc)
now_ts = now if now is not None else time.time()
return max(0.0, when.timestamp() - now_ts)
def backoff_delay(attempt, base=DEFAULT_BASE_DELAY, cap=DEFAULT_MAX_DELAY, rand=random.random):
"""Full-jitter exponential backoff delay in seconds for a 0-indexed *attempt*.
Returns a random value in ``[0, min(cap, base * 2**attempt)]``. Full jitter
spreads retries across the whole window to avoid a thundering herd.
"""
ceiling = min(cap, base * (2 ** attempt))
return rand() * ceiling
def api_request(method, url, auth_header, payload=None, *,
max_retries=None, base_delay=None, max_delay=None,
sleep_func=time.sleep, rand_func=random.random,
now_func=time.time):
"""Make an authenticated JSON request to the Gitea API.
Returns parsed JSON on success, raises on HTTP errors.
Returns parsed JSON on success, raises ``RuntimeError`` on HTTP errors.
On HTTP 429 the request is retried up to *max_retries* times: honoring a
valid ``Retry-After`` header (seconds or HTTP-date) when present, otherwise
using capped jittered exponential backoff. Non-429 errors and successful
responses are unchanged. The ``*_func`` parameters are injection points for
deterministic testing.
"""
if max_retries is None:
max_retries = DEFAULT_MAX_RETRIES
if base_delay is None:
base_delay = DEFAULT_BASE_DELAY
if max_delay is None:
max_delay = DEFAULT_MAX_DELAY
data = json.dumps(payload).encode("utf-8") if payload is not None else None
req = urllib.request.Request(url, data=data, method=method)
req.add_header("Authorization", auth_header)
req.add_header("Content-Type", "application/json")
req.add_header("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
try:
with urllib.request.urlopen(req) as resp:
body = resp.read().decode("utf-8")
return json.loads(body) if body else None
except urllib.error.HTTPError as e:
error_body = e.read().decode("utf-8", errors="replace")
raise RuntimeError(f"HTTP {e.code}: {error_body}") from e
attempt = 0
while True:
try:
with urllib.request.urlopen(req) as resp:
body = resp.read().decode("utf-8")
return json.loads(body) if body else None
except urllib.error.HTTPError as e:
if e.code == 429 and attempt < max_retries:
header = e.headers.get("Retry-After") if e.headers else None
delay = parse_retry_after(header, now=now_func())
if delay is None:
delay = backoff_delay(attempt, base_delay, max_delay, rand_func)
attempt += 1
sleep_func(delay)
continue
error_body = e.read().decode("utf-8", errors="replace")
raise RuntimeError(f"HTTP {e.code}: {error_body}") from e
def repo_api_url(host, org, repo):