109 lines
3 KiB
Python
109 lines
3 KiB
Python
#!/usr/bin/env python3
|
|
"""Generic HTTP client.
|
|
|
|
Thin urllib wrapper with retry-on-rate-limit. No domain knowledge —
|
|
GitHub, Bitbucket, etc. are handled by higher-level modules.
|
|
"""
|
|
|
|
import json
|
|
import time
|
|
from typing import Any, Dict, Optional
|
|
import urllib.request
|
|
import urllib.parse
|
|
from warnings import warn
|
|
|
|
|
|
class HttpResponse:
|
|
def __init__(self, status: int, headers: dict, data: bytes, reason: str):
|
|
self.status_code = status
|
|
self.headers = headers
|
|
self.data = data
|
|
self.reason = reason
|
|
self.text = data.decode("utf-8", errors="replace")
|
|
|
|
def json(self):
|
|
return json.loads(self.data.decode("utf-8"))
|
|
|
|
|
|
def _request(
|
|
url: str,
|
|
method: str = "GET",
|
|
params: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
data: Optional[bytes] = None,
|
|
) -> HttpResponse:
|
|
# TODO: do proper exponential backoff
|
|
backoff = [1, 2, 4]
|
|
|
|
if params:
|
|
query = urllib.parse.urlencode(params)
|
|
url = f"{url}?{query}"
|
|
|
|
req = urllib.request.Request(
|
|
url,
|
|
headers=headers or {},
|
|
method=method,
|
|
data=data,
|
|
)
|
|
|
|
for delay in backoff:
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
status = resp.getcode()
|
|
resp_data = resp.read()
|
|
resp_headers = dict(resp.getheaders())
|
|
|
|
if status == 429:
|
|
warn(f"Rate-limited on {url} (HTTP {status})."
|
|
f" Backing off {delay}s...")
|
|
time.sleep(delay)
|
|
continue
|
|
|
|
return HttpResponse(
|
|
status, resp_headers, resp_data, resp.reason,
|
|
)
|
|
|
|
except urllib.error.HTTPError as e:
|
|
status = e.code
|
|
err_data = e.read()
|
|
err_headers = dict(e.headers.items())
|
|
if status == 429:
|
|
warn(f"Rate-limited on {url} (HTTP {status})."
|
|
f" Backing off {delay}s...")
|
|
time.sleep(delay)
|
|
continue
|
|
return HttpResponse(
|
|
status, err_headers, err_data, e.reason,
|
|
)
|
|
|
|
except urllib.error.URLError as e:
|
|
raise Exception(
|
|
"Network error on %s: %s", url, e,
|
|
) from e
|
|
|
|
# If all retries exhausted, return last error-like response
|
|
return HttpResponse(503, {}, b"", "Service unavailable")
|
|
|
|
|
|
def get(
|
|
url: str,
|
|
params: Optional[Dict[str, Any]] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
) -> HttpResponse:
|
|
return _request(url, method="GET", params=params, headers=headers)
|
|
|
|
|
|
def post(
|
|
url: str,
|
|
data: Optional[bytes] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
) -> HttpResponse:
|
|
return _request(url, method="POST", headers=headers, data=data)
|
|
|
|
|
|
def put(
|
|
url: str,
|
|
data: Optional[bytes] = None,
|
|
headers: Optional[Dict[str, str]] = None,
|
|
) -> HttpResponse:
|
|
return _request(url, method="PUT", headers=headers, data=data)
|