#!/usr/bin/env python3 """Generic HTTP client. Thin urllib wrapper with retry-on-rate-limit. No domain knowledge — GitHub, Bitbucket, etc. are handled by higher-level modules. """ import json import time from typing import Any, Dict, Optional import urllib.request import urllib.parse from warnings import warn class HttpResponse: def __init__(self, status: int, headers: dict, data: bytes, reason: str): self.status_code = status self.headers = headers self.data = data self.reason = reason self.text = data.decode("utf-8", errors="replace") def json(self): return json.loads(self.data.decode("utf-8")) def _request( url: str, method: str = "GET", params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, data: Optional[bytes] = None, ) -> HttpResponse: # TODO: do proper exponential backoff backoff = [1, 2, 4] if params: query = urllib.parse.urlencode(params) url = f"{url}?{query}" req = urllib.request.Request( url, headers=headers or {}, method=method, data=data, ) for delay in backoff: try: with urllib.request.urlopen(req, timeout=30) as resp: status = resp.getcode() resp_data = resp.read() resp_headers = dict(resp.getheaders()) if status == 429: warn(f"Rate-limited on {url} (HTTP {status})." f" Backing off {delay}s...") time.sleep(delay) continue return HttpResponse( status, resp_headers, resp_data, resp.reason, ) except urllib.error.HTTPError as e: status = e.code err_data = e.read() err_headers = dict(e.headers.items()) if status == 429: warn(f"Rate-limited on {url} (HTTP {status})." f" Backing off {delay}s...") time.sleep(delay) continue return HttpResponse( status, err_headers, err_data, e.reason, ) except urllib.error.URLError as e: raise Exception( "Network error on %s: %s", url, e, ) from e # If all retries exhausted, return last error-like response return HttpResponse(503, {}, b"", "Service unavailable") def get( url: str, params: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None, ) -> HttpResponse: return _request(url, method="GET", params=params, headers=headers) def post( url: str, data: Optional[bytes] = None, headers: Optional[Dict[str, str]] = None, ) -> HttpResponse: return _request(url, method="POST", headers=headers, data=data) def put( url: str, data: Optional[bytes] = None, headers: Optional[Dict[str, str]] = None, ) -> HttpResponse: return _request(url, method="PUT", headers=headers, data=data)