migrate sphinxcontrib.h5p.utils

src/byteb4rb1e/utils/http/client.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""Generic HTTP client.
+
+Thin urllib wrapper with retry-on-rate-limit.  No domain knowledge —
+GitHub, Bitbucket, etc. are handled by higher-level modules.
+"""
+
+import json
+import time
+from typing import Any, Dict, Optional
+import urllib.request
+import urllib.parse
+from warnings import warn
+
+
+class HttpResponse:
+    def __init__(self, status: int, headers: dict, data: bytes, reason: str):
+        self.status_code = status
+        self.headers = headers
+        self.data = data
+        self.reason = reason
+        self.text = data.decode("utf-8", errors="replace")
+
+    def json(self):
+        return json.loads(self.data.decode("utf-8"))
+
+
+def _request(
+    url: str,
+    method: str = "GET",
+    params: Optional[Dict[str, Any]] = None,
+    headers: Optional[Dict[str, str]] = None,
+    data: Optional[bytes] = None,
+) -> HttpResponse:
+    # TODO: do proper exponential backoff
+    backoff = [1, 2, 4]
+
+    if params:
+        query = urllib.parse.urlencode(params)
+        url = f"{url}?{query}"
+
+    req = urllib.request.Request(
+        url,
+        headers=headers or {},
+        method=method,
+        data=data,
+    )
+
+    for delay in backoff:
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                status = resp.getcode()
+                resp_data = resp.read()
+                resp_headers = dict(resp.getheaders())
+
+                if status == 429:
+                    warn(f"Rate-limited on {url} (HTTP {status})."
+                         f" Backing off {delay}s...")
+                    time.sleep(delay)
+                    continue
+
+                return HttpResponse(
+                    status, resp_headers, resp_data, resp.reason,
+                )
+
+        except urllib.error.HTTPError as e:
+            status = e.code
+            err_data = e.read()
+            err_headers = dict(e.headers.items())
+            if status == 429:
+                warn(f"Rate-limited on {url} (HTTP {status})."
+                     f" Backing off {delay}s...")
+                time.sleep(delay)
+                continue
+            return HttpResponse(
+                status, err_headers, err_data, e.reason,
+            )
+
+        except urllib.error.URLError as e:
+            raise Exception(
+                "Network error on %s: %s", url, e,
+            ) from e
+
+    # If all retries exhausted, return last error-like response
+    return HttpResponse(503, {}, b"", "Service unavailable")
+
+
+def get(
+    url: str,
+    params: Optional[Dict[str, Any]] = None,
+    headers: Optional[Dict[str, str]] = None,
+) -> HttpResponse:
+    return _request(url, method="GET", params=params, headers=headers)
+
+
+def post(
+    url: str,
+    data: Optional[bytes] = None,
+    headers: Optional[Dict[str, str]] = None,
+) -> HttpResponse:
+    return _request(url, method="POST", headers=headers, data=data)
+
+
+def put(
+    url: str,
+    data: Optional[bytes] = None,
+    headers: Optional[Dict[str, str]] = None,
+) -> HttpResponse:
+    return _request(url, method="PUT", headers=headers, data=data)