From 1ea3b3a24dc757350a07123215169f14fcf8003b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 20 Jun 2025 20:54:32 +0200 Subject: [PATCH 1/6] feat(urllib): init PkgHandler --- src/byteb4rb1e/utils/urllib/__init__.py | 0 src/byteb4rb1e/utils/urllib/request.py | 36 +++++++++++++++++++ .../byteb4rb1e/utils/urllib/test_request.py | 9 +++++ 3 files changed, 45 insertions(+) create mode 100644 src/byteb4rb1e/utils/urllib/__init__.py create mode 100644 src/byteb4rb1e/utils/urllib/request.py create mode 100644 tests/unit/byteb4rb1e/utils/urllib/test_request.py diff --git a/src/byteb4rb1e/utils/urllib/__init__.py b/src/byteb4rb1e/utils/urllib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/byteb4rb1e/utils/urllib/request.py b/src/byteb4rb1e/utils/urllib/request.py new file mode 100644 index 0000000..28e8e8e --- /dev/null +++ b/src/byteb4rb1e/utils/urllib/request.py @@ -0,0 +1,36 @@ +import importlib.resources +from urllib.request import URLError +import urllib.request + + +class PkgHandler(urllib.request.BaseHandler): + """ + """ + def pkg_open(self, req): + pkg_files = importlib.resources.files(req.host) + + raise Exception(sorted(pkg_files.glob('**/*'))) + + try: + fh = list( + pkg_files.glob(req.selector.lstrip('//')) + )[0].open('rb') + except Exception as e: + raise URLError(f'{e.__class__.__name__}: {e}') from e + + fh.seek(0, 2); + size = fh.tell(); + fh.seek(0); + + mtype, _ = mimetypes.guess_type(url) + + headers = email.message_from_string( + 'Content-Type: %s\nContent-Length: %d\n' % + (mtype or 'text/plain', size) + ) + + if not mtype or mtype.starts_with('text/'): + fh.close() + fh = importlib.resources.files(req.host).glob(req.selector)[0].open('r') + + return urllib.request.addinfourl(fh, header) diff --git a/tests/unit/byteb4rb1e/utils/urllib/test_request.py b/tests/unit/byteb4rb1e/utils/urllib/test_request.py new file mode 100644 index 0000000..28548ef --- /dev/null +++ b/tests/unit/byteb4rb1e/utils/urllib/test_request.py @@ -0,0 +1,9 @@ +from byteb4rb1e.utils.urllib.request import PkgHandler + +class TestPkgHandler: + """ + """ + def test_default(self): + """ + """ + pass From 43cdf21d4bb2fa7ff7e54abe78c74bc126753a43 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Fri, 20 Jun 2025 23:16:16 +0200 Subject: [PATCH 2/6] feat(testing.pytest): add subprocess decorator --- .../utils/testing/pytest/__init__.py | 14 ++++++ .../utils/testing/pytest/decorators.py | 47 +++++++++++++++++++ .../utils/testing/pytest/fixtures.py | 6 +-- .../byteb4rb1e/utils/testing/pytest/test_.py | 33 +++++++++++++ .../utils/testing/pytest/test_decorators.py | 21 +++++++++ .../byteb4rb1e/utils/urllib/test_request.py | 4 ++ 6 files changed, 122 insertions(+), 3 deletions(-) create mode 100644 src/byteb4rb1e/utils/testing/pytest/__init__.py create mode 100644 src/byteb4rb1e/utils/testing/pytest/decorators.py create mode 100644 tests/integration/byteb4rb1e/utils/testing/pytest/test_.py create mode 100644 tests/integration/byteb4rb1e/utils/testing/pytest/test_decorators.py diff --git a/src/byteb4rb1e/utils/testing/pytest/__init__.py b/src/byteb4rb1e/utils/testing/pytest/__init__.py new file mode 100644 index 0000000..87e7c10 --- /dev/null +++ b/src/byteb4rb1e/utils/testing/pytest/__init__.py @@ -0,0 +1,14 @@ +import os +from pathlib import Path +from typing import Tuple + + +def get_current_test() -> Tuple[Path, str]: + current_test_env = os.getenv("PYTEST_CURRENT_TEST") + if current_test_env is None: + raise RuntimeError("PYTEST_CURRENT_TEST not set. Must be run under pytest.") + + suite_path, case_name = current_test_env.split('::', 1) + case_name = case_name.split(' ', 1)[0] + return Path(suite_path).resolve(), case_name + diff --git a/src/byteb4rb1e/utils/testing/pytest/decorators.py b/src/byteb4rb1e/utils/testing/pytest/decorators.py new file mode 100644 index 0000000..bf1066b --- /dev/null +++ b/src/byteb4rb1e/utils/testing/pytest/decorators.py @@ -0,0 +1,47 @@ +from functools import wraps +from pathlib import Path +import os +import subprocess +import sys + +from byteb4rb1e.utils.testing.pytest import get_current_test + + +def run_in_subprocess_once(): + """ + A decorator that reruns th test in a subprocess if not already inside one. + Requires pytest to be installed and test to be run by pytest. + + For what? Anything that can't be done in a thread-safe manner, e.g. modifying PYTHON_PATH + """ + def decorator(test_func): + @wraps(test_func) + def wrapper(*args, **kwargs): + if os.environ.get("XPYTEST_INSIDE_SUBPROCESS") == "1": + return test_func(*args, **kwargs) + + suite_path, case_name = get_current_test() + + cmd = [ + sys.executable, + "-m", "pytest", + f"{suite_path}::{case_name}", + ] + + result = subprocess.run( + cmd, + env={**os.environ, "XPYTEST_INSIDE_SUBPROCESS": "1"}, + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(' '.join(cmd)) + print("==== Subprocess stdout ====") + print(result.stdout) + print("==== Subprocess stderr ====") + print(result.stderr) + raise AssertionError(f"Subprocess test failed with exit code {result.returncode}") + return wrapper + return decorator + diff --git a/src/byteb4rb1e/utils/testing/pytest/fixtures.py b/src/byteb4rb1e/utils/testing/pytest/fixtures.py index 7415a22..dc87091 100644 --- a/src/byteb4rb1e/utils/testing/pytest/fixtures.py +++ b/src/byteb4rb1e/utils/testing/pytest/fixtures.py @@ -4,11 +4,11 @@ from typing import Tuple import pytest +from byteb4rb1e.utils.testing.pytest import get_current_test + @pytest.fixture def current_test() -> Tuple[Path, str]: """ """ - suite_path, case_name = os.getenv('PYTEST_CURRENT_TEST').split('::', 1) - case_name = case_name.split(' ', 1)[0] - return Path(suite_path).resolve(), case_name + return get_current_test() diff --git a/tests/integration/byteb4rb1e/utils/testing/pytest/test_.py b/tests/integration/byteb4rb1e/utils/testing/pytest/test_.py new file mode 100644 index 0000000..57a0410 --- /dev/null +++ b/tests/integration/byteb4rb1e/utils/testing/pytest/test_.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.pytest + +from byteb4rb1e.utils.testing.pytest import get_current_test +from byteb4rb1e.utils.testing.pytest.decorators import run_in_subprocess_once + + +class Test_get_current_test: + """ + """ + + def test_default(self): + """ + """ + os.environ['PYTEST_CURRENT_TEST'] = 'foo::bar (something)' + + result = get_current_test() + + assert isinstance(result[0], Path) + assert str(result[0].name) == 'foo' + + assert result[1] == 'bar' + + def test_invalid(self): + """ + """ + del os.environ['PYTEST_CURRENT_TEST'] + with pytest.raises(RuntimeError): + get_current_test() diff --git a/tests/integration/byteb4rb1e/utils/testing/pytest/test_decorators.py b/tests/integration/byteb4rb1e/utils/testing/pytest/test_decorators.py new file mode 100644 index 0000000..7dd20fe --- /dev/null +++ b/tests/integration/byteb4rb1e/utils/testing/pytest/test_decorators.py @@ -0,0 +1,21 @@ +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.pytest + +from byteb4rb1e.utils.testing.pytest.decorators import run_in_subprocess_once + + +@run_in_subprocess_once() +def test_run_in_subprocess_once(tmp_path): + marker = tmp_path / "executed_in_subprocess.txt" + + if marker.exists(): + raise AssertionError("Marker file exists before test logic ran (shouldn't happen in parent process)") + + # Create proof of execution + marker.write_text("Subprocess was here.") + + # Now assert it + assert marker.exists() diff --git a/tests/unit/byteb4rb1e/utils/urllib/test_request.py b/tests/unit/byteb4rb1e/utils/urllib/test_request.py index 28548ef..74cf40d 100644 --- a/tests/unit/byteb4rb1e/utils/urllib/test_request.py +++ b/tests/unit/byteb4rb1e/utils/urllib/test_request.py @@ -1,5 +1,8 @@ +import pytest + from byteb4rb1e.utils.urllib.request import PkgHandler + class TestPkgHandler: """ """ @@ -7,3 +10,4 @@ class TestPkgHandler: """ """ pass + From 24806959bbb4cca59240ffa4391def21bced7bf0 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 21 Jun 2025 00:32:07 +0200 Subject: [PATCH 3/6] feat(testing.pytest): add pkg mock fixture --- .../utils/testing/pytest/fixtures.py | 28 ++++++++++++++++++- .../utils/testing/pytest/test_fixtures.py | 19 ++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/byteb4rb1e/utils/testing/pytest/fixtures.py b/src/byteb4rb1e/utils/testing/pytest/fixtures.py index dc87091..3362010 100644 --- a/src/byteb4rb1e/utils/testing/pytest/fixtures.py +++ b/src/byteb4rb1e/utils/testing/pytest/fixtures.py @@ -1,6 +1,7 @@ import os from pathlib import Path -from typing import Tuple +import sys +from typing import Dict, Tuple, Union import pytest @@ -12,3 +13,28 @@ def current_test() -> Tuple[Path, str]: """ """ return get_current_test() + + +@pytest.fixture +def mock_pkg(tmp_path): + def _create(name: str, files: Dict[str, Union[str, bytes]]): + pkg_path = tmp_path / name.replace('.', os.path.sep) + pkg_path.mkdir(parents=True) + (pkg_path / "__init__.py").touch() + + for fname, content in files.items(): + fpath = (pkg_path / fname) + fpath.parent.mkdir(parents=True, exist_ok=True) + if isinstance(content, str): + fpath.write_text(content) + else: + fpath.write_bytes(content) + + sys.path.insert(0, str(tmp_path)) + return name, pkg_path + + yield _create + + # cleanup sys.path after test + if str(tmp_path) in sys.path: + sys.path.remove(str(tmp_path)) diff --git a/tests/integration/byteb4rb1e/utils/testing/pytest/test_fixtures.py b/tests/integration/byteb4rb1e/utils/testing/pytest/test_fixtures.py index 56302e5..c6b8e1f 100644 --- a/tests/integration/byteb4rb1e/utils/testing/pytest/test_fixtures.py +++ b/tests/integration/byteb4rb1e/utils/testing/pytest/test_fixtures.py @@ -1,10 +1,12 @@ from pathlib import Path +import importlib.resources import pytest pytestmark = pytest.mark.pytest -from byteb4rb1e.utils.testing.pytest.fixtures import current_test +from byteb4rb1e.utils.testing.pytest.decorators import run_in_subprocess_once +from byteb4rb1e.utils.testing.pytest.fixtures import current_test, mock_pkg def test_current_test(current_test): @@ -14,3 +16,18 @@ def test_current_test(current_test): assert str(Path(__file__)) == str(suite_path) assert case_name == "test_current_test" + + +@run_in_subprocess_once() +def test_mock_pkg(mock_pkg): + """ + """ + dummy_data = 'Hello' + + mock_pkg('foobarpkg', { + 'data.txt': dummy_data + }) + + result = next(importlib.resources.files('foobarpkg').glob('data.txt')).read_text() + + assert result == dummy_data From 59713aefb894fe24e81deba795b27d520889a37b Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 21 Jun 2025 00:32:43 +0200 Subject: [PATCH 4/6] feat(urllib.request): add importlib resource handler --- src/byteb4rb1e/utils/urllib/request.py | 20 ++--- .../byteb4rb1e/utils/urllib/test_request.py | 81 ++++++++++++++++++- 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/src/byteb4rb1e/utils/urllib/request.py b/src/byteb4rb1e/utils/urllib/request.py index 28e8e8e..3f9ee20 100644 --- a/src/byteb4rb1e/utils/urllib/request.py +++ b/src/byteb4rb1e/utils/urllib/request.py @@ -1,4 +1,6 @@ +import email import importlib.resources +import mimetypes from urllib.request import URLError import urllib.request @@ -6,15 +8,13 @@ import urllib.request class PkgHandler(urllib.request.BaseHandler): """ """ - def pkg_open(self, req): + def pkg_open(self, req) -> urllib.request.addinfourl: pkg_files = importlib.resources.files(req.host) - raise Exception(sorted(pkg_files.glob('**/*'))) - try: - fh = list( + fh = next( pkg_files.glob(req.selector.lstrip('//')) - )[0].open('rb') + ).open('rb') except Exception as e: raise URLError(f'{e.__class__.__name__}: {e}') from e @@ -22,15 +22,17 @@ class PkgHandler(urllib.request.BaseHandler): size = fh.tell(); fh.seek(0); - mtype, _ = mimetypes.guess_type(url) + mtype, _ = mimetypes.guess_type(req.selector) headers = email.message_from_string( 'Content-Type: %s\nContent-Length: %d\n' % (mtype or 'text/plain', size) ) - if not mtype or mtype.starts_with('text/'): + if not mtype or mtype.startswith('text/'): fh.close() - fh = importlib.resources.files(req.host).glob(req.selector)[0].open('r') + fh = next( + pkg_files.glob(req.selector.lstrip('//')) + ).open('r') - return urllib.request.addinfourl(fh, header) + return urllib.request.addinfourl(fh, headers, None) diff --git a/tests/unit/byteb4rb1e/utils/urllib/test_request.py b/tests/unit/byteb4rb1e/utils/urllib/test_request.py index 74cf40d..5fdf7c0 100644 --- a/tests/unit/byteb4rb1e/utils/urllib/test_request.py +++ b/tests/unit/byteb4rb1e/utils/urllib/test_request.py @@ -1,13 +1,90 @@ +import os.path +import sys +import urllib.request + import pytest +from byteb4rb1e.utils.testing.pytest.decorators import run_in_subprocess_once +from byteb4rb1e.utils.testing.pytest.fixtures import mock_pkg from byteb4rb1e.utils.urllib.request import PkgHandler class TestPkgHandler: """ """ - def test_default(self): + @run_in_subprocess_once() + def test_text(self, mock_pkg): """ """ - pass + _opener: urllib.request.OpenerDirector = urllib.request.build_opener( + PkgHandler() + ) + dummy_data = 'Hello' + + mock_pkg('foobarpkg', { + 'data.txt': dummy_data + }) + + result = _opener.open('pkg://foobarpkg/data.txt').readline() + + assert isinstance(result, str) + assert result == dummy_data + + + @run_in_subprocess_once() + def test_bytes(self, mock_pkg): + """ + """ + _opener: urllib.request.OpenerDirector = urllib.request.build_opener( + PkgHandler() + ) + + dummy_data = b'foobar123' + + mock_pkg('foobarpkg', { + 'data.bin': dummy_data + }) + + result = _opener.open('pkg://foobarpkg/data.bin').readline() + + assert isinstance(result, bytes) + assert result == dummy_data + + + @run_in_subprocess_once() + def test_subdir(self, mock_pkg): + """ + """ + _opener: urllib.request.OpenerDirector = urllib.request.build_opener( + PkgHandler() + ) + + dummy_data = 'foobar123' + + mock_pkg('foobarpkg', { + 'foo/bar/data.txt': dummy_data + }) + + result = _opener.open('pkg://foobarpkg/foo/bar/data.txt').readline() + + assert result == dummy_data + + + @run_in_subprocess_once() + def test_nested_module(self, mock_pkg): + """ + """ + _opener: urllib.request.OpenerDirector = urllib.request.build_opener( + PkgHandler() + ) + + dummy_data = 'foobar123' + + mock_pkg('foo.bar.pkg', { + 'dummy/data.txt': dummy_data + }) + + result = _opener.open('pkg://foo.bar.pkg/dummy/data.txt').readline() + + assert result == dummy_data From 3795ff3e38099e2367828e0e5232b98dd6ade48c Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 21 Jun 2025 00:38:20 +0200 Subject: [PATCH 5/6] chore: add integration test to default tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0a1ec21..5cf7aae 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ configure: configure.ac .venv/bin/python3 -m pip install --upgrade pip .venv/bin/pip install -r requirements-dev.txt -test-reports: test-reports/unit test-reports/static +test-reports: test-reports/unit test-reports/static test-reports/integration test-reports/unit: python3 -m pipenv run -v test-unit From a4fa0839809357573284f2a5e169e909729a4e84 Mon Sep 17 00:00:00 2001 From: Tiara Rodney Date: Sat, 21 Jun 2025 00:39:13 +0200 Subject: [PATCH 6/6] todo(6): done --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 81b72b8..2da03d6 100644 --- a/TODO +++ b/TODO @@ -113,7 +113,7 @@ Description: Implement my custom algorithm for doing rolling hash string search ID: 6 Type: feature Title: implement importlib.resources handler for urllib -Status: in-progress +Status: done Priority: high Created: 2025-06-20 Description: A handler that can be registered with an urllib.request