feature(http): init parser
This commit is contained in:
Tiara Rodney
parent
cc4b567181
commit
db72017810
3 changed files with 398 additions and 0 deletions
296
src/byteb4rb1e/utils/http/parser.py
Normal file
296
src/byteb4rb1e/utils/http/parser.py
Normal file
|
|
@ -0,0 +1,296 @@
|
||||||
|
from __future__ import annotations
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
from typing import Dict, Iterable, List, Optional, Generator, Union
|
||||||
|
|
||||||
|
|
||||||
|
class Node:
|
||||||
|
"""
|
||||||
|
Represents a node in a simple DOM-like tree.
|
||||||
|
|
||||||
|
:param tag: The HTML tag name (e.g., ``"div"``). ``None`` for text nodes.
|
||||||
|
:param attrs: Iterable of ``(key, value)`` attribute pairs.
|
||||||
|
:param parent: Parent :class:`Node` instance.
|
||||||
|
:param text: Text content for text nodes.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
tag: Optional[str] = None,
|
||||||
|
attrs: Optional[Iterable[tuple[str, str]]] = None,
|
||||||
|
parent: Optional["Node"] = None,
|
||||||
|
text: str = "",
|
||||||
|
) -> None:
|
||||||
|
self.tag: Optional[str] = tag
|
||||||
|
self.attrs: Dict[str, str] = dict(attrs or [])
|
||||||
|
self.parent: Optional["Node"] = parent
|
||||||
|
self.children: List["Node"] = []
|
||||||
|
self.text: str = text
|
||||||
|
|
||||||
|
def __repr__(self) -> str:
|
||||||
|
return f"<Node {self.tag} {self.attrs} children={len(self.children)}>"
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
# Tree traversal
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
def iter(self) -> Generator["Node", None, None]:
|
||||||
|
"""
|
||||||
|
Recursively yield all descendant nodes.
|
||||||
|
|
||||||
|
:return: Generator of :class:`Node` objects.
|
||||||
|
"""
|
||||||
|
for child in self.children:
|
||||||
|
yield child
|
||||||
|
yield from child.iter()
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
# DOM-like lookup helpers
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
def get_elements_by_tag_name(self, tag: str) -> List["Node"]:
|
||||||
|
"""
|
||||||
|
Return all descendant elements with the given tag name.
|
||||||
|
|
||||||
|
:param tag: Tag name to match.
|
||||||
|
:return: List of :class:`Node` objects.
|
||||||
|
"""
|
||||||
|
return [n for n in self.iter() if n.tag == tag]
|
||||||
|
|
||||||
|
def get_elements_by_class_name(self, class_name: str) -> List["Node"]:
|
||||||
|
"""
|
||||||
|
Return all descendant elements that contain the given CSS class.
|
||||||
|
|
||||||
|
:param class_name: Class name to match.
|
||||||
|
:return: List of :class:`Node` objects.
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
n
|
||||||
|
for n in self.iter()
|
||||||
|
if "class" in n.attrs and class_name in n.attrs["class"].split()
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_element_by_id(self, element_id: str) -> Optional["Node"]:
|
||||||
|
"""
|
||||||
|
Return the first descendant element with the given ``id`` attribute.
|
||||||
|
|
||||||
|
:param element_id: ID value to match.
|
||||||
|
:return: :class:`Node` or ``None``.
|
||||||
|
"""
|
||||||
|
for n in self.iter():
|
||||||
|
if n.attrs.get("id") == element_id:
|
||||||
|
return n
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_elements_by_attribute(
|
||||||
|
self, attr: str, value: Optional[str] = None
|
||||||
|
) -> List["Node"]:
|
||||||
|
"""
|
||||||
|
Return all descendant elements matching an attribute.
|
||||||
|
|
||||||
|
:param attr: Attribute name.
|
||||||
|
:param value: Optional value to match. If ``None``, only the presence
|
||||||
|
of the attribute is checked.
|
||||||
|
:return: List of :class:`Node` objects.
|
||||||
|
"""
|
||||||
|
if value is None:
|
||||||
|
return [n for n in self.iter() if attr in n.attrs]
|
||||||
|
return [n for n in self.iter() if n.attrs.get(attr) == value]
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
# CSS selector engine (supports chaining)
|
||||||
|
# ----------------------------------------------------------------------
|
||||||
|
def query_selector(self, selector: str) -> Optional["Node"]:
|
||||||
|
"""
|
||||||
|
Return the first element matching a CSS-like selector.
|
||||||
|
|
||||||
|
Supports:
|
||||||
|
- ``tag``
|
||||||
|
- ``.class``
|
||||||
|
- ``#id``
|
||||||
|
|
||||||
|
:param selector: CSS selector string.
|
||||||
|
:return: :class:`Node` or ``None``.
|
||||||
|
"""
|
||||||
|
results = self.query_selector_all(selector)
|
||||||
|
return results[0] if results else None
|
||||||
|
|
||||||
|
def query_selector_all(self, selector: str) -> List["Node"]:
|
||||||
|
"""
|
||||||
|
Return all elements matching a CSS-like selector chain.
|
||||||
|
|
||||||
|
Supports:
|
||||||
|
- ``tag``
|
||||||
|
- ``.class``
|
||||||
|
- ``#id``
|
||||||
|
- descendant chaining: ``div .item span``
|
||||||
|
"""
|
||||||
|
parts = selector.split()
|
||||||
|
current: List[Node] = [self]
|
||||||
|
|
||||||
|
for part in parts:
|
||||||
|
next_nodes: List[Node] = []
|
||||||
|
|
||||||
|
for node in current:
|
||||||
|
# Tag selector
|
||||||
|
if not part.startswith(".") and not part.startswith("#"):
|
||||||
|
if node.tag == part:
|
||||||
|
next_nodes.append(node)
|
||||||
|
next_nodes.extend(node.get_elements_by_tag_name(part))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Class selector
|
||||||
|
if part.startswith("."):
|
||||||
|
cls = part[1:]
|
||||||
|
if "class" in node.attrs and cls in node.attrs["class"].split():
|
||||||
|
next_nodes.append(node)
|
||||||
|
next_nodes.extend(node.get_elements_by_class_name(cls))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# ID selector
|
||||||
|
if part.startswith("#"):
|
||||||
|
ident = part[1:]
|
||||||
|
if node.attrs.get("id") == ident:
|
||||||
|
next_nodes.append(node)
|
||||||
|
found = node.get_element_by_id(ident)
|
||||||
|
if found:
|
||||||
|
next_nodes.append(found)
|
||||||
|
continue
|
||||||
|
|
||||||
|
current = next_nodes
|
||||||
|
|
||||||
|
return current
|
||||||
|
|
||||||
|
def xpath(self, expr: str) -> List["Node"]:
|
||||||
|
"""
|
||||||
|
Very small XPath subset:
|
||||||
|
|
||||||
|
- ``//tag``
|
||||||
|
- ``tag/subtag``
|
||||||
|
- ``//tag[@attr="value"]``
|
||||||
|
|
||||||
|
:param expr: XPath-like expression.
|
||||||
|
:return: List of :class:`Node` objects.
|
||||||
|
"""
|
||||||
|
expr = expr.strip()
|
||||||
|
parts = expr.split("/")
|
||||||
|
current: List[Node] = [self]
|
||||||
|
|
||||||
|
def match(nodes: List[Node], tag: str, attr: Optional[str], val: Optional[str]) -> List[Node]:
|
||||||
|
out: List[Node] = []
|
||||||
|
for n in nodes:
|
||||||
|
candidates = n.iter()
|
||||||
|
for c in candidates:
|
||||||
|
if tag != "*" and c.tag != tag:
|
||||||
|
continue
|
||||||
|
if attr:
|
||||||
|
if c.attrs.get(attr) == val:
|
||||||
|
out.append(c)
|
||||||
|
else:
|
||||||
|
out.append(c)
|
||||||
|
return out
|
||||||
|
|
||||||
|
i = 0
|
||||||
|
while i < len(parts):
|
||||||
|
part = parts[i]
|
||||||
|
if not part:
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# //tag
|
||||||
|
if part.startswith("//"):
|
||||||
|
tag = part[2:]
|
||||||
|
attr = val = None
|
||||||
|
|
||||||
|
if "[" in tag:
|
||||||
|
tag, rest = tag.split("[", 1)
|
||||||
|
rest = rest.rstrip("]")
|
||||||
|
attr, val = rest.split("=")
|
||||||
|
attr = attr.strip("@")
|
||||||
|
val = val.strip('"').strip("'")
|
||||||
|
|
||||||
|
current = match(current, tag, attr, val)
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# tag[@attr="value"]
|
||||||
|
if "[" in part:
|
||||||
|
tag, rest = part.split("[", 1)
|
||||||
|
rest = rest.rstrip("]")
|
||||||
|
attr, val = rest.split("=")
|
||||||
|
attr = attr.strip("@")
|
||||||
|
val = val.strip('"').strip("'")
|
||||||
|
current = match(current, tag, attr, val)
|
||||||
|
else:
|
||||||
|
current = match(current, part, None, None)
|
||||||
|
|
||||||
|
i += 1
|
||||||
|
|
||||||
|
return current
|
||||||
|
|
||||||
|
@property
|
||||||
|
def inner_content(self) -> str:
|
||||||
|
"""
|
||||||
|
Return the concatenated text content of this node and all descendants.
|
||||||
|
|
||||||
|
:return: String containing all text content.
|
||||||
|
"""
|
||||||
|
parts: List[str] = []
|
||||||
|
if self.text:
|
||||||
|
parts.append(self.text)
|
||||||
|
for c in self.children:
|
||||||
|
parts.append(c.inner_content)
|
||||||
|
return "".join(parts)
|
||||||
|
|
||||||
|
def outer_html(self) -> str:
|
||||||
|
"""
|
||||||
|
Reconstruct the HTML for this node and its subtree.
|
||||||
|
|
||||||
|
:return: HTML string.
|
||||||
|
"""
|
||||||
|
if self.tag is None:
|
||||||
|
return self.text
|
||||||
|
|
||||||
|
attrs = "".join(f' {k}="{v}"' for k, v in self.attrs.items())
|
||||||
|
inner = "".join(child.outer_html() for child in self.children)
|
||||||
|
return f"<{self.tag}{attrs}>{inner}</{self.tag}>"
|
||||||
|
|
||||||
|
def pretty(self, indent: int = 0) -> str:
|
||||||
|
"""
|
||||||
|
Return a pretty-printed representation of the DOM tree.
|
||||||
|
|
||||||
|
:param indent: Current indentation level.
|
||||||
|
:return: Multiline string.
|
||||||
|
"""
|
||||||
|
pad = " " * indent
|
||||||
|
if self.tag is None:
|
||||||
|
return f"{pad}{self.text!r}"
|
||||||
|
|
||||||
|
attrs = " ".join(f'{k}="{v}"' for k, v in self.attrs.items())
|
||||||
|
header = f"{pad}<{self.tag} {attrs}>".rstrip()
|
||||||
|
|
||||||
|
lines = [header]
|
||||||
|
for child in self.children:
|
||||||
|
lines.append(child.pretty(indent + 1))
|
||||||
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
class TreeBuilder(HTMLParser):
|
||||||
|
"""
|
||||||
|
HTML parser that constructs a simple DOM-like tree of :class:`Node` objects.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
super().__init__()
|
||||||
|
self.root: Node = Node(tag="__root__")
|
||||||
|
self.current: Node = self.root
|
||||||
|
|
||||||
|
def handle_starttag(self, tag: str, attrs: List[tuple[str, str]]) -> None:
|
||||||
|
node = Node(tag=tag, attrs=attrs, parent=self.current)
|
||||||
|
self.current.children.append(node)
|
||||||
|
self.current = node
|
||||||
|
|
||||||
|
def handle_endtag(self, tag: str) -> None:
|
||||||
|
if self.current.parent is not None:
|
||||||
|
self.current = self.current.parent
|
||||||
|
|
||||||
|
def handle_data(self, data: str) -> None:
|
||||||
|
if data.strip():
|
||||||
|
self.current.children.append(Node(text=data, parent=self.current))
|
||||||
0
tests/unit/byteb4rb1e/utils/http/parser/__init__.py
Normal file
0
tests/unit/byteb4rb1e/utils/http/parser/__init__.py
Normal file
102
tests/unit/byteb4rb1e/utils/http/parser/test_node.py
Normal file
102
tests/unit/byteb4rb1e/utils/http/parser/test_node.py
Normal file
|
|
@ -0,0 +1,102 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from byteb4rb1e.utils.http.parser import Node, TreeBuilder
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def sample_dom():
|
||||||
|
"""
|
||||||
|
Build a small DOM tree for testing:
|
||||||
|
|
||||||
|
<div id="root" class="container">
|
||||||
|
<p class="text">Hello</p>
|
||||||
|
<span class="text highlight">World</span>
|
||||||
|
<div class="box">
|
||||||
|
<span id="inner">Inside</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
html = """
|
||||||
|
<div id="root" class="container">
|
||||||
|
<p class="text">Hello</p>
|
||||||
|
<span class="text highlight">World</span>
|
||||||
|
<div class="box">
|
||||||
|
<span id="inner">Inside</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
parser = TreeBuilder()
|
||||||
|
parser.feed(html)
|
||||||
|
return parser.root.children[0] # the <div id="root">
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetElementsByTagName:
|
||||||
|
def test_find_all_spans(self, sample_dom):
|
||||||
|
spans = sample_dom.get_elements_by_tag_name("span")
|
||||||
|
assert len(spans) == 2
|
||||||
|
assert spans[0].tag == "span"
|
||||||
|
assert spans[1].tag == "span"
|
||||||
|
|
||||||
|
def test_find_no_matches(self, sample_dom):
|
||||||
|
assert sample_dom.get_elements_by_tag_name("table") == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetElementsByClassName:
|
||||||
|
def test_find_single_class(self, sample_dom):
|
||||||
|
items = sample_dom.get_elements_by_class_name("text")
|
||||||
|
assert len(items) == 2
|
||||||
|
|
||||||
|
def test_find_multiple_classes(self, sample_dom):
|
||||||
|
items = sample_dom.get_elements_by_class_name("highlight")
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0].tag == "span"
|
||||||
|
|
||||||
|
def test_no_such_class(self, sample_dom):
|
||||||
|
assert sample_dom.get_elements_by_class_name("missing") == []
|
||||||
|
|
||||||
|
|
||||||
|
class TestGetElementById:
|
||||||
|
def test_find_existing_id(self, sample_dom):
|
||||||
|
node = sample_dom.get_element_by_id("inner")
|
||||||
|
assert node is not None
|
||||||
|
assert node.tag == "span"
|
||||||
|
assert node.inner_content == "Inside"
|
||||||
|
|
||||||
|
def test_missing_id(self, sample_dom):
|
||||||
|
assert sample_dom.get_element_by_id("nope") is None
|
||||||
|
|
||||||
|
|
||||||
|
class TestQuerySelectorAll:
|
||||||
|
def test_class_selector(self, sample_dom):
|
||||||
|
items = sample_dom.query_selector_all(".text")
|
||||||
|
assert len(items) == 2
|
||||||
|
|
||||||
|
def test_id_selector(self, sample_dom):
|
||||||
|
items = sample_dom.query_selector_all("#inner")
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0].inner_content == "Inside"
|
||||||
|
|
||||||
|
def test_tag_selector(self, sample_dom):
|
||||||
|
items = sample_dom.query_selector_all("p")
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0].inner_content == "Hello"
|
||||||
|
|
||||||
|
def test_chained_selector(self, sample_dom):
|
||||||
|
items = sample_dom.query_selector_all("div .highlight")
|
||||||
|
assert len(items) == 1
|
||||||
|
assert items[0].inner_content == "World"
|
||||||
|
|
||||||
|
|
||||||
|
class TestXPath:
|
||||||
|
def test_simple_tag(self, sample_dom):
|
||||||
|
spans = sample_dom.xpath("//span")
|
||||||
|
assert len(spans) == 2
|
||||||
|
|
||||||
|
def test_attribute_match(self, sample_dom):
|
||||||
|
nodes = sample_dom.xpath('//span[@id="inner"]')
|
||||||
|
assert len(nodes) == 1
|
||||||
|
assert nodes[0].inner_content == "Inside"
|
||||||
|
|
||||||
|
def test_nested(self, sample_dom):
|
||||||
|
nodes = sample_dom.xpath("//div[@class='box']")
|
||||||
|
assert len(nodes) == 1
|
||||||
Loading…
Add table
Add a link
Reference in a new issue