), minus ignored elements."""
if isinstance(node, str):
return node
return "" if ignored(node) else "".join(text_of(child) for child in node["children"])
def inline(node, base):
"""Render inline content as Markdown on one line."""
if isinstance(node, str):
return node
if ignored(node):
return ""
tag, attrs = node["tag"], node["attrs"]
if tag == "br":
return " "
if tag == "img":
return attrs.get("alt", "")
inner = "".join(inline(child, base) for child in node["children"])
if tag == "code":
return f"`{collapse(text_of(node))}`" if collapse(text_of(node)) else ""
if tag in ("strong", "b", "em", "i") and collapse(inner):
mark = "**" if tag in ("strong", "b") else "*"
return f"{mark}{collapse(inner)}{mark}"
href = attrs.get("href", "")
if tag == "a" and collapse(inner) and href and not href.startswith(("#", "javascript:")):
return f"[{collapse(inner)}]({urllib.parse.urljoin(base, href)})"
return inner
```
`render()` walks the children of a node. Text and inline elements are collected into a "run" until a block element appears; then the run is flushed as a paragraph. Headings, code blocks, lists, tables and definition lists each get their own branch. Code blocks keep their whitespace; everything else is collapsed.
*Excerpt of `page_to_chunks.py`, lines 149-185:*
``` python
def render(node, base, out):
"""Append Markdown blocks (strings) for the children of `node` to `out`."""
run = []
def flush():
text = collapse("".join(run))
run.clear()
if text:
out.append(text)
for child in node["children"]:
if isinstance(child, str) or child["tag"] not in BLOCKS:
run.append(inline(child, base))
continue
flush()
tag = child["tag"]
if ignored(child) or tag == "hr":
continue
if re.fullmatch(r"h[1-6]", tag):
title = collapse(inline(child, base))
if title:
out.append("#" * int(tag[1]) + " " + title)
elif tag == "pre":
out.append("```
\n" + text_of(child).strip("\n") + "\n
```")
elif tag in ("ul", "ol"):
render_list(child, base, out)
elif tag == "table":
out.append(render_table(child, base))
elif tag == "dt":
out.append(collapse(text_of(child)))
elif tag == "blockquote":
quoted = []
render(child, base, quoted)
out.extend("> " + block.replace("\n", "\n> ") for block in quoted)
else:
render(child, base, out)
flush()
```
Definition terms (`Title¶
Hello bold world.
), minus ignored elements."""
if isinstance(node, str):
return node
return "" if ignored(node) else "".join(text_of(child) for child in node["children"])
def inline(node, base):
"""Render inline content as Markdown on one line."""
if isinstance(node, str):
return node
if ignored(node):
return ""
tag, attrs = node["tag"], node["attrs"]
if tag == "br":
return " "
if tag == "img":
return attrs.get("alt", "")
inner = "".join(inline(child, base) for child in node["children"])
if tag == "code":
return f"`{collapse(text_of(node))}`" if collapse(text_of(node)) else ""
if tag in ("strong", "b", "em", "i") and collapse(inner):
mark = "**" if tag in ("strong", "b") else "*"
return f"{mark}{collapse(inner)}{mark}"
href = attrs.get("href", "")
if tag == "a" and collapse(inner) and href and not href.startswith(("#", "javascript:")):
return f"[{collapse(inner)}]({urllib.parse.urljoin(base, href)})"
return inner
def render_list(node, base, out):
number = 0
for item in node["children"]:
if not isinstance(item, dict) or item["tag"] != "li":
continue
number += 1
marker = f"{number}." if node["tag"] == "ol" else "-"
parts = []
render(item, base, parts)
if parts:
indent = "\n" + " " * (len(marker) + 1)
out.append(marker + " " + indent.join(part.replace("\n", indent) for part in parts))
def render_table(node, base):
rows = []
for row in find_all(node, "tr"):
cells = [collapse(inline(cell, base)).replace("|", "\\|") for cell in row["children"]
if isinstance(cell, dict) and cell["tag"] in ("th", "td")]
if cells:
rows.append(cells)
if not rows:
return ""
width = max(len(row) for row in rows)
rows = [row + [""] * (width - len(row)) for row in rows]
lines = ["| " + " | ".join(rows[0]) + " |", "|" + " --- |" * width]
return "\n".join(lines + ["| " + " | ".join(row) + " |" for row in rows[1:]])
def find_all(node, tag):
for child in node["children"]:
if isinstance(child, dict):
if child["tag"] == tag:
yield child
elif child["tag"] != "table": # do not descend into nested tables
yield from find_all(child, tag)
def render(node, base, out):
"""Append Markdown blocks (strings) for the children of `node` to `out`."""
run = []
def flush():
text = collapse("".join(run))
run.clear()
if text:
out.append(text)
for child in node["children"]:
if isinstance(child, str) or child["tag"] not in BLOCKS:
run.append(inline(child, base))
continue
flush()
tag = child["tag"]
if ignored(child) or tag == "hr":
continue
if re.fullmatch(r"h[1-6]", tag):
title = collapse(inline(child, base))
if title:
out.append("#" * int(tag[1]) + " " + title)
elif tag == "pre":
out.append("```
\n" + text_of(child).strip("\n") + "\n
```")
elif tag in ("ul", "ol"):
render_list(child, base, out)
elif tag == "table":
out.append(render_table(child, base))
elif tag == "dt":
out.append(collapse(text_of(child)))
elif tag == "blockquote":
quoted = []
render(child, base, quoted)
out.extend("> " + block.replace("\n", "\n> ") for block in quoted)
else:
render(child, base, out)
flush()
def to_markdown(html, base):
builder = TreeBuilder()
builder.feed(html)
builder.close()
blocks = []
render(main_content(builder.root), base, blocks)
return [block for block in blocks if block]
def prose(text):
"""Text with link targets and whitespace differences removed."""
return collapse(re.sub(r"\]\([^)]*\)", "]", text))
def fingerprint(text):
return hashlib.sha256(prose(text).encode("utf-8")).hexdigest()[:16]
def chunk(blocks, max_level=3):
"""Group blocks under their nearest heading (h1..h{max_level}); one chunk per section."""
chunks, path, body, seen = [], [], [], {}
def emit():
text = "\n\n".join(body).strip()
body.clear()
if text:
name = " > ".join(path) or "(top)"
seen[name] = seen.get(name, 0) + 1
chunk_id = name if seen[name] == 1 else f"{name} #{seen[name]}"
chunks.append({"id": chunk_id, "words": len(text.split()),
"sha256": hashlib.sha256(text.encode("utf-8")).hexdigest()[:16],
"fingerprint": fingerprint(text), "text": text})
for block in blocks:
heading = re.match(r"(#{1,6}) (.*)", block)
if heading and len(heading.group(1)) <= max_level:
emit()
del path[len(heading.group(1)) - 1:]
path.append(heading.group(2))
body.append(block)
emit()
return chunks
def compare(old, new, rename_threshold=0.6):
"""Classify chunks by id: added, removed, renamed, changed (prose), links-only, unchanged."""
before = {c["id"]: c for c in old}
after = {c["id"]: c for c in new}
result = {"added": sorted(after.keys() - before.keys()),
"removed": sorted(before.keys() - after.keys()),
"renamed": [], "changed": [], "links-only": [], "unchanged": []}
for old_id in list(result["removed"]): # a renamed heading looks like removed + added
scored = [(difflib.SequenceMatcher(None, prose(before[old_id]["text"]),
prose(after[new_id]["text"])).ratio(), new_id)
for new_id in result["added"]]
if scored and max(scored)[0] >= rename_threshold:
ratio, new_id = max(scored)
result["removed"].remove(old_id)
result["added"].remove(new_id)
result["renamed"].append(f"{old_id} -> {new_id} (similarity {ratio:.3f})")
for chunk_id in [c["id"] for c in new if c["id"] in before]:
a, b = before[chunk_id], after[chunk_id]
kind = ("unchanged" if a["sha256"] == b["sha256"] else
"links-only" if a["fingerprint"] == b["fingerprint"] else "changed")
result[kind].append(chunk_id)
return result
def robots_rules(text):
"""Parse robots.txt. urllib.robotparser ends a group at a blank line; RFC 9309 does not."""
rules = urllib.robotparser.RobotFileParser()
rules.parse([line for line in text.splitlines() if line.strip()])
return rules
def fetch(url):
parts = urllib.parse.urlsplit(url)
try:
text = get(f"{parts.scheme}://{parts.netloc}/robots.txt")
except urllib.error.HTTPError as err:
if err.code not in (404, 410):
raise
text = "" # no robots.txt: nothing is disallowed
rules = robots_rules(text)
if not rules.can_fetch(USER_AGENT, url):
sys.exit(f"robots.txt disallows {url}; stopping")
time.sleep(max(1, rules.crawl_delay(USER_AGENT) or 0))
return get(url)
def get(url):
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(request, timeout=30) as response:
return response.read().decode(response.headers.get_content_charset() or "utf-8", "replace")
def main(args):
if args[:1] == ["convert"] and len(args) == 4:
url, md_file, json_file = args[1:]
html = fetch(url)
captured = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
blocks = to_markdown(html, url)
chunks = chunk(blocks)
with open(md_file, "w", encoding="utf-8") as handle:
handle.write("\n\n".join(blocks) + "\n")
with open(json_file, "w", encoding="utf-8") as handle:
json.dump({"url": url, "captured_at": captured, "chunks": chunks}, handle, indent=2)
print(f"{url} at {captured}: {len(html):,} characters of HTML -> "
f"{sum(len(b) for b in blocks):,} characters of Markdown in {len(chunks)} chunks")
elif args[:1] == ["compare"] and len(args) == 3:
with open(args[1], encoding="utf-8") as a, open(args[2], encoding="utf-8") as b:
old, new = json.load(a), json.load(b)
result = compare(old["chunks"], new["chunks"])
print(f"{old['url']} -> {new['url']}")
for kind, ids in result.items():
print(f"{kind:<10} {len(ids):>3}" + "".join(f"\n {i}" for i in ids if kind != "unchanged"))
else:
sys.exit(__doc__)
if __name__ == "__main__":
main(sys.argv[1:])
python
import unittest
from page_to_chunks import chunk, compare, fingerprint, robots_rules, to_markdown
BASE = "https://example.org/docs/page.html"
def md(html):
return to_markdown(html, BASE)
class Markdown(unittest.TestCase):
def test_main_content_only_and_noise_removed(self):
html = """
Title¶
Hello bold
world.
"""
self.assertEqual(md(html), ["# Title", "Hello **bold** world."])
def test_links_become_absolute_and_anchors_become_text(self):
html = 'Call json.dumps( ):
>>> x = 1\n y
One
- a
- b
- Two
JSON Python object
dict a|b
- json.dump(obj, fp)¶
"
"Serialize.
Quoted
First
Second
outside"), ["First", "Second"])
class Chunks(unittest.TestCase):
def test_heading_paths_levels_and_duplicates(self):
blocks = ["intro", "# Guide", "a", "## Setup", "b", "#### Deep", "c", "## Setup", "d", "### Notes", "e"]
result = chunk(blocks)
self.assertEqual([c["id"] for c in result],
["(top)", "Guide", "Guide > Setup", "Guide > Setup #2", "Guide > Setup > Notes"])
self.assertEqual(result[2]["text"], "## Setup\n\nb\n\n#### Deep\n\nc")
def test_fingerprint_ignores_link_targets(self):
self.assertEqual(fingerprint("See [docs](https://x/3.13/a.html)."),
fingerprint("See docs."))
self.assertNotEqual(fingerprint("See [docs](u)."), fingerprint("See [the docs](u)."))
def test_compare(self):
old = chunk(["# A", "same", "# B", "[x](https://v1)", "# C", "old text", "# D", "gone"])
new = chunk(["# A", "same", "# B", "[x](https://v2)", "# C", "new text", "# E", "fresh"])
self.assertEqual(compare(old, new), {"added": ["E"], "removed": ["D"], "renamed": [],
"changed": ["C"], "links-only": ["B"], "unchanged": ["A"]})
def test_renamed_heading_is_paired(self):
body = "The json module can be run from the shell to validate and pretty-print input."
old = chunk(["# Command Line Interface", body])
new = chunk(["# Command-line interface", body + " Also as python -m json."])
result = compare(old, new)
self.assertEqual((result["added"], result["removed"]), ([], []))
self.assertEqual(len(result["renamed"]), 1)
self.assertTrue(result["renamed"][0].startswith("Command Line Interface -> Command-line interface ("))
self.assertEqual(compare(old, new, rename_threshold=0.99)["renamed"], [])
class Robots(unittest.TestCase):
def test_blank_line_does_not_end_the_group(self):
rules = robots_rules("User-agent: *\nDisallow: /dev\n\n# EOL versions\nDisallow: /3.9/\n")
self.assertFalse(rules.can_fetch("page-to-chunks", "https://x/3.9/library/json.html"))
self.assertFalse(rules.can_fetch("page-to-chunks", "https://x/dev/"))
self.assertTrue(rules.can_fetch("page-to-chunks", "https://x/3.14/library/json.html"))
if __name__ == "__main__":
unittest.main()
```
Run the tests with `python -m unittest -v test_page_to_chunks`.
*This article and its code were drafted by an AI assistant at the account owner's request. The code was run against docs.python.org on 27 September 2026 at 13:07:49 and 13:07:51 UTC, and the tests passed the same day. The blank-line handling for `robots.txt` was added later the same day after a review; it makes the same `robots.txt` decisions for both pages, so it does not change the run shown above.*