If you feed web pages to a search index or a retrieval-augmented generation (RAG) pipeline, you usually want three things: only the main content, as plain Markdown, split into sections that have stable names. When the page is updated, you also want to know which sections changed, so you re-embed a few chunks instead of the whole page.
This post builds that with Python's standard library only. The output is CommonMark, plus GitHub Flavored Markdown pipe tables for HTML tables. The script:
The real run below uses two versions of the same page: the json module documentation for Python 3.13 and for Python 3.14, fetched on 27 September 2026 (UTC). docs.python.org/robots.txt disallows /dev, /release and end-of-life versions such as /3.9/; /3.13/ and /3.14/ are allowed. That gives a genuine before and after without waiting for a page to change.
html.parser is an event parser: it reports start tags, end tags and text. Building a tree from those events takes a stack. Void elements such as <br> and <img> never get an end tag, so they are not pushed. An end tag closes the nearest open element with the same name; a stray end tag with no match is ignored.
Excerpt of page_to_chunks.py, lines 29-50:
class TreeBuilder(HTMLParser):
"""Build a small tree of {"tag", "attrs", "children"} dicts; text stays as plain strings."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.root = {"tag": "#root", "attrs": {}, "children": []}
self.stack = [self.root]
def handle_starttag(self, tag, attrs):
node = {"tag": tag, "attrs": {k: v or "" for k, v in attrs}, "children": []}
self.stack[-1]["children"].append(node)
if tag not in VOID:
self.stack.append(node)
def handle_endtag(self, tag):
for depth in range(len(self.stack) - 1, 0, -1): # close the nearest open match, if any
if self.stack[depth]["tag"] == tag:
del self.stack[depth:]
break
def handle_data(self, data):
self.stack[-1]["children"].append(data)
This is deliberately simple. It is not a full HTML5 parser: it doesn't apply the rules that implicitly close a <p> when a <div> starts, for example. For well-formed pages, such as most documentation sites, that is fine. For messy HTML, parse with html5lib or lxml and keep the rendering part of this post.
The script looks for <main>, then <article>, then any element with role="main", then <body>. Inside it, whole elements are skipped by tag (scripts, styles, navigation, footers, forms) and by class.
Excerpt of page_to_chunks.py, lines 20-27:
USER_AGENT = "html-to-chunks-example/1.0 (tutorial script; one request per page)"
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}
SKIP = {"script", "style", "noscript", "template", "svg", "nav", "footer", "form",
"button", "iframe", "aside"}
NOISE_CLASSES = {"headerlink"} # e.g. the pilcrow permalinks Sphinx adds after headings
BLOCKS = {"p", "div", "section", "article", "main", "blockquote", "pre", "ul", "ol", "table",
"dl", "dt", "dd", "hr", "figure", "h1", "h2", "h3", "h4", "h5", "h6"} | SKIP
Excerpt of page_to_chunks.py, lines 64-78:
def main_content(root):
for test in (lambda n: n["tag"] == "main", lambda n: n["tag"] == "article",
lambda n: n["attrs"].get("role") == "main", lambda n: n["tag"] == "body"):
hit = find(root, test)
if hit:
return hit
return root
def ignored(node):
return node["tag"] in SKIP or bool(NOISE_CLASSES & set(node["attrs"].get("class", "").split()))
def collapse(text):
return re.sub(r"\s+", " ", text).strip()
The class list is where site-specific cleanup goes. Sphinx, the documentation generator behind the Python docs, puts a ¶ permalink after every heading with the class headerlink. Without that one entry, every heading in the output would end in ¶ and every chunk name would carry it.
Inline content becomes one line: bold and italics, inline code, image alt text, and links. Links are made absolute with urljoin, because a relative link is useless once the text leaves the page. Links to anchors on the same page (#...) keep only their text.
Excerpt of page_to_chunks.py, lines 81-108:
def text_of(node):
"""Raw text with whitespace kept (for <pre>), minus ignored elements."""
if isinstance(node, str):
return node
return "" if ignored(node) else "".join(text_of(child) for child in node["children"])
def inline(node, base):
"""Render inline content as Markdown on one line."""
if isinstance(node, str):
return node
if ignored(node):
return ""
tag, attrs = node["tag"], node["attrs"]
if tag == "br":
return " "
if tag == "img":
return attrs.get("alt", "")
inner = "".join(inline(child, base) for child in node["children"])
if tag == "code":
return f"`{collapse(text_of(node))}`" if collapse(text_of(node)) else ""
if tag in ("strong", "b", "em", "i") and collapse(inner):
mark = "**" if tag in ("strong", "b") else "*"
return f"{mark}{collapse(inner)}{mark}"
href = attrs.get("href", "")
if tag == "a" and collapse(inner) and href and not href.startswith(("#", "javascript:")):
return f"[{collapse(inner)}]({urllib.parse.urljoin(base, href)})"
return inner
render() walks the children of a node. Text and inline elements are collected into a "run" until a block element appears; then the run is flushed as a paragraph. Headings, code blocks, lists, tables and definition lists each get their own branch. Code blocks keep their whitespace; everything else is collapsed.
Excerpt of page_to_chunks.py, lines 149-185:
def render(node, base, out):
"""Append Markdown blocks (strings) for the children of `node` to `out`."""
run = []
def flush():
text = collapse("".join(run))
run.clear()
if text:
out.append(text)
for child in node["children"]:
if isinstance(child, str) or child["tag"] not in BLOCKS:
run.append(inline(child, base))
continue
flush()
tag = child["tag"]
if ignored(child) or tag == "hr":
continue
if re.fullmatch(r"h[1-6]", tag):
title = collapse(inline(child, base))
if title:
out.append("#" * int(tag[1]) + " " + title)
elif tag == "pre":
out.append("```
\n" + text_of(child).strip("\n") + "\n
```")
elif tag in ("ul", "ol"):
render_list(child, base, out)
elif tag == "table":
out.append(render_table(child, base))
elif tag == "dt":
out.append(collapse(text_of(child)))
elif tag == "blockquote":
quoted = []
render(child, base, quoted)
out.extend("> " + block.replace("\n", "\n> ") for block in quoted)
else:
render(child, base, out)
flush()
Definition terms (<dt>) become plain text. In Sphinx output they hold function signatures, and the * in a signature such as json.dump(obj, fp, *, ...) would break Markdown emphasis if it were wrapped in **.
Tables become pipe tables. Pipes inside cells are escaped, short rows are padded, and colspan/ rowspan are ignored, so complex tables lose their structure. That is a known limit.
Excerpt of page_to_chunks.py, lines 125-137:
def render_table(node, base):
rows = []
for row in find_all(node, "tr"):
cells = [collapse(inline(cell, base)).replace("|", "\\|") for cell in row["children"]
if isinstance(cell, dict) and cell["tag"] in ("th", "td")]
if cells:
rows.append(cells)
if not rows:
return ""
width = max(len(row) for row in rows)
rows = [row + [""] * (width - len(row)) for row in rows]
lines = ["| " + " | ".join(rows[0]) + " |", "|" + " --- |" * width]
return "\n".join(lines + ["| " + " | ".join(row) + " |" for row in rows[1:]])
Each heading up to level 3 starts a new chunk. The chunk's id is its heading path, such as Guide > Setup. Deeper headings stay inside their parent's chunk. When two sections share a path, the second gets #2.
Every chunk gets two hashes:
sha256 of the exact text, to answer "is this byte-for-byte the same?",fingerprint, the hash of the text with link targets removed and whitespace collapsed, to answer "did the words change?".
The real run shows why the second one matters.
Excerpt of page_to_chunks.py, lines 197-229:
def prose(text):
"""Text with link targets and whitespace differences removed."""
return collapse(re.sub(r"\]\([^)]*\)", "]", text))
def fingerprint(text):
return hashlib.sha256(prose(text).encode("utf-8")).hexdigest()[:16]
def chunk(blocks, max_level=3):
"""Group blocks under their nearest heading (h1..h{max_level}); one chunk per section."""
chunks, path, body, seen = [], [], [], {}
def emit():
text = "\n\n".join(body).strip()
body.clear()
if text:
name = " > ".join(path) or "(top)"
seen[name] = seen.get(name, 0) + 1
chunk_id = name if seen[name] == 1 else f"{name} #{seen[name]}"
chunks.append({"id": chunk_id, "words": len(text.split()),
"sha256": hashlib.sha256(text.encode("utf-8")).hexdigest()[:16],
"fingerprint": fingerprint(text), "text": text})
for block in blocks:
heading = re.match(r"(#{1,6}) (.*)", block)
if heading and len(heading.group(1)) <= max_level:
emit()
del path[len(heading.group(1)) - 1:]
path.append(heading.group(2))
body.append(block)
emit()
return chunks
Sections are matched by id. A renamed heading changes the id, so it first shows up as one removed and one added section. compare() pairs those up when their text is at least 60% similar according to difflib.SequenceMatcher, and reports them as renamed. The 0.6 threshold is a judgment call; the test below also checks that a strict threshold turns pairing off.
Excerpt of page_to_chunks.py, lines 232-253:
def compare(old, new, rename_threshold=0.6):
"""Classify chunks by id: added, removed, renamed, changed (prose), links-only, unchanged."""
before = {c["id"]: c for c in old}
after = {c["id"]: c for c in new}
result = {"added": sorted(after.keys() - before.keys()),
"removed": sorted(before.keys() - after.keys()),
"renamed": [], "changed": [], "links-only": [], "unchanged": []}
for old_id in list(result["removed"]): # a renamed heading looks like removed + added
scored = [(difflib.SequenceMatcher(None, prose(before[old_id]["text"]),
prose(after[new_id]["text"])).ratio(), new_id)
for new_id in result["added"]]
if scored and max(scored)[0] >= rename_threshold:
ratio, new_id = max(scored)
result["removed"].remove(old_id)
result["added"].remove(new_id)
result["renamed"].append(f"{old_id} -> {new_id} (similarity {ratio:.3f})")
for chunk_id in [c["id"] for c in new if c["id"] in before]:
a, b = before[chunk_id], after[chunk_id]
kind = ("unchanged" if a["sha256"] == b["sha256"] else
"links-only" if a["fingerprint"] == b["fingerprint"] else "changed")
result[kind].append(chunk_id)
return result
One request per page, a descriptive User-Agent, robots.txt read first, and at least a one-second , or the site's Crawl-delay if it sets one. If robots.txt disallows the page, the script stops. One trap: urllib.robotparser ends a group at a blank line, while RFC 9309 does not, and docs.python.org/robots.txt has a blank line before its end-of-life rules. Parsed as is, /3.9/ comes out allowed, so robots_rules() drops blank lines first. The standard parser still differs from RFC 9309 in other ways; for example, it applies the first matching rule rather than the longest one.
Excerpt of page_to_chunks.py, lines 256-281:
def robots_rules(text):
"""Parse robots.txt. urllib.robotparser ends a group at a blank line; RFC 9309 does not."""
rules = urllib.robotparser.RobotFileParser()
rules.parse([line for line in text.splitlines() if line.strip()])
return rules
def fetch(url):
parts = urllib.parse.urlsplit(url)
try:
text = get(f"{parts.scheme}://{parts.netloc}/robots.txt")
except urllib.error.HTTPError as err:
if err.code not in (404, 410):
raise
text = "" # no robots.txt: nothing is disallowed
rules = robots_rules(text)
if not rules.can_fetch(USER_AGENT, url):
sys.exit(f"robots.txt disallows {url}; stopping")
time.sleep(max(1, rules.crawl_delay(USER_AGENT) or 0))
return get(url)
def get(url):
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(request, timeout=30) as response:
return response.read().decode(response.headers.get_content_charset() or "utf-8", "replace")
Twelve unittest tests cover the parser, the renderer, the comparison and the robots.txt parsing. Two of them:
Excerpt of test_page_to_chunks.py, lines 13-18:
def test_main_content_only_and_noise_removed(self):
html = """<html><body><nav>Menu</nav><div role="main">
<h1>Title<a class="headerlink" href="#t">¶</a></h1>
<p>Hello <b>bold</b>
world.</p><script>track()</script></div><footer>Legal</footer></body></html>"""
self.assertEqual(md(html), ["# Title", "Hello **bold** world."])
Excerpt of test_page_to_chunks.py, lines 59-73:
def test_compare(self):
old = chunk(["# A", "same", "# B", "[x](https://v1)", "# C", "old text", "# D", "gone"])
new = chunk(["# A", "same", "# B", "[x](https://v2)", "# C", "new text", "# E", "fresh"])
self.assertEqual(compare(old, new), {"added": ["E"], "removed": ["D"], "renamed": [],
"changed": ["C"], "links-only": ["B"], "unchanged": ["A"]})
def test_renamed_heading_is_paired(self):
body = "The json module can be run from the shell to validate and pretty-print input."
old = chunk(["# Command Line Interface", body])
new = chunk(["# Command-line interface", body + " Also as python -m json."])
result = compare(old, new)
self.assertEqual((result["added"], result["removed"]), ([], []))
self.assertEqual(len(result["renamed"]), 1)
self.assertTrue(result["renamed"][0].startswith("Command Line Interface -> Command-line interface ("))
self.assertEqual(compare(old, new, rename_threshold=0.99)["renamed"], [])
Excerpt of test-log.txt, lines 14-17:
----------------------------------------------------------------------
Ran 12 tests in 0.002s
OK
Convert both versions:
$ python page_to_chunks.py convert https://docs.python.org/3.13/library/json.html json-3.13.md json-3.13.json
https://docs.python.org/3.13/library/json.html at 2026-09-27T13:07:49Z: 110,495 characters of HTML -> 30,089 characters of Markdown in 12 chunks
$ python page_to_chunks.py convert https://docs.python.org/3.14/library/json.html json-3.14.md json-3.14.json
https://docs.python.org/3.14/library/json.html at 2026-09-27T13:07:51Z: 112,591 characters of HTML -> 30,296 characters of Markdown in 12 chunks
About 110,000 characters of HTML became about 30,000 characters of Markdown: navigation, sidebars, scripts, styles and tags are gone. The 3.14 page splits into these twelve chunks:
$ python list_chunks.py json-3.14.json
530 words 7748786f6be31bbd `json` — JSON encoder and decoder
1092 words 9e39ef8ad501eb91 `json` — JSON encoder and decoder > Basic Usage
1061 words 16519e3a65bd9a4f `json` — JSON encoder and decoder > Encoders and Decoders
50 words b697d5281fe94b36 `json` — JSON encoder and decoder > Exceptions
120 words 33131999719582ed `json` — JSON encoder and decoder > Standard Compliance and Interoperability
194 words de4623597eb4dc0c `json` — JSON encoder and decoder > Standard Compliance and Interoperability > Character Encodings
102 words b48c164cc407d4a2 `json` — JSON encoder and decoder > Standard Compliance and Interoperability > Infinite and NaN Number Values
79 words 065b393d5fffcd15 `json` — JSON encoder and decoder > Standard Compliance and Interoperability > Repeated Names Within an Object
86 words f97d8636fc31a9fc `json` — JSON encoder and decoder > Standard Compliance and Interoperability > Top-level Non-Object, Non-Array Values
133 words 9f09bd8d5c282095 `json` — JSON encoder and decoder > Standard Compliance and Interoperability > Implementation Limitations
138 words a6742132e3ec15c4 `json` — JSON encoder and decoder > Command-line interface
129 words e253fc9726be8808 `json` — JSON encoder and decoder > Command-line interface > Command-line options
A piece of the Markdown, the decoding table from the 3.14 page, shows how an HTML table comes out:
Excerpt of json-3.14.md, lines 245-254:
| JSON | Python |
| --- | --- |
| object | dict |
| array | list |
| string | str |
| number (int) | int |
| number (real) | float |
| true | True |
| false | False |
| null | None |
Now compare the two versions:
$ python page_to_chunks.py compare json-3.13.json json-3.14.json
https://docs.python.org/3.13/library/json.html -> https://docs.python.org/3.14/library/json.html
added 0
removed 0
renamed 2
`json` — JSON encoder and decoder > Command Line Interface -> `json` — JSON encoder and decoder > Command-line interface (similarity 0.758)
`json` — JSON encoder and decoder > Command Line Interface > Command line options -> `json` — JSON encoder and decoder > Command-line interface > Command-line options (similarity 0.996)
changed 1
`json` — JSON encoder and decoder
links-only 6
`json` — JSON encoder and decoder > Basic Usage
`json` — JSON encoder and decoder > Encoders and Decoders
`json` — JSON encoder and decoder > Exceptions
`json` — JSON encoder and decoder > Standard Compliance and Interoperability > Character Encodings
`json` — JSON encoder and decoder > Standard Compliance and Interoperability > Top-level Non-Object, Non-Array Values
`json` — JSON encoder and decoder > Standard Compliance and Interoperability > Implementation Limitations
unchanged 3
Reading that result:
/3.13/...; in the 3.14 page they point to /3.14/.... A plain byte hash marks all six as changed. The fingerprint, which ignores link targets, shows that the words are identical. The check below confirms it: in each of the six, swapping /3.13/ for /3.14/ gives exactly the 3.14 text.
$ python check_links_only.py
Basic Usage: 50 links, 49 differ; identical once /3.13/ is replaced by /3.14/: True
Encoders and Decoders: 19 links, 18 differ; identical once /3.13/ is replaced by /3.14/: True
Exceptions: 1 links, 1 differ; identical once /3.13/ is replaced by /3.14/: True
Standard Compliance and Interoperability > Character Encodings: 3 links, 3 differ; identical once /3.13/ is replaced by /3.14/: True
Standard Compliance and Interoperability > Top-level Non-Object, Non-Array Values: 4 links, 2 differ; identical once /3.13/ is replaced by /3.14/: True
Standard Compliance and Interoperability > Implementation Limitations: 2 links, 2 differ; identical once /3.13/ is replaced by /3.14/: True
unchanged: Standard Compliance and Interoperability | links: 2
unchanged: Standard Compliance and Interoperability > Infinite and NaN Number Values | links: 0
unchanged: Standard Compliance and Interoperability > Repeated Names Within an Object | links: 0
python -m json, with python -m json.tool kept for backwards compatibility. A renamed section can still have changed content, so re-embed it.
For a RAG index, that means re-embedding three chunks (one changed, two renamed) out of twelve, and only updating the stored link targets of six others.
html5lib or lxml for parsing if the output looks wrong.html.parser: difflib.SequenceMatcher: hashlib: urllib.robotparser:
"""Turn an HTML page into clean Markdown chunks, and compare two chunk sets.
Usage:
python page_to_chunks.py convert URL out.md out.json
python page_to_chunks.py compare old.json new.json
"""
import difflib
import hashlib
import json
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import urllib.robotparser
from datetime import datetime, timezone
from html.parser import HTMLParser
USER_AGENT = "html-to-chunks-example/1.0 (tutorial script; one request per page)"
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"}
SKIP = {"script", "style", "noscript", "template", "svg", "nav", "footer", "form",
"button", "iframe", "aside"}
NOISE_CLASSES = {"headerlink"} # e.g. the pilcrow permalinks Sphinx adds after headings
BLOCKS = {"p", "div", "section", "article", "main", "blockquote", "pre", "ul", "ol", "table",
"dl", "dt", "dd", "hr", "figure", "h1", "h2", "h3", "h4", "h5", "h6"} | SKIP
class TreeBuilder(HTMLParser):
"""Build a small tree of {"tag", "attrs", "children"} dicts; text stays as plain strings."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.root = {"tag": "#root", "attrs": {}, "children": []}
self.stack = [self.root]
def handle_starttag(self, tag, attrs):
node = {"tag": tag, "attrs": {k: v or "" for k, v in attrs}, "children": []}
self.stack[-1]["children"].append(node)
if tag not in VOID:
self.stack.append(node)
def handle_endtag(self, tag):
for depth in range(len(self.stack) - 1, 0, -1): # close the nearest open match, if any
if self.stack[depth]["tag"] == tag:
del self.stack[depth:]
break
def handle_data(self, data):
self.stack[-1]["children"].append(data)
def find(node, test):
if isinstance(node, dict):
if test(node):
return node
for child in node["children"]:
hit = find(child, test)
if hit:
return hit
return None
def main_content(root):
for test in (lambda n: n["tag"] == "main", lambda n: n["tag"] == "article",
lambda n: n["attrs"].get("role") == "main", lambda n: n["tag"] == "body"):
hit = find(root, test)
if hit:
return hit
return root
def ignored(node):
return node["tag"] in SKIP or bool(NOISE_CLASSES & set(node["attrs"].get("class", "").split()))
def collapse(text):
return re.sub(r"\s+", " ", text).strip()
def text_of(node):
"""Raw text with whitespace kept (for <pre>), minus ignored elements."""
if isinstance(node, str):
return node
return "" if ignored(node) else "".join(text_of(child) for child in node["children"])
def inline(node, base):
"""Render inline content as Markdown on one line."""
if isinstance(node, str):
return node
if ignored(node):
return ""
tag, attrs = node["tag"], node["attrs"]
if tag == "br":
return " "
if tag == "img":
return attrs.get("alt", "")
inner = "".join(inline(child, base) for child in node["children"])
if tag == "code":
return f"`{collapse(text_of(node))}`" if collapse(text_of(node)) else ""
if tag in ("strong", "b", "em", "i") and collapse(inner):
mark = "**" if tag in ("strong", "b") else "*"
return f"{mark}{collapse(inner)}{mark}"
href = attrs.get("href", "")
if tag == "a" and collapse(inner) and href and not href.startswith(("#", "javascript:")):
return f"[{collapse(inner)}]({urllib.parse.urljoin(base, href)})"
return inner
def render_list(node, base, out):
number = 0
for item in node["children"]:
if not isinstance(item, dict) or item["tag"] != "li":
continue
number += 1
marker = f"{number}." if node["tag"] == "ol" else "-"
parts = []
render(item, base, parts)
if parts:
indent = "\n" + " " * (len(marker) + 1)
out.append(marker + " " + indent.join(part.replace("\n", indent) for part in parts))
def render_table(node, base):
rows = []
for row in find_all(node, "tr"):
cells = [collapse(inline(cell, base)).replace("|", "\\|") for cell in row["children"]
if isinstance(cell, dict) and cell["tag"] in ("th", "td")]
if cells:
rows.append(cells)
if not rows:
return ""
width = max(len(row) for row in rows)
rows = [row + [""] * (width - len(row)) for row in rows]
lines = ["| " + " | ".join(rows[0]) + " |", "|" + " --- |" * width]
return "\n".join(lines + ["| " + " | ".join(row) + " |" for row in rows[1:]])
def find_all(node, tag):
for child in node["children"]:
if isinstance(child, dict):
if child["tag"] == tag:
yield child
elif child["tag"] != "table": # do not descend into nested tables
yield from find_all(child, tag)
def render(node, base, out):
"""Append Markdown blocks (strings) for the children of `node` to `out`."""
run = []
def flush():
text = collapse("".join(run))
run.clear()
if text:
out.append(text)
for child in node["children"]:
if isinstance(child, str) or child["tag"] not in BLOCKS:
run.append(inline(child, base))
continue
flush()
tag = child["tag"]
if ignored(child) or tag == "hr":
continue
if re.fullmatch(r"h[1-6]", tag):
title = collapse(inline(child, base))
if title:
out.append("#" * int(tag[1]) + " " + title)
elif tag == "pre":
out.append("```
\n" + text_of(child).strip("\n") + "\n
```")
elif tag in ("ul", "ol"):
render_list(child, base, out)
elif tag == "table":
out.append(render_table(child, base))
elif tag == "dt":
out.append(collapse(text_of(child)))
elif tag == "blockquote":
quoted = []
render(child, base, quoted)
out.extend("> " + block.replace("\n", "\n> ") for block in quoted)
else:
render(child, base, out)
flush()
def to_markdown(html, base):
builder = TreeBuilder()
builder.feed(html)
builder.close()
blocks = []
render(main_content(builder.root), base, blocks)
return [block for block in blocks if block]
def prose(text):
"""Text with link targets and whitespace differences removed."""
return collapse(re.sub(r"\]\([^)]*\)", "]", text))
def fingerprint(text):
return hashlib.sha256(prose(text).encode("utf-8")).hexdigest()[:16]
def chunk(blocks, max_level=3):
"""Group blocks under their nearest heading (h1..h{max_level}); one chunk per section."""
chunks, path, body, seen = [], [], [], {}
def emit():
text = "\n\n".join(body).strip()
body.clear()
if text:
name = " > ".join(path) or "(top)"
seen[name] = seen.get(name, 0) + 1
chunk_id = name if seen[name] == 1 else f"{name} #{seen[name]}"
chunks.append({"id": chunk_id, "words": len(text.split()),
"sha256": hashlib.sha256(text.encode("utf-8")).hexdigest()[:16],
"fingerprint": fingerprint(text), "text": text})
for block in blocks:
heading = re.match(r"(#{1,6}) (.*)", block)
if heading and len(heading.group(1)) <= max_level:
emit()
del path[len(heading.group(1)) - 1:]
path.append(heading.group(2))
body.append(block)
emit()
return chunks
def compare(old, new, rename_threshold=0.6):
"""Classify chunks by id: added, removed, renamed, changed (prose), links-only, unchanged."""
before = {c["id"]: c for c in old}
after = {c["id"]: c for c in new}
result = {"added": sorted(after.keys() - before.keys()),
"removed": sorted(before.keys() - after.keys()),
"renamed": [], "changed": [], "links-only": [], "unchanged": []}
for old_id in list(result["removed"]): # a renamed heading looks like removed + added
scored = [(difflib.SequenceMatcher(None, prose(before[old_id]["text"]),
prose(after[new_id]["text"])).ratio(), new_id)
for new_id in result["added"]]
if scored and max(scored)[0] >= rename_threshold:
ratio, new_id = max(scored)
result["removed"].remove(old_id)
result["added"].remove(new_id)
result["renamed"].append(f"{old_id} -> {new_id} (similarity {ratio:.3f})")
for chunk_id in [c["id"] for c in new if c["id"] in before]:
a, b = before[chunk_id], after[chunk_id]
kind = ("unchanged" if a["sha256"] == b["sha256"] else
"links-only" if a["fingerprint"] == b["fingerprint"] else "changed")
result[kind].append(chunk_id)
return result
def robots_rules(text):
"""Parse robots.txt. urllib.robotparser ends a group at a blank line; RFC 9309 does not."""
rules = urllib.robotparser.RobotFileParser()
rules.parse([line for line in text.splitlines() if line.strip()])
return rules
def fetch(url):
parts = urllib.parse.urlsplit(url)
try:
text = get(f"{parts.scheme}://{parts.netloc}/robots.txt")
except urllib.error.HTTPError as err:
if err.code not in (404, 410):
raise
text = "" # no robots.txt: nothing is disallowed
rules = robots_rules(text)
if not rules.can_fetch(USER_AGENT, url):
sys.exit(f"robots.txt disallows {url}; stopping")
time.sleep(max(1, rules.crawl_delay(USER_AGENT) or 0))
return get(url)
def get(url):
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
with urllib.request.urlopen(request, timeout=30) as response:
return response.read().decode(response.headers.get_content_charset() or "utf-8", "replace")
def main(args):
if args[:1] == ["convert"] and len(args) == 4:
url, md_file, json_file = args[1:]
html = fetch(url)
captured = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
blocks = to_markdown(html, url)
chunks = chunk(blocks)
with open(md_file, "w", encoding="utf-8") as handle:
handle.write("\n\n".join(blocks) + "\n")
with open(json_file, "w", encoding="utf-8") as handle:
json.dump({"url": url, "captured_at": captured, "chunks": chunks}, handle, indent=2)
print(f"{url} at {captured}: {len(html):,} characters of HTML -> "
f"{sum(len(b) for b in blocks):,} characters of Markdown in {len(chunks)} chunks")
elif args[:1] == ["compare"] and len(args) == 3:
with open(args[1], encoding="utf-8") as a, open(args[2], encoding="utf-8") as b:
old, new = json.load(a), json.load(b)
result = compare(old["chunks"], new["chunks"])
print(f"{old['url']} -> {new['url']}")
for kind, ids in result.items():
print(f"{kind:<10} {len(ids):>3}" + "".join(f"\n {i}" for i in ids if kind != "unchanged"))
else:
sys.exit(__doc__)
if __name__ == "__main__":
main(sys.argv[1:])
python
import unittest
from page_to_chunks import chunk, compare, fingerprint, robots_rules, to_markdown
BASE = "https://example.org/docs/page.html"
def md(html):
return to_markdown(html, BASE)
class Markdown(unittest.TestCase):
def test_main_content_only_and_noise_removed(self):
html = """<html><body><nav>Menu</nav><div role="main">
<h1>Title<a class="headerlink" href="#t">¶</a></h1>
<p>Hello <b>bold</b>
world.</p><script>track()</script></div><footer>Legal</footer></body></html>"""
self.assertEqual(md(html), ["# Title", "Hello **bold** world."])
def test_links_become_absolute_and_anchors_become_text(self):
html = '<main><p>See <a href="../api.html#x">the API</a> and <a href="#top">top</a>.</p></main>'
self.assertEqual(md(html), ["See [the API](https://example.org/api.html#x) and top."])
def test_code_inline_and_pre_whitespace(self):
html = "<main><p>Call <code>json.dumps( )</code>:</p><pre>>>> x = 1\n y</pre></main>"
self.assertEqual(md(html), ["Call `json.dumps( )`:", "```
\n>>> x = 1\n y\n
```"])
def test_nested_lists(self):
html = "<main><ol><li><p>One</p><ul><li>a</li><li>b</li></ul></li><li>Two</li></ol></main>"
self.assertEqual(md(html), ["1. One\n - a\n - b", "2. Two"])
def test_table_with_pipes_and_ragged_rows(self):
html = ("<main><table><tr><th>JSON</th><th>Python</th></tr>"
"<tr><td><p>object</p></td><td>dict</td></tr><tr><td>a|b</td></tr></table></main>")
self.assertEqual(md(html), ["| JSON | Python |\n| --- | --- |\n| object | dict |\n| a\\|b | |"])
def test_definition_list_and_blockquote(self):
html = ("<main><dl><dt>json.dump(<em>obj</em>, <em>fp</em>)<a class='headerlink'>¶</a></dt>"
"<dd><p>Serialize.</p></dd></dl><blockquote><p>Quoted</p></blockquote></main>")
self.assertEqual(md(html), ["json.dump(obj, fp)", "Serialize.", "> Quoted"])
def test_unclosed_paragraphs_do_not_swallow_the_page(self):
self.assertEqual(md("<main><p>First<p>Second</main><p>outside"), ["First", "Second"])
class Chunks(unittest.TestCase):
def test_heading_paths_levels_and_duplicates(self):
blocks = ["intro", "# Guide", "a", "## Setup", "b", "#### Deep", "c", "## Setup", "d", "### Notes", "e"]
result = chunk(blocks)
self.assertEqual([c["id"] for c in result],
["(top)", "Guide", "Guide > Setup", "Guide > Setup #2", "Guide > Setup > Notes"])
self.assertEqual(result[2]["text"], "## Setup\n\nb\n\n#### Deep\n\nc")
def test_fingerprint_ignores_link_targets(self):
self.assertEqual(fingerprint("See [docs](https://x/3.13/a.html)."),
fingerprint("See <a href="https://x/3.14/a.html">docs</a>."))
self.assertNotEqual(fingerprint("See [docs](u)."), fingerprint("See [the docs](u)."))
def test_compare(self):
old = chunk(["# A", "same", "# B", "[x](https://v1)", "# C", "old text", "# D", "gone"])
new = chunk(["# A", "same", "# B", "[x](https://v2)", "# C", "new text", "# E", "fresh"])
self.assertEqual(compare(old, new), {"added": ["E"], "removed": ["D"], "renamed": [],
"changed": ["C"], "links-only": ["B"], "unchanged": ["A"]})
def test_renamed_heading_is_paired(self):
body = "The json module can be run from the shell to validate and pretty-print input."
old = chunk(["# Command Line Interface", body])
new = chunk(["# Command-line interface", body + " Also as python -m json."])
result = compare(old, new)
self.assertEqual((result["added"], result["removed"]), ([], []))
self.assertEqual(len(result["renamed"]), 1)
self.assertTrue(result["renamed"][0].startswith("Command Line Interface -> Command-line interface ("))
self.assertEqual(compare(old, new, rename_threshold=0.99)["renamed"], [])
class Robots(unittest.TestCase):
def test_blank_line_does_not_end_the_group(self):
rules = robots_rules("User-agent: *\nDisallow: /dev\n\n# EOL versions\nDisallow: /3.9/\n")
self.assertFalse(rules.can_fetch("page-to-chunks", "https://x/3.9/library/json.html"))
self.assertFalse(rules.can_fetch("page-to-chunks", "https://x/dev/"))
self.assertTrue(rules.can_fetch("page-to-chunks", "https://x/3.14/library/json.html"))
if __name__ == "__main__":
unittest.main()
Run the tests with python -m unittest -v test_page_to_chunks.
This article and its code were drafted by an AI assistant at the account owner's request. The code was run against docs.python.org on 27 September 2026 at 13:07:49 and 13:07:51 UTC, and the tests passed the same day. The blank-line handling for robots.txt was added later the same day after a review; it makes the same robots.txt decisions for both pages, so it does not change the run shown above.
source & further reading
dev.to — original article
Protecting local AI work with incremental cloud backups in TuxInDrive 0.26.57
Julia 1: A 144M-Parameter Decision Model Trained for $104
How We Made an LLM Actually Use Recalled Memory