Andrew Yim
7b239dda8f
Verified conformant against Google reference bundles (crypto_bitcoin, ga4, stackoverflow). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
130 lines
4.3 KiB
Python
130 lines
4.3 KiB
Python
"""Shared OKF v0.1 parsing utilities (Python standard library only)."""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
RESERVED = {"index.md", "log.md"}
|
|
_LINK_RE = re.compile(r"\[[^\]]*\]\(([^)]+)\)")
|
|
_BLOCK_INDICATORS = {">", "|", ">-", "|-", ">+", "|+"}
|
|
|
|
|
|
class FrontmatterError(ValueError):
|
|
"""Raised when a frontmatter block is present but cannot be parsed."""
|
|
|
|
|
|
def split_frontmatter(text):
|
|
"""Return (raw_frontmatter, body). raw is None if there is no leading '---' block."""
|
|
if not text.startswith("---"):
|
|
return None, text
|
|
lines = text.splitlines()
|
|
if lines[0].strip() != "---":
|
|
return None, text
|
|
for i in range(1, len(lines)):
|
|
if lines[i].strip() == "---":
|
|
return "\n".join(lines[1:i]), "\n".join(lines[i + 1:])
|
|
raise FrontmatterError("opening '---' without a closing '---'")
|
|
|
|
|
|
def _scalar(value):
|
|
value = value.strip()
|
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
return value[1:-1]
|
|
return value
|
|
|
|
|
|
def parse_yaml_subset(raw):
|
|
"""Parse the small YAML subset OKF bundles use in practice.
|
|
|
|
Handles: ``key: value``; inline lists ``key: [a, b]``; block lists
|
|
(``key:`` followed by ``- item`` lines); folded multi-line scalars
|
|
(a value continued on following indented lines); and ``>``/``|`` block
|
|
scalars. Surrounding quotes are stripped from scalar values. This is a
|
|
deliberately small parser — not a full YAML implementation.
|
|
"""
|
|
lines = raw.splitlines()
|
|
n = len(lines)
|
|
meta = {}
|
|
i = 0
|
|
while i < n:
|
|
line = lines[i]
|
|
stripped = line.strip()
|
|
if not stripped or stripped.startswith("#"):
|
|
i += 1
|
|
continue
|
|
if ":" not in line:
|
|
raise FrontmatterError("unparseable frontmatter line: %r" % line)
|
|
key, _, value = line.partition(":")
|
|
key, value = key.strip(), value.strip()
|
|
i += 1
|
|
|
|
# Inline list: key: [a, b]
|
|
if value.startswith("[") and value.endswith("]"):
|
|
inner = value[1:-1].strip()
|
|
meta[key] = [s for s in (_scalar(x) for x in inner.split(",")) if s] if inner else []
|
|
continue
|
|
|
|
# Empty value: either a block list (- item lines) or a continued scalar.
|
|
if value == "":
|
|
items = []
|
|
while i < n and lines[i].strip().startswith("-"):
|
|
items.append(_scalar(lines[i].strip()[1:].strip()))
|
|
i += 1
|
|
if items:
|
|
meta[key] = items
|
|
else:
|
|
cont = []
|
|
i = _collect_indented(lines, i, n, cont)
|
|
meta[key] = " ".join(cont)
|
|
continue
|
|
|
|
# Explicit block scalar indicator (> folded, | literal).
|
|
if value in _BLOCK_INDICATORS:
|
|
cont = []
|
|
i = _collect_indented(lines, i, n, cont)
|
|
meta[key] = ("\n" if value[0] == "|" else " ").join(cont)
|
|
continue
|
|
|
|
# Plain scalar, possibly folded across following indented lines.
|
|
parts = [value]
|
|
i = _collect_indented(lines, i, n, parts, skip_list_items=True)
|
|
meta[key] = _scalar(" ".join(parts))
|
|
return meta
|
|
|
|
|
|
def _collect_indented(lines, i, n, out, skip_list_items=False):
|
|
"""Append stripped indented continuation lines to ``out``; return new index."""
|
|
while i < n and lines[i][:1].isspace() and lines[i].strip():
|
|
if skip_list_items and lines[i].strip().startswith("-"):
|
|
break
|
|
out.append(lines[i].strip())
|
|
i += 1
|
|
return i
|
|
|
|
|
|
def parse_frontmatter(text):
|
|
"""Return (meta, body). meta is None when no frontmatter block is present."""
|
|
raw, body = split_frontmatter(text)
|
|
if raw is None:
|
|
return None, text
|
|
return parse_yaml_subset(raw), body
|
|
|
|
|
|
def iter_concepts(bundle_dir):
|
|
"""Yield Path for every non-reserved .md file under bundle_dir, sorted."""
|
|
root = Path(bundle_dir)
|
|
for path in sorted(root.rglob("*.md")):
|
|
if path.name not in RESERVED:
|
|
yield path
|
|
|
|
|
|
def concept_id(bundle_dir, path):
|
|
"""Concept ID = bundle-relative path with the .md suffix removed."""
|
|
rel = Path(path).relative_to(Path(bundle_dir)).as_posix()
|
|
return rel[:-3] if rel.endswith(".md") else rel
|
|
|
|
|
|
def extract_links(body):
|
|
"""Return the list of link targets from markdown links in body."""
|
|
return _LINK_RE.findall(body)
|