ourdigital/our-claude-skills
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(notion): migrate 31+32 to API 2025-09-03 + add property writes, upsert, anchor-link fix

Browse Source 
Phase 1 — docs alignment:
- Fix path drift in 32 CLAUDE.md (02-notion-writer → 32-notion-writer)
- Align env var to NOTION_API_KEY in 31 docs (NOTION_TOKEN still accepted)
- Document Phase 3 roadmap in 31 (metadata-aware migration, Notion-as-RAG)

Phase 2 — multi-source database support:
- New 32/scripts/_notion_compat.py: client factory, data_source resolution
  with cache, property coercion, find_existing_page (for upsert),
  explain_api_error for friendlier failure messages
- 32 notion_writer.py: drop the 2022-06-28 pin, route schema introspection
  through data_sources.retrieve, build data_source_id parent shape, accept
  arbitrary properties via --properties JSON-or-file flag, add --upsert-by
  for idempotent re-runs
- 32 markdown parser: anchor-link fix — Notion's URL validator rejects
  fragment URLs and relative paths; fragments now render as bold to preserve
  TOC nav intent, relatives drop the link, absolute URLs preserved
- 31 async_organizer.py + schema_migrator.py: same data_sources migration
  with cached resolution; pages.create now uses data_source_id parent
- 16/16 parser tests pass (was 13; +3 link-handling regression guards)

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Yim
2026-04-27 09:57:20 +09:00
parent 2667304bca
commit 144a17c88d
7 changed files with 678 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
custom-skills/31-notion-organizer/code/scripts/schema_migrator.py
View File

@@ -53,6 +53,7 @@ class SchemaMigrator:
"pages_skipped": 0,
"errors": 0,
}
self._data_source_cache: dict[str, str] = {}
@retry(
stop=stop_after_attempt(3),
@@ -63,24 +64,59 @@ class SchemaMigrator:
await asyncio.sleep(REQUEST_DELAY)
return await coro
async def get_schema(self, database_id: str) -> dict:
"""Get database schema."""
async def _resolve_data_source_id(self, target_id: str) -> str:
"""Accept database_id or data_source_id; return data_source_id."""
if target_id in self._data_source_cache:
return self._data_source_cache[target_id]
from notion_client.errors import APIErrorCode, APIResponseError
try:
await self._request(
self.client.data_sources.retrieve(data_source_id=target_id)
)
self._data_source_cache[target_id] = target_id
return target_id
except APIResponseError as exc:
if exc.code != APIErrorCode.ObjectNotFound:
raise
db = await self._request(self.client.databases.retrieve(database_id=target_id))
sources = db.get("data_sources") or []
if not sources:
raise ValueError(f"Database {target_id} has no data sources attached")
if len(sources) > 1:
names = ", ".join(s.get("name", s["id"]) for s in sources)
raise ValueError(
f"Database {target_id} has multiple data sources ({names}); "
"pass a specific data_source_id"
)
ds_id = sources[0]["id"]
self._data_source_cache[target_id] = ds_id
return ds_id
async def get_schema(self, target_id: str) -> dict:
"""Get full schema for a database or data source."""
data_source_id = await self._resolve_data_source_id(target_id)
return await self._request(
self.client.databases.retrieve(database_id=database_id)
self.client.data_sources.retrieve(data_source_id=data_source_id)
)
async def fetch_all_pages(self, database_id: str) -> list[dict]:
"""Fetch all pages from source database."""
async def fetch_all_pages(self, target_id: str) -> list[dict]:
"""Fetch all pages from source data source."""
data_source_id = await self._resolve_data_source_id(target_id)
pages = []
has_more = True
cursor = None
while has_more:
params = {"database_id": database_id, "page_size": 100}
params = {"data_source_id": data_source_id, "page_size": 100}
if cursor:
params["start_cursor"] = cursor
response = await self._request(self.client.databases.query(**params))
response = await self._request(
self.client.data_sources.query(**params)
)
pages.extend(response["results"])
has_more = response.get("has_more", False)
cursor = response.get("next_cursor")
@@ -235,7 +271,10 @@ class SchemaMigrator:
result = await self._request(
self.client.pages.create(
parent={"database_id": target_database_id},
parent={
"type": "data_source_id",
"data_source_id": target_database_id,
},
properties=properties,
)
)
@@ -253,22 +292,26 @@ class SchemaMigrator:
target_db: str,
mapping: dict,
) -> list[dict]:
"""Execute full migration."""
"""Execute full migration. Resolves both IDs to data_source_ids once."""
logger.info("Resolving source and target IDs...")
source_ds = await self._resolve_data_source_id(source_db)
target_ds = await self._resolve_data_source_id(target_db)
logger.info("Fetching schemas...")
source_schema = await self.get_schema(source_db)
target_schema = await self.get_schema(target_db)
source_schema = await self.get_schema(source_ds)
target_schema = await self.get_schema(target_ds)
logger.info(f"Source: {len(source_schema['properties'])} properties")
logger.info(f"Target: {len(target_schema['properties'])} properties")
logger.info("Fetching source pages...")
pages = await self.fetch_all_pages(source_db)
pages = await self.fetch_all_pages(source_ds)
logger.info(f"Found {len(pages)} pages to migrate")
results = []
for page in tqdm(pages, desc="Migrating"):
result = await self.migrate_page(
page, target_db, mapping, source_schema, target_schema
page, target_ds, mapping, source_schema, target_schema
)
results.append(result)