Initial commit — memex
A compounding LLM-maintained knowledge wiki. Synthesis of Andrej Karpathy's persistent-wiki gist and milla-jovovich's mempalace, with an automation layer on top for conversation mining, URL harvesting, human-in-the-loop staging, staleness decay, and hygiene. Includes: - 11 pipeline scripts (extract, summarize, index, harvest, stage, hygiene, maintain, sync, + shared library) - Full docs: README, SETUP, ARCHITECTURE, DESIGN-RATIONALE, CUSTOMIZE - Example CLAUDE.md files (wiki schema + global instructions) tuned for the three-collection qmd setup - 171-test pytest suite (cross-platform, runs in ~1.3s) - MIT licensed
This commit is contained in:
Eric Turner
810
scripts/extract-sessions.py
Executable file
810
scripts/extract-sessions.py
Executable file
@@ -0,0 +1,810 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Extract Claude Code session JSONL files into clean markdown transcripts.
|
||||
|
||||
Phase A of the conversation mining pipeline. Deterministic, no LLM dependency.
|
||||
Handles incremental extraction via byte offset tracking for sessions that span
|
||||
hours or days.
|
||||
|
||||
Usage:
|
||||
python3 extract-sessions.py # Extract all new sessions
|
||||
python3 extract-sessions.py --project mc # Extract one project
|
||||
python3 extract-sessions.py --session 0a543572 # Extract specific session
|
||||
python3 extract-sessions.py --dry-run # Show what would be extracted
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CLAUDE_PROJECTS_DIR = Path(os.environ.get("CLAUDE_PROJECTS_DIR", str(Path.home() / ".claude" / "projects")))
|
||||
WIKI_DIR = Path(os.environ.get("WIKI_DIR", str(Path.home() / "projects" / "wiki")))
|
||||
CONVERSATIONS_DIR = WIKI_DIR / "conversations"
|
||||
MINE_STATE_FILE = WIKI_DIR / ".mine-state.json"
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# CONFIGURE ME — Map Claude project directory suffixes to wiki project codes
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Claude Code stores sessions under ~/.claude/projects/<hashed-path>/. The
|
||||
# directory name is derived from the absolute path of your project, so it
|
||||
# looks like `-Users-alice-projects-myapp` or `-home-alice-projects-myapp`.
|
||||
#
|
||||
# This map tells the extractor which suffix maps to which short wiki code
|
||||
# (the "wing"). More specific suffixes should appear first — the extractor
|
||||
# picks the first match. Everything unmatched goes into `general/`.
|
||||
#
|
||||
# Examples — replace with your own projects:
|
||||
PROJECT_MAP: dict[str, str] = {
|
||||
# More specific suffixes first
|
||||
"projects-wiki": "wiki", # this wiki itself
|
||||
"-claude": "cl", # ~/.claude config repo
|
||||
# Add your real projects here:
|
||||
# "my-webapp": "web",
|
||||
# "my-mobile-app": "mob",
|
||||
# "work-mono-repo": "work",
|
||||
# Catch-all — Claude sessions outside any tracked project
|
||||
"-home": "general",
|
||||
"-Users": "general",
|
||||
}
|
||||
|
||||
# Tool call names to keep full output for
|
||||
KEEP_FULL_OUTPUT_TOOLS = {"Bash", "Skill"}
|
||||
|
||||
# Tool call names to summarize (just note what was accessed)
|
||||
SUMMARIZE_TOOLS = {"Read", "Glob", "Grep"}
|
||||
|
||||
# Tool call names to keep with path + change summary
|
||||
KEEP_CHANGE_TOOLS = {"Edit", "Write"}
|
||||
|
||||
# Tool call names to keep description + result summary
|
||||
KEEP_SUMMARY_TOOLS = {"Agent"}
|
||||
|
||||
# Max lines of Bash output to keep
|
||||
MAX_BASH_OUTPUT_LINES = 200
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State management
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
"""Load mining state from .mine-state.json."""
|
||||
if MINE_STATE_FILE.exists():
|
||||
with open(MINE_STATE_FILE) as f:
|
||||
return json.load(f)
|
||||
return {"sessions": {}, "last_run": None}
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
"""Save mining state to .mine-state.json."""
|
||||
state["last_run"] = datetime.now(timezone.utc).isoformat()
|
||||
with open(MINE_STATE_FILE, "w") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Project mapping
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_project_code(dir_name: str) -> str | None:
|
||||
"""Map a Claude project directory name to a wiki project code.
|
||||
|
||||
Directory names look like: -Users-alice-projects-myapp or -home-alice-projects-myapp
|
||||
"""
|
||||
for suffix, code in PROJECT_MAP.items():
|
||||
if dir_name.endswith(suffix):
|
||||
return code
|
||||
return None
|
||||
|
||||
|
||||
def discover_sessions(
|
||||
project_filter: str | None = None,
|
||||
session_filter: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Discover JSONL session files from Claude projects directory."""
|
||||
sessions = []
|
||||
|
||||
if not CLAUDE_PROJECTS_DIR.exists():
|
||||
print(f"Claude projects directory not found: {CLAUDE_PROJECTS_DIR}", file=sys.stderr)
|
||||
return sessions
|
||||
|
||||
for proj_dir in sorted(CLAUDE_PROJECTS_DIR.iterdir()):
|
||||
if not proj_dir.is_dir():
|
||||
continue
|
||||
|
||||
code = resolve_project_code(proj_dir.name)
|
||||
if code is None:
|
||||
continue
|
||||
|
||||
if project_filter and code != project_filter:
|
||||
continue
|
||||
|
||||
for jsonl_file in sorted(proj_dir.glob("*.jsonl")):
|
||||
session_id = jsonl_file.stem
|
||||
if session_filter and not session_id.startswith(session_filter):
|
||||
continue
|
||||
|
||||
sessions.append({
|
||||
"session_id": session_id,
|
||||
"project": code,
|
||||
"jsonl_path": jsonl_file,
|
||||
"file_size": jsonl_file.stat().st_size,
|
||||
})
|
||||
|
||||
return sessions
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# JSONL parsing and filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_timestamp(obj: dict[str, Any]) -> str | None:
|
||||
"""Get timestamp from a JSONL record."""
|
||||
ts = obj.get("timestamp")
|
||||
if isinstance(ts, str):
|
||||
return ts
|
||||
if isinstance(ts, (int, float)):
|
||||
return datetime.fromtimestamp(ts / 1000, tz=timezone.utc).isoformat()
|
||||
return None
|
||||
|
||||
|
||||
def extract_session_date(obj: dict[str, Any]) -> str:
|
||||
"""Get date string (YYYY-MM-DD) from a JSONL record timestamp."""
|
||||
ts = extract_timestamp(obj)
|
||||
if ts:
|
||||
try:
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d")
|
||||
except (ValueError, TypeError):
|
||||
pass
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def truncate_lines(text: str, max_lines: int) -> str:
|
||||
"""Truncate text to max_lines, adding a note if truncated."""
|
||||
lines = text.splitlines()
|
||||
if len(lines) <= max_lines:
|
||||
return text
|
||||
kept = lines[:max_lines]
|
||||
omitted = len(lines) - max_lines
|
||||
kept.append(f"\n[... {omitted} lines truncated ...]")
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def format_tool_use(name: str, input_data: dict[str, Any]) -> str | None:
|
||||
"""Format a tool_use content block for the transcript."""
|
||||
if name in KEEP_FULL_OUTPUT_TOOLS:
|
||||
if name == "Bash":
|
||||
cmd = input_data.get("command", "")
|
||||
desc = input_data.get("description", "")
|
||||
label = desc if desc else cmd[:100]
|
||||
return f"**[Bash]**: `{label}`"
|
||||
if name == "Skill":
|
||||
skill = input_data.get("skill", "")
|
||||
args = input_data.get("args", "")
|
||||
return f"**[Skill]**: /{skill} {args}".strip()
|
||||
|
||||
if name in SUMMARIZE_TOOLS:
|
||||
if name == "Read":
|
||||
fp = input_data.get("file_path", "?")
|
||||
return f"[Read: {fp}]"
|
||||
if name == "Glob":
|
||||
pattern = input_data.get("pattern", "?")
|
||||
return f"[Glob: {pattern}]"
|
||||
if name == "Grep":
|
||||
pattern = input_data.get("pattern", "?")
|
||||
path = input_data.get("path", "")
|
||||
return f"[Grep: '{pattern}' in {path}]" if path else f"[Grep: '{pattern}']"
|
||||
|
||||
if name in KEEP_CHANGE_TOOLS:
|
||||
if name == "Edit":
|
||||
fp = input_data.get("file_path", "?")
|
||||
old = input_data.get("old_string", "")[:60]
|
||||
return f"**[Edit]**: {fp} — replaced '{old}...'"
|
||||
if name == "Write":
|
||||
fp = input_data.get("file_path", "?")
|
||||
content_len = len(input_data.get("content", ""))
|
||||
return f"**[Write]**: {fp} ({content_len} chars)"
|
||||
|
||||
if name in KEEP_SUMMARY_TOOLS:
|
||||
if name == "Agent":
|
||||
desc = input_data.get("description", "?")
|
||||
return f"**[Agent]**: {desc}"
|
||||
|
||||
if name == "ToolSearch":
|
||||
return None # noise
|
||||
if name == "TaskCreate":
|
||||
subj = input_data.get("subject", "?")
|
||||
return f"[TaskCreate: {subj}]"
|
||||
if name == "TaskUpdate":
|
||||
tid = input_data.get("taskId", "?")
|
||||
status = input_data.get("status", "?")
|
||||
return f"[TaskUpdate: #{tid} → {status}]"
|
||||
|
||||
# Default: note the tool was called
|
||||
return f"[{name}]"
|
||||
|
||||
|
||||
def format_tool_result(
|
||||
tool_name: str | None,
|
||||
content: Any,
|
||||
is_error: bool = False,
|
||||
) -> str | None:
|
||||
"""Format a tool_result content block for the transcript."""
|
||||
text = ""
|
||||
if isinstance(content, str):
|
||||
text = content
|
||||
elif isinstance(content, list):
|
||||
parts = []
|
||||
for item in content:
|
||||
if isinstance(item, dict) and item.get("type") == "text":
|
||||
parts.append(item.get("text", ""))
|
||||
text = "\n".join(parts)
|
||||
|
||||
if not text.strip():
|
||||
return None
|
||||
|
||||
if is_error:
|
||||
return f"**[ERROR]**:\n```\n{truncate_lines(text, MAX_BASH_OUTPUT_LINES)}\n```"
|
||||
|
||||
if tool_name in KEEP_FULL_OUTPUT_TOOLS:
|
||||
return f"```\n{truncate_lines(text, MAX_BASH_OUTPUT_LINES)}\n```"
|
||||
|
||||
if tool_name in SUMMARIZE_TOOLS:
|
||||
# Just note the result size
|
||||
line_count = len(text.splitlines())
|
||||
char_count = len(text)
|
||||
return f"[→ {line_count} lines, {char_count} chars]"
|
||||
|
||||
if tool_name in KEEP_CHANGE_TOOLS:
|
||||
return None # The tool_use already captured what changed
|
||||
|
||||
if tool_name in KEEP_SUMMARY_TOOLS:
|
||||
# Keep a summary of agent results
|
||||
summary = text[:300]
|
||||
if len(text) > 300:
|
||||
summary += "..."
|
||||
return f"> {summary}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_content_blocks(
|
||||
content: list[dict[str, Any]],
|
||||
role: str,
|
||||
tool_id_to_name: dict[str, str],
|
||||
) -> list[str]:
|
||||
"""Parse content blocks from a message into transcript lines."""
|
||||
parts: list[str] = []
|
||||
|
||||
for block in content:
|
||||
block_type = block.get("type")
|
||||
|
||||
if block_type == "text":
|
||||
text = block.get("text", "").strip()
|
||||
if not text:
|
||||
continue
|
||||
# Skip system-reminder content
|
||||
if "<system-reminder>" in text:
|
||||
# Strip system reminder tags and their content
|
||||
text = re.sub(
|
||||
r"<system-reminder>.*?</system-reminder>",
|
||||
"",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
).strip()
|
||||
# Skip local-command noise
|
||||
if text.startswith("<local-command"):
|
||||
continue
|
||||
if text:
|
||||
parts.append(text)
|
||||
|
||||
elif block_type == "thinking":
|
||||
# Skip thinking blocks
|
||||
continue
|
||||
|
||||
elif block_type == "tool_use":
|
||||
tool_name = block.get("name", "unknown")
|
||||
tool_id = block.get("id", "")
|
||||
input_data = block.get("input", {})
|
||||
tool_id_to_name[tool_id] = tool_name
|
||||
formatted = format_tool_use(tool_name, input_data)
|
||||
if formatted:
|
||||
parts.append(formatted)
|
||||
|
||||
elif block_type == "tool_result":
|
||||
tool_id = block.get("tool_use_id", "")
|
||||
tool_name = tool_id_to_name.get(tool_id)
|
||||
is_error = block.get("is_error", False)
|
||||
result_content = block.get("content", "")
|
||||
formatted = format_tool_result(tool_name, result_content, is_error)
|
||||
if formatted:
|
||||
parts.append(formatted)
|
||||
|
||||
return parts
|
||||
|
||||
|
||||
def process_jsonl(
|
||||
jsonl_path: Path,
|
||||
byte_offset: int = 0,
|
||||
) -> tuple[list[str], dict[str, Any]]:
|
||||
"""Process a JSONL session file and return transcript lines + metadata.
|
||||
|
||||
Args:
|
||||
jsonl_path: Path to the JSONL file
|
||||
byte_offset: Start reading from this byte position (for incremental)
|
||||
|
||||
Returns:
|
||||
Tuple of (transcript_lines, metadata_dict)
|
||||
"""
|
||||
transcript_lines: list[str] = []
|
||||
metadata: dict[str, Any] = {
|
||||
"first_date": None,
|
||||
"last_date": None,
|
||||
"message_count": 0,
|
||||
"human_messages": 0,
|
||||
"assistant_messages": 0,
|
||||
"git_branch": None,
|
||||
"new_byte_offset": 0,
|
||||
}
|
||||
|
||||
# Map tool_use IDs to tool names for correlating results
|
||||
tool_id_to_name: dict[str, str] = {}
|
||||
|
||||
# Track when a command/skill was just invoked so the next user message
|
||||
# (the skill prompt injection) gets labeled correctly
|
||||
last_command_name: str | None = None
|
||||
|
||||
with open(jsonl_path, "rb") as f:
|
||||
if byte_offset > 0:
|
||||
f.seek(byte_offset)
|
||||
|
||||
for raw_line in f:
|
||||
try:
|
||||
obj = json.loads(raw_line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
record_type = obj.get("type")
|
||||
|
||||
# Skip non-message types
|
||||
if record_type not in ("user", "assistant"):
|
||||
continue
|
||||
|
||||
msg = obj.get("message", {})
|
||||
role = msg.get("role", record_type)
|
||||
content = msg.get("content", "")
|
||||
|
||||
# Track metadata
|
||||
date = extract_session_date(obj)
|
||||
if metadata["first_date"] is None:
|
||||
metadata["first_date"] = date
|
||||
metadata["last_date"] = date
|
||||
metadata["message_count"] += 1
|
||||
|
||||
if not metadata["git_branch"]:
|
||||
metadata["git_branch"] = obj.get("gitBranch")
|
||||
|
||||
if role == "user":
|
||||
metadata["human_messages"] += 1
|
||||
elif role == "assistant":
|
||||
metadata["assistant_messages"] += 1
|
||||
|
||||
# Process content
|
||||
if isinstance(content, str):
|
||||
text = content.strip()
|
||||
# Skip system-reminder and local-command noise
|
||||
if "<system-reminder>" in text:
|
||||
text = re.sub(
|
||||
r"<system-reminder>.*?</system-reminder>",
|
||||
"",
|
||||
text,
|
||||
flags=re.DOTALL,
|
||||
).strip()
|
||||
if text.startswith("<local-command"):
|
||||
continue
|
||||
if text.startswith("<command-name>/exit"):
|
||||
continue
|
||||
|
||||
# Detect command/skill invocation: <command-name>/foo</command-name>
|
||||
cmd_match = re.search(
|
||||
r"<command-name>/([^<]+)</command-name>", text,
|
||||
)
|
||||
if cmd_match:
|
||||
last_command_name = cmd_match.group(1)
|
||||
# Keep just a brief note about the command invocation
|
||||
transcript_lines.append(
|
||||
f"**Human**: /{last_command_name}"
|
||||
)
|
||||
transcript_lines.append("")
|
||||
continue
|
||||
|
||||
# Detect skill prompt injection (large structured text after a command)
|
||||
if (
|
||||
last_command_name
|
||||
and role == "user"
|
||||
and len(text) > 500
|
||||
):
|
||||
# This is the skill's injected prompt — summarize it
|
||||
transcript_lines.append(
|
||||
f"[Skill prompt: /{last_command_name} — {len(text)} chars]"
|
||||
)
|
||||
transcript_lines.append("")
|
||||
last_command_name = None
|
||||
continue
|
||||
|
||||
# Also detect skill prompts by content pattern (catches cases
|
||||
# where the command-name message wasn't separate, or where the
|
||||
# prompt arrives without a preceding command-name tag)
|
||||
if (
|
||||
role == "user"
|
||||
and len(text) > 500
|
||||
and re.match(
|
||||
r"^##\s*(Tracking|Step|Context|Instructions|Overview|Goal)",
|
||||
text,
|
||||
)
|
||||
):
|
||||
# Structured skill prompt — try to extract command name
|
||||
cmd_in_text = re.search(
|
||||
r'--command\s+"([^"]+)"', text,
|
||||
)
|
||||
prompt_label = cmd_in_text.group(1) if cmd_in_text else (last_command_name or "unknown")
|
||||
transcript_lines.append(
|
||||
f"[Skill prompt: /{prompt_label} — {len(text)} chars]"
|
||||
)
|
||||
transcript_lines.append("")
|
||||
last_command_name = None
|
||||
continue
|
||||
|
||||
last_command_name = None # Reset after non-matching message
|
||||
|
||||
if text:
|
||||
label = "**Human**" if role == "user" else "**Assistant**"
|
||||
transcript_lines.append(f"{label}: {text}")
|
||||
transcript_lines.append("")
|
||||
|
||||
elif isinstance(content, list):
|
||||
# Check if this is a skill prompt in list form
|
||||
is_skill_prompt = False
|
||||
skill_prompt_name = last_command_name
|
||||
if role == "user":
|
||||
for block in content:
|
||||
if block.get("type") == "text":
|
||||
block_text = block.get("text", "").strip()
|
||||
# Detect by preceding command name
|
||||
if last_command_name and len(block_text) > 500:
|
||||
is_skill_prompt = True
|
||||
break
|
||||
# Detect by content pattern (## Tracking, etc.)
|
||||
if (
|
||||
len(block_text) > 500
|
||||
and re.match(
|
||||
r"^##\s*(Tracking|Step|Context|Instructions|Overview|Goal)",
|
||||
block_text,
|
||||
)
|
||||
):
|
||||
is_skill_prompt = True
|
||||
# Try to extract command name from content
|
||||
cmd_in_text = re.search(
|
||||
r'--command\s+"([^"]+)"', block_text,
|
||||
)
|
||||
if cmd_in_text:
|
||||
skill_prompt_name = cmd_in_text.group(1)
|
||||
break
|
||||
|
||||
if is_skill_prompt:
|
||||
total_len = sum(
|
||||
len(b.get("text", ""))
|
||||
for b in content
|
||||
if b.get("type") == "text"
|
||||
)
|
||||
label = skill_prompt_name or "unknown"
|
||||
transcript_lines.append(
|
||||
f"[Skill prompt: /{label} — {total_len} chars]"
|
||||
)
|
||||
transcript_lines.append("")
|
||||
last_command_name = None
|
||||
continue
|
||||
|
||||
last_command_name = None
|
||||
|
||||
parts = parse_content_blocks(content, role, tool_id_to_name)
|
||||
if parts:
|
||||
# Determine if this is a tool result message (user role but
|
||||
# contains only tool_result blocks — these are tool outputs,
|
||||
# not human input)
|
||||
has_only_tool_results = all(
|
||||
b.get("type") in ("tool_result",)
|
||||
for b in content
|
||||
if b.get("type") != "text" or b.get("text", "").strip()
|
||||
) and any(b.get("type") == "tool_result" for b in content)
|
||||
|
||||
if has_only_tool_results:
|
||||
# Tool results — no speaker label, just the formatted output
|
||||
for part in parts:
|
||||
transcript_lines.append(part)
|
||||
elif role == "user":
|
||||
# Check if there's actual human text (not just tool results)
|
||||
has_human_text = any(
|
||||
b.get("type") == "text"
|
||||
and b.get("text", "").strip()
|
||||
and "<system-reminder>" not in b.get("text", "")
|
||||
for b in content
|
||||
)
|
||||
label = "**Human**" if has_human_text else "**Assistant**"
|
||||
if len(parts) == 1:
|
||||
transcript_lines.append(f"{label}: {parts[0]}")
|
||||
else:
|
||||
transcript_lines.append(f"{label}:")
|
||||
for part in parts:
|
||||
transcript_lines.append(part)
|
||||
else:
|
||||
label = "**Assistant**"
|
||||
if len(parts) == 1:
|
||||
transcript_lines.append(f"{label}: {parts[0]}")
|
||||
else:
|
||||
transcript_lines.append(f"{label}:")
|
||||
for part in parts:
|
||||
transcript_lines.append(part)
|
||||
transcript_lines.append("")
|
||||
|
||||
metadata["new_byte_offset"] = f.tell()
|
||||
|
||||
return transcript_lines, metadata
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Markdown generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_frontmatter(
|
||||
session_id: str,
|
||||
project: str,
|
||||
date: str,
|
||||
message_count: int,
|
||||
git_branch: str | None = None,
|
||||
) -> str:
|
||||
"""Build YAML frontmatter for a conversation markdown file."""
|
||||
lines = [
|
||||
"---",
|
||||
f"title: Session {session_id[:8]}",
|
||||
"type: conversation",
|
||||
f"project: {project}",
|
||||
f"date: {date}",
|
||||
f"session_id: {session_id}",
|
||||
f"messages: {message_count}",
|
||||
"status: extracted",
|
||||
]
|
||||
if git_branch:
|
||||
lines.append(f"git_branch: {git_branch}")
|
||||
lines.append("---")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_new_conversation(
|
||||
output_path: Path,
|
||||
session_id: str,
|
||||
project: str,
|
||||
transcript_lines: list[str],
|
||||
metadata: dict[str, Any],
|
||||
) -> None:
|
||||
"""Write a new conversation markdown file."""
|
||||
date = metadata["first_date"] or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
frontmatter = build_frontmatter(
|
||||
session_id=session_id,
|
||||
project=project,
|
||||
date=date,
|
||||
message_count=metadata["message_count"],
|
||||
git_branch=metadata.get("git_branch"),
|
||||
)
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_path, "w") as f:
|
||||
f.write(frontmatter)
|
||||
f.write("\n\n## Transcript\n\n")
|
||||
f.write("\n".join(transcript_lines))
|
||||
f.write("\n")
|
||||
|
||||
|
||||
def append_to_conversation(
|
||||
output_path: Path,
|
||||
transcript_lines: list[str],
|
||||
new_message_count: int,
|
||||
) -> None:
|
||||
"""Append new transcript content to an existing conversation file.
|
||||
|
||||
Updates the message count in frontmatter and appends new transcript lines.
|
||||
"""
|
||||
content = output_path.read_text()
|
||||
|
||||
# Update message count in frontmatter
|
||||
content = re.sub(
|
||||
r"^messages: \d+$",
|
||||
f"messages: {new_message_count}",
|
||||
content,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
|
||||
# Add last_updated
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
if "last_updated:" in content:
|
||||
content = re.sub(
|
||||
r"^last_updated: .+$",
|
||||
f"last_updated: {today}",
|
||||
content,
|
||||
count=1,
|
||||
flags=re.MULTILINE,
|
||||
)
|
||||
else:
|
||||
content = content.replace(
|
||||
"\nstatus: extracted",
|
||||
f"\nlast_updated: {today}\nstatus: extracted",
|
||||
)
|
||||
|
||||
# Append new transcript
|
||||
with open(output_path, "w") as f:
|
||||
f.write(content)
|
||||
if not content.endswith("\n"):
|
||||
f.write("\n")
|
||||
f.write("\n".join(transcript_lines))
|
||||
f.write("\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main extraction logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_session(
|
||||
session_info: dict[str, Any],
|
||||
state: dict[str, Any],
|
||||
dry_run: bool = False,
|
||||
) -> bool:
|
||||
"""Extract a single session. Returns True if work was done."""
|
||||
session_id = session_info["session_id"]
|
||||
project = session_info["project"]
|
||||
jsonl_path = session_info["jsonl_path"]
|
||||
file_size = session_info["file_size"]
|
||||
|
||||
# Check state for prior extraction
|
||||
session_state = state["sessions"].get(session_id, {})
|
||||
last_offset = session_state.get("byte_offset", 0)
|
||||
|
||||
# Skip if no new content
|
||||
if file_size <= last_offset:
|
||||
return False
|
||||
|
||||
is_incremental = last_offset > 0
|
||||
|
||||
if dry_run:
|
||||
mode = "append" if is_incremental else "new"
|
||||
new_bytes = file_size - last_offset
|
||||
print(f" [{mode}] {project}/{session_id[:8]} — {new_bytes:,} new bytes")
|
||||
return True
|
||||
|
||||
# Parse the JSONL
|
||||
transcript_lines, metadata = process_jsonl(jsonl_path, byte_offset=last_offset)
|
||||
|
||||
if not transcript_lines:
|
||||
# Update offset even if no extractable content
|
||||
state["sessions"][session_id] = {
|
||||
"project": project,
|
||||
"byte_offset": metadata["new_byte_offset"],
|
||||
"message_count": session_state.get("message_count", 0),
|
||||
"last_extracted": datetime.now(timezone.utc).isoformat(),
|
||||
"summarized_through_msg": session_state.get("summarized_through_msg", 0),
|
||||
}
|
||||
return False
|
||||
|
||||
# Determine output path
|
||||
date = metadata["first_date"] or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
if is_incremental:
|
||||
# Use existing output file
|
||||
output_file = session_state.get("output_file", "")
|
||||
output_path = WIKI_DIR / output_file if output_file else None
|
||||
else:
|
||||
output_path = None
|
||||
|
||||
if output_path is None or not output_path.exists():
|
||||
filename = f"{date}-{session_id[:8]}.md"
|
||||
output_path = CONVERSATIONS_DIR / project / filename
|
||||
|
||||
# Write or append
|
||||
total_messages = session_state.get("message_count", 0) + metadata["message_count"]
|
||||
|
||||
if is_incremental and output_path.exists():
|
||||
append_to_conversation(output_path, transcript_lines, total_messages)
|
||||
print(f" [append] {project}/{output_path.name} — +{metadata['message_count']} messages")
|
||||
else:
|
||||
write_new_conversation(output_path, session_id, project, transcript_lines, metadata)
|
||||
print(f" [new] {project}/{output_path.name} — {metadata['message_count']} messages")
|
||||
|
||||
# Update state
|
||||
state["sessions"][session_id] = {
|
||||
"project": project,
|
||||
"output_file": str(output_path.relative_to(WIKI_DIR)),
|
||||
"byte_offset": metadata["new_byte_offset"],
|
||||
"message_count": total_messages,
|
||||
"last_extracted": datetime.now(timezone.utc).isoformat(),
|
||||
"summarized_through_msg": session_state.get("summarized_through_msg", 0),
|
||||
}
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract Claude Code sessions into markdown transcripts",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--project",
|
||||
help="Only extract sessions for this project code (e.g., mc, if, lp)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--session",
|
||||
help="Only extract this specific session (prefix match on session ID)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would be extracted without writing files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Re-extract from the beginning, ignoring saved byte offsets",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
state = load_state()
|
||||
|
||||
if args.force:
|
||||
# Reset all byte offsets
|
||||
for sid in state["sessions"]:
|
||||
state["sessions"][sid]["byte_offset"] = 0
|
||||
|
||||
# Discover sessions
|
||||
sessions = discover_sessions(
|
||||
project_filter=args.project,
|
||||
session_filter=args.session,
|
||||
)
|
||||
|
||||
if not sessions:
|
||||
print("No sessions found matching filters.")
|
||||
return
|
||||
|
||||
print(f"Found {len(sessions)} session(s) to check...")
|
||||
if args.dry_run:
|
||||
print("DRY RUN — no files will be written\n")
|
||||
|
||||
extracted = 0
|
||||
for session_info in sessions:
|
||||
if extract_session(session_info, state, dry_run=args.dry_run):
|
||||
extracted += 1
|
||||
|
||||
if extracted == 0:
|
||||
print("No new content to extract.")
|
||||
else:
|
||||
print(f"\nExtracted {extracted} session(s).")
|
||||
|
||||
if not args.dry_run:
|
||||
save_state(state)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
118
scripts/mine-conversations.sh
Executable file
118
scripts/mine-conversations.sh
Executable file
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# mine-conversations.sh — Top-level orchestrator for conversation mining pipeline
|
||||
#
|
||||
# Chains: Extract (Python) → Summarize (llama.cpp) → Index (Python)
|
||||
#
|
||||
# Usage:
|
||||
# mine-conversations.sh # Full pipeline
|
||||
# mine-conversations.sh --extract-only # Phase A only (no LLM)
|
||||
# mine-conversations.sh --summarize-only # Phase B only (requires llama-server)
|
||||
# mine-conversations.sh --index-only # Phase C only
|
||||
# mine-conversations.sh --project mc # Filter to one project
|
||||
# mine-conversations.sh --dry-run # Show what would be done
|
||||
|
||||
# Resolve script location first so sibling scripts are found regardless of WIKI_DIR
|
||||
SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WIKI_DIR="${WIKI_DIR:-$(dirname "${SCRIPTS_DIR}")}"
|
||||
LOG_FILE="${SCRIPTS_DIR}/.mine.log"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
EXTRACT=true
|
||||
SUMMARIZE=true
|
||||
INDEX=true
|
||||
PROJECT=""
|
||||
DRY_RUN=""
|
||||
EXTRA_ARGS=()
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--extract-only)
|
||||
SUMMARIZE=false
|
||||
INDEX=false
|
||||
shift
|
||||
;;
|
||||
--summarize-only)
|
||||
EXTRACT=false
|
||||
INDEX=false
|
||||
shift
|
||||
;;
|
||||
--index-only)
|
||||
EXTRACT=false
|
||||
SUMMARIZE=false
|
||||
shift
|
||||
;;
|
||||
--project)
|
||||
PROJECT="$2"
|
||||
shift 2
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN="--dry-run"
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
EXTRA_ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
log() {
|
||||
local msg
|
||||
msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||||
echo "${msg}" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
mkdir -p "${WIKI_DIR}/scripts"
|
||||
|
||||
log "=== Conversation mining started ==="
|
||||
|
||||
# Phase A: Extract
|
||||
if [[ "${EXTRACT}" == true ]]; then
|
||||
log "Phase A: Extracting sessions..."
|
||||
local_args=()
|
||||
if [[ -n "${PROJECT}" ]]; then
|
||||
local_args+=(--project "${PROJECT}")
|
||||
fi
|
||||
if [[ -n "${DRY_RUN}" ]]; then
|
||||
local_args+=(--dry-run)
|
||||
fi
|
||||
python3 "${SCRIPTS_DIR}/extract-sessions.py" "${local_args[@]}" "${EXTRA_ARGS[@]}" 2>&1 | tee -a "${LOG_FILE}"
|
||||
fi
|
||||
|
||||
# Phase B: Summarize
|
||||
if [[ "${SUMMARIZE}" == true ]]; then
|
||||
log "Phase B: Summarizing conversations..."
|
||||
local_args=()
|
||||
if [[ -n "${PROJECT}" ]]; then
|
||||
local_args+=(--project "${PROJECT}")
|
||||
fi
|
||||
if [[ -n "${DRY_RUN}" ]]; then
|
||||
local_args+=(--dry-run)
|
||||
fi
|
||||
python3 "${SCRIPTS_DIR}/summarize-conversations.py" "${local_args[@]}" "${EXTRA_ARGS[@]}" 2>&1 | tee -a "${LOG_FILE}"
|
||||
fi
|
||||
|
||||
# Phase C: Index
|
||||
if [[ "${INDEX}" == true ]]; then
|
||||
log "Phase C: Updating index and context..."
|
||||
local_args=()
|
||||
if [[ -z "${DRY_RUN}" ]]; then
|
||||
local_args+=(--reindex)
|
||||
fi
|
||||
python3 "${SCRIPTS_DIR}/update-conversation-index.py" "${local_args[@]}" 2>&1 | tee -a "${LOG_FILE}"
|
||||
fi
|
||||
|
||||
log "=== Conversation mining complete ==="
|
||||
40
scripts/mine-prompt-v2.md
Normal file
40
scripts/mine-prompt-v2.md
Normal file
@@ -0,0 +1,40 @@
|
||||
You analyze AI coding assistant conversation transcripts and produce structured JSON summaries.
|
||||
|
||||
Read the transcript, then output a single JSON object. No markdown fencing. No explanation. Just JSON.
|
||||
|
||||
REQUIRED JSON STRUCTURE:
|
||||
|
||||
{"trivial":false,"title":"...","summary":"...","halls":["fact"],"topics":["firebase-emulator","docker-compose"],"decisions":["..."],"discoveries":["..."],"preferences":["..."],"advice":["..."],"events":["..."],"tooling":["..."],"key_exchanges":[{"human":"...","assistant":"..."}],"related_topics":["..."]}
|
||||
|
||||
FIELD RULES:
|
||||
|
||||
title: 3-8 word descriptive title. NOT "Session XYZ". Describe what happened.
|
||||
|
||||
summary: 2-3 sentences. What the human wanted. What the assistant did. What was the outcome.
|
||||
|
||||
topics: REQUIRED. 1-4 kebab-case tags for the main subjects. Examples: firebase-emulator, blue-green-deploy, ci-pipeline, docker-hardening, database-migration, api-key-management, git-commit, test-failures.
|
||||
|
||||
halls: Which knowledge types are present. Pick from: fact, discovery, preference, advice, event, tooling.
|
||||
- fact = decisions made, config changed, choices locked in
|
||||
- discovery = root causes, bugs found, breakthroughs
|
||||
- preference = user working style or preferences
|
||||
- advice = recommendations, lessons learned
|
||||
- event = deployments, incidents, milestones
|
||||
- tooling = scripts used, commands run, failures encountered
|
||||
|
||||
decisions: State each decision as a fact. "Added restart policy to firebase service."
|
||||
discoveries: State root cause clearly. "npm install failed because working directory was wrong."
|
||||
preferences: Only if explicitly expressed. Usually empty.
|
||||
advice: Recommendations made during the session.
|
||||
events: Notable milestones or incidents.
|
||||
tooling: Scripts, commands, and tools used. Note failures especially.
|
||||
|
||||
key_exchanges: 1-3 most important moments. Paraphrase to 1 sentence each.
|
||||
|
||||
related_topics: Secondary tags for cross-referencing to other wiki pages.
|
||||
|
||||
trivial: Set true ONLY if < 3 meaningful exchanges and no decisions or discoveries.
|
||||
|
||||
OMIT empty arrays — if no preferences were expressed, use "preferences": [].
|
||||
|
||||
Output ONLY valid JSON. No markdown. No explanation.
|
||||
646
scripts/summarize-conversations.py
Executable file
646
scripts/summarize-conversations.py
Executable file
@@ -0,0 +1,646 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize extracted conversation transcripts via LLM.
|
||||
|
||||
Phase B of the conversation mining pipeline. Sends transcripts to a local
|
||||
llama-server or Claude Code CLI for classification, summarization, and
|
||||
key exchange selection.
|
||||
|
||||
Handles chunking and incremental summarization.
|
||||
|
||||
Usage:
|
||||
python3 summarize-conversations.py # All unsummarized (local LLM)
|
||||
python3 summarize-conversations.py --claude # Use claude -p (haiku/sonnet)
|
||||
python3 summarize-conversations.py --claude --long 300 # Sonnet threshold: 300 msgs
|
||||
python3 summarize-conversations.py --project mc # One project only
|
||||
python3 summarize-conversations.py --file path.md # One file
|
||||
python3 summarize-conversations.py --dry-run # Show what would be done
|
||||
|
||||
Claude mode uses Haiku for short conversations (<= threshold) and Sonnet
|
||||
for longer ones. Threshold default: 200 messages.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Force unbuffered output for background/pipe usage
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
sys.stderr.reconfigure(line_buffering=True)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WIKI_DIR = Path(os.environ.get("WIKI_DIR", str(Path.home() / "projects" / "wiki")))
|
||||
CONVERSATIONS_DIR = WIKI_DIR / "conversations"
|
||||
MINE_STATE_FILE = WIKI_DIR / ".mine-state.json"
|
||||
# Prompt file lives next to this script, not in $WIKI_DIR
|
||||
MINE_PROMPT_FILE = Path(__file__).resolve().parent / "mine-prompt-v2.md"
|
||||
|
||||
# Local LLM defaults (llama-server)
|
||||
AI_BASE_URL = "http://localhost:8080/v1"
|
||||
AI_MODEL = "Phi-4-14B-Q4_K_M"
|
||||
AI_TOKEN = "dummy"
|
||||
AI_TIMEOUT = 180
|
||||
AI_TEMPERATURE = 0.3
|
||||
|
||||
# Claude CLI defaults
|
||||
CLAUDE_HAIKU_MODEL = "haiku"
|
||||
CLAUDE_SONNET_MODEL = "sonnet"
|
||||
CLAUDE_LONG_THRESHOLD = 200 # messages — above this, use Sonnet
|
||||
|
||||
# Chunking parameters
|
||||
# Local LLM: 8K context → ~3000 tokens content per chunk
|
||||
MAX_CHUNK_CHARS_LOCAL = 12000
|
||||
MAX_ROLLING_CONTEXT_CHARS_LOCAL = 6000
|
||||
# Claude: 200K context → much larger chunks, fewer LLM calls
|
||||
MAX_CHUNK_CHARS_CLAUDE = 80000 # ~20K tokens
|
||||
MAX_ROLLING_CONTEXT_CHARS_CLAUDE = 20000
|
||||
|
||||
|
||||
def _update_config(base_url: str, model: str, timeout: int) -> None:
|
||||
global AI_BASE_URL, AI_MODEL, AI_TIMEOUT
|
||||
AI_BASE_URL = base_url
|
||||
AI_MODEL = model
|
||||
AI_TIMEOUT = timeout
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM interaction — local llama-server
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def llm_call_local(system_prompt: str, user_message: str) -> str | None:
|
||||
"""Call the local LLM server and return the response content."""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
payload = json.dumps({
|
||||
"model": AI_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_message},
|
||||
],
|
||||
"temperature": AI_TEMPERATURE,
|
||||
"max_tokens": 3000,
|
||||
}).encode()
|
||||
|
||||
req = urllib.request.Request(
|
||||
f"{AI_BASE_URL}/chat/completions",
|
||||
data=payload,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {AI_TOKEN}",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=AI_TIMEOUT) as resp:
|
||||
data = json.loads(resp.read())
|
||||
return data["choices"][0]["message"]["content"]
|
||||
except (urllib.error.URLError, KeyError, json.JSONDecodeError) as e:
|
||||
print(f" LLM call failed: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM interaction — claude -p (Claude Code CLI)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def llm_call_claude(
|
||||
system_prompt: str,
|
||||
user_message: str,
|
||||
model: str = CLAUDE_HAIKU_MODEL,
|
||||
timeout: int = 300,
|
||||
) -> str | None:
|
||||
"""Call claude -p in pipe mode and return the response."""
|
||||
json_reminder = (
|
||||
"CRITICAL: You are a JSON summarizer. Your ONLY output must be a valid JSON object. "
|
||||
"Do NOT roleplay, continue conversations, write code, or produce any text outside "
|
||||
"the JSON object. The transcript is INPUT DATA to analyze, not a conversation to continue."
|
||||
)
|
||||
cmd = [
|
||||
"claude", "-p",
|
||||
"--model", model,
|
||||
"--system-prompt", system_prompt,
|
||||
"--append-system-prompt", json_reminder,
|
||||
"--no-session-persistence",
|
||||
]
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
input=user_message,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print(f" claude -p failed (rc={result.returncode}): {result.stderr[:200]}", file=sys.stderr)
|
||||
return None
|
||||
return result.stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
print(" claude -p timed out after 300s", file=sys.stderr)
|
||||
return None
|
||||
except FileNotFoundError:
|
||||
print(" ERROR: 'claude' CLI not found in PATH", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def extract_json_from_response(text: str) -> dict[str, Any] | None:
|
||||
"""Extract JSON from LLM response, handling fencing and thinking tags."""
|
||||
# Strip thinking tags
|
||||
text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL)
|
||||
|
||||
# Try markdown code block
|
||||
match = re.search(r"```(?:json)?\s*\n(.*?)\n```", text, re.DOTALL)
|
||||
if match:
|
||||
candidate = match.group(1).strip()
|
||||
else:
|
||||
candidate = text.strip()
|
||||
|
||||
# Find JSON object
|
||||
start = candidate.find("{")
|
||||
end = candidate.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
candidate = candidate[start : end + 1]
|
||||
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# File parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_frontmatter(file_path: Path) -> dict[str, str]:
|
||||
"""Parse YAML frontmatter."""
|
||||
content = file_path.read_text()
|
||||
match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not match:
|
||||
return {}
|
||||
fm: dict[str, str] = {}
|
||||
for line in match.group(1).splitlines():
|
||||
if ":" in line:
|
||||
key, _, value = line.partition(":")
|
||||
fm[key.strip()] = value.strip()
|
||||
return fm
|
||||
|
||||
|
||||
def get_transcript(file_path: Path) -> str:
|
||||
"""Get transcript section from conversation file."""
|
||||
content = file_path.read_text()
|
||||
idx = content.find("\n## Transcript\n")
|
||||
if idx < 0:
|
||||
return ""
|
||||
return content[idx + len("\n## Transcript\n") :]
|
||||
|
||||
|
||||
def get_existing_summary(file_path: Path) -> str:
|
||||
"""Get existing summary sections (between frontmatter end and transcript)."""
|
||||
content = file_path.read_text()
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
return ""
|
||||
after_fm = parts[2]
|
||||
idx = after_fm.find("## Transcript")
|
||||
if idx < 0:
|
||||
return ""
|
||||
return after_fm[:idx].strip()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Chunking
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def chunk_text(text: str, max_chars: int) -> list[str]:
|
||||
"""Split text into chunks, breaking at paragraph boundaries."""
|
||||
if len(text) <= max_chars:
|
||||
return [text]
|
||||
|
||||
chunks: list[str] = []
|
||||
current = ""
|
||||
|
||||
for line in text.splitlines(keepends=True):
|
||||
if len(current) + len(line) > max_chars and current:
|
||||
chunks.append(current)
|
||||
current = line
|
||||
else:
|
||||
current += line
|
||||
|
||||
if current:
|
||||
chunks.append(current)
|
||||
|
||||
return chunks
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summarization
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def select_claude_model(file_path: Path, long_threshold: int) -> str:
|
||||
"""Pick haiku or sonnet based on message count."""
|
||||
fm = parse_frontmatter(file_path)
|
||||
try:
|
||||
msg_count = int(fm.get("messages", "0"))
|
||||
except ValueError:
|
||||
msg_count = 0
|
||||
if msg_count > long_threshold:
|
||||
return CLAUDE_SONNET_MODEL
|
||||
return CLAUDE_HAIKU_MODEL
|
||||
|
||||
|
||||
def summarize_file(
|
||||
file_path: Path,
|
||||
system_prompt: str,
|
||||
dry_run: bool = False,
|
||||
use_claude: bool = False,
|
||||
long_threshold: int = CLAUDE_LONG_THRESHOLD,
|
||||
) -> bool:
|
||||
"""Summarize a single conversation file. Returns True on success."""
|
||||
transcript = get_transcript(file_path)
|
||||
if not transcript.strip():
|
||||
print(f" [skip] {file_path.name} — no transcript")
|
||||
return False
|
||||
|
||||
existing_summary = get_existing_summary(file_path)
|
||||
is_incremental = "## Summary" in existing_summary
|
||||
|
||||
# Pick chunk sizes based on provider
|
||||
if use_claude:
|
||||
max_chunk = MAX_CHUNK_CHARS_CLAUDE
|
||||
max_rolling = MAX_ROLLING_CONTEXT_CHARS_CLAUDE
|
||||
else:
|
||||
max_chunk = MAX_CHUNK_CHARS_LOCAL
|
||||
max_rolling = MAX_ROLLING_CONTEXT_CHARS_LOCAL
|
||||
|
||||
chunks = chunk_text(transcript, max_chunk)
|
||||
num_chunks = len(chunks)
|
||||
|
||||
# Pick model for claude mode
|
||||
claude_model = ""
|
||||
if use_claude:
|
||||
claude_model = select_claude_model(file_path, long_threshold)
|
||||
|
||||
if dry_run:
|
||||
mode = "incremental" if is_incremental else "new"
|
||||
model_info = f", model={claude_model}" if use_claude else ""
|
||||
print(f" [dry-run] {file_path.name} — {num_chunks} chunk(s) ({mode}{model_info})")
|
||||
return True
|
||||
|
||||
model_label = f" [{claude_model}]" if use_claude else ""
|
||||
print(f" [summarize] {file_path.name} — {num_chunks} chunk(s)"
|
||||
f"{' (incremental)' if is_incremental else ''}{model_label}")
|
||||
|
||||
rolling_context = ""
|
||||
if is_incremental:
|
||||
rolling_context = f"EXISTING SUMMARY (extend, do not repeat):\n{existing_summary}\n\n"
|
||||
|
||||
final_json: dict[str, Any] | None = None
|
||||
start_time = time.time()
|
||||
|
||||
for i, chunk in enumerate(chunks, 1):
|
||||
if rolling_context:
|
||||
user_msg = (
|
||||
f"{rolling_context}\n\n"
|
||||
f"NEW CONVERSATION CONTENT (chunk {i}/{num_chunks}):\n{chunk}"
|
||||
)
|
||||
else:
|
||||
user_msg = f"CONVERSATION TRANSCRIPT (chunk {i}/{num_chunks}):\n{chunk}"
|
||||
|
||||
if i == num_chunks:
|
||||
user_msg += "\n\nThis is the FINAL chunk. Produce the complete JSON summary now."
|
||||
else:
|
||||
user_msg += "\n\nMore chunks follow. Produce a PARTIAL summary JSON for what you've seen so far."
|
||||
|
||||
# Call the appropriate LLM (with retry on parse failure)
|
||||
max_attempts = 2
|
||||
parsed = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
if use_claude:
|
||||
# Longer timeout for sonnet / multi-chunk conversations
|
||||
call_timeout = 600 if claude_model == CLAUDE_SONNET_MODEL else 300
|
||||
response = llm_call_claude(system_prompt, user_msg,
|
||||
model=claude_model, timeout=call_timeout)
|
||||
else:
|
||||
response = llm_call_local(system_prompt, user_msg)
|
||||
|
||||
if not response:
|
||||
print(f" [error] LLM call failed on chunk {i}/{num_chunks} (attempt {attempt})")
|
||||
if attempt < max_attempts:
|
||||
continue
|
||||
return False
|
||||
|
||||
parsed = extract_json_from_response(response)
|
||||
if parsed:
|
||||
break
|
||||
|
||||
print(f" [warn] JSON parse failed on chunk {i}/{num_chunks} (attempt {attempt})")
|
||||
if attempt < max_attempts:
|
||||
print(f" Retrying...")
|
||||
else:
|
||||
# Log first 200 chars for debugging
|
||||
print(f" Response preview: {response[:200]}", file=sys.stderr)
|
||||
|
||||
if not parsed:
|
||||
print(f" [error] JSON parse failed on chunk {i}/{num_chunks} after {max_attempts} attempts")
|
||||
return False
|
||||
|
||||
final_json = parsed
|
||||
|
||||
# Build rolling context for next chunk
|
||||
partial_summary = parsed.get("summary", "")
|
||||
if partial_summary:
|
||||
rolling_context = f"PARTIAL SUMMARY SO FAR:\n{partial_summary}"
|
||||
decisions = parsed.get("decisions", [])
|
||||
if decisions:
|
||||
rolling_context += "\n\nKEY DECISIONS:\n" + "\n".join(
|
||||
f"- {d}" for d in decisions[:5]
|
||||
)
|
||||
if len(rolling_context) > max_rolling:
|
||||
rolling_context = rolling_context[:max_rolling] + "..."
|
||||
|
||||
if not final_json:
|
||||
print(f" [error] No summary produced")
|
||||
return False
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# Apply the summary to the file
|
||||
apply_summary(file_path, final_json)
|
||||
|
||||
halls = final_json.get("halls", [])
|
||||
topics = final_json.get("topics", [])
|
||||
status = "trivial" if final_json.get("trivial") else "summarized"
|
||||
|
||||
print(
|
||||
f" [done] {file_path.name} — {status}, "
|
||||
f"halls=[{', '.join(halls)}], "
|
||||
f"topics=[{', '.join(topics)}] "
|
||||
f"({elapsed:.0f}s)"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def apply_summary(file_path: Path, summary_json: dict[str, Any]) -> None:
|
||||
"""Apply LLM summary to the conversation markdown file."""
|
||||
content = file_path.read_text()
|
||||
|
||||
# Parse existing frontmatter
|
||||
fm_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not fm_match:
|
||||
return
|
||||
|
||||
fm_lines = fm_match.group(1).splitlines()
|
||||
|
||||
# Find transcript
|
||||
transcript_idx = content.find("\n## Transcript\n")
|
||||
transcript_section = content[transcript_idx:] if transcript_idx >= 0 else ""
|
||||
|
||||
# Update frontmatter
|
||||
is_trivial = summary_json.get("trivial", False)
|
||||
new_status = "trivial" if is_trivial else "summarized"
|
||||
title = summary_json.get("title", "Untitled Session")
|
||||
halls = summary_json.get("halls", [])
|
||||
topics = summary_json.get("topics", [])
|
||||
related = summary_json.get("related_topics", [])
|
||||
|
||||
fm_dict: dict[str, str] = {}
|
||||
fm_key_order: list[str] = []
|
||||
for line in fm_lines:
|
||||
if ":" in line:
|
||||
key = line.partition(":")[0].strip()
|
||||
val = line.partition(":")[2].strip()
|
||||
fm_dict[key] = val
|
||||
fm_key_order.append(key)
|
||||
|
||||
fm_dict["title"] = title
|
||||
fm_dict["status"] = new_status
|
||||
if halls:
|
||||
fm_dict["halls"] = "[" + ", ".join(halls) + "]"
|
||||
if topics:
|
||||
fm_dict["topics"] = "[" + ", ".join(topics) + "]"
|
||||
if related:
|
||||
fm_dict["related"] = "[" + ", ".join(related) + "]"
|
||||
|
||||
# Add new keys
|
||||
for key in ["halls", "topics", "related"]:
|
||||
if key in fm_dict and key not in fm_key_order:
|
||||
fm_key_order.append(key)
|
||||
|
||||
new_fm = "\n".join(f"{k}: {fm_dict[k]}" for k in fm_key_order if k in fm_dict)
|
||||
|
||||
# Build summary sections
|
||||
sections: list[str] = []
|
||||
|
||||
summary_text = summary_json.get("summary", "")
|
||||
if summary_text:
|
||||
sections.append(f"## Summary\n\n{summary_text}")
|
||||
|
||||
for hall_name, hall_label in [
|
||||
("decisions", "Decisions (hall: fact)"),
|
||||
("discoveries", "Discoveries (hall: discovery)"),
|
||||
("preferences", "Preferences (hall: preference)"),
|
||||
("advice", "Advice (hall: advice)"),
|
||||
("events", "Events (hall: event)"),
|
||||
("tooling", "Tooling (hall: tooling)"),
|
||||
]:
|
||||
items = summary_json.get(hall_name, [])
|
||||
if items:
|
||||
lines = [f"## {hall_label}\n"]
|
||||
for item in items:
|
||||
lines.append(f"- {item}")
|
||||
sections.append("\n".join(lines))
|
||||
|
||||
exchanges = summary_json.get("key_exchanges", [])
|
||||
if exchanges:
|
||||
lines = ["## Key Exchanges\n"]
|
||||
for ex in exchanges:
|
||||
if isinstance(ex, dict):
|
||||
human = ex.get("human", "")
|
||||
assistant = ex.get("assistant", "")
|
||||
lines.append(f"> **Human**: {human}")
|
||||
lines.append(">")
|
||||
lines.append(f"> **Assistant**: {assistant}")
|
||||
lines.append("")
|
||||
elif isinstance(ex, str):
|
||||
lines.append(f"- {ex}")
|
||||
sections.append("\n".join(lines))
|
||||
|
||||
# Assemble
|
||||
output = f"---\n{new_fm}\n---\n\n"
|
||||
if sections:
|
||||
output += "\n\n".join(sections) + "\n\n---\n"
|
||||
output += transcript_section
|
||||
if not output.endswith("\n"):
|
||||
output += "\n"
|
||||
|
||||
file_path.write_text(output)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def find_files_to_summarize(
|
||||
project_filter: str | None = None,
|
||||
file_filter: str | None = None,
|
||||
) -> list[Path]:
|
||||
"""Find conversation files needing summarization."""
|
||||
if file_filter:
|
||||
p = Path(file_filter)
|
||||
if p.exists():
|
||||
return [p]
|
||||
p = WIKI_DIR / file_filter
|
||||
if p.exists():
|
||||
return [p]
|
||||
return []
|
||||
|
||||
search_dir = CONVERSATIONS_DIR
|
||||
if project_filter:
|
||||
search_dir = CONVERSATIONS_DIR / project_filter
|
||||
|
||||
files: list[Path] = []
|
||||
for md_file in sorted(search_dir.rglob("*.md")):
|
||||
if md_file.name in ("index.md", ".gitkeep"):
|
||||
continue
|
||||
fm = parse_frontmatter(md_file)
|
||||
if fm.get("status") == "extracted":
|
||||
files.append(md_file)
|
||||
|
||||
return files
|
||||
|
||||
|
||||
def update_mine_state(session_id: str, msg_count: int) -> None:
|
||||
"""Update summarized_through_msg in mine state."""
|
||||
if not MINE_STATE_FILE.exists():
|
||||
return
|
||||
try:
|
||||
with open(MINE_STATE_FILE) as f:
|
||||
state = json.load(f)
|
||||
if session_id in state.get("sessions", {}):
|
||||
state["sessions"][session_id]["summarized_through_msg"] = msg_count
|
||||
with open(MINE_STATE_FILE, "w") as f:
|
||||
json.dump(state, f, indent=2)
|
||||
except (json.JSONDecodeError, KeyError):
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Summarize conversation transcripts")
|
||||
parser.add_argument("--project", help="Only summarize this project code")
|
||||
parser.add_argument("--file", help="Summarize a specific file")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show what would be done")
|
||||
parser.add_argument(
|
||||
"--claude", action="store_true",
|
||||
help="Use claude -p instead of local LLM (haiku for short, sonnet for long)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--long", type=int, default=CLAUDE_LONG_THRESHOLD, metavar="N",
|
||||
help=f"Message count threshold for sonnet (default: {CLAUDE_LONG_THRESHOLD})",
|
||||
)
|
||||
parser.add_argument("--ai-url", default=AI_BASE_URL)
|
||||
parser.add_argument("--ai-model", default=AI_MODEL)
|
||||
parser.add_argument("--ai-timeout", type=int, default=AI_TIMEOUT)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Update module-level config from args (local LLM only)
|
||||
_update_config(args.ai_url, args.ai_model, args.ai_timeout)
|
||||
|
||||
# Load system prompt
|
||||
if not MINE_PROMPT_FILE.exists():
|
||||
print(f"ERROR: Prompt not found: {MINE_PROMPT_FILE}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
system_prompt = MINE_PROMPT_FILE.read_text()
|
||||
|
||||
# Find files
|
||||
files = find_files_to_summarize(args.project, args.file)
|
||||
if not files:
|
||||
print("No conversations need summarization.")
|
||||
return
|
||||
|
||||
provider = "claude -p" if args.claude else f"local ({AI_MODEL})"
|
||||
print(f"Found {len(files)} conversation(s) to summarize. Provider: {provider}")
|
||||
|
||||
if args.dry_run:
|
||||
for f in files:
|
||||
summarize_file(f, system_prompt, dry_run=True,
|
||||
use_claude=args.claude, long_threshold=args.long)
|
||||
return
|
||||
|
||||
# Check provider availability
|
||||
if args.claude:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["claude", "--version"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
print("ERROR: 'claude' CLI not working", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print(f"Claude CLI: {result.stdout.strip()}")
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
print("ERROR: 'claude' CLI not found in PATH", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
health_url = AI_BASE_URL.replace("/v1", "/health")
|
||||
try:
|
||||
urllib.request.urlopen(health_url, timeout=5)
|
||||
except urllib.error.URLError:
|
||||
print(f"ERROR: LLM server not responding at {health_url}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
processed = 0
|
||||
errors = 0
|
||||
total_start = time.time()
|
||||
|
||||
for i, f in enumerate(files, 1):
|
||||
print(f"\n[{i}/{len(files)}]", end=" ")
|
||||
try:
|
||||
if summarize_file(f, system_prompt, use_claude=args.claude,
|
||||
long_threshold=args.long):
|
||||
processed += 1
|
||||
|
||||
# Update mine state
|
||||
fm = parse_frontmatter(f)
|
||||
sid = fm.get("session_id", "")
|
||||
msgs = fm.get("messages", "0")
|
||||
if sid:
|
||||
try:
|
||||
update_mine_state(sid, int(msgs))
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
errors += 1
|
||||
except Exception as e:
|
||||
print(f" [crash] {f.name} — {e}", file=sys.stderr)
|
||||
errors += 1
|
||||
|
||||
elapsed = time.time() - total_start
|
||||
print(f"\nDone. Summarized: {processed}, Errors: {errors}, Time: {elapsed:.0f}s")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
476
scripts/update-conversation-index.py
Executable file
476
scripts/update-conversation-index.py
Executable file
@@ -0,0 +1,476 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Update conversation index and context files from summarized conversations.
|
||||
|
||||
Phase C of the conversation mining pipeline. Reads all conversation markdown
|
||||
files and regenerates:
|
||||
- conversations/index.md — catalog organized by project
|
||||
- context/wake-up.md — world briefing from recent conversations
|
||||
- context/active-concerns.md — current blockers and open threads
|
||||
|
||||
Usage:
|
||||
python3 update-conversation-index.py
|
||||
python3 update-conversation-index.py --reindex # Also triggers qmd update
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WIKI_DIR = Path(os.environ.get("WIKI_DIR", str(Path.home() / "projects" / "wiki")))
|
||||
CONVERSATIONS_DIR = WIKI_DIR / "conversations"
|
||||
CONTEXT_DIR = WIKI_DIR / "context"
|
||||
INDEX_FILE = CONVERSATIONS_DIR / "index.md"
|
||||
WAKEUP_FILE = CONTEXT_DIR / "wake-up.md"
|
||||
CONCERNS_FILE = CONTEXT_DIR / "active-concerns.md"
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# CONFIGURE ME — Project code to display name mapping
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Every project code you use in `extract-sessions.py`'s PROJECT_MAP should
|
||||
# have a display name here. The conversation index groups conversations by
|
||||
# these codes and renders them under sections named by the display name.
|
||||
#
|
||||
# Examples — replace with your own:
|
||||
PROJECT_NAMES: dict[str, str] = {
|
||||
"wiki": "WIKI — This Wiki",
|
||||
"cl": "CL — Claude Config",
|
||||
# "web": "WEB — My Webapp",
|
||||
# "mob": "MOB — My Mobile App",
|
||||
# "work": "WORK — Day Job",
|
||||
"general": "General — Cross-Project",
|
||||
}
|
||||
|
||||
# Order for display — put your most-active projects first
|
||||
PROJECT_ORDER = [
|
||||
# "work", "web", "mob",
|
||||
"wiki", "cl", "general",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Frontmatter parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_frontmatter(file_path: Path) -> dict[str, str]:
|
||||
"""Parse YAML frontmatter from a markdown file."""
|
||||
fm: dict[str, str] = {}
|
||||
content = file_path.read_text()
|
||||
|
||||
# Find frontmatter between --- markers
|
||||
match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL)
|
||||
if not match:
|
||||
return fm
|
||||
|
||||
for line in match.group(1).splitlines():
|
||||
if ":" in line:
|
||||
key, _, value = line.partition(":")
|
||||
fm[key.strip()] = value.strip()
|
||||
|
||||
return fm
|
||||
|
||||
|
||||
def get_summary_line(file_path: Path) -> str:
|
||||
"""Extract the first sentence of the Summary section."""
|
||||
content = file_path.read_text()
|
||||
match = re.search(r"## Summary\n\n(.+?)(?:\n\n|\n##)", content, re.DOTALL)
|
||||
if match:
|
||||
summary = match.group(1).strip()
|
||||
# First sentence
|
||||
first_sentence = summary.split(". ")[0]
|
||||
if not first_sentence.endswith("."):
|
||||
first_sentence += "."
|
||||
# Truncate if too long
|
||||
if len(first_sentence) > 120:
|
||||
first_sentence = first_sentence[:117] + "..."
|
||||
return first_sentence
|
||||
return "No summary available."
|
||||
|
||||
|
||||
def get_decisions(file_path: Path) -> list[str]:
|
||||
"""Extract decisions from a conversation file."""
|
||||
content = file_path.read_text()
|
||||
decisions: list[str] = []
|
||||
match = re.search(r"## Decisions.*?\n(.*?)(?:\n##|\n---|\Z)", content, re.DOTALL)
|
||||
if match:
|
||||
for line in match.group(1).strip().splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("- "):
|
||||
decisions.append(line[2:])
|
||||
return decisions
|
||||
|
||||
|
||||
def get_discoveries(file_path: Path) -> list[str]:
|
||||
"""Extract discoveries from a conversation file."""
|
||||
content = file_path.read_text()
|
||||
discoveries: list[str] = []
|
||||
match = re.search(r"## Discoveries.*?\n(.*?)(?:\n##|\n---|\Z)", content, re.DOTALL)
|
||||
if match:
|
||||
for line in match.group(1).strip().splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("- "):
|
||||
discoveries.append(line[2:])
|
||||
return discoveries
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conversation discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def discover_conversations() -> dict[str, list[dict[str, Any]]]:
|
||||
"""Discover all conversation files organized by project."""
|
||||
by_project: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
|
||||
for project_dir in sorted(CONVERSATIONS_DIR.iterdir()):
|
||||
if not project_dir.is_dir():
|
||||
continue
|
||||
|
||||
project_code = project_dir.name
|
||||
if project_code not in PROJECT_NAMES:
|
||||
continue
|
||||
|
||||
for md_file in sorted(project_dir.glob("*.md"), reverse=True):
|
||||
if md_file.name == ".gitkeep":
|
||||
continue
|
||||
|
||||
fm = parse_frontmatter(md_file)
|
||||
status = fm.get("status", "extracted")
|
||||
|
||||
entry = {
|
||||
"file": md_file,
|
||||
"relative": md_file.relative_to(CONVERSATIONS_DIR),
|
||||
"title": fm.get("title", md_file.stem),
|
||||
"date": fm.get("date", "unknown"),
|
||||
"status": status,
|
||||
"messages": fm.get("messages", "0"),
|
||||
"halls": fm.get("halls", ""),
|
||||
"topics": fm.get("topics", ""),
|
||||
"project": project_code,
|
||||
}
|
||||
|
||||
by_project[project_code].append(entry)
|
||||
|
||||
return by_project
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_index(by_project: dict[str, list[dict[str, Any]]]) -> str:
|
||||
"""Generate the conversations/index.md content."""
|
||||
total = sum(len(convos) for convos in by_project.values())
|
||||
summarized = sum(
|
||||
1
|
||||
for convos in by_project.values()
|
||||
for c in convos
|
||||
if c["status"] == "summarized"
|
||||
)
|
||||
trivial = sum(
|
||||
1
|
||||
for convos in by_project.values()
|
||||
for c in convos
|
||||
if c["status"] == "trivial"
|
||||
)
|
||||
extracted = total - summarized - trivial
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"title: Conversation Index",
|
||||
"type: index",
|
||||
f"last_updated: {datetime.now(timezone.utc).strftime('%Y-%m-%d')}",
|
||||
"---",
|
||||
"",
|
||||
"# Conversation Index",
|
||||
"",
|
||||
f"Mined conversations from Claude Code sessions, organized by project (wing).",
|
||||
"",
|
||||
f"**{total} conversations** — {summarized} summarized, {extracted} pending, {trivial} trivial.",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
|
||||
for project_code in PROJECT_ORDER:
|
||||
convos = by_project.get(project_code, [])
|
||||
display_name = PROJECT_NAMES.get(project_code, project_code.upper())
|
||||
|
||||
lines.append(f"## {display_name}")
|
||||
lines.append("")
|
||||
|
||||
if not convos:
|
||||
lines.append("_No conversations mined yet._")
|
||||
lines.append("")
|
||||
continue
|
||||
|
||||
# Show summarized first, then extracted, skip trivial from listing
|
||||
shown = 0
|
||||
for c in convos:
|
||||
if c["status"] == "trivial":
|
||||
continue
|
||||
|
||||
status_tag = ""
|
||||
if c["status"] == "extracted":
|
||||
status_tag = " _(pending summary)_"
|
||||
|
||||
# Get summary line if summarized
|
||||
summary_text = ""
|
||||
if c["status"] == "summarized":
|
||||
summary_text = f" — {get_summary_line(c['file'])}"
|
||||
|
||||
lines.append(
|
||||
f"- [{c['title']}]({c['relative']})"
|
||||
f" ({c['date']}, {c['messages']} msgs)"
|
||||
f"{summary_text}{status_tag}"
|
||||
)
|
||||
shown += 1
|
||||
|
||||
trivial_count = len(convos) - shown
|
||||
if trivial_count > 0:
|
||||
lines.append(f"\n_{trivial_count} trivial session(s) not listed._")
|
||||
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Context generation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def generate_wakeup(by_project: dict[str, list[dict[str, Any]]]) -> str:
|
||||
"""Generate context/wake-up.md from recent conversations."""
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
# Determine activity level per project
|
||||
project_activity: dict[str, dict[str, Any]] = {}
|
||||
for code in PROJECT_ORDER:
|
||||
convos = by_project.get(code, [])
|
||||
summarized = [c for c in convos if c["status"] == "summarized"]
|
||||
|
||||
if summarized:
|
||||
latest = max(summarized, key=lambda c: c["date"])
|
||||
last_date = latest["date"]
|
||||
# Simple activity heuristic: sessions in last 7 days = active
|
||||
try:
|
||||
dt = datetime.strptime(last_date, "%Y-%m-%d")
|
||||
days_ago = (datetime.now() - dt).days
|
||||
if days_ago <= 7:
|
||||
status = "Active"
|
||||
elif days_ago <= 30:
|
||||
status = "Quiet"
|
||||
else:
|
||||
status = "Inactive"
|
||||
except ValueError:
|
||||
status = "Unknown"
|
||||
last_date = "—"
|
||||
else:
|
||||
# Check extracted-only
|
||||
if convos:
|
||||
latest = max(convos, key=lambda c: c["date"])
|
||||
last_date = latest["date"]
|
||||
status = "Active" if latest["date"] >= today[:7] else "Quiet"
|
||||
else:
|
||||
status = "—"
|
||||
last_date = "—"
|
||||
|
||||
project_activity[code] = {
|
||||
"status": status,
|
||||
"last_date": last_date,
|
||||
"count": len(convos),
|
||||
}
|
||||
|
||||
# Gather recent decisions across all projects
|
||||
recent_decisions: list[tuple[str, str, str]] = [] # (date, project, decision)
|
||||
for code, convos in by_project.items():
|
||||
for c in convos:
|
||||
if c["status"] != "summarized":
|
||||
continue
|
||||
for decision in get_decisions(c["file"]):
|
||||
recent_decisions.append((c["date"], code, decision))
|
||||
|
||||
recent_decisions.sort(key=lambda x: x[0], reverse=True)
|
||||
recent_decisions = recent_decisions[:10] # Top 10 most recent
|
||||
|
||||
# Gather recent discoveries
|
||||
recent_discoveries: list[tuple[str, str, str]] = []
|
||||
for code, convos in by_project.items():
|
||||
for c in convos:
|
||||
if c["status"] != "summarized":
|
||||
continue
|
||||
for disc in get_discoveries(c["file"]):
|
||||
recent_discoveries.append((c["date"], code, disc))
|
||||
|
||||
recent_discoveries.sort(key=lambda x: x[0], reverse=True)
|
||||
recent_discoveries = recent_discoveries[:5]
|
||||
|
||||
lines = [
|
||||
"---",
|
||||
"title: Wake-Up Briefing",
|
||||
"type: context",
|
||||
f"last_updated: {today}",
|
||||
"---",
|
||||
"",
|
||||
"# Wake-Up Briefing",
|
||||
"",
|
||||
"Auto-generated world state for AI session context.",
|
||||
"",
|
||||
"## Active Projects",
|
||||
"",
|
||||
"| Code | Project | Status | Last Activity | Sessions |",
|
||||
"|------|---------|--------|---------------|----------|",
|
||||
]
|
||||
|
||||
for code in PROJECT_ORDER:
|
||||
if code == "general":
|
||||
continue # Skip general from roster
|
||||
info = project_activity.get(code, {"status": "—", "last_date": "—", "count": 0})
|
||||
display = PROJECT_NAMES.get(code, code).split(" — ")[1] if " — " in PROJECT_NAMES.get(code, "") else code
|
||||
lines.append(
|
||||
f"| {code.upper()} | {display} | {info['status']} | {info['last_date']} | {info['count']} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
|
||||
if recent_decisions:
|
||||
lines.append("## Recent Decisions")
|
||||
lines.append("")
|
||||
for date, proj, decision in recent_decisions[:7]:
|
||||
lines.append(f"- **[{proj.upper()}]** {decision} ({date})")
|
||||
lines.append("")
|
||||
|
||||
if recent_discoveries:
|
||||
lines.append("## Recent Discoveries")
|
||||
lines.append("")
|
||||
for date, proj, disc in recent_discoveries[:5]:
|
||||
lines.append(f"- **[{proj.upper()}]** {disc} ({date})")
|
||||
lines.append("")
|
||||
|
||||
if not recent_decisions and not recent_discoveries:
|
||||
lines.append("## Recent Decisions")
|
||||
lines.append("")
|
||||
lines.append("_Populated after summarization runs._")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_concerns(by_project: dict[str, list[dict[str, Any]]]) -> str:
|
||||
"""Generate context/active-concerns.md from recent conversations."""
|
||||
today = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
# For now, this is a template that gets populated as summaries accumulate.
|
||||
# Future enhancement: parse "blockers", "open questions" from summaries.
|
||||
lines = [
|
||||
"---",
|
||||
"title: Active Concerns",
|
||||
"type: context",
|
||||
f"last_updated: {today}",
|
||||
"---",
|
||||
"",
|
||||
"# Active Concerns",
|
||||
"",
|
||||
"Auto-generated from recent conversations. Current blockers, deadlines, and open questions.",
|
||||
"",
|
||||
]
|
||||
|
||||
# Count recent activity to give a sense of what's hot
|
||||
active_projects: list[tuple[str, int]] = []
|
||||
for code in PROJECT_ORDER:
|
||||
convos = by_project.get(code, [])
|
||||
recent = [c for c in convos if c["date"] >= today[:7]] # This month
|
||||
if recent:
|
||||
active_projects.append((code, len(recent)))
|
||||
|
||||
if active_projects:
|
||||
active_projects.sort(key=lambda x: x[1], reverse=True)
|
||||
lines.append("## Current Focus Areas")
|
||||
lines.append("")
|
||||
for code, count in active_projects[:5]:
|
||||
display = PROJECT_NAMES.get(code, code)
|
||||
lines.append(f"- **{display}** — {count} session(s) this month")
|
||||
lines.append("")
|
||||
|
||||
lines.extend([
|
||||
"## Blockers",
|
||||
"",
|
||||
"_Populated from conversation analysis._",
|
||||
"",
|
||||
"## Open Questions",
|
||||
"",
|
||||
"_Populated from conversation analysis._",
|
||||
"",
|
||||
])
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Update conversation index and context files",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reindex",
|
||||
action="store_true",
|
||||
help="Also trigger qmd update and embed after updating files",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
# Discover all conversations
|
||||
by_project = discover_conversations()
|
||||
|
||||
total = sum(len(v) for v in by_project.values())
|
||||
print(f"Found {total} conversation(s) across {len(by_project)} projects.")
|
||||
|
||||
# Generate and write index
|
||||
index_content = generate_index(by_project)
|
||||
INDEX_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
INDEX_FILE.write_text(index_content)
|
||||
print(f"Updated {INDEX_FILE.relative_to(WIKI_DIR)}")
|
||||
|
||||
# Generate and write context files (create dir if needed)
|
||||
WAKEUP_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
wakeup_content = generate_wakeup(by_project)
|
||||
WAKEUP_FILE.write_text(wakeup_content)
|
||||
print(f"Updated {WAKEUP_FILE.relative_to(WIKI_DIR)}")
|
||||
|
||||
concerns_content = generate_concerns(by_project)
|
||||
CONCERNS_FILE.write_text(concerns_content)
|
||||
print(f"Updated {CONCERNS_FILE.relative_to(WIKI_DIR)}")
|
||||
|
||||
# Optionally trigger qmd reindex
|
||||
if args.reindex:
|
||||
print("Triggering qmd reindex...")
|
||||
try:
|
||||
subprocess.run(["qmd", "update"], check=True, capture_output=True)
|
||||
subprocess.run(["qmd", "embed"], check=True, capture_output=True)
|
||||
print("qmd index updated.")
|
||||
except FileNotFoundError:
|
||||
print("qmd not found — skipping reindex.", file=sys.stderr)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"qmd reindex failed: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
878
scripts/wiki-harvest.py
Executable file
878
scripts/wiki-harvest.py
Executable file
@@ -0,0 +1,878 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Harvest external reference URLs from summarized conversations into the wiki.
|
||||
|
||||
Scans summarized conversation transcripts for URLs, classifies them, fetches
|
||||
the content, stores the raw source under raw/harvested/, and optionally calls
|
||||
`claude -p` to compile each raw file into a staging/ wiki page.
|
||||
|
||||
Usage:
|
||||
python3 scripts/wiki-harvest.py # Process all summarized conversations
|
||||
python3 scripts/wiki-harvest.py --project mc # One project only
|
||||
python3 scripts/wiki-harvest.py --file PATH # One conversation file
|
||||
python3 scripts/wiki-harvest.py --dry-run # Show what would be harvested
|
||||
python3 scripts/wiki-harvest.py --no-compile # Fetch only, skip claude -p compile step
|
||||
python3 scripts/wiki-harvest.py --limit 10 # Cap number of URLs processed
|
||||
|
||||
State is persisted in .harvest-state.json; existing URLs are deduplicated.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Force unbuffered output for pipe usage
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
sys.stderr.reconfigure(line_buffering=True)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
WIKI_DIR = Path(os.environ.get("WIKI_DIR", str(Path.home() / "projects" / "wiki")))
|
||||
CONVERSATIONS_DIR = WIKI_DIR / "conversations"
|
||||
RAW_HARVESTED_DIR = WIKI_DIR / "raw" / "harvested"
|
||||
STAGING_DIR = WIKI_DIR / "staging"
|
||||
INDEX_FILE = WIKI_DIR / "index.md"
|
||||
CLAUDE_MD = WIKI_DIR / "CLAUDE.md"
|
||||
HARVEST_STATE_FILE = WIKI_DIR / ".harvest-state.json"
|
||||
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
# CONFIGURE ME — URL classification rules
|
||||
# ════════════════════════════════════════════════════════════════════════════
|
||||
#
|
||||
# Type D: always skip. Add your own internal/ephemeral/personal domains here.
|
||||
# Patterns use `re.search` so unanchored suffixes like `\.example\.com$` work.
|
||||
# Private IPs (10.x, 172.16-31.x, 192.168.x, 127.x) are detected separately.
|
||||
SKIP_DOMAIN_PATTERNS = [
|
||||
# Generic: ephemeral / personal / chat / internal
|
||||
r"\.atlassian\.net$",
|
||||
r"^app\.asana\.com$",
|
||||
r"^(www\.)?slack\.com$",
|
||||
r"\.slack\.com$",
|
||||
r"^(www\.)?discord\.com$",
|
||||
r"^localhost$",
|
||||
r"^0\.0\.0\.0$",
|
||||
r"^mail\.google\.com$",
|
||||
r"^calendar\.google\.com$",
|
||||
r"^docs\.google\.com$",
|
||||
r"^drive\.google\.com$",
|
||||
r"^.+\.local$",
|
||||
r"^.+\.internal$",
|
||||
# Add your own internal domains below, for example:
|
||||
# r"\.mycompany\.com$",
|
||||
# r"^git\.mydomain\.com$",
|
||||
]
|
||||
|
||||
# Type C — issue trackers / Q&A; only harvest if topic touches existing wiki
|
||||
C_TYPE_URL_PATTERNS = [
|
||||
r"^https?://github\.com/[^/]+/[^/]+/issues/\d+",
|
||||
r"^https?://github\.com/[^/]+/[^/]+/pull/\d+",
|
||||
r"^https?://github\.com/[^/]+/[^/]+/discussions/\d+",
|
||||
r"^https?://(www\.)?stackoverflow\.com/questions/\d+",
|
||||
r"^https?://(www\.)?serverfault\.com/questions/\d+",
|
||||
r"^https?://(www\.)?superuser\.com/questions/\d+",
|
||||
r"^https?://.+\.stackexchange\.com/questions/\d+",
|
||||
]
|
||||
|
||||
# Asset/image extensions to filter out
|
||||
ASSET_EXTENSIONS = {
|
||||
".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", ".bmp",
|
||||
".css", ".js", ".mjs", ".woff", ".woff2", ".ttf", ".eot",
|
||||
".mp4", ".webm", ".mov", ".mp3", ".wav",
|
||||
".zip", ".tar", ".gz", ".bz2",
|
||||
}
|
||||
|
||||
# URL regex — HTTP(S), stops at whitespace, brackets, and common markdown delimiters
|
||||
URL_REGEX = re.compile(
|
||||
r"https?://[^\s<>\"')\]}\\|`]+",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Claude CLI models
|
||||
CLAUDE_HAIKU_MODEL = "haiku"
|
||||
CLAUDE_SONNET_MODEL = "sonnet"
|
||||
SONNET_CONTENT_THRESHOLD = 20_000 # chars — larger than this → sonnet
|
||||
|
||||
# Fetch behavior
|
||||
FETCH_DELAY_SECONDS = 2
|
||||
MAX_FAILED_ATTEMPTS = 3
|
||||
MIN_CONTENT_LENGTH = 100
|
||||
FETCH_TIMEOUT = 45
|
||||
|
||||
# HTML-leak detection — content containing any of these is treated as a failed extraction
|
||||
HTML_LEAK_MARKERS = ["<div", "<script", "<nav", "<header", "<footer"]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# State management
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def load_state() -> dict[str, Any]:
|
||||
defaults: dict[str, Any] = {
|
||||
"harvested_urls": {},
|
||||
"skipped_urls": {},
|
||||
"failed_urls": {},
|
||||
"rejected_urls": {},
|
||||
"last_run": None,
|
||||
}
|
||||
if HARVEST_STATE_FILE.exists():
|
||||
try:
|
||||
with open(HARVEST_STATE_FILE) as f:
|
||||
state = json.load(f)
|
||||
for k, v in defaults.items():
|
||||
state.setdefault(k, v)
|
||||
return state
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
return defaults
|
||||
|
||||
|
||||
def save_state(state: dict[str, Any]) -> None:
|
||||
state["last_run"] = datetime.now(timezone.utc).isoformat()
|
||||
tmp = HARVEST_STATE_FILE.with_suffix(".json.tmp")
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(state, f, indent=2, sort_keys=True)
|
||||
tmp.replace(HARVEST_STATE_FILE)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# URL extraction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def extract_urls_from_file(file_path: Path) -> list[str]:
|
||||
"""Extract all HTTP(S) URLs from a conversation markdown file.
|
||||
|
||||
Filters:
|
||||
- Asset URLs (images, CSS, JS, fonts, media, archives)
|
||||
- URLs shorter than 20 characters
|
||||
- Duplicates within the same file
|
||||
"""
|
||||
try:
|
||||
text = file_path.read_text(errors="replace")
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
seen: set[str] = set()
|
||||
urls: list[str] = []
|
||||
|
||||
for match in URL_REGEX.finditer(text):
|
||||
url = match.group(0).rstrip(".,;:!?") # strip trailing sentence punctuation
|
||||
# Drop trailing markdown/code artifacts
|
||||
while url and url[-1] in "()[]{}\"'":
|
||||
url = url[:-1]
|
||||
if len(url) < 20:
|
||||
continue
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError:
|
||||
continue
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
continue
|
||||
path_lower = parsed.path.lower()
|
||||
if any(path_lower.endswith(ext) for ext in ASSET_EXTENSIONS):
|
||||
continue
|
||||
if url in seen:
|
||||
continue
|
||||
seen.add(url)
|
||||
urls.append(url)
|
||||
|
||||
return urls
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# URL classification
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _is_private_ip(host: str) -> bool:
|
||||
"""Return True if host is an RFC1918 or loopback IP literal."""
|
||||
if not re.match(r"^\d+\.\d+\.\d+\.\d+$", host):
|
||||
return False
|
||||
parts = [int(p) for p in host.split(".")]
|
||||
if parts[0] == 10:
|
||||
return True
|
||||
if parts[0] == 127:
|
||||
return True
|
||||
if parts[0] == 172 and 16 <= parts[1] <= 31:
|
||||
return True
|
||||
if parts[0] == 192 and parts[1] == 168:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def classify_url(url: str) -> str:
|
||||
"""Classify a URL as 'harvest' (A/B), 'check' (C), or 'skip' (D)."""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError:
|
||||
return "skip"
|
||||
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host:
|
||||
return "skip"
|
||||
|
||||
if _is_private_ip(host):
|
||||
return "skip"
|
||||
|
||||
for pattern in SKIP_DOMAIN_PATTERNS:
|
||||
if re.search(pattern, host):
|
||||
return "skip"
|
||||
|
||||
for pattern in C_TYPE_URL_PATTERNS:
|
||||
if re.match(pattern, url):
|
||||
return "check"
|
||||
|
||||
return "harvest"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Filename derivation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def slugify(text: str) -> str:
|
||||
text = text.lower()
|
||||
text = re.sub(r"[^a-z0-9]+", "-", text)
|
||||
return text.strip("-")
|
||||
|
||||
|
||||
def raw_filename_for_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
host = parsed.netloc.lower().replace("www.", "")
|
||||
path = parsed.path.rstrip("/")
|
||||
host_slug = slugify(host)
|
||||
path_slug = slugify(path) if path else "index"
|
||||
# Truncate overly long names
|
||||
if len(path_slug) > 80:
|
||||
path_slug = path_slug[:80].rstrip("-")
|
||||
return f"{host_slug}-{path_slug}.md"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fetch cascade
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_fetch_command(cmd: list[str], timeout: int = FETCH_TIMEOUT) -> tuple[bool, str]:
|
||||
"""Run a fetch command and return (success, output)."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
return False, result.stderr.strip() or "non-zero exit"
|
||||
return True, result.stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "timeout"
|
||||
except FileNotFoundError as e:
|
||||
return False, f"command not found: {e}"
|
||||
except OSError as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def validate_content(content: str) -> bool:
|
||||
if not content or len(content.strip()) < MIN_CONTENT_LENGTH:
|
||||
return False
|
||||
low = content.lower()
|
||||
if any(marker in low for marker in HTML_LEAK_MARKERS):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def fetch_with_trafilatura(url: str) -> tuple[bool, str]:
|
||||
ok, out = run_fetch_command(
|
||||
["trafilatura", "-u", url, "--markdown", "--no-comments", "--precision"]
|
||||
)
|
||||
if ok and validate_content(out):
|
||||
return True, out
|
||||
return False, out if not ok else "content validation failed"
|
||||
|
||||
|
||||
def fetch_with_crawl4ai(url: str, stealth: bool = False) -> tuple[bool, str]:
|
||||
cmd = ["crwl", url, "-o", "markdown-fit"]
|
||||
if stealth:
|
||||
cmd += [
|
||||
"-b", "headless=true,user_agent_mode=random",
|
||||
"-c", "magic=true,scan_full_page=true,page_timeout=20000",
|
||||
]
|
||||
else:
|
||||
cmd += ["-c", "page_timeout=15000"]
|
||||
ok, out = run_fetch_command(cmd, timeout=90)
|
||||
if ok and validate_content(out):
|
||||
return True, out
|
||||
return False, out if not ok else "content validation failed"
|
||||
|
||||
|
||||
def fetch_from_conversation(url: str, conversation_file: Path) -> tuple[bool, str]:
|
||||
"""Fallback: scrape a block of content near where the URL appears in the transcript.
|
||||
|
||||
If the assistant fetched the URL during the session, some portion of the
|
||||
content is likely inline in the transcript.
|
||||
"""
|
||||
try:
|
||||
text = conversation_file.read_text(errors="replace")
|
||||
except OSError:
|
||||
return False, "cannot read conversation file"
|
||||
|
||||
idx = text.find(url)
|
||||
if idx == -1:
|
||||
return False, "url not found in conversation"
|
||||
|
||||
# Grab up to 2000 chars after the URL mention
|
||||
snippet = text[idx : idx + 2000]
|
||||
if not validate_content(snippet):
|
||||
return False, "snippet failed validation"
|
||||
return True, snippet
|
||||
|
||||
|
||||
def fetch_cascade(url: str, conversation_file: Path) -> tuple[bool, str, str]:
|
||||
"""Attempt the full fetch cascade. Returns (success, content, method_used)."""
|
||||
ok, out = fetch_with_trafilatura(url)
|
||||
if ok:
|
||||
return True, out, "trafilatura"
|
||||
|
||||
ok, out = fetch_with_crawl4ai(url, stealth=False)
|
||||
if ok:
|
||||
return True, out, "crawl4ai"
|
||||
|
||||
ok, out = fetch_with_crawl4ai(url, stealth=True)
|
||||
if ok:
|
||||
return True, out, "crawl4ai-stealth"
|
||||
|
||||
ok, out = fetch_from_conversation(url, conversation_file)
|
||||
if ok:
|
||||
return True, out, "conversation-fallback"
|
||||
|
||||
return False, out, "failed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Raw file storage
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def content_hash(content: str) -> str:
|
||||
return "sha256:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def write_raw_file(
|
||||
url: str,
|
||||
content: str,
|
||||
method: str,
|
||||
discovered_in: Path,
|
||||
) -> Path:
|
||||
RAW_HARVESTED_DIR.mkdir(parents=True, exist_ok=True)
|
||||
filename = raw_filename_for_url(url)
|
||||
out_path = RAW_HARVESTED_DIR / filename
|
||||
# Collision: append short hash
|
||||
if out_path.exists():
|
||||
suffix = hashlib.sha256(url.encode()).hexdigest()[:8]
|
||||
out_path = RAW_HARVESTED_DIR / f"{out_path.stem}-{suffix}.md"
|
||||
|
||||
rel_discovered = discovered_in.relative_to(WIKI_DIR)
|
||||
frontmatter = [
|
||||
"---",
|
||||
f"source_url: {url}",
|
||||
f"fetched_date: {datetime.now(timezone.utc).date().isoformat()}",
|
||||
f"fetch_method: {method}",
|
||||
f"discovered_in: {rel_discovered}",
|
||||
f"content_hash: {content_hash(content)}",
|
||||
"---",
|
||||
"",
|
||||
]
|
||||
out_path.write_text("\n".join(frontmatter) + content.strip() + "\n")
|
||||
return out_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AI compilation via claude -p
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
COMPILE_PROMPT_TEMPLATE = """You are compiling a raw harvested source document into the LLM wiki at {wiki_dir}.
|
||||
|
||||
The wiki schema and conventions are defined in CLAUDE.md. The wiki has four
|
||||
content directories: patterns/ (how), decisions/ (why), environments/ (where),
|
||||
concepts/ (what). All pages require YAML frontmatter with title, type,
|
||||
confidence, sources, related, last_compiled, last_verified.
|
||||
|
||||
IMPORTANT: Do NOT include `status`, `origin`, `staged_*`, `target_path`,
|
||||
`modifies`, `harvest_source`, or `compilation_notes` fields in your page
|
||||
frontmatter — the harvest script injects those automatically.
|
||||
|
||||
The raw source material is below. Decide what to do with it and emit the
|
||||
result as a single JSON object on stdout (nothing else). Valid actions:
|
||||
|
||||
- "new_page" — create a new wiki page
|
||||
- "update_page" — update an existing wiki page (add source, merge content)
|
||||
- "both" — create a new page AND update an existing one
|
||||
- "skip" — content isn't substantive enough to warrant a wiki page
|
||||
|
||||
JSON schema:
|
||||
|
||||
{{
|
||||
"action": "new_page" | "update_page" | "both" | "skip",
|
||||
"compilation_notes": "1-3 sentences explaining what you did and why",
|
||||
"new_page": {{
|
||||
"directory": "patterns" | "decisions" | "environments" | "concepts",
|
||||
"filename": "kebab-case-name.md",
|
||||
"content": "full markdown including frontmatter"
|
||||
}},
|
||||
"update_page": {{
|
||||
"path": "patterns/existing-page.md",
|
||||
"content": "full updated markdown including frontmatter"
|
||||
}}
|
||||
}}
|
||||
|
||||
Omit "new_page" if not applicable; omit "update_page" if not applicable. If
|
||||
action is "skip", omit both. Do NOT include any prose outside the JSON.
|
||||
|
||||
Wiki index (so you know what pages exist):
|
||||
|
||||
{wiki_index}
|
||||
|
||||
Raw harvested source:
|
||||
|
||||
{raw_content}
|
||||
|
||||
Conversation context (the working session where this URL was cited):
|
||||
|
||||
{conversation_context}
|
||||
"""
|
||||
|
||||
|
||||
def call_claude_compile(
|
||||
raw_path: Path,
|
||||
raw_content: str,
|
||||
conversation_file: Path,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Invoke `claude -p` to compile the raw source into a staging wiki page."""
|
||||
|
||||
# Pick model by size
|
||||
model = CLAUDE_SONNET_MODEL if len(raw_content) > SONNET_CONTENT_THRESHOLD else CLAUDE_HAIKU_MODEL
|
||||
|
||||
try:
|
||||
wiki_index = INDEX_FILE.read_text()[:20_000]
|
||||
except OSError:
|
||||
wiki_index = ""
|
||||
|
||||
try:
|
||||
conversation_context = conversation_file.read_text(errors="replace")[:8_000]
|
||||
except OSError:
|
||||
conversation_context = ""
|
||||
|
||||
prompt = COMPILE_PROMPT_TEMPLATE.format(
|
||||
wiki_dir=str(WIKI_DIR),
|
||||
wiki_index=wiki_index,
|
||||
raw_content=raw_content[:40_000],
|
||||
conversation_context=conversation_context,
|
||||
)
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["claude", "-p", "--model", model, "--output-format", "text", prompt],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print(" [warn] claude CLI not found — skipping compilation", file=sys.stderr)
|
||||
return None
|
||||
except subprocess.TimeoutExpired:
|
||||
print(" [warn] claude -p timed out", file=sys.stderr)
|
||||
return None
|
||||
|
||||
if result.returncode != 0:
|
||||
print(f" [warn] claude -p failed: {result.stderr.strip()[:200]}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# Extract JSON from output (may be wrapped in fences)
|
||||
output = result.stdout.strip()
|
||||
match = re.search(r"\{.*\}", output, re.DOTALL)
|
||||
if not match:
|
||||
print(f" [warn] no JSON found in claude output ({len(output)} chars)", file=sys.stderr)
|
||||
return None
|
||||
try:
|
||||
return json.loads(match.group(0))
|
||||
except json.JSONDecodeError as e:
|
||||
print(f" [warn] JSON parse failed: {e}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
STAGING_INJECT_TEMPLATE = (
|
||||
"---\n"
|
||||
"origin: automated\n"
|
||||
"status: pending\n"
|
||||
"staged_date: {staged_date}\n"
|
||||
"staged_by: wiki-harvest\n"
|
||||
"target_path: {target_path}\n"
|
||||
"{modifies_line}"
|
||||
"harvest_source: {source_url}\n"
|
||||
"compilation_notes: {compilation_notes}\n"
|
||||
)
|
||||
|
||||
|
||||
def _inject_staging_frontmatter(
|
||||
content: str,
|
||||
source_url: str,
|
||||
target_path: str,
|
||||
compilation_notes: str,
|
||||
modifies: str | None,
|
||||
) -> str:
|
||||
"""Insert staging metadata after the opening --- fence of the AI-generated content."""
|
||||
# Strip existing status/origin/staged fields the AI may have added
|
||||
content = re.sub(r"^(status|origin|staged_\w+|target_path|modifies|harvest_source|compilation_notes):.*\n", "", content, flags=re.MULTILINE)
|
||||
|
||||
modifies_line = f"modifies: {modifies}\n" if modifies else ""
|
||||
# Collapse multi-line compilation notes to single line for safe YAML
|
||||
clean_notes = compilation_notes.replace("\n", " ").replace("\r", " ").strip()
|
||||
injection = STAGING_INJECT_TEMPLATE.format(
|
||||
staged_date=datetime.now(timezone.utc).date().isoformat(),
|
||||
target_path=target_path,
|
||||
modifies_line=modifies_line,
|
||||
source_url=source_url,
|
||||
compilation_notes=clean_notes or "(none provided)",
|
||||
)
|
||||
|
||||
if content.startswith("---\n"):
|
||||
return injection + content[4:]
|
||||
# AI forgot the fence — prepend full frontmatter
|
||||
return injection + "---\n" + content
|
||||
|
||||
|
||||
def _unique_staging_path(base: Path) -> Path:
|
||||
"""Append a short hash if the target already exists."""
|
||||
if not base.exists():
|
||||
return base
|
||||
suffix = hashlib.sha256(str(base).encode() + str(time.time()).encode()).hexdigest()[:6]
|
||||
return base.with_stem(f"{base.stem}-{suffix}")
|
||||
|
||||
|
||||
def apply_compile_result(
|
||||
result: dict[str, Any],
|
||||
source_url: str,
|
||||
raw_path: Path,
|
||||
) -> list[Path]:
|
||||
"""Write the AI compilation result into staging/. Returns paths written."""
|
||||
written: list[Path] = []
|
||||
action = result.get("action", "skip")
|
||||
if action == "skip":
|
||||
return written
|
||||
|
||||
notes = result.get("compilation_notes", "")
|
||||
|
||||
# New page
|
||||
new_page = result.get("new_page") or {}
|
||||
if action in ("new_page", "both") and new_page.get("filename") and new_page.get("content"):
|
||||
directory = new_page.get("directory", "patterns")
|
||||
filename = new_page["filename"]
|
||||
target_rel = f"{directory}/{filename}"
|
||||
dest = _unique_staging_path(STAGING_DIR / target_rel)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = _inject_staging_frontmatter(
|
||||
new_page["content"],
|
||||
source_url=source_url,
|
||||
target_path=target_rel,
|
||||
compilation_notes=notes,
|
||||
modifies=None,
|
||||
)
|
||||
dest.write_text(content)
|
||||
written.append(dest)
|
||||
|
||||
# Update to existing page
|
||||
update_page = result.get("update_page") or {}
|
||||
if action in ("update_page", "both") and update_page.get("path") and update_page.get("content"):
|
||||
target_rel = update_page["path"]
|
||||
dest = _unique_staging_path(STAGING_DIR / target_rel)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
content = _inject_staging_frontmatter(
|
||||
update_page["content"],
|
||||
source_url=source_url,
|
||||
target_path=target_rel,
|
||||
compilation_notes=notes,
|
||||
modifies=target_rel,
|
||||
)
|
||||
dest.write_text(content)
|
||||
written.append(dest)
|
||||
|
||||
return written
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wiki topic coverage check (for C-type URLs)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def wiki_covers_topic(url: str) -> bool:
|
||||
"""Quick heuristic: check if any wiki page mentions terms from the URL path.
|
||||
|
||||
Used for C-type URLs (GitHub issues, SO questions) — only harvest if the
|
||||
wiki already covers the topic.
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
# Derive candidate keywords from path
|
||||
path_terms = [t for t in re.split(r"[/\-_]+", parsed.path.lower()) if len(t) >= 4]
|
||||
if not path_terms:
|
||||
return False
|
||||
|
||||
# Try qmd search if available; otherwise fall back to a simple grep
|
||||
query = " ".join(path_terms[:5])
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["qmd", "search", query, "--json", "-n", "3"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip():
|
||||
try:
|
||||
data = json.loads(result.stdout)
|
||||
hits = data.get("results") if isinstance(data, dict) else data
|
||||
return bool(hits)
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Conversation discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_frontmatter(file_path: Path) -> dict[str, str]:
|
||||
fm: dict[str, str] = {}
|
||||
try:
|
||||
text = file_path.read_text(errors="replace")
|
||||
except OSError:
|
||||
return fm
|
||||
if not text.startswith("---\n"):
|
||||
return fm
|
||||
end = text.find("\n---\n", 4)
|
||||
if end == -1:
|
||||
return fm
|
||||
for line in text[4:end].splitlines():
|
||||
if ":" in line:
|
||||
key, _, value = line.partition(":")
|
||||
fm[key.strip()] = value.strip()
|
||||
return fm
|
||||
|
||||
|
||||
def discover_summarized_conversations(
|
||||
project_filter: str | None = None,
|
||||
file_filter: str | None = None,
|
||||
) -> list[Path]:
|
||||
if file_filter:
|
||||
path = Path(file_filter)
|
||||
if not path.is_absolute():
|
||||
path = WIKI_DIR / path
|
||||
return [path] if path.exists() else []
|
||||
|
||||
files: list[Path] = []
|
||||
for project_dir in sorted(CONVERSATIONS_DIR.iterdir()):
|
||||
if not project_dir.is_dir():
|
||||
continue
|
||||
if project_filter and project_dir.name != project_filter:
|
||||
continue
|
||||
for md in sorted(project_dir.glob("*.md")):
|
||||
fm = parse_frontmatter(md)
|
||||
if fm.get("status") == "summarized":
|
||||
files.append(md)
|
||||
return files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main pipeline
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def process_url(
|
||||
url: str,
|
||||
conversation_file: Path,
|
||||
state: dict[str, Any],
|
||||
dry_run: bool,
|
||||
compile_enabled: bool,
|
||||
) -> str:
|
||||
"""Process a single URL. Returns a short status tag for logging."""
|
||||
|
||||
rel_conv = str(conversation_file.relative_to(WIKI_DIR))
|
||||
today = datetime.now(timezone.utc).date().isoformat()
|
||||
|
||||
# Already harvested?
|
||||
if url in state["harvested_urls"]:
|
||||
entry = state["harvested_urls"][url]
|
||||
if rel_conv not in entry.get("seen_in", []):
|
||||
entry.setdefault("seen_in", []).append(rel_conv)
|
||||
return "dup-harvested"
|
||||
|
||||
# Already rejected by AI?
|
||||
if url in state["rejected_urls"]:
|
||||
return "dup-rejected"
|
||||
|
||||
# Previously skipped?
|
||||
if url in state["skipped_urls"]:
|
||||
return "dup-skipped"
|
||||
|
||||
# Previously failed too many times?
|
||||
if url in state["failed_urls"]:
|
||||
if state["failed_urls"][url].get("attempts", 0) >= MAX_FAILED_ATTEMPTS:
|
||||
return "dup-failed"
|
||||
|
||||
# Classify
|
||||
classification = classify_url(url)
|
||||
if classification == "skip":
|
||||
state["skipped_urls"][url] = {
|
||||
"reason": "domain-skip-list",
|
||||
"first_seen": today,
|
||||
}
|
||||
return "skip-domain"
|
||||
|
||||
if classification == "check":
|
||||
if not wiki_covers_topic(url):
|
||||
state["skipped_urls"][url] = {
|
||||
"reason": "c-type-no-wiki-match",
|
||||
"first_seen": today,
|
||||
}
|
||||
return "skip-c-type"
|
||||
|
||||
if dry_run:
|
||||
return f"would-harvest ({classification})"
|
||||
|
||||
# Fetch
|
||||
print(f" [fetch] {url}")
|
||||
ok, content, method = fetch_cascade(url, conversation_file)
|
||||
time.sleep(FETCH_DELAY_SECONDS)
|
||||
|
||||
if not ok:
|
||||
entry = state["failed_urls"].setdefault(url, {
|
||||
"first_seen": today,
|
||||
"attempts": 0,
|
||||
})
|
||||
entry["attempts"] += 1
|
||||
entry["last_attempt"] = today
|
||||
entry["reason"] = content[:200] if content else "unknown"
|
||||
return f"fetch-failed ({method})"
|
||||
|
||||
# Save raw file
|
||||
raw_path = write_raw_file(url, content, method, conversation_file)
|
||||
rel_raw = str(raw_path.relative_to(WIKI_DIR))
|
||||
|
||||
state["harvested_urls"][url] = {
|
||||
"first_seen": today,
|
||||
"seen_in": [rel_conv],
|
||||
"raw_file": rel_raw,
|
||||
"wiki_pages": [],
|
||||
"status": "raw",
|
||||
"fetch_method": method,
|
||||
"last_checked": today,
|
||||
}
|
||||
|
||||
# Compile via claude -p
|
||||
if compile_enabled:
|
||||
print(f" [compile] {rel_raw}")
|
||||
result = call_claude_compile(raw_path, content, conversation_file)
|
||||
if result is None:
|
||||
state["harvested_urls"][url]["status"] = "raw-compile-failed"
|
||||
return f"raw-saved ({method}) compile-failed"
|
||||
|
||||
action = result.get("action", "skip")
|
||||
if action == "skip":
|
||||
state["rejected_urls"][url] = {
|
||||
"reason": result.get("compilation_notes", "AI rejected"),
|
||||
"rejected_date": today,
|
||||
}
|
||||
# Remove from harvested; keep raw file for audit
|
||||
state["harvested_urls"].pop(url, None)
|
||||
return f"rejected ({method})"
|
||||
|
||||
written = apply_compile_result(result, url, raw_path)
|
||||
state["harvested_urls"][url]["status"] = "compiled"
|
||||
state["harvested_urls"][url]["wiki_pages"] = [
|
||||
str(p.relative_to(WIKI_DIR)) for p in written
|
||||
]
|
||||
return f"compiled ({method}) → {len(written)} staging file(s)"
|
||||
|
||||
return f"raw-saved ({method})"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--project", help="Only process this project (wing) directory")
|
||||
parser.add_argument("--file", help="Only process this conversation file")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Classify and report without fetching")
|
||||
parser.add_argument("--no-compile", action="store_true", help="Fetch raw only; skip claude -p compile")
|
||||
parser.add_argument("--limit", type=int, default=0, help="Stop after N new URLs processed (0 = no limit)")
|
||||
args = parser.parse_args()
|
||||
|
||||
files = discover_summarized_conversations(args.project, args.file)
|
||||
print(f"Scanning {len(files)} summarized conversation(s) for URLs...")
|
||||
|
||||
state = load_state()
|
||||
stats: dict[str, int] = {}
|
||||
processed_new = 0
|
||||
|
||||
for file_path in files:
|
||||
urls = extract_urls_from_file(file_path)
|
||||
if not urls:
|
||||
continue
|
||||
rel = file_path.relative_to(WIKI_DIR)
|
||||
print(f"\n[{rel}] {len(urls)} URL(s)")
|
||||
|
||||
for url in urls:
|
||||
status = process_url(
|
||||
url,
|
||||
file_path,
|
||||
state,
|
||||
dry_run=args.dry_run,
|
||||
compile_enabled=not args.no_compile,
|
||||
)
|
||||
stats[status] = stats.get(status, 0) + 1
|
||||
print(f" [{status}] {url}")
|
||||
|
||||
# Persist state after each non-dry URL
|
||||
if not args.dry_run and not status.startswith("dup-"):
|
||||
processed_new += 1
|
||||
save_state(state)
|
||||
|
||||
if args.limit and processed_new >= args.limit:
|
||||
print(f"\nLimit reached ({args.limit}); stopping.")
|
||||
save_state(state)
|
||||
_print_summary(stats)
|
||||
return 0
|
||||
|
||||
if not args.dry_run:
|
||||
save_state(state)
|
||||
|
||||
_print_summary(stats)
|
||||
return 0
|
||||
|
||||
|
||||
def _print_summary(stats: dict[str, int]) -> None:
|
||||
print("\nSummary:")
|
||||
for status, count in sorted(stats.items()):
|
||||
print(f" {status}: {count}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1587
scripts/wiki-hygiene.py
Executable file
1587
scripts/wiki-hygiene.py
Executable file
File diff suppressed because it is too large
Load Diff
198
scripts/wiki-maintain.sh
Executable file
198
scripts/wiki-maintain.sh
Executable file
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# wiki-maintain.sh — Top-level orchestrator for wiki maintenance.
|
||||
#
|
||||
# Chains the three maintenance scripts in the correct order:
|
||||
# 1. wiki-harvest.py (URL harvesting from summarized conversations)
|
||||
# 2. wiki-hygiene.py (quick or full hygiene checks)
|
||||
# 3. qmd update && qmd embed (reindex after changes)
|
||||
#
|
||||
# Usage:
|
||||
# wiki-maintain.sh # Harvest + quick hygiene
|
||||
# wiki-maintain.sh --full # Harvest + full hygiene (LLM-powered)
|
||||
# wiki-maintain.sh --harvest-only # URL harvesting only
|
||||
# wiki-maintain.sh --hygiene-only # Quick hygiene only
|
||||
# wiki-maintain.sh --hygiene-only --full # Full hygiene only
|
||||
# wiki-maintain.sh --dry-run # Show what would run (no writes)
|
||||
# wiki-maintain.sh --no-compile # Harvest without claude -p compilation step
|
||||
# wiki-maintain.sh --no-reindex # Skip qmd update/embed after
|
||||
#
|
||||
# Log file: scripts/.maintain.log (rotated manually)
|
||||
|
||||
# Resolve script location first so we can find sibling scripts regardless of
|
||||
# how WIKI_DIR is set. WIKI_DIR defaults to the parent of scripts/ but may be
|
||||
# overridden for tests or alternate installs.
|
||||
SCRIPTS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WIKI_DIR="${WIKI_DIR:-$(dirname "${SCRIPTS_DIR}")}"
|
||||
LOG_FILE="${SCRIPTS_DIR}/.maintain.log"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Argument parsing
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
FULL_MODE=false
|
||||
HARVEST_ONLY=false
|
||||
HYGIENE_ONLY=false
|
||||
DRY_RUN=false
|
||||
NO_COMPILE=false
|
||||
NO_REINDEX=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--full) FULL_MODE=true; shift ;;
|
||||
--harvest-only) HARVEST_ONLY=true; shift ;;
|
||||
--hygiene-only) HYGIENE_ONLY=true; shift ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
--no-compile) NO_COMPILE=true; shift ;;
|
||||
--no-reindex) NO_REINDEX=true; shift ;;
|
||||
-h|--help)
|
||||
sed -n '3,20p' "$0" | sed 's/^# \?//'
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "${HARVEST_ONLY}" == "true" && "${HYGIENE_ONLY}" == "true" ]]; then
|
||||
echo "--harvest-only and --hygiene-only are mutually exclusive" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Logging
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
log() {
|
||||
local ts
|
||||
ts="$(date '+%Y-%m-%d %H:%M:%S')"
|
||||
printf '[%s] %s\n' "${ts}" "$*"
|
||||
}
|
||||
|
||||
section() {
|
||||
echo ""
|
||||
log "━━━ $* ━━━"
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Sanity checks
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
if [[ ! -d "${WIKI_DIR}" ]]; then
|
||||
echo "Wiki directory not found: ${WIKI_DIR}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
for req in python3 qmd; do
|
||||
if ! command -v "${req}" >/dev/null 2>&1; then
|
||||
if [[ "${req}" == "qmd" && "${NO_REINDEX}" == "true" ]]; then
|
||||
continue # qmd not required if --no-reindex
|
||||
fi
|
||||
echo "Required command not found: ${req}" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Pipeline
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
START_TS="$(date '+%s')"
|
||||
section "wiki-maintain.sh starting"
|
||||
log "mode: $(${FULL_MODE} && echo full || echo quick)"
|
||||
log "harvest: $(${HYGIENE_ONLY} && echo skipped || echo enabled)"
|
||||
log "hygiene: $(${HARVEST_ONLY} && echo skipped || echo enabled)"
|
||||
log "reindex: $(${NO_REINDEX} && echo skipped || echo enabled)"
|
||||
log "dry-run: ${DRY_RUN}"
|
||||
log "wiki: ${WIKI_DIR}"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Phase 1: Harvest
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
if [[ "${HYGIENE_ONLY}" != "true" ]]; then
|
||||
section "Phase 1: URL harvesting"
|
||||
harvest_args=()
|
||||
${DRY_RUN} && harvest_args+=(--dry-run)
|
||||
${NO_COMPILE} && harvest_args+=(--no-compile)
|
||||
|
||||
if python3 "${SCRIPTS_DIR}/wiki-harvest.py" "${harvest_args[@]}"; then
|
||||
log "harvest completed"
|
||||
else
|
||||
log "[error] harvest failed (exit $?) — continuing to hygiene"
|
||||
fi
|
||||
else
|
||||
section "Phase 1: URL harvesting (skipped)"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Phase 2: Hygiene
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
if [[ "${HARVEST_ONLY}" != "true" ]]; then
|
||||
section "Phase 2: Hygiene checks"
|
||||
hygiene_args=()
|
||||
if ${FULL_MODE}; then
|
||||
hygiene_args+=(--full)
|
||||
fi
|
||||
${DRY_RUN} && hygiene_args+=(--dry-run)
|
||||
|
||||
if python3 "${SCRIPTS_DIR}/wiki-hygiene.py" "${hygiene_args[@]}"; then
|
||||
log "hygiene completed"
|
||||
else
|
||||
log "[error] hygiene failed (exit $?) — continuing to reindex"
|
||||
fi
|
||||
else
|
||||
section "Phase 2: Hygiene checks (skipped)"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Phase 3: qmd reindex
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
if [[ "${NO_REINDEX}" != "true" && "${DRY_RUN}" != "true" ]]; then
|
||||
section "Phase 3: qmd reindex"
|
||||
|
||||
if qmd update 2>&1 | sed 's/^/ /'; then
|
||||
log "qmd update completed"
|
||||
else
|
||||
log "[error] qmd update failed (exit $?)"
|
||||
fi
|
||||
|
||||
if qmd embed 2>&1 | sed 's/^/ /'; then
|
||||
log "qmd embed completed"
|
||||
else
|
||||
log "[warn] qmd embed failed or produced warnings"
|
||||
fi
|
||||
else
|
||||
section "Phase 3: qmd reindex (skipped)"
|
||||
fi
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Summary
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
END_TS="$(date '+%s')"
|
||||
DURATION=$((END_TS - START_TS))
|
||||
section "wiki-maintain.sh finished in ${DURATION}s"
|
||||
|
||||
# Report the most recent hygiene reports, if any. Use `if` statements (not
|
||||
# `[[ ]] && action`) because under `set -e` a false test at end-of-script
|
||||
# becomes the process exit status.
|
||||
if [[ -d "${WIKI_DIR}/reports" ]]; then
|
||||
latest_fixed="$(ls -t "${WIKI_DIR}"/reports/hygiene-*-fixed.md 2>/dev/null | head -n 1 || true)"
|
||||
latest_review="$(ls -t "${WIKI_DIR}"/reports/hygiene-*-needs-review.md 2>/dev/null | head -n 1 || true)"
|
||||
if [[ -n "${latest_fixed}" ]]; then
|
||||
log "latest fixed report: $(basename "${latest_fixed}")"
|
||||
fi
|
||||
if [[ -n "${latest_review}" ]]; then
|
||||
log "latest review report: $(basename "${latest_review}")"
|
||||
fi
|
||||
fi
|
||||
|
||||
exit 0
|
||||
639
scripts/wiki-staging.py
Executable file
639
scripts/wiki-staging.py
Executable file
@@ -0,0 +1,639 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Human-in-the-loop staging pipeline for wiki content.
|
||||
|
||||
Pure file operations — no LLM calls. Moves pages between staging/ and the live
|
||||
wiki, updates indexes, rewrites cross-references, and tracks rejections in
|
||||
.harvest-state.json.
|
||||
|
||||
Usage:
|
||||
python3 scripts/wiki-staging.py --list # List pending items
|
||||
python3 scripts/wiki-staging.py --list --json # JSON output
|
||||
python3 scripts/wiki-staging.py --stats # Summary by type and age
|
||||
python3 scripts/wiki-staging.py --promote PATH # Approve one page
|
||||
python3 scripts/wiki-staging.py --reject PATH --reason "..." # Reject with reason
|
||||
python3 scripts/wiki-staging.py --promote-all # Approve everything
|
||||
python3 scripts/wiki-staging.py --review # Interactive approval loop
|
||||
python3 scripts/wiki-staging.py --sync # Rebuild staging/index.md
|
||||
|
||||
PATH may be relative to the wiki root (e.g. `staging/patterns/foo.md`) or absolute.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Import shared helpers
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
from wiki_lib import ( # noqa: E402
|
||||
ARCHIVE_DIR,
|
||||
CONVERSATIONS_DIR,
|
||||
HARVEST_STATE_FILE,
|
||||
INDEX_FILE,
|
||||
LIVE_CONTENT_DIRS,
|
||||
REPORTS_DIR,
|
||||
STAGING_DIR,
|
||||
STAGING_INDEX,
|
||||
WIKI_DIR,
|
||||
WikiPage,
|
||||
iter_live_pages,
|
||||
iter_staging_pages,
|
||||
parse_date,
|
||||
parse_page,
|
||||
today,
|
||||
write_page,
|
||||
)
|
||||
|
||||
sys.stdout.reconfigure(line_buffering=True)
|
||||
sys.stderr.reconfigure(line_buffering=True)
|
||||
|
||||
# Fields stripped from frontmatter on promotion (staging-only metadata)
|
||||
STAGING_ONLY_FIELDS = [
|
||||
"status",
|
||||
"staged_date",
|
||||
"staged_by",
|
||||
"target_path",
|
||||
"modifies",
|
||||
"compilation_notes",
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Discovery
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def list_pending() -> list[WikiPage]:
|
||||
pages = [p for p in iter_staging_pages() if p.path.name != "index.md"]
|
||||
return pages
|
||||
|
||||
|
||||
def page_summary(page: WikiPage) -> dict[str, Any]:
|
||||
rel = str(page.path.relative_to(WIKI_DIR))
|
||||
fm = page.frontmatter
|
||||
target = fm.get("target_path") or _infer_target_path(page)
|
||||
staged = parse_date(fm.get("staged_date"))
|
||||
age = (today() - staged).days if staged else None
|
||||
return {
|
||||
"path": rel,
|
||||
"title": fm.get("title", page.path.stem),
|
||||
"type": fm.get("type", _infer_type(page)),
|
||||
"status": fm.get("status", "pending"),
|
||||
"origin": fm.get("origin", "automated"),
|
||||
"staged_by": fm.get("staged_by", "unknown"),
|
||||
"staged_date": str(staged) if staged else None,
|
||||
"age_days": age,
|
||||
"target_path": target,
|
||||
"modifies": fm.get("modifies"),
|
||||
"compilation_notes": fm.get("compilation_notes", ""),
|
||||
}
|
||||
|
||||
|
||||
def _infer_target_path(page: WikiPage) -> str:
|
||||
"""Derive a target path when target_path isn't set in frontmatter."""
|
||||
try:
|
||||
rel = page.path.relative_to(STAGING_DIR)
|
||||
except ValueError:
|
||||
return str(page.path.relative_to(WIKI_DIR))
|
||||
return str(rel)
|
||||
|
||||
|
||||
def _infer_type(page: WikiPage) -> str:
|
||||
"""Infer type from the directory name when frontmatter doesn't specify it."""
|
||||
parts = page.path.relative_to(STAGING_DIR).parts
|
||||
if len(parts) >= 2 and parts[0] in LIVE_CONTENT_DIRS:
|
||||
return parts[0].rstrip("s") # 'patterns' → 'pattern'
|
||||
return "unknown"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main index update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _remove_from_main_index(rel_path: str) -> None:
|
||||
if not INDEX_FILE.exists():
|
||||
return
|
||||
text = INDEX_FILE.read_text()
|
||||
lines = text.splitlines(keepends=True)
|
||||
pattern = re.compile(rf"^- \[.+\]\({re.escape(rel_path)}\) ")
|
||||
new_lines = [line for line in lines if not pattern.match(line)]
|
||||
if len(new_lines) != len(lines):
|
||||
INDEX_FILE.write_text("".join(new_lines))
|
||||
|
||||
|
||||
def _add_to_main_index(rel_path: str, title: str, summary: str = "") -> None:
|
||||
"""Append a new entry under the appropriate section. Best-effort — operator may re-order later."""
|
||||
if not INDEX_FILE.exists():
|
||||
return
|
||||
text = INDEX_FILE.read_text()
|
||||
# Avoid duplicates
|
||||
if f"]({rel_path})" in text:
|
||||
return
|
||||
entry = f"- [{title}]({rel_path})"
|
||||
if summary:
|
||||
entry += f" — {summary}"
|
||||
entry += "\n"
|
||||
# Insert at the end of the first matching section
|
||||
ptype = rel_path.split("/")[0]
|
||||
section_headers = {
|
||||
"patterns": "## Patterns",
|
||||
"decisions": "## Decisions",
|
||||
"concepts": "## Concepts",
|
||||
"environments": "## Environments",
|
||||
}
|
||||
header = section_headers.get(ptype)
|
||||
if header and header in text:
|
||||
# Find the header and append before the next ## header or EOF
|
||||
idx = text.find(header)
|
||||
next_header = text.find("\n## ", idx + len(header))
|
||||
if next_header == -1:
|
||||
next_header = len(text)
|
||||
# Find the last non-empty line in the section
|
||||
section = text[idx:next_header]
|
||||
last_nl = section.rfind("\n", 0, len(section) - 1) + 1
|
||||
INDEX_FILE.write_text(text[: idx + last_nl] + entry + text[idx + last_nl :])
|
||||
else:
|
||||
INDEX_FILE.write_text(text.rstrip() + "\n" + entry)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Staging index update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def regenerate_staging_index() -> None:
|
||||
STAGING_DIR.mkdir(parents=True, exist_ok=True)
|
||||
pending = list_pending()
|
||||
|
||||
lines = [
|
||||
"# Staging — Pending Wiki Content",
|
||||
"",
|
||||
"Content awaiting human review. These pages were generated by automated scripts",
|
||||
"and need approval before joining the live wiki.",
|
||||
"",
|
||||
"**Review options**:",
|
||||
"- Browse in Obsidian and move files manually (then run `scripts/wiki-staging.py --sync`)",
|
||||
"- Run `python3 scripts/wiki-staging.py --list` for a summary",
|
||||
"- Start a Claude session: \"let's review what's in staging\"",
|
||||
"",
|
||||
f"**{len(pending)} pending item(s)** as of {today().isoformat()}",
|
||||
"",
|
||||
"## Pending Items",
|
||||
"",
|
||||
]
|
||||
|
||||
if not pending:
|
||||
lines.append("_No pending items._")
|
||||
else:
|
||||
lines.append("| Page | Type | Source | Staged | Age | Target |")
|
||||
lines.append("|------|------|--------|--------|-----|--------|")
|
||||
for page in pending:
|
||||
s = page_summary(page)
|
||||
title = s["title"]
|
||||
rel_in_staging = str(page.path.relative_to(STAGING_DIR))
|
||||
age = f"{s['age_days']}d" if s["age_days"] is not None else "—"
|
||||
staged = s["staged_date"] or "—"
|
||||
lines.append(
|
||||
f"| [{title}]({rel_in_staging}) | {s['type']} | "
|
||||
f"{s['staged_by']} | {staged} | {age} | `{s['target_path']}` |"
|
||||
)
|
||||
|
||||
STAGING_INDEX.write_text("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cross-reference rewriting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _rewrite_cross_references(old_path: str, new_path: str) -> int:
|
||||
"""Rewrite links and `related:` entries across the wiki."""
|
||||
targets: list[Path] = [INDEX_FILE]
|
||||
for sub in LIVE_CONTENT_DIRS:
|
||||
targets.extend((WIKI_DIR / sub).glob("*.md"))
|
||||
if STAGING_DIR.exists():
|
||||
for sub in LIVE_CONTENT_DIRS:
|
||||
targets.extend((STAGING_DIR / sub).glob("*.md"))
|
||||
if ARCHIVE_DIR.exists():
|
||||
for sub in LIVE_CONTENT_DIRS:
|
||||
targets.extend((ARCHIVE_DIR / sub).glob("*.md"))
|
||||
|
||||
count = 0
|
||||
old_esc = re.escape(old_path)
|
||||
link_patterns = [
|
||||
(re.compile(rf"\]\({old_esc}\)"), f"]({new_path})"),
|
||||
(re.compile(rf"\]\(\.\./{old_esc}\)"), f"](../{new_path})"),
|
||||
]
|
||||
related_patterns = [
|
||||
(re.compile(rf"^(\s*-\s*){old_esc}$", re.MULTILINE), rf"\g<1>{new_path}"),
|
||||
]
|
||||
for target in targets:
|
||||
if not target.exists():
|
||||
continue
|
||||
try:
|
||||
text = target.read_text()
|
||||
except OSError:
|
||||
continue
|
||||
new_text = text
|
||||
for pat, repl in link_patterns + related_patterns:
|
||||
new_text = pat.sub(repl, new_text)
|
||||
if new_text != text:
|
||||
target.write_text(new_text)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Promote
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def promote(page: WikiPage, dry_run: bool = False) -> Path | None:
|
||||
summary = page_summary(page)
|
||||
target_rel = summary["target_path"]
|
||||
target_path = WIKI_DIR / target_rel
|
||||
|
||||
modifies = summary["modifies"]
|
||||
if modifies:
|
||||
# This is an update to an existing page. Merge: keep staging content,
|
||||
# preserve the live page's origin if it was manual.
|
||||
live_path = WIKI_DIR / modifies
|
||||
if not live_path.exists():
|
||||
print(
|
||||
f" [warn] modifies target {modifies} does not exist — treating as new page",
|
||||
file=sys.stderr,
|
||||
)
|
||||
modifies = None
|
||||
else:
|
||||
live_page = parse_page(live_path)
|
||||
if live_page:
|
||||
# Warn if live page has been updated since staging
|
||||
live_compiled = parse_date(live_page.frontmatter.get("last_compiled"))
|
||||
staged = parse_date(page.frontmatter.get("staged_date"))
|
||||
if live_compiled and staged and live_compiled > staged:
|
||||
print(
|
||||
f" [warn] live page {modifies} was updated ({live_compiled}) "
|
||||
f"after staging ({staged}) — human should verify merge",
|
||||
file=sys.stderr,
|
||||
)
|
||||
# Preserve origin from live if it was manual
|
||||
if live_page.frontmatter.get("origin") == "manual":
|
||||
page.frontmatter["origin"] = "manual"
|
||||
|
||||
rel_src = str(page.path.relative_to(WIKI_DIR))
|
||||
|
||||
if dry_run:
|
||||
action = "update" if modifies else "new page"
|
||||
print(f" [dry-run] promote {rel_src} → {target_rel} ({action})")
|
||||
return target_path
|
||||
|
||||
# Clean frontmatter — strip staging-only fields
|
||||
new_fm = {k: v for k, v in page.frontmatter.items() if k not in STAGING_ONLY_FIELDS}
|
||||
new_fm.setdefault("origin", "automated")
|
||||
new_fm["last_verified"] = today().isoformat()
|
||||
if "last_compiled" not in new_fm:
|
||||
new_fm["last_compiled"] = today().isoformat()
|
||||
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
old_path = page.path
|
||||
page.path = target_path
|
||||
page.frontmatter = new_fm
|
||||
write_page(page)
|
||||
old_path.unlink()
|
||||
|
||||
# Rewrite cross-references: staging/... → target_rel
|
||||
rel_staging = str(old_path.relative_to(WIKI_DIR))
|
||||
_rewrite_cross_references(rel_staging, target_rel)
|
||||
|
||||
# Update main index
|
||||
summary_text = page.body.strip().splitlines()[0] if page.body.strip() else ""
|
||||
_add_to_main_index(target_rel, new_fm.get("title", page.path.stem), summary_text[:120])
|
||||
|
||||
# Regenerate staging index
|
||||
regenerate_staging_index()
|
||||
|
||||
# Log to hygiene report (append a line)
|
||||
_append_log(f"promote | {rel_staging} → {target_rel}" + (f" (modifies {modifies})" if modifies else ""))
|
||||
return target_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reject
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def reject(page: WikiPage, reason: str, dry_run: bool = False) -> None:
|
||||
rel = str(page.path.relative_to(WIKI_DIR))
|
||||
if dry_run:
|
||||
print(f" [dry-run] reject {rel} — {reason}")
|
||||
return
|
||||
|
||||
# Record in harvest-state if this came from URL harvesting
|
||||
_record_rejection_in_harvest_state(page, reason)
|
||||
|
||||
# Delete the file
|
||||
page.path.unlink()
|
||||
|
||||
# Regenerate staging index
|
||||
regenerate_staging_index()
|
||||
|
||||
_append_log(f"reject | {rel} — {reason}")
|
||||
print(f" [rejected] {rel}")
|
||||
|
||||
|
||||
def _record_rejection_in_harvest_state(page: WikiPage, reason: str) -> None:
|
||||
"""If the staged page came from wiki-harvest, add the source URL to rejected_urls."""
|
||||
if not HARVEST_STATE_FILE.exists():
|
||||
return
|
||||
# Look for the source URL in frontmatter (harvest_source) or in sources field
|
||||
source_url = page.frontmatter.get("harvest_source")
|
||||
if not source_url:
|
||||
sources = page.frontmatter.get("sources") or []
|
||||
if isinstance(sources, list):
|
||||
for src in sources:
|
||||
src_str = str(src)
|
||||
# If src is a raw/harvested/... file, look up its source_url
|
||||
if "raw/harvested/" in src_str:
|
||||
raw_path = WIKI_DIR / src_str
|
||||
if raw_path.exists():
|
||||
raw_page = parse_page(raw_path)
|
||||
if raw_page:
|
||||
source_url = raw_page.frontmatter.get("source_url")
|
||||
break
|
||||
|
||||
if not source_url:
|
||||
return
|
||||
|
||||
try:
|
||||
with open(HARVEST_STATE_FILE) as f:
|
||||
state = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return
|
||||
|
||||
state.setdefault("rejected_urls", {})[source_url] = {
|
||||
"reason": reason,
|
||||
"rejected_date": today().isoformat(),
|
||||
}
|
||||
# Remove from harvested_urls if present
|
||||
state.get("harvested_urls", {}).pop(source_url, None)
|
||||
|
||||
with open(HARVEST_STATE_FILE, "w") as f:
|
||||
json.dump(state, f, indent=2, sort_keys=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _append_log(line: str) -> None:
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
log = REPORTS_DIR / f"staging-{today().isoformat()}.log"
|
||||
with open(log, "a") as f:
|
||||
f.write(f"{line}\n")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Path resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def resolve_page(raw_path: str) -> WikiPage | None:
|
||||
path = Path(raw_path)
|
||||
if not path.is_absolute():
|
||||
# Accept "staging/..." or just "patterns/foo.md" (assumes staging)
|
||||
if not raw_path.startswith("staging/") and raw_path.split("/", 1)[0] in LIVE_CONTENT_DIRS:
|
||||
path = STAGING_DIR / raw_path
|
||||
else:
|
||||
path = WIKI_DIR / raw_path
|
||||
if not path.exists():
|
||||
print(f" [error] not found: {path}", file=sys.stderr)
|
||||
return None
|
||||
return parse_page(path)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Commands
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cmd_list(as_json: bool = False) -> int:
|
||||
pending = list_pending()
|
||||
if as_json:
|
||||
data = [page_summary(p) for p in pending]
|
||||
print(json.dumps(data, indent=2))
|
||||
return 0
|
||||
|
||||
if not pending:
|
||||
print("No pending items in staging.")
|
||||
return 0
|
||||
|
||||
print(f"{len(pending)} pending item(s):\n")
|
||||
for p in pending:
|
||||
s = page_summary(p)
|
||||
age = f"{s['age_days']}d" if s["age_days"] is not None else "—"
|
||||
marker = " (update)" if s["modifies"] else ""
|
||||
print(f" {s['path']}{marker}")
|
||||
print(f" title: {s['title']}")
|
||||
print(f" type: {s['type']}")
|
||||
print(f" source: {s['staged_by']}")
|
||||
print(f" staged: {s['staged_date']} ({age} old)")
|
||||
print(f" target: {s['target_path']}")
|
||||
if s["modifies"]:
|
||||
print(f" modifies: {s['modifies']}")
|
||||
if s["compilation_notes"]:
|
||||
notes = s["compilation_notes"][:100]
|
||||
print(f" notes: {notes}")
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_stats() -> int:
|
||||
pending = list_pending()
|
||||
total = len(pending)
|
||||
if total == 0:
|
||||
print("No pending items in staging.")
|
||||
return 0
|
||||
|
||||
by_type: dict[str, int] = {}
|
||||
by_source: dict[str, int] = {}
|
||||
ages: list[int] = []
|
||||
updates = 0
|
||||
|
||||
for p in pending:
|
||||
s = page_summary(p)
|
||||
by_type[s["type"]] = by_type.get(s["type"], 0) + 1
|
||||
by_source[s["staged_by"]] = by_source.get(s["staged_by"], 0) + 1
|
||||
if s["age_days"] is not None:
|
||||
ages.append(s["age_days"])
|
||||
if s["modifies"]:
|
||||
updates += 1
|
||||
|
||||
print(f"Total pending: {total}")
|
||||
print(f"Updates (modifies existing): {updates}")
|
||||
print(f"New pages: {total - updates}")
|
||||
print()
|
||||
print("By type:")
|
||||
for t, n in sorted(by_type.items()):
|
||||
print(f" {t}: {n}")
|
||||
print()
|
||||
print("By source:")
|
||||
for s, n in sorted(by_source.items()):
|
||||
print(f" {s}: {n}")
|
||||
if ages:
|
||||
print()
|
||||
print(f"Age (days): min={min(ages)}, max={max(ages)}, avg={sum(ages)//len(ages)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_promote(path_arg: str, dry_run: bool) -> int:
|
||||
page = resolve_page(path_arg)
|
||||
if not page:
|
||||
return 1
|
||||
result = promote(page, dry_run=dry_run)
|
||||
if result and not dry_run:
|
||||
print(f" [promoted] {result.relative_to(WIKI_DIR)}")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_reject(path_arg: str, reason: str, dry_run: bool) -> int:
|
||||
page = resolve_page(path_arg)
|
||||
if not page:
|
||||
return 1
|
||||
reject(page, reason, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_promote_all(dry_run: bool) -> int:
|
||||
pending = list_pending()
|
||||
if not pending:
|
||||
print("No pending items.")
|
||||
return 0
|
||||
print(f"Promoting {len(pending)} page(s)...")
|
||||
for p in pending:
|
||||
promote(p, dry_run=dry_run)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_review() -> int:
|
||||
"""Interactive review loop. Prompts approve/reject/skip for each pending item."""
|
||||
pending = list_pending()
|
||||
if not pending:
|
||||
print("No pending items.")
|
||||
return 0
|
||||
|
||||
print(f"Reviewing {len(pending)} pending item(s). (a)pprove / (r)eject / (s)kip / (q)uit\n")
|
||||
for p in pending:
|
||||
s = page_summary(p)
|
||||
print(f"━━━ {s['path']} ━━━")
|
||||
print(f" {s['title']} ({s['type']})")
|
||||
print(f" from: {s['staged_by']} ({s['staged_date']})")
|
||||
print(f" target: {s['target_path']}")
|
||||
if s["modifies"]:
|
||||
print(f" updates: {s['modifies']}")
|
||||
if s["compilation_notes"]:
|
||||
print(f" notes: {s['compilation_notes'][:150]}")
|
||||
# Show first few lines of body
|
||||
first_lines = [ln for ln in p.body.strip().splitlines() if ln.strip()][:3]
|
||||
for ln in first_lines:
|
||||
print(f" │ {ln[:100]}")
|
||||
print()
|
||||
|
||||
while True:
|
||||
try:
|
||||
answer = input(" [a/r/s/q] > ").strip().lower()
|
||||
except EOFError:
|
||||
return 0
|
||||
if answer in ("a", "approve"):
|
||||
promote(p)
|
||||
break
|
||||
if answer in ("r", "reject"):
|
||||
try:
|
||||
reason = input(" reason > ").strip()
|
||||
except EOFError:
|
||||
return 0
|
||||
reject(p, reason or "no reason given")
|
||||
break
|
||||
if answer in ("s", "skip"):
|
||||
break
|
||||
if answer in ("q", "quit"):
|
||||
return 0
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_sync() -> int:
|
||||
"""Reconcile staging index after manual operations (Obsidian moves, deletions).
|
||||
|
||||
Also detects pages that were manually moved out of staging without going through
|
||||
the promotion flow and reports them.
|
||||
"""
|
||||
print("Regenerating staging index...")
|
||||
regenerate_staging_index()
|
||||
|
||||
# Detect pages in live directories with status: pending (manual promotion without cleanup)
|
||||
leaked: list[Path] = []
|
||||
for page in iter_live_pages():
|
||||
if str(page.frontmatter.get("status", "")) == "pending":
|
||||
leaked.append(page.path)
|
||||
|
||||
if leaked:
|
||||
print("\n[warn] live pages still marked status: pending — fix manually:")
|
||||
for p in leaked:
|
||||
print(f" {p.relative_to(WIKI_DIR)}")
|
||||
|
||||
pending = list_pending()
|
||||
print(f"\n{len(pending)} pending item(s) in staging.")
|
||||
return 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Wiki staging pipeline")
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument("--list", action="store_true", help="List pending items")
|
||||
group.add_argument("--stats", action="store_true", help="Summary stats")
|
||||
group.add_argument("--promote", metavar="PATH", help="Approve a pending page")
|
||||
group.add_argument("--reject", metavar="PATH", help="Reject a pending page")
|
||||
group.add_argument("--promote-all", action="store_true", help="Promote every pending page")
|
||||
group.add_argument("--review", action="store_true", help="Interactive approval loop")
|
||||
group.add_argument("--sync", action="store_true", help="Regenerate staging index & detect drift")
|
||||
|
||||
parser.add_argument("--json", action="store_true", help="JSON output for --list")
|
||||
parser.add_argument("--reason", default="", help="Rejection reason for --reject")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Show what would happen")
|
||||
args = parser.parse_args()
|
||||
|
||||
STAGING_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if args.list:
|
||||
return cmd_list(as_json=args.json)
|
||||
if args.stats:
|
||||
return cmd_stats()
|
||||
if args.promote:
|
||||
return cmd_promote(args.promote, args.dry_run)
|
||||
if args.reject:
|
||||
if not args.reason:
|
||||
print("--reject requires --reason", file=sys.stderr)
|
||||
return 2
|
||||
return cmd_reject(args.reject, args.reason, args.dry_run)
|
||||
if args.promote_all:
|
||||
return cmd_promote_all(args.dry_run)
|
||||
if args.review:
|
||||
return cmd_review()
|
||||
if args.sync:
|
||||
return cmd_sync()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
230
scripts/wiki-sync.sh
Executable file
230
scripts/wiki-sync.sh
Executable file
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# wiki-sync.sh — Auto-commit, pull, resolve conflicts, push, reindex
|
||||
#
|
||||
# Designed to run via cron on both work and home machines.
|
||||
# Safe to run frequently — no-ops when nothing has changed.
|
||||
#
|
||||
# Usage:
|
||||
# wiki-sync.sh # Full sync (commit + pull + push + reindex)
|
||||
# wiki-sync.sh --commit # Only commit local changes
|
||||
# wiki-sync.sh --pull # Only pull remote changes
|
||||
# wiki-sync.sh --push # Only push local commits
|
||||
# wiki-sync.sh --reindex # Only rebuild qmd index
|
||||
# wiki-sync.sh --status # Show sync status (no changes)
|
||||
|
||||
WIKI_DIR="${WIKI_DIR:-${HOME}/projects/wiki}"
|
||||
LOG_FILE="${WIKI_DIR}/scripts/.sync.log"
|
||||
LOCK_FILE="/tmp/wiki-sync.lock"
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
log() {
|
||||
local msg
|
||||
msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||||
echo "${msg}" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
die() {
|
||||
log "ERROR: $*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
if [[ -f "${LOCK_FILE}" ]]; then
|
||||
local pid
|
||||
pid=$(cat "${LOCK_FILE}" 2>/dev/null || echo "")
|
||||
if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
|
||||
die "Another sync is running (pid ${pid})"
|
||||
fi
|
||||
rm -f "${LOCK_FILE}"
|
||||
fi
|
||||
echo $$ > "${LOCK_FILE}"
|
||||
trap 'rm -f "${LOCK_FILE}"' EXIT
|
||||
}
|
||||
|
||||
# --- Operations ---
|
||||
|
||||
do_commit() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
# Check for uncommitted changes (staged + unstaged + untracked)
|
||||
if git diff --quiet && git diff --cached --quiet && [[ -z "$(git ls-files --others --exclude-standard)" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local hostname
|
||||
hostname=$(hostname -s 2>/dev/null || echo "unknown")
|
||||
|
||||
git add -A
|
||||
git commit -m "$(cat <<EOF
|
||||
wiki: auto-sync from ${hostname}
|
||||
|
||||
Automatic commit of wiki changes detected by cron.
|
||||
EOF
|
||||
)" 2>/dev/null || true
|
||||
|
||||
log "Committed local changes from ${hostname}"
|
||||
}
|
||||
|
||||
do_pull() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
# Fetch first to check if there's anything to pull
|
||||
git fetch origin main 2>/dev/null || die "Failed to fetch from origin"
|
||||
|
||||
local local_head remote_head
|
||||
local_head=$(git rev-parse HEAD)
|
||||
remote_head=$(git rev-parse origin/main)
|
||||
|
||||
if [[ "${local_head}" == "${remote_head}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Pull with rebase to keep history linear
|
||||
# If conflicts occur, resolve markdown files by keeping both sides
|
||||
if ! git pull --rebase origin main 2>/dev/null; then
|
||||
log "Conflicts detected, attempting auto-resolution..."
|
||||
resolve_conflicts
|
||||
fi
|
||||
|
||||
log "Pulled remote changes"
|
||||
}
|
||||
|
||||
resolve_conflicts() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
local conflicted
|
||||
conflicted=$(git diff --name-only --diff-filter=U 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "${conflicted}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
if [[ "${file}" == *.md ]]; then
|
||||
# For markdown: accept both sides (union merge)
|
||||
# Remove conflict markers, keep all content
|
||||
if [[ -f "${file}" ]]; then
|
||||
sed -i.bak \
|
||||
-e '/^<<<<<<< /d' \
|
||||
-e '/^=======/d' \
|
||||
-e '/^>>>>>>> /d' \
|
||||
"${file}"
|
||||
rm -f "${file}.bak"
|
||||
git add "${file}"
|
||||
log "Auto-resolved conflict in ${file} (kept both sides)"
|
||||
fi
|
||||
else
|
||||
# For non-markdown: keep ours (local version wins)
|
||||
git checkout --ours "${file}" 2>/dev/null
|
||||
git add "${file}"
|
||||
log "Auto-resolved conflict in ${file} (kept local)"
|
||||
fi
|
||||
done <<< "${conflicted}"
|
||||
|
||||
# Continue the rebase
|
||||
git rebase --continue 2>/dev/null || git commit --no-edit 2>/dev/null || true
|
||||
}
|
||||
|
||||
do_push() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
# Check if we have commits to push
|
||||
local ahead
|
||||
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo "0")
|
||||
|
||||
if [[ "${ahead}" -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
git push origin main 2>/dev/null || die "Failed to push to origin"
|
||||
log "Pushed ${ahead} commit(s) to origin"
|
||||
}
|
||||
|
||||
do_reindex() {
|
||||
if ! command -v qmd &>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if qmd collection exists
|
||||
if ! qmd collection list 2>/dev/null | grep -q "wiki"; then
|
||||
qmd collection add "${WIKI_DIR}" --name wiki 2>/dev/null
|
||||
fi
|
||||
|
||||
qmd update 2>/dev/null
|
||||
qmd embed 2>/dev/null
|
||||
log "Rebuilt qmd index"
|
||||
}
|
||||
|
||||
do_status() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
echo "=== Wiki Sync Status ==="
|
||||
echo "Directory: ${WIKI_DIR}"
|
||||
echo "Branch: $(git branch --show-current)"
|
||||
echo "Remote: $(git remote get-url origin)"
|
||||
echo ""
|
||||
|
||||
# Local changes
|
||||
local changes
|
||||
changes=$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "Uncommitted changes: ${changes}"
|
||||
|
||||
# Ahead/behind
|
||||
git fetch origin main 2>/dev/null
|
||||
local ahead behind
|
||||
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo "0")
|
||||
behind=$(git rev-list --count HEAD..origin/main 2>/dev/null || echo "0")
|
||||
echo "Ahead of remote: ${ahead}"
|
||||
echo "Behind remote: ${behind}"
|
||||
|
||||
# qmd status
|
||||
if command -v qmd &>/dev/null; then
|
||||
echo ""
|
||||
echo "qmd: installed"
|
||||
qmd collection list 2>/dev/null | grep wiki || echo "qmd: wiki collection not found"
|
||||
else
|
||||
echo ""
|
||||
echo "qmd: not installed"
|
||||
fi
|
||||
|
||||
# Last sync
|
||||
if [[ -f "${LOG_FILE}" ]]; then
|
||||
echo ""
|
||||
echo "Last sync log entries:"
|
||||
tail -5 "${LOG_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Main ---
|
||||
|
||||
main() {
|
||||
local mode="${1:-full}"
|
||||
|
||||
mkdir -p "${WIKI_DIR}/scripts"
|
||||
|
||||
# Status doesn't need a lock
|
||||
if [[ "${mode}" == "--status" ]]; then
|
||||
do_status
|
||||
return 0
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
case "${mode}" in
|
||||
--commit) do_commit ;;
|
||||
--pull) do_pull ;;
|
||||
--push) do_push ;;
|
||||
--reindex) do_reindex ;;
|
||||
full|*)
|
||||
do_commit
|
||||
do_pull
|
||||
do_push
|
||||
do_reindex
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
211
scripts/wiki_lib.py
Normal file
211
scripts/wiki_lib.py
Normal file
@@ -0,0 +1,211 @@
|
||||
"""Shared helpers for wiki maintenance scripts.
|
||||
|
||||
Provides frontmatter parsing/serialization, WikiPage dataclass, and common
|
||||
constants used by wiki-hygiene.py, wiki-staging.py, and wiki-harvest.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Wiki root — override via WIKI_DIR env var for tests / alternate installs
|
||||
WIKI_DIR = Path(os.environ.get("WIKI_DIR", str(Path.home() / "projects" / "wiki")))
|
||||
INDEX_FILE = WIKI_DIR / "index.md"
|
||||
STAGING_DIR = WIKI_DIR / "staging"
|
||||
STAGING_INDEX = STAGING_DIR / "index.md"
|
||||
ARCHIVE_DIR = WIKI_DIR / "archive"
|
||||
ARCHIVE_INDEX = ARCHIVE_DIR / "index.md"
|
||||
REPORTS_DIR = WIKI_DIR / "reports"
|
||||
CONVERSATIONS_DIR = WIKI_DIR / "conversations"
|
||||
HARVEST_STATE_FILE = WIKI_DIR / ".harvest-state.json"
|
||||
|
||||
LIVE_CONTENT_DIRS = ["patterns", "decisions", "concepts", "environments"]
|
||||
|
||||
FM_FENCE = "---\n"
|
||||
|
||||
|
||||
@dataclass
|
||||
class WikiPage:
|
||||
path: Path
|
||||
frontmatter: dict[str, Any]
|
||||
fm_raw: str
|
||||
body: str
|
||||
fm_start: int
|
||||
|
||||
|
||||
def today() -> date:
|
||||
return datetime.now(timezone.utc).date()
|
||||
|
||||
|
||||
def parse_date(value: Any) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
s = str(value).strip()
|
||||
try:
|
||||
return datetime.strptime(s, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_page(path: Path) -> WikiPage | None:
|
||||
"""Parse a markdown page with YAML frontmatter. Returns None if no frontmatter."""
|
||||
try:
|
||||
text = path.read_text()
|
||||
except OSError:
|
||||
return None
|
||||
if not text.startswith(FM_FENCE):
|
||||
return None
|
||||
end = text.find("\n---\n", 4)
|
||||
if end == -1:
|
||||
return None
|
||||
fm_raw = text[4:end]
|
||||
body = text[end + 5 :]
|
||||
fm = parse_yaml_lite(fm_raw)
|
||||
return WikiPage(path=path, frontmatter=fm, fm_raw=fm_raw, body=body, fm_start=end + 5)
|
||||
|
||||
|
||||
def parse_yaml_lite(text: str) -> dict[str, Any]:
|
||||
"""Parse a subset of YAML used in wiki frontmatter.
|
||||
|
||||
Supports:
|
||||
- key: value
|
||||
- key: [a, b, c]
|
||||
- key:
|
||||
- a
|
||||
- b
|
||||
"""
|
||||
result: dict[str, Any] = {}
|
||||
lines = text.splitlines()
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
i += 1
|
||||
continue
|
||||
m = re.match(r"^([\w_-]+):\s*(.*)$", line)
|
||||
if not m:
|
||||
i += 1
|
||||
continue
|
||||
key, rest = m.group(1), m.group(2).strip()
|
||||
if rest == "":
|
||||
items: list[str] = []
|
||||
j = i + 1
|
||||
while j < len(lines) and re.match(r"^\s+-\s+", lines[j]):
|
||||
items.append(re.sub(r"^\s+-\s+", "", lines[j]).strip())
|
||||
j += 1
|
||||
if items:
|
||||
result[key] = items
|
||||
i = j
|
||||
continue
|
||||
result[key] = ""
|
||||
i += 1
|
||||
continue
|
||||
if rest.startswith("[") and rest.endswith("]"):
|
||||
inner = rest[1:-1].strip()
|
||||
if inner:
|
||||
result[key] = [x.strip().strip('"').strip("'") for x in inner.split(",")]
|
||||
else:
|
||||
result[key] = []
|
||||
i += 1
|
||||
continue
|
||||
result[key] = rest.strip('"').strip("'")
|
||||
i += 1
|
||||
return result
|
||||
|
||||
|
||||
# Canonical frontmatter key order for serialization
|
||||
PREFERRED_KEY_ORDER = [
|
||||
"title", "type", "confidence",
|
||||
"status", "origin",
|
||||
"last_compiled", "last_verified",
|
||||
"staged_date", "staged_by", "target_path", "modifies", "compilation_notes",
|
||||
"archived_date", "archived_reason", "original_path",
|
||||
"sources", "related",
|
||||
]
|
||||
|
||||
|
||||
def serialize_frontmatter(fm: dict[str, Any]) -> str:
|
||||
"""Serialize a frontmatter dict back to YAML in the wiki's canonical style."""
|
||||
out_lines: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for key in PREFERRED_KEY_ORDER:
|
||||
if key in fm:
|
||||
out_lines.append(_format_fm_entry(key, fm[key]))
|
||||
seen.add(key)
|
||||
for key in sorted(fm.keys()):
|
||||
if key in seen:
|
||||
continue
|
||||
out_lines.append(_format_fm_entry(key, fm[key]))
|
||||
return "\n".join(out_lines)
|
||||
|
||||
|
||||
def _format_fm_entry(key: str, value: Any) -> str:
|
||||
if isinstance(value, list):
|
||||
if not value:
|
||||
return f"{key}: []"
|
||||
lines = [f"{key}:"]
|
||||
for item in value:
|
||||
lines.append(f" - {item}")
|
||||
return "\n".join(lines)
|
||||
return f"{key}: {value}"
|
||||
|
||||
|
||||
def write_page(page: WikiPage, new_fm: dict[str, Any] | None = None, new_body: str | None = None) -> None:
|
||||
fm = new_fm if new_fm is not None else page.frontmatter
|
||||
body = new_body if new_body is not None else page.body
|
||||
fm_yaml = serialize_frontmatter(fm)
|
||||
text = f"---\n{fm_yaml}\n---\n{body}"
|
||||
page.path.write_text(text)
|
||||
|
||||
|
||||
def iter_live_pages() -> list[WikiPage]:
|
||||
pages: list[WikiPage] = []
|
||||
for sub in LIVE_CONTENT_DIRS:
|
||||
for md in sorted((WIKI_DIR / sub).glob("*.md")):
|
||||
page = parse_page(md)
|
||||
if page:
|
||||
pages.append(page)
|
||||
return pages
|
||||
|
||||
|
||||
def iter_staging_pages() -> list[WikiPage]:
|
||||
pages: list[WikiPage] = []
|
||||
if not STAGING_DIR.exists():
|
||||
return pages
|
||||
for sub in LIVE_CONTENT_DIRS:
|
||||
d = STAGING_DIR / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for md in sorted(d.glob("*.md")):
|
||||
page = parse_page(md)
|
||||
if page:
|
||||
pages.append(page)
|
||||
return pages
|
||||
|
||||
|
||||
def iter_archived_pages() -> list[WikiPage]:
|
||||
pages: list[WikiPage] = []
|
||||
if not ARCHIVE_DIR.exists():
|
||||
return pages
|
||||
for sub in LIVE_CONTENT_DIRS:
|
||||
d = ARCHIVE_DIR / sub
|
||||
if not d.exists():
|
||||
continue
|
||||
for md in sorted(d.glob("*.md")):
|
||||
page = parse_page(md)
|
||||
if page:
|
||||
pages.append(page)
|
||||
return pages
|
||||
|
||||
|
||||
def page_content_hash(page: WikiPage) -> str:
|
||||
"""Hash of page body only (excludes frontmatter) so mechanical frontmatter fixes don't churn the hash."""
|
||||
return "sha256:" + hashlib.sha256(page.body.strip().encode("utf-8")).hexdigest()
|
||||
Reference in New Issue
Block a user