A compounding LLM-maintained knowledge wiki. Synthesis of Andrej Karpathy's persistent-wiki gist and milla-jovovich's mempalace, with an automation layer on top for conversation mining, URL harvesting, human-in-the-loop staging, staleness decay, and hygiene. Includes: - 11 pipeline scripts (extract, summarize, index, harvest, stage, hygiene, maintain, sync, + shared library) - Full docs: README, SETUP, ARCHITECTURE, DESIGN-RATIONALE, CUSTOMIZE - Example CLAUDE.md files (wiki schema + global instructions) tuned for the three-collection qmd setup - 171-test pytest suite (cross-platform, runs in ~1.3s) - MIT licensed
231 lines
5.4 KiB
Bash
Executable File
231 lines
5.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# wiki-sync.sh — Auto-commit, pull, resolve conflicts, push, reindex
|
|
#
|
|
# Designed to run via cron on both work and home machines.
|
|
# Safe to run frequently — no-ops when nothing has changed.
|
|
#
|
|
# Usage:
|
|
# wiki-sync.sh # Full sync (commit + pull + push + reindex)
|
|
# wiki-sync.sh --commit # Only commit local changes
|
|
# wiki-sync.sh --pull # Only pull remote changes
|
|
# wiki-sync.sh --push # Only push local commits
|
|
# wiki-sync.sh --reindex # Only rebuild qmd index
|
|
# wiki-sync.sh --status # Show sync status (no changes)
|
|
|
|
WIKI_DIR="${WIKI_DIR:-${HOME}/projects/wiki}"
|
|
LOG_FILE="${WIKI_DIR}/scripts/.sync.log"
|
|
LOCK_FILE="/tmp/wiki-sync.lock"
|
|
|
|
# --- Helpers ---
|
|
|
|
log() {
|
|
local msg
|
|
msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
|
echo "${msg}" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
die() {
|
|
log "ERROR: $*"
|
|
exit 1
|
|
}
|
|
|
|
acquire_lock() {
|
|
if [[ -f "${LOCK_FILE}" ]]; then
|
|
local pid
|
|
pid=$(cat "${LOCK_FILE}" 2>/dev/null || echo "")
|
|
if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
|
|
die "Another sync is running (pid ${pid})"
|
|
fi
|
|
rm -f "${LOCK_FILE}"
|
|
fi
|
|
echo $$ > "${LOCK_FILE}"
|
|
trap 'rm -f "${LOCK_FILE}"' EXIT
|
|
}
|
|
|
|
# --- Operations ---
|
|
|
|
do_commit() {
|
|
cd "${WIKI_DIR}"
|
|
|
|
# Check for uncommitted changes (staged + unstaged + untracked)
|
|
if git diff --quiet && git diff --cached --quiet && [[ -z "$(git ls-files --others --exclude-standard)" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
local hostname
|
|
hostname=$(hostname -s 2>/dev/null || echo "unknown")
|
|
|
|
git add -A
|
|
git commit -m "$(cat <<EOF
|
|
wiki: auto-sync from ${hostname}
|
|
|
|
Automatic commit of wiki changes detected by cron.
|
|
EOF
|
|
)" 2>/dev/null || true
|
|
|
|
log "Committed local changes from ${hostname}"
|
|
}
|
|
|
|
do_pull() {
|
|
cd "${WIKI_DIR}"
|
|
|
|
# Fetch first to check if there's anything to pull
|
|
git fetch origin main 2>/dev/null || die "Failed to fetch from origin"
|
|
|
|
local local_head remote_head
|
|
local_head=$(git rev-parse HEAD)
|
|
remote_head=$(git rev-parse origin/main)
|
|
|
|
if [[ "${local_head}" == "${remote_head}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
# Pull with rebase to keep history linear
|
|
# If conflicts occur, resolve markdown files by keeping both sides
|
|
if ! git pull --rebase origin main 2>/dev/null; then
|
|
log "Conflicts detected, attempting auto-resolution..."
|
|
resolve_conflicts
|
|
fi
|
|
|
|
log "Pulled remote changes"
|
|
}
|
|
|
|
resolve_conflicts() {
|
|
cd "${WIKI_DIR}"
|
|
|
|
local conflicted
|
|
conflicted=$(git diff --name-only --diff-filter=U 2>/dev/null || echo "")
|
|
|
|
if [[ -z "${conflicted}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
while IFS= read -r file; do
|
|
if [[ "${file}" == *.md ]]; then
|
|
# For markdown: accept both sides (union merge)
|
|
# Remove conflict markers, keep all content
|
|
if [[ -f "${file}" ]]; then
|
|
sed -i.bak \
|
|
-e '/^<<<<<<< /d' \
|
|
-e '/^=======/d' \
|
|
-e '/^>>>>>>> /d' \
|
|
"${file}"
|
|
rm -f "${file}.bak"
|
|
git add "${file}"
|
|
log "Auto-resolved conflict in ${file} (kept both sides)"
|
|
fi
|
|
else
|
|
# For non-markdown: keep ours (local version wins)
|
|
git checkout --ours "${file}" 2>/dev/null
|
|
git add "${file}"
|
|
log "Auto-resolved conflict in ${file} (kept local)"
|
|
fi
|
|
done <<< "${conflicted}"
|
|
|
|
# Continue the rebase
|
|
git rebase --continue 2>/dev/null || git commit --no-edit 2>/dev/null || true
|
|
}
|
|
|
|
do_push() {
|
|
cd "${WIKI_DIR}"
|
|
|
|
# Check if we have commits to push
|
|
local ahead
|
|
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo "0")
|
|
|
|
if [[ "${ahead}" -eq 0 ]]; then
|
|
return 0
|
|
fi
|
|
|
|
git push origin main 2>/dev/null || die "Failed to push to origin"
|
|
log "Pushed ${ahead} commit(s) to origin"
|
|
}
|
|
|
|
do_reindex() {
|
|
if ! command -v qmd &>/dev/null; then
|
|
return 0
|
|
fi
|
|
|
|
# Check if qmd collection exists
|
|
if ! qmd collection list 2>/dev/null | grep -q "wiki"; then
|
|
qmd collection add "${WIKI_DIR}" --name wiki 2>/dev/null
|
|
fi
|
|
|
|
qmd update 2>/dev/null
|
|
qmd embed 2>/dev/null
|
|
log "Rebuilt qmd index"
|
|
}
|
|
|
|
do_status() {
|
|
cd "${WIKI_DIR}"
|
|
|
|
echo "=== Wiki Sync Status ==="
|
|
echo "Directory: ${WIKI_DIR}"
|
|
echo "Branch: $(git branch --show-current)"
|
|
echo "Remote: $(git remote get-url origin)"
|
|
echo ""
|
|
|
|
# Local changes
|
|
local changes
|
|
changes=$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')
|
|
echo "Uncommitted changes: ${changes}"
|
|
|
|
# Ahead/behind
|
|
git fetch origin main 2>/dev/null
|
|
local ahead behind
|
|
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo "0")
|
|
behind=$(git rev-list --count HEAD..origin/main 2>/dev/null || echo "0")
|
|
echo "Ahead of remote: ${ahead}"
|
|
echo "Behind remote: ${behind}"
|
|
|
|
# qmd status
|
|
if command -v qmd &>/dev/null; then
|
|
echo ""
|
|
echo "qmd: installed"
|
|
qmd collection list 2>/dev/null | grep wiki || echo "qmd: wiki collection not found"
|
|
else
|
|
echo ""
|
|
echo "qmd: not installed"
|
|
fi
|
|
|
|
# Last sync
|
|
if [[ -f "${LOG_FILE}" ]]; then
|
|
echo ""
|
|
echo "Last sync log entries:"
|
|
tail -5 "${LOG_FILE}"
|
|
fi
|
|
}
|
|
|
|
# --- Main ---
|
|
|
|
main() {
|
|
local mode="${1:-full}"
|
|
|
|
mkdir -p "${WIKI_DIR}/scripts"
|
|
|
|
# Status doesn't need a lock
|
|
if [[ "${mode}" == "--status" ]]; then
|
|
do_status
|
|
return 0
|
|
fi
|
|
|
|
acquire_lock
|
|
|
|
case "${mode}" in
|
|
--commit) do_commit ;;
|
|
--pull) do_pull ;;
|
|
--push) do_push ;;
|
|
--reindex) do_reindex ;;
|
|
full|*)
|
|
do_commit
|
|
do_pull
|
|
do_push
|
|
do_reindex
|
|
;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|