Initial commit — memex
A compounding LLM-maintained knowledge wiki. Synthesis of Andrej Karpathy's persistent-wiki gist and milla-jovovich's mempalace, with an automation layer on top for conversation mining, URL harvesting, human-in-the-loop staging, staleness decay, and hygiene. Includes: - 11 pipeline scripts (extract, summarize, index, harvest, stage, hygiene, maintain, sync, + shared library) - Full docs: README, SETUP, ARCHITECTURE, DESIGN-RATIONALE, CUSTOMIZE - Example CLAUDE.md files (wiki schema + global instructions) tuned for the three-collection qmd setup - 171-test pytest suite (cross-platform, runs in ~1.3s) - MIT licensed
This commit is contained in:
Eric Turner
230
scripts/wiki-sync.sh
Executable file
230
scripts/wiki-sync.sh
Executable file
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# wiki-sync.sh — Auto-commit, pull, resolve conflicts, push, reindex
|
||||
#
|
||||
# Designed to run via cron on both work and home machines.
|
||||
# Safe to run frequently — no-ops when nothing has changed.
|
||||
#
|
||||
# Usage:
|
||||
# wiki-sync.sh # Full sync (commit + pull + push + reindex)
|
||||
# wiki-sync.sh --commit # Only commit local changes
|
||||
# wiki-sync.sh --pull # Only pull remote changes
|
||||
# wiki-sync.sh --push # Only push local commits
|
||||
# wiki-sync.sh --reindex # Only rebuild qmd index
|
||||
# wiki-sync.sh --status # Show sync status (no changes)
|
||||
|
||||
WIKI_DIR="${WIKI_DIR:-${HOME}/projects/wiki}"
|
||||
LOG_FILE="${WIKI_DIR}/scripts/.sync.log"
|
||||
LOCK_FILE="/tmp/wiki-sync.lock"
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
log() {
|
||||
local msg
|
||||
msg="[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
||||
echo "${msg}" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
die() {
|
||||
log "ERROR: $*"
|
||||
exit 1
|
||||
}
|
||||
|
||||
acquire_lock() {
|
||||
if [[ -f "${LOCK_FILE}" ]]; then
|
||||
local pid
|
||||
pid=$(cat "${LOCK_FILE}" 2>/dev/null || echo "")
|
||||
if [[ -n "${pid}" ]] && kill -0 "${pid}" 2>/dev/null; then
|
||||
die "Another sync is running (pid ${pid})"
|
||||
fi
|
||||
rm -f "${LOCK_FILE}"
|
||||
fi
|
||||
echo $$ > "${LOCK_FILE}"
|
||||
trap 'rm -f "${LOCK_FILE}"' EXIT
|
||||
}
|
||||
|
||||
# --- Operations ---
|
||||
|
||||
do_commit() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
# Check for uncommitted changes (staged + unstaged + untracked)
|
||||
if git diff --quiet && git diff --cached --quiet && [[ -z "$(git ls-files --others --exclude-standard)" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
local hostname
|
||||
hostname=$(hostname -s 2>/dev/null || echo "unknown")
|
||||
|
||||
git add -A
|
||||
git commit -m "$(cat <<EOF
|
||||
wiki: auto-sync from ${hostname}
|
||||
|
||||
Automatic commit of wiki changes detected by cron.
|
||||
EOF
|
||||
)" 2>/dev/null || true
|
||||
|
||||
log "Committed local changes from ${hostname}"
|
||||
}
|
||||
|
||||
do_pull() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
# Fetch first to check if there's anything to pull
|
||||
git fetch origin main 2>/dev/null || die "Failed to fetch from origin"
|
||||
|
||||
local local_head remote_head
|
||||
local_head=$(git rev-parse HEAD)
|
||||
remote_head=$(git rev-parse origin/main)
|
||||
|
||||
if [[ "${local_head}" == "${remote_head}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Pull with rebase to keep history linear
|
||||
# If conflicts occur, resolve markdown files by keeping both sides
|
||||
if ! git pull --rebase origin main 2>/dev/null; then
|
||||
log "Conflicts detected, attempting auto-resolution..."
|
||||
resolve_conflicts
|
||||
fi
|
||||
|
||||
log "Pulled remote changes"
|
||||
}
|
||||
|
||||
resolve_conflicts() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
local conflicted
|
||||
conflicted=$(git diff --name-only --diff-filter=U 2>/dev/null || echo "")
|
||||
|
||||
if [[ -z "${conflicted}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
if [[ "${file}" == *.md ]]; then
|
||||
# For markdown: accept both sides (union merge)
|
||||
# Remove conflict markers, keep all content
|
||||
if [[ -f "${file}" ]]; then
|
||||
sed -i.bak \
|
||||
-e '/^<<<<<<< /d' \
|
||||
-e '/^=======/d' \
|
||||
-e '/^>>>>>>> /d' \
|
||||
"${file}"
|
||||
rm -f "${file}.bak"
|
||||
git add "${file}"
|
||||
log "Auto-resolved conflict in ${file} (kept both sides)"
|
||||
fi
|
||||
else
|
||||
# For non-markdown: keep ours (local version wins)
|
||||
git checkout --ours "${file}" 2>/dev/null
|
||||
git add "${file}"
|
||||
log "Auto-resolved conflict in ${file} (kept local)"
|
||||
fi
|
||||
done <<< "${conflicted}"
|
||||
|
||||
# Continue the rebase
|
||||
git rebase --continue 2>/dev/null || git commit --no-edit 2>/dev/null || true
|
||||
}
|
||||
|
||||
do_push() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
# Check if we have commits to push
|
||||
local ahead
|
||||
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo "0")
|
||||
|
||||
if [[ "${ahead}" -eq 0 ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
git push origin main 2>/dev/null || die "Failed to push to origin"
|
||||
log "Pushed ${ahead} commit(s) to origin"
|
||||
}
|
||||
|
||||
do_reindex() {
|
||||
if ! command -v qmd &>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
# Check if qmd collection exists
|
||||
if ! qmd collection list 2>/dev/null | grep -q "wiki"; then
|
||||
qmd collection add "${WIKI_DIR}" --name wiki 2>/dev/null
|
||||
fi
|
||||
|
||||
qmd update 2>/dev/null
|
||||
qmd embed 2>/dev/null
|
||||
log "Rebuilt qmd index"
|
||||
}
|
||||
|
||||
do_status() {
|
||||
cd "${WIKI_DIR}"
|
||||
|
||||
echo "=== Wiki Sync Status ==="
|
||||
echo "Directory: ${WIKI_DIR}"
|
||||
echo "Branch: $(git branch --show-current)"
|
||||
echo "Remote: $(git remote get-url origin)"
|
||||
echo ""
|
||||
|
||||
# Local changes
|
||||
local changes
|
||||
changes=$(git status --porcelain 2>/dev/null | wc -l | tr -d ' ')
|
||||
echo "Uncommitted changes: ${changes}"
|
||||
|
||||
# Ahead/behind
|
||||
git fetch origin main 2>/dev/null
|
||||
local ahead behind
|
||||
ahead=$(git rev-list --count origin/main..HEAD 2>/dev/null || echo "0")
|
||||
behind=$(git rev-list --count HEAD..origin/main 2>/dev/null || echo "0")
|
||||
echo "Ahead of remote: ${ahead}"
|
||||
echo "Behind remote: ${behind}"
|
||||
|
||||
# qmd status
|
||||
if command -v qmd &>/dev/null; then
|
||||
echo ""
|
||||
echo "qmd: installed"
|
||||
qmd collection list 2>/dev/null | grep wiki || echo "qmd: wiki collection not found"
|
||||
else
|
||||
echo ""
|
||||
echo "qmd: not installed"
|
||||
fi
|
||||
|
||||
# Last sync
|
||||
if [[ -f "${LOG_FILE}" ]]; then
|
||||
echo ""
|
||||
echo "Last sync log entries:"
|
||||
tail -5 "${LOG_FILE}"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Main ---
|
||||
|
||||
main() {
|
||||
local mode="${1:-full}"
|
||||
|
||||
mkdir -p "${WIKI_DIR}/scripts"
|
||||
|
||||
# Status doesn't need a lock
|
||||
if [[ "${mode}" == "--status" ]]; then
|
||||
do_status
|
||||
return 0
|
||||
fi
|
||||
|
||||
acquire_lock
|
||||
|
||||
case "${mode}" in
|
||||
--commit) do_commit ;;
|
||||
--pull) do_pull ;;
|
||||
--push) do_push ;;
|
||||
--reindex) do_reindex ;;
|
||||
full|*)
|
||||
do_commit
|
||||
do_pull
|
||||
do_push
|
||||
do_reindex
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user