maintain.rs
Rust · 314 lines · 13.2 KB
//! `brain maintain`: the nightly upkeep, safe to run unattended. It never edits entries; it only
//! reports and writes advisory review notes:
//! 1. brings the search index up to date;
//! 2. finds stale entries and has the librarian review them against the code changes;
//! 3. lists links that point at missing entries, and near-duplicate entries;
//! 4. has the librarian check pairs of closely related entries for contradictions.
//! Verdicts are cached per pair of entry versions, so each pair is judged once.
//!
//! The summary is printed and saved to `reviews/maintain-latest.md` (shown by `brain doctor`).
use crate::index::Index;
use crate::librarian::{self, Librarian};
use crate::links::Graph;
use crate::store::{Brain, Entry};
use crate::ui;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::{BTreeMap, BTreeSet};
use std::fs;
/// Contradictions count only at or above this confidence.
const CONFIDENCE_FLOOR: f64 = 0.7;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Verdict {
/// Content hashes of the two entries when judged; a newer version of either means re-judge.
pub hashes: [String; 2],
pub contradicts: bool,
pub confidence: f64,
pub note: String,
pub checked: String,
}
pub struct Contradiction {
pub other: String,
pub note: String,
}
fn cache_path(brain: &Brain) -> std::path::PathBuf {
brain.home.join("reviews").join("contradictions.json")
}
fn load_cache(brain: &Brain) -> BTreeMap<String, Verdict> {
fs::read_to_string(cache_path(brain)).ok().and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default()
}
fn save_cache(brain: &Brain, c: &BTreeMap<String, Verdict>) {
let _ = fs::create_dir_all(brain.home.join("reviews"));
let _ = fs::write(cache_path(brain), serde_json::to_string_pretty(c).unwrap_or_default());
}
fn hash(e: &Entry) -> String {
hex::encode(Sha256::digest(e.raw.as_bytes()))[..16].to_string()
}
fn pair_key(a: &str, b: &str) -> String {
if a < b { format!("{a}|{b}") } else { format!("{b}|{a}") }
}
/// Current, confident contradictions involving `id` (both entries unchanged since judged).
pub fn contradictions_for(brain: &Brain, id: &str) -> Vec<Contradiction> {
contradictions_in(brain, &Graph::build(brain), id)
}
/// As `contradictions_for`, with the link graph already built. Retired entries have none.
fn contradictions_in(brain: &Brain, graph: &Graph, id: &str) -> Vec<Contradiction> {
if graph.is_retired(id) {
return vec![];
}
let Ok(me) = brain.load(id) else { return vec![] };
let mine = hash(&me);
let mut out = vec![];
for (key, v) in load_cache(brain) {
let Some((a, b)) = key.split_once('|') else { continue };
if !(v.contradicts && v.confidence >= CONFIDENCE_FLOOR) || (a != id && b != id) {
continue;
}
let other = if a == id { b } else { a };
if graph.is_retired(other) {
continue;
}
let Ok(o) = brain.load(other) else { continue };
let (ha, hb) = if a == id { (mine.clone(), hash(&o)) } else { (hash(&o), mine.clone()) };
if v.hashes == [ha, hb] {
out.push(Contradiction { other: other.to_string(), note: librarian::clean_note(&v.note).chars().take(200).collect() });
}
}
out
}
/// How many current, confident contradictions the cache holds (for `brain doctor`).
pub fn open_contradictions(brain: &Brain) -> usize {
let graph = Graph::build(brain);
let ids: BTreeSet<String> = brain.all_ids().unwrap_or_default().into_iter().collect();
let mut keys = BTreeSet::new();
for id in &ids {
for c in contradictions_in(brain, &graph, id) {
keys.insert(pair_key(id, &c.other));
}
}
keys.len()
}
pub fn run(brain: &Brain, limit: usize, no_librarian: bool) -> Result<()> {
println!("{} {}", ui::wordmark("maintain"), ui::dim(&format!("{} · reports only, never edits entries", chrono::Local::now().format("%Y-%m-%d %H:%M"))));
let mut report: Vec<String> = vec![];
let mut line = |s: String| {
println!(" {s}");
report.push(format!("- {}", strip_ansi(&s)));
};
// 1. index
let mut ix = Index::open(brain)?;
let n = ix.sync(brain)?;
line(format!("{} index up to date ({n} re-indexed)", ui::green("✓")));
// 2. links, retired, stale
let graph = Graph::build(brain);
let active: Vec<String> = brain.all_ids()?.into_iter().filter(|id| !graph.is_retired(id)).collect();
if graph.broken.is_empty() {
line(format!("{} no broken links", ui::green("✓")));
} else {
line(format!("{} {} broken link(s) (see `brain check`)", ui::yellow("⚠"), graph.broken.len()));
}
if !graph.retired.is_empty() {
line(format!("{} {} retired or superseded (hidden from search)", ui::dim("·"), graph.retired.len()));
}
let mut stale: Vec<Entry> = vec![];
for id in &active {
if let Ok(e) = brain.load(id)
&& !brain.stale_refs(&e.meta).is_empty()
{
stale.push(e);
}
}
if stale.is_empty() {
line(format!("{} nothing stale", ui::green("✓")));
} else {
line(format!("{} {} stale (see `brain stale --all`)", ui::yellow("⚠"), stale.len()));
}
// 3. near-duplicates and contradiction candidates, from whole-entry similarity
let (dup_at, _, related_at) = brain.config.embeddings.thresholds();
let mut dups = BTreeSet::new();
let mut pairs: Vec<(f32, String, String)> = vec![];
let kinds: BTreeMap<String, String> =
active.iter().filter_map(|id| brain.load(id).ok().map(|e| (id.clone(), e.meta.kind.clone()))).collect();
let active_set: BTreeSet<&String> = active.iter().collect();
for id in &active {
let Some(v) = ix.doc_vec(id) else { continue };
if v.is_empty() {
continue;
}
// Look past retired neighbours so they can't crowd out active ones.
let near = ix.nearest(&v, id, usize::MAX)?.into_iter().filter(|n| active_set.contains(&n.0) && n.3 >= related_at).take(6);
for (other, _, _, sim) in near {
let key = pair_key(id, &other);
if sim >= dup_at && kinds.get(id) == kinds.get(&other) {
dups.insert(key.clone());
}
// Near-duplicates are checked too: they can differ by exactly one critical value.
if !pairs.iter().any(|p| pair_key(&p.1, &p.2) == key) {
pairs.push((sim, id.clone(), other));
}
}
}
if dups.is_empty() {
line(format!("{} no near-duplicates", ui::green("✓")));
} else {
line(format!("{} {} near-duplicate pair(s): merge them, or retire one", ui::yellow("⚠"), dups.len()));
for d in &dups {
line(format!(" {}", d.replace('|', " ≈ ")));
}
}
// 4. the librarian: reviews of stale entries, then contradiction checks
let lib = Librarian::new(brain);
if no_librarian || !lib.configured() {
line(format!("{} librarian skipped ({})", ui::dim("·"), if no_librarian { "--no-librarian" } else { "none configured" }));
} else {
// Entries whose review failed for this exact version are skipped until they change, so one
// entry the librarian can't handle doesn't use up every night's budget.
let to_review: Vec<Entry> = stale
.into_iter()
.filter(|e| Librarian::current_review(brain, e).is_none() && !failed_before(brain, e))
.take(limit)
.collect();
let mut cache = load_cache(brain);
pairs.sort_by(|a, b| b.0.total_cmp(&a.0));
let mut to_judge = vec![];
for (_, a, b) in &pairs {
if to_review.len() + to_judge.len() >= limit {
break;
}
let (Ok(ea), Ok(eb)) = (brain.load(a), brain.load(b)) else { continue };
let key = pair_key(a, b);
let hashes = if a < b { [hash(&ea), hash(&eb)] } else { [hash(&eb), hash(&ea)] };
if cache.get(&key).is_some_and(|v| v.hashes == hashes) {
continue;
}
to_judge.push((key, hashes, ea, eb));
}
if to_review.is_empty() && to_judge.is_empty() {
line(format!("{} librarian: nothing new to review", ui::green("✓")));
} else {
match librarian::LmSession::start(brain, &lib, &brain.config.librarian.model) {
Err(e) => line(format!("{} librarian unavailable: {e:#}", ui::yellow("⚠"))),
Ok(_session) => {
if !to_review.is_empty() {
let versions: BTreeMap<String, String> = to_review.iter().map(|e| (e.id.clone(), version(brain, e))).collect();
let results = lib.review_all(to_review);
for (id, r) in &results {
if let Err(e) = r {
// Error text can echo a server or model reply: flatten and cap it.
let why = librarian::clean_note(&format!("{e:#}")).chars().take(120).collect::<String>();
line(format!("{} review of {id} failed: {why}", ui::yellow("⚠")));
if let Some(v) = versions.get(id) {
let _ = fs::write(failed_path(brain, id), format!("{v} {}", crate::store::today()));
}
}
}
let ok = results.iter().filter(|r| r.1.is_ok()).count();
let needs = results.iter().filter(|r| r.1.as_ref().is_ok_and(|v| v.verdict == "needs_update")).count();
line(format!("{} reviewed {ok} stale entr{} ({needs} need an update)", ui::green("✓"), if ok == 1 { "y" } else { "ies" }));
}
let mut judged = 0;
for (key, hashes, ea, eb) in to_judge {
match lib.contradiction(&ea, &eb) {
Ok((contradicts, confidence, note)) => {
judged += 1;
cache.insert(
key,
Verdict { hashes, contradicts, confidence, note, checked: crate::store::today() },
);
}
Err(e) => eprintln!(" (contradiction check {} / {} skipped: {e:#})", ea.id, eb.id),
}
}
save_cache(brain, &cache);
if judged > 0 {
line(format!("{} checked {judged} pair(s) of related entries for contradictions", ui::green("✓")));
}
}
}
}
}
let open = open_contradictions(brain);
if open == 0 {
line(format!("{} no known contradictions", ui::green("✓")));
} else {
line(format!("{} {open} possible contradiction(s): `brain get <id>` shows them; update or retire one side", ui::yellow("⚠")));
let cache = load_cache(brain);
for (key, v) in cache.iter().filter(|(_, v)| v.contradicts && v.confidence >= CONFIDENCE_FLOOR) {
if let Some((a, b)) = key.split_once('|')
&& contradictions_in(brain, &graph, a).iter().any(|c| c.other == b)
{
// Ids only: the model's note is free text, read deliberately with `brain get`.
let _ = v;
line(format!(" {a} ≠ {b}"));
}
}
}
let _ = fs::create_dir_all(brain.home.join("reviews"));
let _ = fs::write(
brain.home.join("reviews").join("maintain-latest.md"),
format!("# brain maintain, {}\n\n{}\n", chrono::Local::now().format("%Y-%m-%d %H:%M"), report.join("\n")),
);
Ok(())
}
/// What a review depends on: the entry text and its referenced files' current state.
fn version(brain: &Brain, e: &Entry) -> String {
let mut h = Sha256::new();
h.update(e.raw.as_bytes());
for r in &e.meta.refs {
h.update(brain.ref_hash_now(r).unwrap_or_default().as_bytes());
}
hex::encode(h.finalize())
}
fn failed_path(brain: &Brain, id: &str) -> std::path::PathBuf {
brain.home.join("reviews").join(format!("{id}.failed"))
}
/// A failed review of this exact version, within the last 3 days. Older failures are retried, so
/// a passing outage (a timeout, a 503) doesn't skip an entry for good.
fn failed_before(brain: &Brain, e: &Entry) -> bool {
let Ok(s) = fs::read_to_string(failed_path(brain, &e.id)) else { return false };
let mut parts = s.split_whitespace();
let (Some(v), Some(day)) = (parts.next(), parts.next()) else { return false };
let recent = chrono::NaiveDate::parse_from_str(day, "%Y-%m-%d")
.ok()
.is_some_and(|d| (chrono::Local::now().date_naive() - d).num_days() < 3);
v == version(brain, e) && recent
}
fn strip_ansi(s: &str) -> String {
let mut out = String::new();
let mut esc = false;
for c in s.chars() {
if esc {
if c.is_ascii_alphabetic() {
esc = false;
}
} else if c == '\x1b' {
esc = true;
} else {
out.push(c);
}
}
out
}
