store.rs
Rust · 1,087 lines · 43.3 KB
//! The brain's files: entries (markdown + YAML frontmatter, tracked in git),
//! proposals, librarian reviews, projects.toml and config.toml.
use anyhow::{anyhow, bail, Context, Result};
use fs2::FileExt;
use regex::Regex;
use serde::{Deserialize, Deserializer, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::sync::LazyLock;
// ------------------------------------------------------------------ kinds
pub struct Kind {
pub name: &'static str,
pub about: &'static str,
/// (heading, hint, required)
pub sections: &'static [(&'static str, &'static str, bool)],
}
pub const KINDS: &[Kind] = &[
Kind {
name: "system",
about: "How something was built (inventory, item system, AI, networking, a site feature), so another agent can reuse or follow it.",
sections: &[
("Goal", "What it does and why it exists.", true),
("Approach", "How it works: main classes/modules, data flow, the key design choices and why.", true),
("Key files", "The files to read first, one line each on what they hold. List them in `refs` too.", true),
("How to reuse", "What another project would copy, adapt or call; dependencies it needs.", false),
("Pitfalls", "What went wrong while building it, or is easy to get wrong.", false),
],
},
Kind {
name: "workflow",
about: "How to do a multi-step task: a cinematic, a trailer, store screenshots, a build or release.",
sections: &[
("Goal", "What the finished result is.", true),
("Steps", "Numbered steps, with the exact commands, scripts and settings.", true),
("Tools", "Programs, scripts, packages, settings used.", false),
("What made it right", "Feedback {user} gave and what fixed it; what separates good from bad here.", false),
("Pitfalls", "Mistakes made along the way.", false),
],
},
Kind {
name: "fix",
about: "A problem that took real digging, and how it was solved.",
sections: &[
("Problem", "The symptom, with the exact error text.", true),
("Cause", "The root cause.", false),
("Solution", "What fixed it, and why it works.", true),
("What didn't work", "Attempts that failed, so nobody repeats them.", false),
],
},
Kind {
name: "gotcha",
about: "A trap that is easy to fall into.",
sections: &[
("What happens", "The trap and its symptom.", true),
("Avoid it", "What to do instead.", true),
],
},
Kind {
name: "decision",
about: "A choice that was made and why, so nobody re-litigates or silently reverses it.",
sections: &[
("Context", "What prompted the decision.", false),
("Decision", "What was decided.", true),
("Why", "The reasons, including {user}'s preferences.", true),
("Alternatives", "What was considered and rejected.", false),
],
},
Kind {
name: "runbook",
about: "An exact operational procedure (deploying a site, backups, server tasks). Changes are proposals reviewed by {reviewer}. A runbook says HOW, never WHETHER: approval for deploys and other outward actions only ever comes from {user}, live.",
sections: &[
("Purpose", "What this procedure does.", true),
("Preconditions", "What must be true first, including who must approve.", false),
("Steps", "Exact commands, in order.", true),
("Verify", "How to confirm it worked.", true),
("Rollback", "How to undo it.", false),
],
},
Kind {
name: "overview",
about: "One per project: what it is, its stack, layout, main systems, and how it relates to other projects.",
sections: &[
("Stack", "Engine/framework and versions, key packages and services.", true),
("Layout", "Where things live in the repo.", true),
("Key systems", "The main systems, linking their `system` entries by id.", false),
("Conventions", "Rules this project follows (style, commits, deploys).", false),
("Related projects", "What it shares with other projects.", false),
],
},
];
pub fn kind(name: &str) -> Option<&'static Kind> {
KINDS.iter().find(|k| k.name == name)
}
/// Fill `{user}` / `{reviewer}` in built-in text with this brain's identity settings.
pub fn personalize(text: &str, id: &Identity) -> String {
text.replace("{user}", &id.user).replace("{reviewer}", &id.reviewer)
}
pub fn template(k: &Kind, project: &str) -> String {
let mut s = format!(
"---\ntitle: <one line a searcher would recognise>\nkind: {}\nproject: {}\ntags: [tag-one, tag-two]\n\
engine: [<e.g. rust 1.90, axum 0.8, next.js 16>]\ngenre: []\nplatform: []\n\
verified: false\nverified_how: <how it was confirmed: built, tested, played, deployed>\n\
refs:\n - {}:<path/inside/repo>\nrelated: []\n---\n",
k.name,
if project.is_empty() { "<project>" } else { project },
if project.is_empty() { "<project>" } else { project },
);
for (h, hint, req) in k.sections {
s.push_str(&format!("\n## {h}\n\n<{hint}{}>\n", if *req { "" } else { " (optional)" }));
}
s
}
// ------------------------------------------------------------------ meta
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
pub struct RefState {
pub hash: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub commit: String,
}
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
pub struct Meta {
#[serde(default)]
pub title: String,
#[serde(default)]
pub kind: String,
#[serde(default)]
pub project: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub owner: String,
#[serde(default, deserialize_with = "tag_list", skip_serializing_if = "Vec::is_empty")]
pub tags: Vec<String>,
#[serde(default, deserialize_with = "plain_list", skip_serializing_if = "Vec::is_empty")]
pub engine: Vec<String>,
#[serde(default, deserialize_with = "tag_list", skip_serializing_if = "Vec::is_empty")]
pub genre: Vec<String>,
#[serde(default, deserialize_with = "tag_list", skip_serializing_if = "Vec::is_empty")]
pub platform: Vec<String>,
#[serde(default, deserialize_with = "boolish")]
pub verified: bool,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub verified_how: String,
#[serde(default, deserialize_with = "plain_list", skip_serializing_if = "Vec::is_empty")]
pub refs: Vec<String>,
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
pub ref_state: BTreeMap<String, RefState>,
#[serde(default, deserialize_with = "plain_list", skip_serializing_if = "Vec::is_empty")]
pub related: Vec<String>,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub supersedes: String,
/// `retired` hides the entry from normal search and lists (it stays readable and in git).
#[serde(default, skip_serializing_if = "String::is_empty")]
pub status: String,
/// Why it was retired, and the entry that replaces it, if any.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub retired_why: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub replaced_by: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub author: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub created: String,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub updated: String,
#[serde(flatten)]
pub extra: BTreeMap<String, serde_yaml::Value>,
}
fn yaml_strings(v: serde_yaml::Value) -> Vec<String> {
use serde_yaml::Value as V;
match v {
V::Null => vec![],
V::String(s) => s.split(',').map(str::to_string).collect(),
V::Sequence(xs) => xs
.into_iter()
.map(|x| match x {
V::String(s) => s,
other => serde_yaml::to_string(&other).unwrap_or_default().trim().to_string(),
})
.collect(),
other => vec![serde_yaml::to_string(&other).unwrap_or_default().trim().to_string()],
}
.into_iter()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.collect()
}
fn plain_list<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<String>, D::Error> {
Ok(yaml_strings(serde_yaml::Value::deserialize(d)?))
}
pub fn norm_tag(s: &str) -> String {
let s = s.trim().trim_start_matches('#').to_lowercase();
s.split_whitespace().collect::<Vec<_>>().join("-")
}
fn tag_list<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<String>, D::Error> {
Ok(yaml_strings(serde_yaml::Value::deserialize(d)?).iter().map(|s| norm_tag(s)).collect())
}
fn boolish<'de, D: Deserializer<'de>>(d: D) -> Result<bool, D::Error> {
use serde_yaml::Value as V;
Ok(match V::deserialize(d)? {
V::Bool(b) => b,
V::String(s) => matches!(s.trim().to_lowercase().as_str(), "yes" | "y" | "true" | "verified"),
_ => false,
})
}
#[derive(Clone, Debug)]
pub struct Entry {
pub id: String,
pub meta: Meta,
pub body: String,
pub raw: String,
}
pub fn split_frontmatter(text: &str) -> Result<(Meta, String)> {
let t = text.trim_start_matches('\u{feff}');
let rest = t
.strip_prefix("---")
.ok_or_else(|| anyhow!("entry must start with a '---' YAML frontmatter block"))?;
let end = rest
.find("\n---")
.ok_or_else(|| anyhow!("frontmatter is not closed with '---'"))?;
let meta: Meta = serde_yaml::from_str(&rest[..end]).context(
"frontmatter is not valid YAML (quote any value that contains `: ` or starts with a special \
character, e.g. verified_how: \"live 2026-10-29: works\")",
)?;
let after = &rest[end + 4..];
let body = after.split_once('\n').map(|x| x.1).unwrap_or("").trim().to_string();
Ok((meta, body))
}
pub fn render(meta: &Meta, body: &str) -> String {
format!("---\n{}---\n\n{}\n", serde_yaml::to_string(meta).unwrap_or_default(), body.trim())
}
/// `## Heading` sections of a body, in order.
pub fn sections(body: &str) -> Vec<(String, String)> {
let mut out: Vec<(String, String)> = vec![];
let mut cur = (String::new(), String::new());
for line in body.lines() {
if let Some(h) = line.strip_prefix("## ") {
if !cur.0.is_empty() || !cur.1.trim().is_empty() {
out.push(std::mem::take(&mut cur));
}
cur.0 = h.trim().to_string();
} else {
cur.1.push_str(line);
cur.1.push('\n');
}
}
if !cur.0.is_empty() || !cur.1.trim().is_empty() {
out.push(cur);
}
out.into_iter().map(|(h, t)| (h, t.trim().to_string())).collect()
}
// ------------------------------------------------------------------ secrets
static SECRET_RES: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
[
(r"-----BEGIN [A-Z ]*PRIVATE KEY-----", "a PEM private key"),
(r"(?i)(private|secret|mnemonic|seed)[^\n]{0,40}\b(0x)?[0-9a-f]{64}\b", "a hex private key"),
(r"\bsk-(ant-|proj-)?[A-Za-z0-9_-]{20,}", "an API key (sk-...)"),
(r"\b(ghp|gho|ghu|ghs|github_pat)_[A-Za-z0-9_]{20,}", "a GitHub token"),
(r"\bAKIA[0-9A-Z]{16}\b", "an AWS access key"),
(r"(?i)AccountKey=[A-Za-z0-9+/=]{20,}", "an Azure storage connection string"),
(r"(?i)SharedAccessSignature\s|[?&]sig=[A-Za-z0-9%+/=]{20,}", "an Azure SAS token"), // leak-scan: allow
(r"(?i)(api[_-]?key|subscription[_-]?key|ocp-apim)[^\n]{0,20}\b[0-9a-f]{32}\b", "an Azure API key"),
(r"\bxox[abpr]-[A-Za-z0-9-]{10,}", "a Slack token"),
(r"\beyJ[A-Za-z0-9_-]{15,}\.eyJ[A-Za-z0-9_-]{15,}\.[A-Za-z0-9_-]{10,}", "a JWT"),
(r"(?i)\b(authorization\s*:\s*)?(bearer|basic)\s+[A-Za-z0-9._~+/=-]{20,}", "an Authorization header"),
(r"(?i)\b[a-z][a-z0-9+.-]*://[^\s/:@<>]+:[^\s/@<>$]{6,}@", "a password inside a URL"),
]
.into_iter()
.map(|(r, what)| (Regex::new(r).unwrap(), what))
.collect()
});
/// `password = x`, `"api_key": "x y z"`, `client_secret: x`, `AZURE_CLIENT_SECRET=x`, and YAML
/// `password: |` blocks. Whether the value is really a secret is decided by `secret_value`.
const SECRET_KEYS: &str = r"(?i)(password|passwd|passphrase|pwd|api[_-]?key|secret[_-]access[_-]key|secret|auth[_-]?token|access[_-]?token|refresh[_-]?token|private[_-]?key|credentials?)";
static ASSIGNMENT: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(&format!(
r#"{SECRET_KEYS}["']?[ \t]*[:=][ \t]*(?:"([^"\n]{{1,200}})"|'([^'\n]{{1,200}})'|[|>][-+]?[ \t]*\r?\n[ \t]+([^\n]{{1,200}})|([^\s"'`<>,;]{{1,200}}))"#
))
.unwrap()
});
/// Names of where a secret lives, not secrets: `$DEPLOY_KEY`, `$password`, `${TOKEN}`, `$(pass show x)`,
/// `%APPDATA%`, `{{ secrets.X }}`, `********`, `xxxx`, `<your key>`, `0xYOUR_KEY`, `your-api-key`.
static REFERENCE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(r"^(\$[A-Za-z_][A-Za-z0-9_]*|\$\{.*|\$\(.*|%[A-Za-z_][A-Za-z0-9_]*%.*|\{\{.*|\*+|[xX]{4,}|<.*>|(?i)(0x)?your[_-].*)$").unwrap()
});
/// Code, judged only for unquoted values: type names (`String`, `Option<…>`, `SecretString`,
/// `config::Secret`), field paths (`self.password`, `cfg.api_key.clone()`), calls and lookups
/// (`env::var(`, `os.environ[`). A quoted value is a literal, so these never excuse it.
static CODE: LazyLock<Regex> = LazyLock::new(|| {
Regex::new(concat!(
// A fixed list of type names: PascalCase in general isn't excused (`CorrectHorseBatteryStaple`).
r"^(String|Str|Option|Secret|SecretString|SecretBox|SecretVec|Zeroizing|Vec|Bytes|Box|Arc|Rc",
r"|Credentials?|Password|PasswordHash|Token|Key|ApiKey)$|^[A-Z][A-Za-z]*(::[A-Za-z_]\w*)+$",
// Field paths start with a lowercase receiver (`self.password`, `cfg.api_key`), never digits
// (checked below): `Summer.Winter` and `welcome.home2025` aren't code.
r"|^[a-z_]+(\.[A-Za-z_][A-Za-z0-9_]*)+(\(\))?\??$",
// Calls and lookups go through a path (`env::var(`, `os.environ[`, `Math.random(`), so
// `Welcome[2024]` or `hunter(22` aren't excused.
r"|^[A-Za-z_]\w*(\.|::)[A-Za-z0-9_.:]*[(\[]",
// A plain function call whose argument (if any) starts like code: `require_env(`,
// `env_first(env_prefix`; `hunter(22x` isn't one.
r"|^[a-z_]+\(([a-z_&*]|$)",
))
.unwrap()
});
/// Also fetch()'s `credentials` options.
const PLACEHOLDER_WORDS: [&str; 15] = [
"example", "placeholder", "redacted", "required", "optional", "string", "none", "null", "true", "false", "undefined",
"unset", "same-origin", "include", "omit",
];
/// Prose, not a passphrase: `secret: "stored in the vault"`.
const PROSE_WORDS: [&str; 26] = [
"the", "a", "an", "in", "is", "are", "to", "of", "and", "or", "from", "for", "with", "see", "use", "set", "stored",
"via", "your", "this", "that", "it", "not", "on", "at", "by",
];
/// Whether an assigned value is really a secret. `quoted`: it was written as a literal ("…", '…'
/// or a YAML block), so only references, placeholders and plain prose are excused.
fn secret_value(key: &str, v: &str, quoted: bool) -> bool {
let v = v.trim().trim_end_matches([',', ';', '`']);
let lower = v.to_ascii_lowercase();
if v.chars().count() < 6 || REFERENCE.is_match(v) || PLACEHOLDER_WORDS.contains(&lower.as_str()) {
return false;
}
if quoted {
// Prose has spaces, a common little word, and no digits (`"me and the 7 dwarfs"` is a secret).
// Never for passwords: a passphrase is often ordinary words (`"winter is coming"`).
let password_key = ["password", "passwd", "passphrase", "pwd"].contains(&key.to_ascii_lowercase().as_str());
let prose = !password_key
&& v.contains(' ')
&& !v.chars().any(|c| c.is_ascii_digit())
&& lower.split_whitespace().any(|w| PROSE_WORDS.contains(&w));
return !prose;
}
let v = v.trim_end_matches([')', '.']);
let lower = v.to_ascii_lowercase();
let digits = v.chars().any(|c| c.is_ascii_digit());
// Code shapes; a field path with digits in it isn't excused (calls and lookups are).
if CODE.is_match(v) && (!digits || v.contains(['(', '['])) {
return false;
}
// A variable or field named after what it holds: the key itself (`password: password,`,
// `api_key=api_key`) or a name ending in it (`db_password`), with no digits:
// `mypassword` and `Summer2024_password` are not excused.
let key = key.to_ascii_lowercase().replace('-', "_");
let named = !digits
&& v.chars().all(|c| c.is_ascii_alphabetic() || c == '_')
&& (lower == key || lower.ends_with(&format!("_{key}")));
!named
}
/// What kind of secret the text seems to contain, if any.
pub fn secret_kind(text: &str) -> Option<&'static str> {
if let Some((_, what)) = SECRET_RES.iter().find(|(re, _)| re.is_match(text)) {
return Some(what);
}
ASSIGNMENT
.captures_iter(text)
.any(|c| (2..=5).any(|i| c.get(i).is_some_and(|m| secret_value(&c[1], m.as_str(), i < 5))))
.then_some("a password/key assignment")
}
pub fn secret_check(text: &str) -> Result<()> {
{
if let Some(what) = secret_kind(text) {
bail!(
"this looks like it contains {what}. Never put secrets in the brain: name the env var, \
file or vault entry that holds it instead (e.g. `$DEPLOY_KEY`, `~/.config/x/secret`)."
);
}
}
Ok(())
}
// ------------------------------------------------------------------ projects & config
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
pub struct Project {
pub path: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub stack: Vec<String>,
/// Session names this project's agent uses, e.g. "payments-api" (sessions get suffixes like -93).
#[serde(default)]
pub session: String,
}
/// Who the agents work for, and who reviews runbook changes. Filled in by `brain setup`.
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(default)]
pub struct Identity {
/// The person the agents work for, as agents should refer to them (e.g. "Alex").
pub user: String,
/// Who accepts runbook proposals (e.g. "the ops agent", "Alex").
pub reviewer: String,
/// Route runbook changes through `brain proposals` for review.
pub review_runbooks: bool,
}
impl Default for Identity {
fn default() -> Self {
Identity { user: "the user".into(), reviewer: "the reviewer".into(), review_runbooks: true }
}
}
/// Embeddings for search-by-meaning. Without them, search is keyword-only.
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(default)]
pub struct Embeddings {
/// `local` (CPU model, needs the local-embeddings build feature), `azure`, `openai`
/// (any OpenAI-compatible server, e.g. LM Studio), or `none` (keyword search only).
pub provider: String,
/// Azure: the resource endpoint (https://<resource>.openai.azure.com). openai: the base URL incl. /v1.
pub url: String,
/// Azure: the embedding deployment name. openai: the model id.
pub model: String,
/// `azure` (managed identity, else `az login`), `env:VAR` (API key in VAR), `command:<cmd>`, or `none`.
pub auth: String,
/// Hosts besides Azure's own domains allowed to receive the Azure sign-in token (a company gateway).
pub token_hosts: Vec<String>,
/// Similarity thresholds; unset = defaults for the provider (models score differently).
pub duplicate_at: Option<f32>,
pub weak_below: Option<f32>,
pub related_at: Option<f32>,
}
impl Embeddings {
/// (duplicate, weak-match, related) similarity thresholds. bge (local) scores run high;
/// OpenAI/Azure text-embedding-3 scores run lower, so they get lower defaults.
pub fn thresholds(&self) -> (f32, f32, f32) {
let (d, w, r) = if self.provider == "local" { (0.88, 0.55, 0.60) } else { (0.85, 0.35, 0.45) };
(self.duplicate_at.unwrap_or(d), self.weak_below.unwrap_or(w), self.related_at.unwrap_or(r))
}
}
impl Default for Embeddings {
fn default() -> Self {
Embeddings {
provider: if cfg!(feature = "local-embeddings") { "local" } else { "none" }.into(),
url: String::new(),
model: String::new(),
auth: "none".into(),
token_hosts: vec![],
duplicate_at: None,
weak_below: None,
related_at: None,
}
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(default)]
pub struct Librarian {
/// `lmstudio` (starts LM Studio as needed), `azure`, `openai`, or `none`.
pub provider: String,
/// LM Studio/openai: base URL incl. /v1. Azure: the resource endpoint.
pub url: String,
/// Same values as embeddings.auth.
pub auth: String,
/// Same as embeddings.token_hosts.
pub token_hosts: Vec<String>,
/// Model for the nightly/manual reviews: the strongest, since nobody waits on it.
pub model: String,
/// Model for `--deep` search, which an agent waits on: fast.
pub quick_model: String,
pub concurrency: usize,
pub timeout_secs: u64,
/// The LM Studio CLI, used to start the server and load the model when needed.
pub lms: String,
/// Context the model is loaded with; an entry plus a diff fits in 16k (the default 262k costs ~8 GB more).
pub context_length: u32,
}
impl Default for Librarian {
fn default() -> Self {
Librarian {
provider: "none".into(),
url: "http://127.0.0.1:1234/v1".into(),
auth: "none".into(),
token_hosts: vec![],
model: "google/gemma-4-31b".into(),
quick_model: "google/gemma-4-26b-a4b".into(),
concurrency: 1,
timeout_secs: 600,
lms: "~/.lmstudio/bin/lms".into(),
context_length: 16384,
}
}
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Hooks {
/// Check for newly stale entries on every Nth message to a session (1 = every message).
pub prompt_every: usize,
}
impl Default for Hooks {
fn default() -> Self {
Hooks { prompt_every: 1 }
}
}
/// What is masked before text goes to a model that isn't on this machine.
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(default)]
pub struct Privacy {
/// Mask emails, IP addresses and home-folder user names in text sent to remote models.
pub mask: bool,
/// Extra regular expressions to mask (e.g. account-number shapes, internal host names).
pub patterns: Vec<String>,
}
impl Default for Privacy {
fn default() -> Self {
Privacy { mask: true, patterns: vec![] }
}
}
#[derive(Serialize, Deserialize, Clone, Default, Debug)]
pub struct Config {
#[serde(default)]
pub identity: Identity,
#[serde(default)]
pub embeddings: Embeddings,
#[serde(default)]
pub librarian: Librarian,
#[serde(default)]
pub hooks: Hooks,
#[serde(default)]
pub privacy: Privacy,
}
/// The user's home directory (HOME, or USERPROFILE on Windows).
pub fn home_dir() -> PathBuf {
std::env::var_os("HOME")
.or_else(|| std::env::var_os("USERPROFILE"))
.map(PathBuf::from)
.unwrap_or_default()
}
/// Where per-user data lives: ~/.local/share on Unix, %LOCALAPPDATA% on Windows.
pub fn data_dir() -> PathBuf {
if cfg!(windows)
&& let Some(d) = std::env::var_os("LOCALAPPDATA") {
return PathBuf::from(d);
}
std::env::var_os("XDG_DATA_HOME").filter(|v| !v.is_empty()).map(PathBuf::from).unwrap_or_else(|| home_dir().join(".local/share"))
}
/// Where caches live: ~/.cache on Unix, %LOCALAPPDATA% on Windows.
pub fn cache_dir() -> PathBuf {
if cfg!(windows)
&& let Some(d) = std::env::var_os("LOCALAPPDATA") {
return PathBuf::from(d);
}
std::env::var_os("XDG_CACHE_HOME").filter(|v| !v.is_empty()).map(PathBuf::from).unwrap_or_else(|| home_dir().join(".cache"))
}
/// Create a folder only its owner can open (0700 on Unix; Windows profile folders are already private).
pub fn create_private_dir(dir: &Path) {
let _ = fs::create_dir_all(dir);
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let _ = fs::set_permissions(dir, fs::Permissions::from_mode(0o700));
}
}
/// Where Azure tokens are cached between runs: the private cache folder, outside the brain, so a
/// copy or backup of the brain never carries a live token.
pub fn auth_cache_path() -> PathBuf {
app_cache_dir().join("auth-cache.json")
}
/// A folder named after the product. Installs from before the rename (Giga Brain) keep their
/// folder: whichever of the new and old folders actually holds data (`markers`) is used, the new
/// one winning only when it holds data or the old one doesn't exist. An empty new folder can't hide
/// an existing brain, cache, token or model.
fn named_dir(root: &Path, suffix: &str, markers: &[&str]) -> PathBuf {
let new = root.join(format!("big-brain{suffix}"));
let old = root.join(format!("giga-brain{suffix}"));
// Real data: a marker file, or a marker folder with something in it (not just an empty folder).
let has_data = |d: &Path| {
markers.iter().any(|m| {
let p = d.join(m);
if p.is_dir() { fs::read_dir(&p).is_ok_and(|mut r| r.next().is_some()) } else { p.is_file() }
})
};
if has_data(&new) {
new
} else if has_data(&old) || (old.exists() && !new.exists()) {
old
} else {
new
}
}
/// Both a pre-rename and a current brain folder hold entries: say so (`brain doctor`).
pub fn split_brain_folders() -> Option<(PathBuf, PathBuf)> {
if std::env::var_os("BRAIN_HOME").is_some_and(|v| !v.is_empty()) {
return None;
}
let (new, old) = (data_dir().join("big-brain"), data_dir().join("giga-brain"));
(new.join("entries").is_dir() && old.join("entries").is_dir()).then_some((new, old))
}
const DATA_MARKERS: &[&str] = &["entries"];
const CACHE_MARKERS: &[&str] = &["auth-cache.json", "models"];
/// Big Brain's own cache folder (embedding model, Azure tokens). Never inside the brain: on
/// Windows the data and cache roots are both %LOCALAPPDATA%, so the cache gets its own folder there.
pub fn app_cache_dir() -> PathBuf {
if cfg!(windows) && std::env::var_os("LOCALAPPDATA").is_some() {
return named_dir(&cache_dir(), "-cache", CACHE_MARKERS);
}
named_dir(&cache_dir(), "", CACHE_MARKERS)
}
pub fn expand(p: &str) -> PathBuf {
if let Some(rest) = p.strip_prefix("~/").or_else(|| p.strip_prefix("~\\")) {
return home_dir().join(rest);
}
PathBuf::from(p)
}
// ------------------------------------------------------------------ the brain
pub struct Brain {
pub home: PathBuf,
pub projects: BTreeMap<String, Project>,
pub config: Config,
}
#[derive(Debug, Clone, PartialEq)]
pub enum RefStatus {
Fresh,
Changed,
Missing,
Untracked,
Unresolved(String),
}
impl RefStatus {
pub fn label(&self) -> String {
match self {
RefStatus::Fresh => "ok".into(),
RefStatus::Changed => "⚠ changed since this entry was written".into(),
RefStatus::Missing => "⚠ missing (moved or deleted)".into(),
RefStatus::Untracked => "not tracked (written before ref tracking)".into(),
RefStatus::Unresolved(why) => format!("⚠ {why}"),
}
}
pub fn is_stale(&self) -> bool {
matches!(self, RefStatus::Changed | RefStatus::Missing | RefStatus::Unresolved(_))
}
}
pub const GITIGNORE: &str = "index.db\nindex.db-*\n.models/\nmodels/\n.lock\nreviews/\nhook-state/\n.auth-cache*\nauth-cache*\n.edit/\n";
impl Brain {
pub fn home() -> PathBuf {
std::env::var_os("BRAIN_HOME")
.filter(|v| !v.is_empty())
.map(PathBuf::from)
.unwrap_or_else(|| named_dir(&data_dir(), "", DATA_MARKERS))
}
pub fn open() -> Result<Brain> {
let home = Self::home();
if !home.join("entries").is_dir() {
bail!("no brain at {} — run `brain setup` first", home.display());
}
// Brains made by older versions: make sure newer private files stay out of git.
let gi = home.join(".gitignore");
if let Ok(cur) = fs::read_to_string(&gi) {
let missing: Vec<&str> = GITIGNORE.lines().filter(|l| !cur.lines().any(|c| c == *l)).collect();
if !missing.is_empty() {
let _ = fs::write(&gi, format!("{}\n{}\n", cur.trim_end(), missing.join("\n")));
}
}
// Tokens used to be cached inside the brain folder; they now live in the cache folder.
// (and on Windows, 0.3.0 previews cached them as auth-cache.json in the same folder)
let _ = fs::remove_file(home.join(".auth-cache.json"));
if home != app_cache_dir() {
let _ = fs::remove_file(home.join("auth-cache.json"));
}
let projects = match fs::read_to_string(home.join("projects.toml")) {
Ok(s) => {
#[derive(Deserialize)]
struct F {
#[serde(default)]
projects: BTreeMap<String, Project>,
}
toml::from_str::<F>(&s).context("projects.toml")?.projects
}
Err(_) => BTreeMap::new(),
};
let config = match fs::read_to_string(home.join("config.toml")) {
Ok(s) => toml::from_str(&s).context("config.toml")?,
Err(_) => Config::default(),
};
Ok(Brain { home, projects, config })
}
pub fn init() -> Result<PathBuf> {
let home = Self::home();
// The brain holds notes about private systems: only its owner can read it.
create_private_dir(&home);
for d in ["entries", "proposals", "reviews", "hook-state"] {
fs::create_dir_all(home.join(d))?;
}
let write_new = |name: &str, content: &str| -> Result<()> {
let p = home.join(name);
if !p.exists() {
fs::write(p, content)?;
}
Ok(())
};
write_new(".gitignore", GITIGNORE)?;
write_new("projects.toml", "# Projects the brain knows. Refs are written as `<project>:<path inside it>`.\n")?;
write_new(
"config.toml",
&toml::to_string_pretty(&Config::default())?,
)?;
if !home.join(".git").exists() {
git(&home, &["init", "-q"])?;
git(&home, &["config", "user.name", "Big Brain"])?;
git(&home, &["config", "user.email", "brain@localhost"])?;
git(&home, &["add", "-A"])?;
git(&home, &["commit", "-q", "-m", "Big Brain: initial layout"])?;
}
Ok(home)
}
pub fn entries_dir(&self) -> PathBuf {
self.home.join("entries")
}
pub fn lock(&self) -> Result<fs::File> {
let f = fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(self.home.join(".lock"))?;
f.lock_exclusive()?;
Ok(f)
}
pub fn entry_path(&self, id: &str) -> Result<PathBuf> {
if !valid_id(id) {
bail!("bad entry id {id:?}");
}
Ok(self.entries_dir().join(format!("{id}.md")))
}
pub fn load(&self, id: &str) -> Result<Entry> {
let path = self.entry_path(id)?;
let raw = fs::read_to_string(&path).map_err(|_| anyhow!("no entry {id:?} (try `brain search`)"))?;
let (meta, body) = split_frontmatter(&raw).with_context(|| format!("entry {id}"))?;
Ok(Entry { id: id.to_string(), meta, body, raw })
}
pub fn all_ids(&self) -> Result<Vec<String>> {
let mut ids: Vec<String> = fs::read_dir(self.entries_dir())?
.filter_map(|e| e.ok())
.filter_map(|e| {
let p = e.path();
if p.extension()? != "md" {
return None;
}
Some(p.file_stem()?.to_string_lossy().to_string())
})
.collect();
ids.sort();
Ok(ids)
}
/// Project whose path contains `dir` (the deepest match), e.g. the cwd of an agent.
pub fn project_for_dir(&self, dir: &Path) -> Option<String> {
let dir = dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf());
self.projects
.iter()
.filter_map(|(name, p)| {
let root = expand(&p.path);
let root = root.canonicalize().unwrap_or(root);
dir.starts_with(&root).then(|| (root.components().count(), name.clone()))
})
.max()
.map(|(_, n)| n)
}
/// `project:path/in/repo` (or an absolute / ~ path) → (project root if any, absolute path).
pub fn resolve_ref(&self, r: &str) -> Result<(Option<PathBuf>, PathBuf), String> {
let r = r.split('#').next().unwrap_or(r).trim();
if Path::new(r).is_absolute() || r.starts_with("~/") || r.starts_with("~\\") || has_drive_prefix(r) {
return self.resolve_abs_ref(r);
}
let (proj, rel) = r
.split_once(':')
.ok_or_else(|| format!("ref {r:?} should look like `<project>:<path>`"))?;
let p = self
.projects
.get(proj)
.ok_or_else(|| format!("unknown project {proj:?} in ref (see `brain projects`)"))?;
let rel = rel.trim_start_matches(['/', '\\']);
if Path::new(rel).components().any(|c| matches!(c, std::path::Component::ParentDir)) {
return Err(format!("ref {r:?} must stay inside the project (no `..`)"));
}
let root = expand(&p.path);
let joined = root.join(rel);
// A drive prefix (Windows) or a symlink can lead outside the project: check the real path.
if !joined.starts_with(&root) {
return Err(format!("ref {r:?} must stay inside the project"));
}
if let (Ok(real), Ok(real_root)) = (joined.canonicalize(), root.canonicalize())
&& !real.starts_with(&real_root) {
return Err(format!("ref {r:?} points outside the project (symlink)"));
}
Ok((Some(root), joined))
}
/// An absolute or `~/` ref is allowed only inside a registered project (agents open refs, so a
/// ref must never point them at keys or tokens elsewhere on the machine).
fn resolve_abs_ref(&self, r: &str) -> Result<(Option<PathBuf>, PathBuf), String> {
let abs = expand(r);
if !abs.is_absolute()
|| abs.components().any(|c| matches!(c, std::path::Component::ParentDir | std::path::Component::CurDir))
{
return Err(format!("ref {r:?} must be a plain path inside a project (no `..`)"));
}
let real = abs.canonicalize().unwrap_or_else(|_| abs.clone());
let best = self
.projects
.values()
.filter_map(|p| {
let root = expand(&p.path);
let real_root = root.canonicalize().unwrap_or_else(|_| root.clone());
(abs.starts_with(&root) && real.starts_with(&real_root)).then(|| (root.components().count(), root))
})
.max();
match best {
Some((_, root)) => Ok((Some(root), abs)),
None => Err(format!(
"ref {r:?} is outside every registered project: write it as `<project>:<path>` \
(add the folder with `brain project add`) or mention the path in the text instead"
)),
}
}
/// Current fingerprint of a ref: content hash (files) or git tree listing (dirs).
pub fn ref_state_now(&self, r: &str) -> Result<RefState, RefStatus> {
let (_, abs) = self.resolve_ref(r).map_err(RefStatus::Unresolved)?;
let hash = if abs.is_file() {
let bytes = fs::read(&abs).map_err(|_| RefStatus::Missing)?;
hex::encode(Sha256::digest(&bytes))
} else if abs.is_dir() {
let listing = Command::new("git")
.arg("-C")
.arg(&abs)
.args(["ls-files", "-s", "-m", "."])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| o.stdout)
.unwrap_or_default();
if listing.is_empty() {
return Err(RefStatus::Untracked);
}
hex::encode(Sha256::digest(&listing))
} else {
return Err(RefStatus::Missing);
};
let (dir, name) = if abs.is_dir() {
(abs.clone(), ".".to_string())
} else {
(
abs.parent().map(Path::to_path_buf).unwrap_or_default(),
abs.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(),
)
};
let commit = Command::new("git")
.arg("-C")
.arg(&dir)
.args(["--literal-pathspecs", "log", "-1", "--format=%H", "--", &name])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_default();
Ok(RefState { hash, commit })
}
/// Content fingerprint only (no git history lookup): what staleness checks need.
pub fn ref_hash_now(&self, r: &str) -> Result<String, RefStatus> {
let (_, abs) = self.resolve_ref(r).map_err(RefStatus::Unresolved)?;
if abs.is_file() {
let bytes = fs::read(&abs).map_err(|_| RefStatus::Missing)?;
return Ok(hex::encode(Sha256::digest(&bytes)));
}
self.ref_state_now(r).map(|s| s.hash)
}
pub fn ref_status(&self, meta: &Meta, r: &str) -> RefStatus {
match (self.ref_hash_now(r), meta.ref_state.get(r)) {
(Err(s), _) => s,
(Ok(_), None) => RefStatus::Untracked,
(Ok(now), Some(then)) if now == then.hash => RefStatus::Fresh,
(Ok(_), Some(_)) => RefStatus::Changed,
}
}
pub fn stale_refs(&self, meta: &Meta) -> Vec<(String, RefStatus)> {
meta.refs
.iter()
.map(|r| (r.clone(), self.ref_status(meta, r)))
.filter(|(_, s)| s.is_stale())
.collect()
}
/// Snapshot every ref's current state into the frontmatter.
pub fn stamp_refs(&self, meta: &mut Meta) -> Result<()> {
let mut bad = vec![];
meta.ref_state.clear();
for r in &meta.refs {
match self.ref_state_now(r) {
Ok(s) => {
meta.ref_state.insert(r.clone(), s);
}
Err(RefStatus::Untracked) => {}
Err(s) => bad.push(format!(" {r}: {}", s.label())),
}
}
if !bad.is_empty() {
bail!("these refs don't resolve to existing files:\n{}", bad.join("\n"));
}
Ok(())
}
/// Validate an entry before it is written.
pub fn validate(&self, meta: &Meta, body: &str) -> Result<()> {
let mut errs = vec![];
if meta.title.trim().is_empty() {
errs.push("frontmatter needs a `title`".to_string());
}
let k = kind(&meta.kind);
if k.is_none() {
errs.push(format!(
"`kind` must be one of: {}",
KINDS.iter().map(|k| k.name).collect::<Vec<_>>().join(", ")
));
}
if meta.project.trim().is_empty() {
errs.push("frontmatter needs a `project`".into());
} else if !self.projects.contains_key(&meta.project) && meta.project != "general" {
errs.push(format!(
"unknown project {:?}: use one of {} or `general`, or add it with `brain project add`",
meta.project,
self.projects.keys().cloned().collect::<Vec<_>>().join(", ")
));
}
if let Some(k) = k {
let have: Vec<String> = sections(body).into_iter().map(|(h, _)| h.to_lowercase()).collect();
for (h, _, req) in k.sections {
if *req && !have.iter().any(|x| x == &h.to_lowercase()) {
errs.push(format!("a `{}` entry needs a `## {h}` section (see `brain template {}`)", k.name, k.name));
}
}
}
if body.trim().len() < 40 {
errs.push("the body is nearly empty".into());
}
if !errs.is_empty() {
bail!("{}", errs.join("\n"));
}
secret_check(&format!("{}\n{}", serde_yaml::to_string(meta).unwrap_or_default(), body))
}
pub fn commit(&self, paths: &[&Path], message: &str, author: &str) -> Result<()> {
let rel: Vec<String> = paths
.iter()
.map(|p| p.strip_prefix(&self.home).unwrap_or(p).to_string_lossy().to_string())
.collect();
let mut add = vec!["add", "-A", "--"];
add.extend(rel.iter().map(String::as_str));
git(&self.home, &add)?;
// Names come from --as / $BRAIN_AGENT; keep them to characters git accepts in an author.
let clean: String = author.chars().filter(|c| c.is_alphanumeric() || " ._-".contains(*c)).take(60).collect();
let author = format!("--author={} <agent@localhost>", if clean.trim().is_empty() { "unknown" } else { clean.trim() });
let mut c = vec!["commit", "-q", "-m", message, &author, "--"];
c.extend(rel.iter().map(String::as_str));
git(&self.home, &c)
}
}
/// Run git in the brain's own repo. Commit signing and git hooks are off for it: they come from the
/// user's global config (often enforced on work machines) and would block or fail every brain write.
pub fn git(dir: &Path, args: &[&str]) -> Result<()> {
let o = Command::new("git")
.arg("-C")
.arg(dir)
.args(["-c", "commit.gpgsign=false", "-c", "tag.gpgsign=false"])
.arg("-c")
.arg(format!("core.hooksPath={}", dir.join(".no-git-hooks").display()))
.args(args)
.output()?;
if !o.status.success() {
bail!(
"git {} failed: {}",
args.first().unwrap_or(&""),
String::from_utf8_lossy(&o.stderr).trim()
);
}
Ok(())
}
pub fn git_out(dir: &Path, args: &[&str]) -> String {
Command::new("git")
.arg("-C")
.arg(dir)
.args(args)
.output()
.map(|o| String::from_utf8_lossy(&o.stdout).to_string())
.unwrap_or_default()
}
/// Windows `C:x` / `C:\\x` (Path::is_absolute misses drive-relative ones). Not a drive elsewhere:
/// there `x:path` is a project ref.
fn has_drive_prefix(r: &str) -> bool {
let b = r.as_bytes();
cfg!(windows) && b.len() >= 2 && b[0].is_ascii_alphabetic() && b[1] == b':'
}
pub fn valid_id(id: &str) -> bool {
!id.is_empty()
&& id.len() <= 120
&& id.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-')
&& !id.starts_with('-')
}
pub fn slugify(s: &str) -> String {
let mut out = String::new();
for c in s.to_lowercase().chars() {
if c.is_ascii_alphanumeric() {
out.push(c);
} else if !out.ends_with('-') {
out.push('-');
}
}
let out = out.trim_matches('-');
// Cut long titles at a word boundary, not mid-word.
let out = if out.len() > 60 {
let cut = &out[..60];
cut.rfind('-').map(|i| &cut[..i]).unwrap_or(cut)
} else {
out
};
let out = out.trim_end_matches('-').to_string();
if out.is_empty() { "entry".into() } else { out }
}
pub fn today() -> String {
chrono::Local::now().format("%Y-%m-%d").to_string()
}
