use std::fs::OpenOptions; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result, bail}; use fs4::fs_std::FileExt; use rayon::prelude::*; use rusqlite::{Connection, OpenFlags}; use crate::fsutil::{Measurement, human_size, measure, remove}; use crate::policy::{Intent, Policy}; use crate::progress::{Reporter, plural}; use crate::reclaim::{CacheRow, Category, EntryKey, Findings, Provider, Reclaimable, Retained}; /// A tracked artifact as the global cache database describes it, resolved to /// the path it occupies but not yet checked against the disk. struct TrackedRow { path: PathBuf, /// The size cargo recorded, if it recorded one. size: Option, timestamp: i64, row: CacheRow, } /// Cargo's own record of when each cached artifact was last used. const GLOBAL_CACHE_DB: &str = ".global-cache"; /// The lock file cargo holds while mutating its caches. const PACKAGE_CACHE_LOCK: &str = ".package-cache"; /// How many cached artifacts go by between progress reports. const PROGRESS_EVERY: usize = 500; /// A cargo home directory: `$CARGO_HOME`, usually `~/.cargo`. #[derive(Debug, Clone)] pub struct CargoHome { root: PathBuf, } impl CargoHome { pub fn open() -> Result { let root = home::cargo_home().context("locating CARGO_HOME")?; Ok(Self::at(root)) } pub fn at(root: PathBuf) -> Self { Self { root } } pub fn root(&self) -> &Path { &self.root } pub fn db_path(&self) -> PathBuf { self.root.join(GLOBAL_CACHE_DB) } /// Whether cargo has ever tracked cache usage here. pub fn tracks_usage(&self) -> bool { self.db_path().is_file() } /// Take the same exclusive lock cargo takes before touching its caches. /// /// Without this, a concurrent `cargo build` could be downloading a crate /// into the very directory being deleted. The lock file and protocol are /// cargo's, not ours; we are just another participant. fn lock(&self) -> Result { let path = self.root.join(PACKAGE_CACHE_LOCK); let file = OpenOptions::new() .read(true) .write(true) .create(true) .open(&path) .with_context(|| format!("opening {}", path.display()))?; FileExt::lock_exclusive(&file).with_context(|| format!("locking {}", path.display()))?; Ok(CacheLock { file }) } } /// Cargo's package cache lock, released on drop. struct CacheLock { file: std::fs::File, } impl Drop for CacheLock { fn drop(&mut self) { let _ = FileExt::unlock(&self.file); } } /// The reclaimable contents of a cargo home, as tracked by its global cache DB. /// /// Everything here is driven by cargo's `.global-cache` database rather than by /// file timestamps. That database is the only honest record of when a cached /// artifact was last *used*: the mtime of an unpacked source directory says /// when it was extracted, which can be months before the last build that /// depended on it. Ageing against mtime would mean deleting things in daily use. /// /// The registry index is deliberately not reclaimable. Cargo refreshes it on /// demand and it is small; deleting it buys almost nothing. pub struct GlobalCache { home: CargoHome, } impl GlobalCache { pub fn new(home: CargoHome) -> Self { Self { home } } fn open_db(&self, flags: OpenFlags) -> Result { Connection::open_with_flags(self.home.db_path(), flags) .with_context(|| format!("opening {}", self.home.db_path().display())) } /// Sort tracked rows into what may go and what stays. /// /// The ages and sizes both come out of the database, so an entry can be /// judged without touching the disk at all. The only reason to go near it /// is to check the files are still there — a tracked artifact with no files /// is a stale row, and counting it would promise bytes that are not there /// to free. /// /// A survey pays for that check, because a person is reading the totals. A /// clean does not: it is a few thousand seeks to confirm records that are /// almost always right, and nothing downstream needs them. Eligible rows /// get deleted either way — deletion tolerates a file that has already gone /// and reports what it actually removed, so the "reclaimed" figure stays /// exact regardless. Only the count of what was held back can drift, by /// including rows whose files went missing behind cargo's back. fn entries(&self, category: Category, rows: Vec, policy: &Policy) -> Findings { let rows = match policy.intent() { // Independent stats, none waiting on another, so spread the waiting // across the pool. `into_par_iter` preserves row order, keeping the // report stable from run to run. Intent::Survey => rows .into_par_iter() .filter(|tracked| tracked.path.exists()) .collect(), Intent::Clean => rows, }; let mut findings = Findings::default(); for tracked in rows { let last_used = unix_seconds(tracked.timestamp); if policy.is_eligible(category, last_used) { findings.reclaimable.push(Reclaimable { category, size: self.size_of(&tracked), path: tracked.path, last_used, key: EntryKey::CacheRow(tracked.row), }); } else { // Same bargain as a target directory: a clean does not price up // what it is keeping. Most rows would cost nothing to size, // since cargo recorded it, and a clean could report those for // free — but not the ones cargo left blank, which need a walk. // Sizing whichever happened to be cheap would make the report // depend on what cargo bothered to write down. One rule // instead: a clean counts what it keeps, a survey measures it. let measured = match policy.intent() { Intent::Survey => Some(Measurement { size: self.size_of(&tracked), newest: last_used, }), Intent::Clean => None, }; findings.retained.push(Retained { category, path: tracked.path, measured, }); } } findings } /// What a tracked artifact occupies. /// /// Cargo records sizes as it populates the cache, which saves walking /// several thousand directories just to print a total. Only the rows it /// left blank cost a walk — and that walk parallelises internally, so it /// happens out here rather than inside a `par_iter`, which would starve it /// of the very threads it needs. fn size_of(&self, tracked: &TrackedRow) -> u64 { match tracked.size { Some(size) if size >= 0 => size as u64, _ => measure(&tracked.path).map(|m| m.size).unwrap_or(0), } } fn scan_registry_crates(&self, db: &Connection, policy: &Policy) -> Result { let mut stmt = db.prepare( "SELECT i.name, c.registry_id, c.name, c.size, c.timestamp FROM registry_crate c JOIN registry_index i ON i.id = c.registry_id", )?; let rows = stmt .query_map([], |row| { let index: String = row.get(0)?; let registry_id: i64 = row.get(1)?; let name: String = row.get(2)?; Ok(TrackedRow { // The tracked name already carries the `.crate` extension. path: self .home .root .join("registry/cache") .join(index) .join(&name), size: row.get(3)?, timestamp: row.get(4)?, row: CacheRow::RegistryCrate { registry_id, name }, }) })? .collect::>>()?; Ok(self.entries(Category::RegistryCrate, rows, policy)) } fn scan_registry_srcs(&self, db: &Connection, policy: &Policy) -> Result { let mut stmt = db.prepare( "SELECT i.name, s.registry_id, s.name, s.size, s.timestamp FROM registry_src s JOIN registry_index i ON i.id = s.registry_id", )?; let rows = stmt .query_map([], |row| { let index: String = row.get(0)?; let registry_id: i64 = row.get(1)?; let name: String = row.get(2)?; Ok(TrackedRow { path: self.home.root.join("registry/src").join(index).join(&name), size: row.get(3)?, timestamp: row.get(4)?, row: CacheRow::RegistrySrc { registry_id, name }, }) })? .collect::>>()?; Ok(self.entries(Category::RegistrySrc, rows, policy)) } fn scan_git_checkouts(&self, db: &Connection, policy: &Policy) -> Result { let mut stmt = db.prepare( "SELECT d.name, c.git_id, c.name, c.size, c.timestamp FROM git_checkout c JOIN git_db d ON d.id = c.git_id", )?; let rows = stmt .query_map([], |row| { let db_name: String = row.get(0)?; let git_id: i64 = row.get(1)?; let name: String = row.get(2)?; Ok(TrackedRow { path: self .home .root .join("git/checkouts") .join(db_name) .join(&name), size: row.get(3)?, timestamp: row.get(4)?, row: CacheRow::GitCheckout { git_id, name }, }) })? .collect::>>()?; Ok(self.entries(Category::GitCheckout, rows, policy)) } /// Bare git clones, aged against their checkouts as well as themselves. /// /// A checkout is derived from its db, and cargo's schema cascades a db's /// deletion to its checkouts. So a db is only as idle as its liveliest /// checkout: taking `MAX` here is what stops an old clone from being /// deleted out from under a working tree that was used this morning. fn scan_git_dbs(&self, db: &Connection, policy: &Policy) -> Result { let mut stmt = db.prepare( "SELECT d.id, d.name, MAX(d.timestamp, COALESCE(MAX(c.timestamp), 0)) FROM git_db d LEFT JOIN git_checkout c ON c.git_id = d.id GROUP BY d.id, d.name, d.timestamp", )?; let rows = stmt .query_map([], |row| { let id: i64 = row.get(0)?; let name: String = row.get(1)?; Ok(TrackedRow { path: self.home.root.join("git/db").join(&name), // Cargo records no size for a bare clone, so it is measured. size: None, timestamp: row.get(2)?, row: CacheRow::GitDb { id }, }) })? .collect::>>()?; Ok(self.entries(Category::GitDb, rows, policy)) } /// Every path a cache row owns on disk. /// /// Deleting a git db strands its checkouts — they are worthless without the /// clone they came from, and cargo's schema drops their rows by cascade — /// so they go at the same time. fn owned_paths(&self, item: &Reclaimable, db: &Connection) -> Result> { let EntryKey::CacheRow(CacheRow::GitDb { id }) = &item.key else { return Ok(vec![item.path.clone()]); }; let mut stmt = db.prepare("SELECT name FROM git_db WHERE id = ?1")?; let mut paths = vec![item.path.clone()]; let names = stmt.query_map([id], |row| row.get::<_, String>(0))?; for name in names { paths.push(self.home.root.join("git/checkouts").join(name?)); } Ok(paths) } } impl Provider for GlobalCache { fn scan(&self, policy: &Policy, progress: &Reporter) -> Result { let mut findings = Findings::default(); if !self.home.tracks_usage() { return Ok(findings); } // Instant for a clean, several thousand seeks for a survey — so it is // worth naming either way, and worth naming before rather than after. progress.phase("Examining the cargo home:"); progress.start(self.home.root().display()); let db = self.open_db(OpenFlags::SQLITE_OPEN_READ_ONLY)?; // A category nobody asked for is not queried at all; each one skipped // is a few thousand `exists` calls not made. if policy.wants(Category::RegistryCrate) { findings.absorb(self.scan_registry_crates(&db, policy)?); } if policy.wants(Category::RegistrySrc) { findings.absorb(self.scan_registry_srcs(&db, policy)?); } if policy.wants(Category::GitCheckout) { findings.absorb(self.scan_git_checkouts(&db, policy)?); } if policy.wants(Category::GitDb) { findings.absorb(self.scan_git_dbs(&db, policy)?); } progress.finish(format_args!( "{} to reclaim, {} held back", human_size( findings .reclaimable .iter() .map(|item| item.size) .sum::() ), findings.retained.len() )); Ok(findings) } fn remove(&mut self, items: &[Reclaimable], progress: &Reporter) -> Result> { if items.is_empty() { return Ok(Vec::new()); } progress.phase(format_args!( "Deleting {} cached {} from the cargo home:", items.len(), plural(items.len(), "artifact", "artifacts") )); let _lock = self.home.lock()?; let mut db = self.open_db(OpenFlags::SQLITE_OPEN_READ_WRITE)?; let tx = db.transaction()?; let mut removed = Vec::with_capacity(items.len()); for (done, item) in items.iter().enumerate() { // Thousands of small deletions: each one is far too dull to narrate, // and on a cold spinning disk they are collectively slow enough to // need something said. A count every so often is the compromise — // proof of progress without a line per crate. if done > 0 && done.is_multiple_of(PROGRESS_EVERY) { progress.line(format_args!("{done}/{} deleted", items.len())); } let EntryKey::CacheRow(row) = &item.key else { bail!( "{} is not tracked by the global cache and cannot be removed from here", item.path.display() ); }; // Files first, then the row. A row still standing over deleted // files just makes cargo refetch something it already would have; // the reverse leaves bytes on disk that nothing will ever reclaim, // because cargo has forgotten they exist. let mut existed = false; for path in self.owned_paths(item, &tx)? { existed |= remove(&path)?; } delete_row(&tx, row)?; // The row goes either way — it tracks something that is now // definitely absent. But bytes are only counted if there were any. if existed { removed.push(item.clone()); } } tx.commit()?; progress.line(format_args!("{}/{} deleted", removed.len(), items.len())); Ok(removed) } } fn delete_row(db: &Connection, row: &CacheRow) -> Result<()> { match row { CacheRow::RegistryCrate { registry_id, name } => db.execute( "DELETE FROM registry_crate WHERE registry_id = ?1 AND name = ?2", rusqlite::params![registry_id, name], ), CacheRow::RegistrySrc { registry_id, name } => db.execute( "DELETE FROM registry_src WHERE registry_id = ?1 AND name = ?2", rusqlite::params![registry_id, name], ), CacheRow::GitCheckout { git_id, name } => db.execute( "DELETE FROM git_checkout WHERE git_id = ?1 AND name = ?2", rusqlite::params![git_id, name], ), // Checkout rows go by cascade, matching the files removed alongside. CacheRow::GitDb { id } => { db.execute("DELETE FROM git_db WHERE id = ?1", rusqlite::params![id]) } } .with_context(|| format!("deleting cache row {row:?}"))?; Ok(()) } /// Cargo stores timestamps as whole seconds since the epoch. fn unix_seconds(seconds: i64) -> SystemTime { if seconds >= 0 { SystemTime::UNIX_EPOCH + Duration::from_secs(seconds as u64) } else { SystemTime::UNIX_EPOCH - Duration::from_secs(seconds.unsigned_abs()) } } #[cfg(test)] mod tests { use super::*; use std::fs; use tempfile::TempDir; /// Cargo's schema, shared with the end-to-end tests so both agree on what /// they are testing against. fn create_schema(db: &Connection) { db.execute_batch(include_str!("../fixtures/global-cache-schema.sql")) .unwrap(); } const INDEX: &str = "index.crates.io-1949cf8c6b5b557f"; const GIT_DB: &str = "badgebuilder-e22ab1d71dcafcb2"; /// The size cargo recorded for the crate tarball, which is deliberately /// nothing like the size of the file the fixture actually writes. const TRACKED_CRATE_SIZE: u64 = 4096; /// The payload written into the unpacked source directory, big enough that /// a measured size is unmistakably it and not directory overhead. const SRC_PAYLOAD: usize = 10_000; struct Fixture { _tmp: TempDir, home: CargoHome, } impl Fixture { /// A cargo home with one crate, one src, one git db and one checkout. fn new() -> Self { let tmp = TempDir::new().unwrap(); let root = tmp.path().to_path_buf(); let write = |rel: &str, contents: &[u8]| { let path = root.join(rel); fs::create_dir_all(path.parent().unwrap()).unwrap(); fs::write(&path, contents).unwrap(); }; let dir = |rel: &str, contents: &[u8]| { fs::create_dir_all(root.join(rel)).unwrap(); fs::write(root.join(rel).join("file"), contents).unwrap(); }; write( &format!("registry/cache/{INDEX}/serde-1.0.0.crate"), b"tiny", ); dir( &format!("registry/src/{INDEX}/serde-1.0.0"), &vec![b'x'; SRC_PAYLOAD], ); dir(&format!("git/db/{GIT_DB}"), b"contents"); dir(&format!("git/checkouts/{GIT_DB}/bb58e65"), b"contents"); let db = Connection::open(root.join(GLOBAL_CACHE_DB)).unwrap(); create_schema(&db); db.execute( "INSERT INTO registry_index (id, name, timestamp) VALUES (2, ?1, 100)", [INDEX], ) .unwrap(); db.execute( "INSERT INTO registry_crate VALUES (2, 'serde-1.0.0.crate', ?1, 1000)", [TRACKED_CRATE_SIZE], ) .unwrap(); db.execute( "INSERT INTO registry_src VALUES (2, 'serde-1.0.0', NULL, 2000)", [], ) .unwrap(); db.execute( "INSERT INTO git_db (id, name, timestamp) VALUES (5, ?1, 3000)", [GIT_DB], ) .unwrap(); db.execute( "INSERT INTO git_checkout VALUES (5, 'bb58e65', NULL, 4000)", [], ) .unwrap(); drop(db); Self { _tmp: tmp, home: CargoHome::at(root), } } fn cache(&self) -> GlobalCache { GlobalCache::new(self.home.clone()) } fn root(&self) -> &Path { self.home.root() } fn rows(&self, table: &str) -> i64 { let db = Connection::open(self.home.db_path()).unwrap(); db.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |r| r.get(0)) .unwrap() } } /// A policy that finds everything eligible, so scans return it all as /// reclaimable and these tests can talk about paths and sizes. fn take_everything() -> Policy { let dir = TempDir::new().unwrap(); let path = dir.path().join("grunk.toml"); fs::write(&path, "min_age = \"0s\"\n").unwrap(); Policy::new( &crate::config::Config::load_from(&path).unwrap(), &crate::policy::Selection::default(), crate::policy::Intent::Survey, ) } fn find(items: &[Reclaimable], category: Category) -> &Reclaimable { items .iter() .find(|i| i.category == category) .unwrap_or_else(|| panic!("no {category} entry in {items:#?}")) } #[test] fn scan_maps_every_row_to_its_path() { let fixture = Fixture::new(); let items = fixture .cache() .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; assert_eq!(items.len(), 4, "{items:#?}"); assert_eq!( find(&items, Category::RegistryCrate).path, fixture .root() .join(format!("registry/cache/{INDEX}/serde-1.0.0.crate")) ); assert_eq!( find(&items, Category::RegistrySrc).path, fixture .root() .join(format!("registry/src/{INDEX}/serde-1.0.0")) ); assert_eq!( find(&items, Category::GitCheckout).path, fixture .root() .join(format!("git/checkouts/{GIT_DB}/bb58e65")) ); assert_eq!( find(&items, Category::GitDb).path, fixture.root().join(format!("git/db/{GIT_DB}")) ); } #[test] fn scan_prefers_the_tracked_size_and_measures_when_absent() { let fixture = Fixture::new(); let items = fixture .cache() .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; // Cargo recorded a size, so it is used rather than walking the file — // which is why this reads 4096 and not the four bytes actually written. assert_eq!( find(&items, Category::RegistryCrate).size, TRACKED_CRATE_SIZE ); // Cargo recorded no size for the unpacked sources, so they get measured // off the disk. The payload dominates, plus a little for the directory // itself, which occupies space too. let measured = find(&items, Category::RegistrySrc).size; assert!( (SRC_PAYLOAD as u64..SRC_PAYLOAD as u64 + 4096).contains(&measured), "expected roughly the {SRC_PAYLOAD}-byte payload, got {measured}" ); } #[test] fn scan_uses_cargos_timestamps() { let fixture = Fixture::new(); let items = fixture .cache() .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; assert_eq!( find(&items, Category::RegistryCrate).last_used, unix_seconds(1000) ); assert_eq!( find(&items, Category::RegistrySrc).last_used, unix_seconds(2000) ); } #[test] fn a_git_db_is_as_recent_as_its_newest_checkout() { // The db row says 3000, but its checkout was used at 4000. Ageing the // db at 3000 would let a threshold delete a clone whose working tree is // in active use. let fixture = Fixture::new(); let items = fixture .cache() .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; assert_eq!(find(&items, Category::GitDb).last_used, unix_seconds(4000)); } #[test] fn scan_skips_rows_whose_files_are_gone() { let fixture = Fixture::new(); fs::remove_file( fixture .root() .join(format!("registry/cache/{INDEX}/serde-1.0.0.crate")), ) .unwrap(); let items = fixture .cache() .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; assert!(!items.iter().any(|i| i.category == Category::RegistryCrate)); assert_eq!(items.len(), 3); } #[test] fn a_home_without_tracking_has_nothing_to_offer() { let tmp = TempDir::new().unwrap(); let home = CargoHome::at(tmp.path().to_path_buf()); assert!(!home.tracks_usage()); let findings = GlobalCache::new(home) .scan(&take_everything(), &Reporter::silent()) .unwrap(); assert!(findings.reclaimable.is_empty()); assert!(findings.retained.is_empty()); } #[test] fn removing_takes_the_files_and_the_row_together() { let fixture = Fixture::new(); let mut cache = fixture.cache(); let items = cache .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; let crate_entry = find(&items, Category::RegistryCrate).clone(); let removed = cache .remove(std::slice::from_ref(&crate_entry), &Reporter::silent()) .unwrap(); assert_eq!(removed.len(), 1); assert_eq!(removed[0].size, 4096); assert!(!crate_entry.path.exists()); assert_eq!( fixture.rows("registry_crate"), 0, "the row must go with the file, or cargo still believes it is there" ); // Nothing else was touched. assert_eq!(fixture.rows("registry_src"), 1); assert_eq!(fixture.rows("git_db"), 1); } #[test] fn removing_a_git_db_takes_its_checkouts_too() { let fixture = Fixture::new(); let mut cache = fixture.cache(); let items = cache .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; let git_db = find(&items, Category::GitDb).clone(); let checkout = find(&items, Category::GitCheckout).path.clone(); cache .remove(std::slice::from_ref(&git_db), &Reporter::silent()) .unwrap(); assert!(!git_db.path.exists()); assert!( !checkout.exists(), "a checkout is worthless without the clone it came from" ); assert!( !fixture .root() .join(format!("git/checkouts/{GIT_DB}")) .exists() ); assert_eq!(fixture.rows("git_db"), 0); assert_eq!( fixture.rows("git_checkout"), 0, "checkout rows should have gone by cascade" ); } #[test] fn removing_nothing_touches_nothing() { let fixture = Fixture::new(); let removed = fixture.cache().remove(&[], &Reporter::silent()).unwrap(); assert!(removed.is_empty()); assert_eq!(fixture.rows("registry_crate"), 1); } #[test] fn removing_everything_empties_the_home_and_the_db() { let fixture = Fixture::new(); let mut cache = fixture.cache(); let items = cache .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; let removed = cache.remove(&items, &Reporter::silent()).unwrap(); assert_eq!(removed.len(), 4); for item in &items { assert!(!item.path.exists(), "{} survived", item.path.display()); } assert_eq!(fixture.rows("registry_crate"), 0); assert_eq!(fixture.rows("registry_src"), 0); assert_eq!(fixture.rows("git_db"), 0); assert_eq!(fixture.rows("git_checkout"), 0); // The index itself is never reclaimed. assert_eq!(fixture.rows("registry_index"), 1); } #[test] fn scanning_again_after_a_full_clean_finds_nothing() { let fixture = Fixture::new(); let mut cache = fixture.cache(); let items = cache .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable; cache.remove(&items, &Reporter::silent()).unwrap(); assert!( cache .scan(&take_everything(), &Reporter::silent()) .unwrap() .reclaimable .is_empty() ); } }