use std::fs; use std::io; use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result, anyhow}; use jwalk::{Parallelism, WalkDirGeneric}; /// The signature line every cache directory tag starts with. /// /// See . Cargo writes one of these into every /// target directory and into `registry/` and `git/` in the cargo home. const CACHEDIR_TAG_SIGNATURE: &str = "Signature: 8a477f597d28d172789f06886806bc55"; /// The size of a tree and the most recent moment anything in it changed. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Measurement { pub size: u64, pub newest: SystemTime, } /// What one directory entry contributes to its tree's measurement. type EntryFacts = Option<(u64, SystemTime)>; /// How to spread a walk across threads, given where the caller is standing. /// /// A walk spawns its directory reads onto rayon's global pool. If the caller is /// *already* running on that pool — inside a `par_iter`, say — those tasks /// queue behind the very work waiting on them, and jwalk gives up and yields an /// empty tree rather than deadlocking. Silently: an empty tree measures zero /// bytes and takes its age from the root directory alone, which is to say it /// looks like a huge stale target that is safe to delete. It is not. /// /// So a nested walk runs serially. It is slower than it could be, and it is /// right, and nothing about the caller has to remember this. fn parallelism() -> Parallelism { if rayon::current_thread_index().is_some() { Parallelism::Serial } else { Parallelism::RayonDefaultPool { busy_timeout: Duration::from_secs(30), } } } /// Total up a path's bytes and find its newest modification time in one pass. /// /// Both numbers come from the same walk because callers always want both: the /// size to report and the mtime to age against. Unreadable entries are skipped /// rather than failing the walk — a directory we cannot read is a directory we /// cannot delete either, and the rest of the tree is still worth measuring. /// /// This is the whole cost of looking at a target directory, and it is one /// `stat` per file — a large target runs to six figures of them. The work is /// almost pure I/O wait, so the stats happen inside `process_read_dir`, which /// jwalk runs on a thread pool as it reads each directory. That parallelises /// the waiting rather than just the directory reads. pub fn measure(path: &Path) -> Result { // A cutoff in the far future is one nothing can be newer than, so the walk // never stops early and always has a measurement to return. walk(path, SystemTime::now() + FAR_FUTURE)? .ok_or_else(|| anyhow!("{} changed while it was being measured", path.display())) } /// Long enough that no file's mtime plausibly reaches it, short enough not to /// overflow anyone's clock arithmetic. const FAR_FUTURE: Duration = Duration::from_secs(100 * 365 * 24 * 60 * 60); /// Measure a tree, unless something in it is newer than `cutoff`. /// /// Returns `None` as soon as it meets an entry modified after `cutoff`: the /// tree is in use, and no one is going to need its size. Otherwise it walks to /// the end and returns the full measurement. /// /// This asymmetry is the point. Deciding a target is too new to touch takes one /// recent file, and a target being actively built is full of them — so the /// question costs a few stats instead of a walk of the whole tree. The case /// where nothing is newer is the case where the walk runs to completion, and /// there the size comes for free from a tree that is about to be deleted /// anyway. Cheap when the answer is "leave it", thorough when it is "delete /// it", which is the way round that matters. pub fn measure_if_older_than(path: &Path, cutoff: SystemTime) -> Result> { walk(path, cutoff) } fn walk(path: &Path, cutoff: SystemTime) -> Result> { let root_meta = fs::symlink_metadata(path) .with_context(|| format!("reading metadata for {}", path.display()))?; let mut size = 0; let mut newest = root_meta.modified().unwrap_or(SystemTime::UNIX_EPOCH); if newest > cutoff { return Ok(None); } if root_meta.is_file() { return Ok(Some(Measurement { size: root_meta.len(), newest, })); } let walk = WalkDirGeneric::<((), EntryFacts)>::new(path) .skip_hidden(false) .follow_links(false) .parallelism(parallelism()) .process_read_dir(|_depth, _path, _state, children| { for child in children.iter_mut().flatten() { // Not `metadata()`: a symlink must weigh what the link weighs, // not what it points at, which may be outside the tree // entirely — or be this tree, counted twice. child.client_state = fs::symlink_metadata(child.path()).ok().map(|meta| { ( meta.len(), meta.modified().unwrap_or(SystemTime::UNIX_EPOCH), ) }); } }); for entry in walk.into_iter().flatten() { let Some((len, modified)) = entry.client_state else { continue; }; if modified > cutoff { // Dropping the iterator stops the walk; whatever jwalk has already // read ahead is wasted, and that is the price of reading ahead. return Ok(None); } size += len; if modified > newest { newest = modified; } } Ok(Some(Measurement { size, newest })) } /// Delete a file or directory tree, treating an already-absent path as success. /// /// Reports whether anything was actually there. Callers total up what they /// freed from what this says went, so a path that had already vanished /// contributes nothing — which is what lets a caller skip checking in advance /// whether its records match the disk, and still report an exact figure. pub fn remove(path: &Path) -> Result { let meta = match fs::symlink_metadata(path) { Ok(meta) => meta, Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(false), Err(err) => { return Err(err).with_context(|| format!("reading metadata for {}", path.display())); } }; let result = if meta.is_dir() { fs::remove_dir_all(path) } else { fs::remove_file(path) }; match result { Ok(()) => Ok(true), // Something else got there first, which is still the outcome we wanted. Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(false), Err(err) => Err(err).with_context(|| format!("removing {}", path.display())), } } /// Whether a directory carries a cache directory tag. pub fn has_cachedir_tag(dir: &Path) -> bool { let Ok(contents) = fs::read_to_string(dir.join("CACHEDIR.TAG")) else { return false; }; contents.starts_with(CACHEDIR_TAG_SIGNATURE) } /// The cache of `rustc`'s version probe cargo keeps at a target directory's root. /// /// Cargo has written this since long before it began tagging target directories, /// so it recognises an old build the [`CACHEDIR_TAG_SIGNATURE`] check would miss. const RUSTC_INFO_CACHE: &str = ".rustc_info.json"; /// Whether cargo has built into `dir`, making its contents regenerable. /// /// This is the question that decides whether a directory is grunk's to delete. /// It is not "is this cargo's target directory" — cargo metadata answers *where* /// build output goes, but a project that has never been built has a `target` /// path pointing at whatever a person happened to leave there. What matters is /// whether cargo has actually produced output, and two markers settle it, each /// written by cargo and by nothing else that would land here: the cache tag /// modern cargo stamps on the directory, and the rustc-info cache every cargo /// since has left at its root. Old targets predating the tag carry only the /// second, which is exactly why the tag alone is not enough. pub fn is_cargo_build_output(dir: &Path) -> bool { has_cachedir_tag(dir) || dir.join(RUSTC_INFO_CACHE).is_file() } /// Whether `dir` is a cargo target directory. /// /// Two things have to hold: cargo has built into the directory, and its parent /// is a cargo package. Neither alone is enough — a neighbouring `Cargo.toml` /// says nothing about what the directory is, and cargo's build markers turn up /// under other tools too (`.pytest_cache` writes the same cache tag). Together /// they are decisive. pub fn is_cargo_target_dir(dir: &Path) -> bool { is_cargo_build_output(dir) && dir .parent() .is_some_and(|parent| parent.join("Cargo.toml").is_file()) } /// Expand a leading `~` to the user's home directory. /// /// Paths in the config file are written by hand, so they get the shell's /// courtesy even though no shell has been near them. pub fn expand_tilde(path: &Path) -> PathBuf { let Ok(rest) = path.strip_prefix("~") else { return path.to_path_buf(); }; match dirs::home_dir() { Some(home) => home.join(rest), None => path.to_path_buf(), } } /// Render a byte count the way a human reads a disk usage report. pub fn human_size(bytes: u64) -> String { const UNITS: [&str; 6] = ["B", "KiB", "MiB", "GiB", "TiB", "PiB"]; let mut value = bytes as f64; let mut unit = 0; while value >= 1024.0 && unit < UNITS.len() - 1 { value /= 1024.0; unit += 1; } if unit == 0 { format!("{bytes} B") } else { format!("{value:.1} {}", UNITS[unit]) } } #[cfg(test)] mod tests { use super::*; use std::fs::File; use std::io::Write; use tempfile::TempDir; fn write(path: &Path, contents: &str) { if let Some(parent) = path.parent() { fs::create_dir_all(parent).unwrap(); } File::create(path) .unwrap() .write_all(contents.as_bytes()) .unwrap(); } fn cargo_tag() -> &'static str { "Signature: 8a477f597d28d172789f06886806bc55\n\ # This file is a cache directory tag created by cargo.\n" } #[test] fn human_size_scales_and_keeps_bytes_exact() { assert_eq!(human_size(0), "0 B"); assert_eq!(human_size(999), "999 B"); assert_eq!(human_size(1024), "1.0 KiB"); assert_eq!(human_size(1536), "1.5 KiB"); assert_eq!(human_size(3 * 1024 * 1024 * 1024), "3.0 GiB"); } #[test] fn measure_totals_a_tree() { let tmp = TempDir::new().unwrap(); write(&tmp.path().join("a.txt"), "0123456789"); write(&tmp.path().join("nested/b.txt"), "01234"); let measured = measure(tmp.path()).unwrap(); assert!( measured.size >= 15, "expected at least the file bytes, got {}", measured.size ); } #[test] fn measure_counts_the_directory_itself() { // A directory occupies space of its own, and `du` counts it. Pinning it // against the directory's own metadata rather than a literal keeps this // honest across filesystems, which disagree about the number. let tmp = TempDir::new().unwrap(); let empty = tmp.path().join("empty"); fs::create_dir(&empty).unwrap(); assert_eq!( measure(&empty).unwrap().size, fs::symlink_metadata(&empty).unwrap().len() ); } #[test] fn measure_counts_every_directory_in_the_tree() { let tmp = TempDir::new().unwrap(); let root = tmp.path().join("root"); let nested = root.join("nested"); fs::create_dir_all(&nested).unwrap(); fs::write(nested.join("file.txt"), "0123456789").unwrap(); let expected = fs::symlink_metadata(&root).unwrap().len() + fs::symlink_metadata(&nested).unwrap().len() + 10; assert_eq!(measure(&root).unwrap().size, expected); } #[test] fn measure_is_correct_when_called_from_inside_a_parallel_iterator() { // Regression: `measure` spawns its directory reads onto rayon's global // pool. Called from inside a `par_iter` on that same pool, those tasks // starved and the walk yielded nothing at all — reporting a 34 GiB // target as 0 bytes, aged from its own directory, i.e. stale enough to // delete. Enough trees to fill the pool, each deep enough to need // several directory reads. use rayon::prelude::*; let tmp = TempDir::new().unwrap(); let trees: Vec = (0..64) .map(|t| { let root = tmp.path().join(format!("tree{t}")); for d in 0..8 { let dir = root.join(format!("d{d}")); for f in 0..8 { write(&dir.join(format!("f{f}.txt")), "0123456789"); } } root }) .collect(); let expected: Vec = trees.iter().map(|t| measure(t).unwrap().size).collect(); assert!( expected.iter().all(|&size| size >= 640), "fixture should hold 64 files of 10 bytes per tree: {expected:?}" ); let in_parallel: Vec = trees.par_iter().map(|t| measure(t).unwrap().size).collect(); assert_eq!( in_parallel, expected, "measuring inside a par_iter must agree with measuring outside one" ); } #[test] fn measure_reports_a_files_own_size() { let tmp = TempDir::new().unwrap(); let file = tmp.path().join("a.txt"); write(&file, "0123456789"); assert_eq!(measure(&file).unwrap().size, 10); } #[test] fn target_dir_needs_both_a_tag_and_a_manifest() { let tmp = TempDir::new().unwrap(); let project = tmp.path().join("proj"); let target = project.join("target"); fs::create_dir_all(&target).unwrap(); // Tag but no manifest: not a target dir. write(&target.join("CACHEDIR.TAG"), cargo_tag()); assert!(!is_cargo_target_dir(&target)); // Manifest as well: now it is. write(&project.join("Cargo.toml"), "[package]\n"); assert!(is_cargo_target_dir(&target)); } #[test] fn manifest_beside_an_unbuilt_target_is_not_a_target_dir() { let tmp = TempDir::new().unwrap(); let project = tmp.path().join("proj"); write(&project.join("Cargo.toml"), "[package]\n"); // A `target` with a manifest beside it but nothing cargo ever wrote: // the path where output would go, holding data cargo did not make. fs::create_dir_all(project.join("target")).unwrap(); assert!(!is_cargo_target_dir(&project.join("target"))); } #[test] fn an_old_target_is_recognised_by_its_rustc_info_alone() { // Cargo tagged target directories only from 1.42 on. A build older than // that carries `.rustc_info.json` but no `CACHEDIR.TAG`, and is every // bit as regenerable — so it must still read as a target directory. let tmp = TempDir::new().unwrap(); let project = tmp.path().join("proj"); let target = project.join("target"); write(&project.join("Cargo.toml"), "[package]\n"); write(&target.join(".rustc_info.json"), "{}"); assert!(!has_cachedir_tag(&target)); assert!(is_cargo_build_output(&target)); assert!(is_cargo_target_dir(&target)); } #[test] fn build_output_is_either_marker_and_hand_made_data_is_neither() { let tmp = TempDir::new().unwrap(); let tagged = tmp.path().join("tagged"); write(&tagged.join("CACHEDIR.TAG"), cargo_tag()); assert!(is_cargo_build_output(&tagged)); let old = tmp.path().join("old"); write(&old.join(".rustc_info.json"), "{}"); assert!(is_cargo_build_output(&old)); // A directory a person filled themselves carries neither marker. let handmade = tmp.path().join("handmade"); write(&handmade.join("precious.txt"), "do not delete"); assert!(!is_cargo_build_output(&handmade)); } #[test] fn a_foreign_cache_tag_is_not_a_target_dir() { // pytest writes the same signature into its own cache directory. let tmp = TempDir::new().unwrap(); let project = tmp.path().join("proj"); write(&project.join("Cargo.toml"), "[package]\n"); let cache = project.join(".pytest_cache"); write( &cache.join("CACHEDIR.TAG"), "Signature: 8a477f597d28d172789f06886806bc55\n\ # This file is a cache directory tag created by pytest.\n", ); // The tag check passes, so only the caller's target-shaped naming // separates these; `has_cachedir_tag` is deliberately permissive. assert!(has_cachedir_tag(&cache)); } #[test] fn remove_tolerates_a_missing_path() { let tmp = TempDir::new().unwrap(); assert!(remove(&tmp.path().join("never-existed")).is_ok()); } #[test] fn remove_deletes_files_and_trees() { let tmp = TempDir::new().unwrap(); let file = tmp.path().join("a.txt"); write(&file, "x"); remove(&file).unwrap(); assert!(!file.exists()); let tree = tmp.path().join("tree"); write(&tree.join("deep/b.txt"), "x"); remove(&tree).unwrap(); assert!(!tree.exists()); } }