manifest.rs
raw
use std::{
collections::HashSet,
io::Read,
path::PathBuf,
time::{SystemTime, UNIX_EPOCH},
};
use crate::{ChunkId, Error, LockId, PruneId, Repository, SnapshotId};
/// Whether a snapshot is a backup's finished word, or only what one had
/// stored by the time it was interrupted.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum Coverage {
/// Everything the backup that published this meant to record, as it
/// read them. The run it came from is over — whether it walked the
/// whole tree or deliberately stopped short of it.
#[default]
Complete,
/// Only the objects a run had stored when it published this, while
/// it was still going. Restoring one yields a subset of the tree —
/// the point of it is that a backup interrupted halfway is still
/// worth something — and a finished snapshot supersedes it.
Partial,
}
impl Coverage {
/// How this claim enters a snapshot's id; see
/// [`Repository::snapshot_id`].
pub(crate) fn tag(self) -> &'static [u8] {
match self {
Coverage::Complete => b"complete",
Coverage::Partial => b"partial",
}
}
}
/// A backup: metadata plus the chunk list of its manifest.
///
/// The manifest itself — the stream of [`Entry`] values describing every
/// backed-up filesystem object — is stored through the chunk store like
/// file content, so mostly-unchanged manifests deduplicate across
/// snapshots. The snapshot record is the small, encrypted root object
/// that makes the manifest findable.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Snapshot {
pub id: SnapshotId,
/// Seconds since the Unix epoch at which the snapshot was completed.
pub created: u64,
/// The root directory that was backed up.
#[serde(with = "crate::os_path")]
pub root: PathBuf,
/// The chunks holding the manifest stream, in order.
pub manifest: Vec<ChunkId>,
/// Number of entries recorded in the manifest.
///
/// An upper bound on the objects the snapshot covers rather than an
/// exact count: a backup interrupted mid-directory records that
/// directory's children again when it resumes, and reading the
/// manifest yields each path once regardless.
pub entries: u64,
/// Total plaintext bytes of file content covered by the snapshot.
pub content_bytes: u64,
/// Whether the backup that produced this snapshot ran to completion.
///
/// Defaulted for the sake of records written before partial
/// snapshots existed, and [`Coverage::Complete`] is the right
/// reading of those: back then a snapshot was only ever stored once
/// the whole tree had been.
#[serde(default)]
pub coverage: Coverage,
}
impl Snapshot {
/// A snapshot record for a manifest already stored via
/// [`Repository::store_manifest`], stamped with the current time.
///
/// Compute `id` with [`Repository::snapshot_id`] so that redoing an
/// interrupted backup converges on one snapshot object instead of
/// producing duplicates.
pub fn new(
id: SnapshotId,
root: PathBuf,
manifest: Vec<ChunkId>,
entries: u64,
content_bytes: u64,
) -> Snapshot {
Snapshot::with_coverage(
id,
root,
manifest,
entries,
content_bytes,
Coverage::Complete,
)
}
/// A snapshot record for what a backup had stored by the time it
/// was written — published as a run goes and again when one stands
/// down, so there is always something to restore from. Otherwise
/// exactly like [`Snapshot::new`]: the manifest is real, the chunks
/// it names are in the repository, and restoring it works — it
/// simply does not cover the whole tree.
pub fn partial(
id: SnapshotId,
root: PathBuf,
manifest: Vec<ChunkId>,
entries: u64,
content_bytes: u64,
) -> Snapshot {
Snapshot::with_coverage(
id,
root,
manifest,
entries,
content_bytes,
Coverage::Partial,
)
}
/// The same backup over a narrowed manifest, as an excision leaves
/// it.
///
/// Everything the record says about the run that made it is kept —
/// when it ran, what it covered, how completely — because none of
/// that has changed. What the manifest holds has. Derive `id` with
/// [`Repository::rewritten_snapshot_id`], which keeps two snapshots
/// two even when an excision leaves them the same manifest.
pub fn rewritten(
&self,
id: SnapshotId,
manifest: Vec<ChunkId>,
entries: u64,
content_bytes: u64,
) -> Snapshot {
Snapshot {
id,
created: self.created,
root: self.root.clone(),
manifest,
entries,
content_bytes,
coverage: self.coverage,
}
}
fn with_coverage(
id: SnapshotId,
root: PathBuf,
manifest: Vec<ChunkId>,
entries: u64,
content_bytes: u64,
coverage: Coverage,
) -> Snapshot {
Snapshot {
id,
created: now(),
root,
manifest,
entries,
content_bytes,
coverage,
}
}
}
/// What one prune left behind: enough to audit the repository's
/// history, and — far more importantly — an object whose existence tells
/// every local chunk cache that something has been removed.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct PruneRecord {
pub id: PruneId,
/// Seconds since the Unix epoch at which the prune began.
pub started: u64,
/// The snapshots it removed.
pub removed: Vec<SnapshotId>,
/// Where it ran, for the benefit of whoever reads the history.
pub host: String,
}
impl PruneRecord {
pub fn new(removed: Vec<SnapshotId>, host: impl Into<String>) -> PruneRecord {
PruneRecord {
id: PruneId::generate(),
started: now(),
removed,
host: host.into(),
}
}
}
/// A running backup's claim on the repository: it holds chunks that no
/// snapshot names yet, so a prune would sweep them out from under it.
///
/// Taken when a backup starts and released once everything it has
/// recorded is published — which is when it finishes, and also when it
/// suspends gracefully, since suspending publishes a partial snapshot.
/// A run that was killed leaves its lock behind, which is exactly the
/// state that should stop a prune.
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Lock {
pub id: LockId,
/// Seconds since the Unix epoch at which the lock was taken.
pub taken: u64,
pub host: String,
pub process: u32,
/// The tree being backed up, so a stale lock can be recognised.
#[serde(with = "crate::os_path")]
pub root: PathBuf,
}
impl Lock {
pub fn new(root: PathBuf, host: impl Into<String>) -> Lock {
Lock {
id: LockId::generate(),
taken: now(),
host: host.into(),
process: std::process::id(),
root,
}
}
}
fn now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|elapsed| elapsed.as_secs())
.unwrap_or(0)
}
/// One filesystem object recorded in a snapshot manifest.
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub struct Entry {
/// Path relative to the snapshot root.
#[serde(with = "crate::os_path")]
pub path: PathBuf,
pub kind: EntryKind,
/// Unix permission bits, where the platform has them.
pub mode: Option<u32>,
/// Modification time in nanoseconds since the Unix epoch.
pub mtime: Option<i64>,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)]
pub enum EntryKind {
File {
/// The file's content as chunk store references, in order.
chunks: Vec<ChunkId>,
/// Plaintext length in bytes.
len: u64,
},
Directory,
Symlink {
#[serde(with = "crate::os_path")]
target: PathBuf,
},
}
/// Adapts an iterator of entries into the CBOR byte stream that gets
/// chunked and stored as the manifest.
pub(crate) struct EntryStream<I> {
entries: I,
buffer: Vec<u8>,
position: usize,
}
impl<I> EntryStream<I> {
pub(crate) fn new(entries: I) -> EntryStream<I> {
EntryStream {
entries,
buffer: Vec::new(),
position: 0,
}
}
}
impl<I> Read for EntryStream<I>
where
I: Iterator<Item = Entry>,
{
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
while self.position == self.buffer.len() {
let Some(entry) = self.entries.next() else {
return Ok(0);
};
self.buffer.clear();
self.position = 0;
ciborium::into_writer(&entry, &mut self.buffer).map_err(std::io::Error::other)?;
}
let available = &self.buffer[self.position..];
let count = available.len().min(out.len());
out[..count].copy_from_slice(&available[..count]);
self.position += count;
Ok(count)
}
}
/// Presents a manifest's chunk list as one continuous byte stream,
/// fetching and decrypting each chunk as the reader reaches it.
struct ChunkChain<'r> {
repository: &'r Repository,
ids: std::vec::IntoIter<ChunkId>,
current: Vec<u8>,
position: usize,
}
impl ChunkChain<'_> {
/// Ensures at least one unread byte is buffered, reporting whether
/// the stream has ended instead. This lets the manifest reader
/// distinguish a clean end-of-stream from a truncated entry.
fn has_data(&mut self) -> std::io::Result<bool> {
while self.position == self.current.len() {
let Some(id) = self.ids.next() else {
return Ok(false);
};
self.current = self
.repository
.load_chunk(id)
.map_err(std::io::Error::other)?;
self.position = 0;
}
Ok(true)
}
}
impl Read for ChunkChain<'_> {
fn read(&mut self, out: &mut [u8]) -> std::io::Result<usize> {
if !self.has_data()? {
return Ok(0);
}
let available = &self.current[self.position..];
let count = available.len().min(out.len());
out[..count].copy_from_slice(&available[..count]);
self.position += count;
Ok(count)
}
}
/// Iterator over the entries of a stored manifest, created by
/// [`Repository::manifest_entries`].
pub struct ManifestEntries<'r> {
chain: ChunkChain<'r>,
failed: bool,
seen: HashSet<[u8; 16]>,
}
impl<'r> ManifestEntries<'r> {
pub(crate) fn new(repository: &'r Repository, manifest: Vec<ChunkId>) -> ManifestEntries<'r> {
ManifestEntries {
chain: ChunkChain {
repository,
ids: manifest.into_iter(),
current: Vec::new(),
position: 0,
},
failed: false,
seen: HashSet::new(),
}
}
}
/// Yields each recorded object once, in the order it was recorded.
///
/// A manifest is an append-only log and may name the same path more than
/// once: a backup that is suspended mid-directory leaves that directory
/// to be scanned again on resume, and its segment is already stored by
/// then. The first occurrence is the definitive one — any of them is a
/// valid observation of the object as it was read, and taking the first
/// consistently is what stops a path recorded as a file and later as a
/// directory from being restored as both.
impl Iterator for ManifestEntries<'_> {
type Item = Result<Entry, Error>;
fn next(&mut self) -> Option<Self::Item> {
loop {
if self.failed {
return None;
}
match self.chain.has_data() {
Ok(true) => {}
Ok(false) => return None,
Err(err) => {
self.failed = true;
return Some(Err(err.into()));
}
}
let entry: Entry = match ciborium::from_reader(&mut self.chain) {
Ok(entry) => entry,
Err(err) => {
self.failed = true;
return Some(Err(err.into()));
}
};
if self.seen.insert(path_digest(&entry.path)) {
return Some(Ok(entry));
}
}
}
}
/// A fixed-size digest of a path for the deduplication set: 128 bits of
/// SHA-256, small enough to hold millions in memory and wide enough that
/// a collision (which would silently drop an entry) will not happen.
fn path_digest(path: &std::path::Path) -> [u8; 16] {
use sha2::Digest as _;
let digest = sha2::Sha256::digest(path.as_os_str().as_encoded_bytes());
digest[..16].try_into().expect("SHA-256 yields 32 bytes")
}