crash_tolerance.rs raw

//! The end-to-end proof of the defining requirement: the real `beeping`
//! binary is SIGKILLed at random points during a backup, over and over,
//! and the eventually-completed backup must restore byte-for-byte,
//! metadata and all.
//!
//! Everything runs on synthetic trees under a temp directory.

mod common;

use std::{
    path::Path,
    process::{Command, Stdio},
    time::Duration,
};

use common::{assert_trees_equal, deterministic_bytes};

const PASSWORD: &str = "crash-test-password";

fn beeping(dir: &Path) -> Command {
    let mut command = Command::new(env!("CARGO_BIN_EXE_beeping"));

    command
        .current_dir(dir)
        .env("BEEPING_PASSWORD", PASSWORD)
        .stdout(Stdio::piped())
        .stderr(Stdio::null());

    command
}

fn run_to_completion(mut command: Command, what: &str) -> String {
    let output = command.output().unwrap();

    assert!(
        output.status.success(),
        "{what} failed: {}",
        String::from_utf8_lossy(&output.stdout)
    );

    String::from_utf8(output.stdout).unwrap()
}

#[test]
fn backup_survives_repeated_sigkill() {
    let dir = tempfile::tempdir().unwrap();
    let root = dir.path().join("tree");

    // A tree big enough that a backup takes meaningfully longer than the
    // kill delays below
    for outer in 0..30 {
        let sub = root.join(format!("dir-{outer:02}"));
        std::fs::create_dir_all(&sub).unwrap();

        for file in 0..25 {
            std::fs::write(
                sub.join(format!("file-{file:02}.dat")),
                deterministic_bytes(30_000, (outer * 1000 + file) as u64),
            )
            .unwrap();
        }
    }

    #[cfg(unix)]
    std::os::unix::fs::symlink("dir-00/file-00.dat", root.join("link")).unwrap();

    run_to_completion(
        {
            let mut cmd = beeping(dir.path());
            cmd.args(["init", "--repository", "repo"]);
            cmd
        },
        "init",
    );

    let backup_args = [
        "backup",
        "tree",
        "--repository",
        "repo",
        "--state-dir",
        "state",
        "--plain",
    ];

    // Kill the backup at a spread of points: during repository
    // unlocking, mid-scan, mid-chunking, and (as delays grow) possibly
    // during finalization.
    let mut completed = None;

    for round in 0..12u64 {
        let mut child = {
            let mut cmd = beeping(dir.path());
            cmd.args(backup_args);
            cmd.spawn().unwrap()
        };

        std::thread::sleep(Duration::from_millis(150 + round * 120));

        // SIGKILL: no signal handler, no cleanup, no goodbye
        let killed = child.kill().is_ok();
        let status = child.wait().unwrap();

        if !killed || status.success() {
            // It finished before the kill landed
            let mut stdout = String::new();
            use std::io::Read as _;
            if let Some(mut pipe) = child.stdout.take() {
                let _ = pipe.read_to_string(&mut stdout);
            }

            if status.success() {
                completed = Some(stdout);
                break;
            }
        }
    }

    // However the kill rounds went, a final undisturbed run completes
    // the backup.
    let stdout = match completed {
        Some(stdout) => stdout,
        None => run_to_completion(
            {
                let mut cmd = beeping(dir.path());
                cmd.args(backup_args);
                cmd
            },
            "final backup",
        ),
    };

    let snapshot = stdout
        .lines()
        .find_map(|line| line.strip_prefix("done: snapshot "))
        .unwrap_or_else(|| panic!("no snapshot id in output: {stdout}"))
        .trim()
        .to_string();

    // The tree never changed, so the restored result must match it
    // exactly — contents, symlinks, modes, and mtimes.
    run_to_completion(
        {
            let mut cmd = beeping(dir.path());
            cmd.args([
                "restore",
                &snapshot,
                "restored",
                "--repository",
                "repo",
                "--state-dir",
                "restore-state",
                "--plain",
            ]);
            cmd
        },
        "restore",
    );

    assert_trees_equal(&root, &dir.path().join("restored"));
}

#[test]
fn restore_survives_repeated_sigkill() {
    let dir = tempfile::tempdir().unwrap();
    let root = dir.path().join("tree");

    for outer in 0..20 {
        let sub = root.join(format!("dir-{outer:02}"));
        std::fs::create_dir_all(&sub).unwrap();

        for file in 0..20 {
            std::fs::write(
                sub.join(format!("file-{file:02}.dat")),
                deterministic_bytes(40_000, (outer * 77 + file) as u64),
            )
            .unwrap();
        }
    }

    run_to_completion(
        {
            let mut cmd = beeping(dir.path());
            cmd.args(["init", "--repository", "repo"]);
            cmd
        },
        "init",
    );

    let stdout = run_to_completion(
        {
            let mut cmd = beeping(dir.path());
            cmd.args([
                "backup",
                "tree",
                "--repository",
                "repo",
                "--state-dir",
                "state",
                "--plain",
            ]);
            cmd
        },
        "backup",
    );

    let snapshot = stdout
        .lines()
        .find_map(|line| line.strip_prefix("done: snapshot "))
        .unwrap()
        .trim()
        .to_string();

    let restore_args = [
        "restore",
        &snapshot,
        "restored",
        "--repository",
        "repo",
        "--state-dir",
        "restore-state",
        "--plain",
    ];

    for round in 0..8u64 {
        let mut child = {
            let mut cmd = beeping(dir.path());
            cmd.args(restore_args);
            cmd.spawn().unwrap()
        };

        std::thread::sleep(Duration::from_millis(150 + round * 130));

        let _ = child.kill();
        let status = child.wait().unwrap();

        if status.success() {
            break;
        }
    }

    run_to_completion(
        {
            let mut cmd = beeping(dir.path());
            cmd.args(restore_args);
            cmd
        },
        "final restore",
    );

    assert_trees_equal(&root, &dir.path().join("restored"));
}