use std::{io::Write as _, path::PathBuf}; use super::*; fn queue_path(dir: &tempfile::TempDir) -> PathBuf { dir.path().join("queue") } #[test] fn roundtrip_fifo() { let dir = tempfile::tempdir().unwrap(); let queue: PersistentQueue = PersistentQueue::open(queue_path(&dir)).unwrap(); assert!(queue.is_empty()); assert!(queue.get().unwrap().is_none()); queue.put(&String::from("one")).unwrap(); queue.put(&String::from("two")).unwrap(); queue.put(&String::from("three")).unwrap(); assert!(!queue.is_empty()); assert_eq!(queue.pending(), 3); let lease = queue.get().unwrap().unwrap(); assert_eq!(&*lease, "one"); assert_eq!(queue.pending(), 2); assert_eq!(queue.in_flight(), 1); assert_eq!(lease.ack(), "one"); assert_eq!(queue.in_flight(), 0); assert_eq!(queue.get().unwrap().unwrap().ack(), "two"); assert_eq!(queue.get().unwrap().unwrap().ack(), "three"); assert!(queue.is_empty()); assert!(queue.get().unwrap().is_none()); } #[test] fn dropped_lease_is_redelivered() { let dir = tempfile::tempdir().unwrap(); let queue: PersistentQueue = PersistentQueue::open(queue_path(&dir)).unwrap(); queue.put(&1).unwrap(); queue.put(&2).unwrap(); let lease = queue.get().unwrap().unwrap(); assert_eq!(*lease, 1); drop(lease); assert_eq!(queue.pending(), 2); // The returned value is delivered again, ahead of the rest let lease = queue.get().unwrap().unwrap(); assert_eq!(*lease, 1); lease.release(); assert_eq!(queue.get().unwrap().unwrap().ack(), 1); assert_eq!(queue.get().unwrap().unwrap().ack(), 2); assert!(queue.is_empty()); } #[test] fn survives_reopen() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); { let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); queue.put(&1).unwrap(); queue.put(&2).unwrap(); queue.put(&3).unwrap(); assert_eq!(queue.get().unwrap().unwrap().ack(), 1); // Dropping the queue releases the lock and performs a final sync } let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); assert_eq!(queue.pending(), 2); assert_eq!(queue.get().unwrap().unwrap().ack(), 2); assert_eq!(queue.get().unwrap().unwrap().ack(), 3); assert!(queue.is_empty()); } #[test] fn double_open_is_refused() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); let _queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); match PersistentQueue::::open(&path) { Err(Error::QueueLocked) => {} other => panic!("expected Error::QueueLocked, got {other:?}"), } } #[test] fn unsynced_acks_redeliver_after_crash() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); queue.put(&1).unwrap(); queue.put(&2).unwrap(); queue.sync().unwrap(); queue.get().unwrap().unwrap().ack(); // Copying the file while the queue is open simulates a crash: the // in-memory state is discarded and only the on-disk state survives. // The ack has not been synced, so it must not have reached the disk. let crashed = dir.path().join("crashed"); std::fs::copy(&path, &crashed).unwrap(); let recovered: PersistentQueue = PersistentQueue::open(&crashed).unwrap(); assert_eq!(recovered.pending(), 2); assert_eq!(recovered.get().unwrap().unwrap().ack(), 1); assert_eq!(recovered.get().unwrap().unwrap().ack(), 2); // After a sync the ack is durable and survives the same "crash" queue.sync().unwrap(); let synced = dir.path().join("synced"); std::fs::copy(&path, &synced).unwrap(); let recovered: PersistentQueue = PersistentQueue::open(&synced).unwrap(); assert_eq!(recovered.pending(), 1); assert_eq!(recovered.get().unwrap().unwrap().ack(), 2); } #[test] fn torn_tail_is_discarded() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); { let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); queue.put(&1).unwrap(); queue.put(&2).unwrap(); } // Garbage after the last record, as left by an interrupted append let mut file = File::options().append(true).open(&path).unwrap(); file.write_all(&[0xAB; 7]).unwrap(); drop(file); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); assert_eq!(queue.pending(), 2); assert_eq!(queue.get().unwrap().unwrap().ack(), 1); assert_eq!(queue.get().unwrap().unwrap().ack(), 2); // The truncated file remains fully usable queue.put(&3).unwrap(); assert_eq!(queue.get().unwrap().unwrap().ack(), 3); } #[test] fn oversized_length_in_tail_is_discarded() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); { let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); queue.put(&1).unwrap(); } // A record header whose length points past the end of the file, as // left by a crash between writing the header and the payload let mut file = File::options().append(true).open(&path).unwrap(); RecordHeader { state: STATE_LIVE, length: 1000, crc: 0, } .store(&mut file) .unwrap(); drop(file); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); assert_eq!(queue.pending(), 1); assert_eq!(queue.get().unwrap().unwrap().ack(), 1); } #[test] fn corrupt_payload_is_discarded() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); { let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); queue.put(&String::from("aaaa")).unwrap(); queue.put(&String::from("bbbb")).unwrap(); queue.put(&String::from("cccc")).unwrap(); } // Flip a bit near the end of the file, inside the last payload let mut contents = std::fs::read(&path).unwrap(); let target = contents.len() - 2; contents[target] ^= 0x01; std::fs::write(&path, &contents).unwrap(); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); assert_eq!(queue.pending(), 2); assert_eq!(queue.get().unwrap().unwrap().ack(), "aaaa"); assert_eq!(queue.get().unwrap().unwrap().ack(), "bbbb"); assert!(queue.is_empty()); } #[test] fn non_queue_file_is_rejected() { let dir = tempfile::tempdir().unwrap(); let short = dir.path().join("short"); std::fs::write(&short, b"hello").unwrap(); assert!(matches!( PersistentQueue::::open(&short), Err(Error::InvalidFile) )); let long = dir.path().join("long"); std::fs::write(&long, [0x55; 64]).unwrap(); assert!(matches!( PersistentQueue::::open(&long), Err(Error::InvalidFile) )); } #[test] fn drained_queue_file_is_reset() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); for _ in 0..10 { queue.put(&"x".repeat(1000)).unwrap(); } while let Some(lease) = queue.get().unwrap() { lease.ack(); } queue.sync().unwrap(); assert_eq!( std::fs::metadata(&path).unwrap().len(), FILE_HEADER_BYTES, "a fully drained queue file should shrink back to just its header" ); queue.put(&String::from("again")).unwrap(); assert_eq!(queue.get().unwrap().unwrap().ack(), "again"); } #[test] fn compaction_shrinks_file_and_preserves_leases() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); let big = "x".repeat(8192); for _ in 0..200 { queue.put(&big).unwrap(); } queue.put(&String::from("marker")).unwrap(); for _ in 0..200 { let lease = queue.get().unwrap().unwrap(); assert_eq!(lease.len(), 8192); lease.ack(); } // Hold a lease across the compaction to exercise offset remapping let marker = queue.get().unwrap().unwrap(); assert_eq!(&*marker, "marker"); let before = std::fs::metadata(&path).unwrap().len(); assert!(before > 200 * 8192); queue.sync().unwrap(); let after = std::fs::metadata(&path).unwrap().len(); assert!( after < 10_000, "compaction should have shrunk the file, but it is {after} bytes" ); assert_eq!(marker.ack(), "marker"); queue.sync().unwrap(); assert_eq!( std::fs::metadata(&path).unwrap().len(), FILE_HEADER_BYTES, "acking the last value and syncing should reset the file" ); queue.put(&String::from("after")).unwrap(); assert_eq!(queue.get().unwrap().unwrap().ack(), "after"); } #[test] fn compaction_remaps_requeued_records() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); let big = "x".repeat(8192); for _ in 0..200 { queue.put(&big).unwrap(); } queue.put(&String::from("x-requeued")).unwrap(); queue.put(&String::from("y-leased")).unwrap(); for _ in 0..200 { queue.get().unwrap().unwrap().ack(); } let lease_x = queue.get().unwrap().unwrap(); assert_eq!(&*lease_x, "x-requeued"); let lease_y = queue.get().unwrap().unwrap(); assert_eq!(&*lease_y, "y-leased"); lease_x.release(); queue.sync().unwrap(); assert!(std::fs::metadata(&path).unwrap().len() < 10_000); // The requeued record survived the compaction and is delivered first let lease_x = queue.get().unwrap().unwrap(); assert_eq!(&*lease_x, "x-requeued"); assert_eq!(lease_x.ack(), "x-requeued"); assert_eq!(lease_y.ack(), "y-leased"); queue.sync().unwrap(); assert!(queue.is_empty()); assert_eq!(std::fs::metadata(&path).unwrap().len(), FILE_HEADER_BYTES); } #[test] fn structured_values_roundtrip() { #[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)] struct Job { id: u64, path: PathBuf, chunks: Vec<[u8; 4]>, } let dir = tempfile::tempdir().unwrap(); let queue: PersistentQueue = PersistentQueue::open(queue_path(&dir)).unwrap(); let job = Job { id: 42, path: PathBuf::from("/some/where"), chunks: vec![[1, 2, 3, 4], [5, 6, 7, 8]], }; queue.put(&job).unwrap(); assert_eq!(queue.get().unwrap().unwrap().ack(), job); } /// Retaining keeps the order of what survives and loses nothing else, /// and what it leaves behind is what a later session finds. #[test] fn retaining_drops_the_rest_and_keeps_the_order() { let dir = tempfile::tempdir().unwrap(); let path = queue_path(&dir); { let queue: PersistentQueue = PersistentQueue::open(&path).unwrap(); for value in 1..=9 { queue.put(&value).unwrap(); } assert_eq!(queue.retain(|value| value % 3 != 0).unwrap(), 3); assert_eq!(queue.pending(), 6); assert_eq!(queue.peek(9).unwrap(), vec![1, 2, 4, 5, 7, 8]); // Keeping everything is not the same as doing nothing, but it // must look like it from the outside assert_eq!(queue.retain(|_| true).unwrap(), 0); assert_eq!(queue.peek(9).unwrap(), vec![1, 2, 4, 5, 7, 8]); queue.sync().unwrap(); } let reopened: PersistentQueue = PersistentQueue::open(&path).unwrap(); assert_eq!(reopened.peek(9).unwrap(), vec![1, 2, 4, 5, 7, 8]); // And an empty queue is an ordinary case, not an edge one assert_eq!(reopened.retain(|_| false).unwrap(), 6); assert_eq!(reopened.retain(|_| false).unwrap(), 0); assert!(reopened.is_empty()); } /// Inspecting walks everything the queue holds without taking any of it. #[test] fn inspecting_walks_the_queue_without_taking_it() { let dir = tempfile::tempdir().unwrap(); let queue: PersistentQueue = PersistentQueue::open(queue_path(&dir)).unwrap(); for value in 1..=5 { queue.put(&value).unwrap(); } let mut seen = Vec::new(); queue.inspect(|value| seen.push(*value)).unwrap(); assert_eq!(seen, vec![1, 2, 3, 4, 5]); assert_eq!(queue.pending(), 5, "inspecting delivers nothing"); } /// Peeking says what the next deliveries will be without becoming one: /// the values stay pending, and in the order `get` will hand them over. #[test] fn peeking_names_the_front_without_taking_it() { let dir = tempfile::tempdir().unwrap(); let queue: PersistentQueue = PersistentQueue::open(queue_path(&dir)).unwrap(); assert!(queue.peek(4).unwrap().is_empty(), "nothing queued yet"); for value in 1..=5 { queue.put(&value).unwrap(); } assert_eq!(queue.peek(3).unwrap(), vec![1, 2, 3]); assert_eq!( queue.peek(50).unwrap(), vec![1, 2, 3, 4, 5], "asking for more than there is" ); assert_eq!(queue.pending(), 5, "peeking delivers nothing"); // What is out on a lease is no longer at the front, and what is // acknowledged never comes back to it let lease = queue.get().unwrap().unwrap(); assert_eq!(queue.peek(2).unwrap(), vec![2, 3]); lease.ack(); assert_eq!(queue.peek(2).unwrap(), vec![2, 3]); // A dropped lease is redelivered ahead of the rest, and peeking says // so before it happens let lease = queue.get().unwrap().unwrap(); assert_eq!(*lease, 2); let held = queue.get().unwrap().unwrap(); assert_eq!(*held, 3); drop(lease); assert_eq!(queue.peek(3).unwrap(), vec![2, 4, 5]); assert_eq!(queue.get().unwrap().unwrap().ack(), 2); assert_eq!(queue.peek(3).unwrap(), vec![4, 5]); // Peeking is still honest across a reopen, where everything // unacknowledged becomes deliverable again held.release(); drop(queue); let queue: PersistentQueue = PersistentQueue::open(queue_path(&dir)).unwrap(); assert_eq!(queue.peek(4).unwrap(), vec![3, 4, 5]); }