use std::fs::File; use std::error::Error; use std::io::{Read, Write, ErrorKind}; use std::collections::{HashMap, BTreeMap}; use std::string::FromUtf8Error; use num_enum::{TryFromPrimitive, TryFromPrimitiveError}; use std::fmt::{Debug, Display}; use chrono::DateTime; use thiserror::Error; #[derive(TryFromPrimitive,Debug,PartialEq,Eq,Clone,Copy,PartialOrd,Ord)] #[repr(u16)] pub enum CommandType { Unspec = 0, Subvol = 1, Snapshot = 2, MkFile = 3, MkDir = 4, MkNod = 5, MkFifo = 6, MkSock = 7, Symlink = 8, Rename = 9, Link = 10, Unlink = 11, RmDir = 12, SetXAttr = 13, RemoveXAttr = 14, Write = 15, Clone = 16, Truncate = 17, Chmod = 18, Chown = 19, UTimes = 20, End = 21, UpdateExtent = 22, FAllocate = 23, FileAttr = 24, EncodedWrite = 25, } #[derive(Debug,Clone)] pub enum Command { Unspec, Subvol { path: String, uuid: UUID, transid: u64 }, Snapshot { clone_transid: u64, clone_uuid: UUID, uuid: UUID, path: String, transid: u64 }, MkFile { path: String, inode: u64 }, MkDir { path: String, inode: u64 }, MkNod { path: String, mode: u64, rdev: u64 }, MkFifo { path: String, inode: u64 }, MkSock { path: String, inode: u64 }, Symlink { path: String, inode: u64, link: String }, Rename { from: String, to: String }, Link { path: String, link: String }, Unlink { path: String }, RmDir { path: String }, SetXAttr, RemoveXAttr, Write { path: String, offset: u64, data: Vec }, Clone { path: String, offset: u64, clone_path: String, clone_uuid: UUID, clone_len: u64, clone_transid: u64, clone_offset: u64 }, Truncate { path: String, size: u64 }, Chmod { path: String, mode: u64 }, Chown { path: String, uid: u64, gid: u64 }, UTimes { path: String, atime: Time, mtime: Time, ctime: Time }, End, UpdateExtent { path: String, offset: u64, size: u64 }, FAllocate { path: String, mode: u32, offset: u64, size: u64 }, FileAttr, EncodedWrite, } #[derive(TryFromPrimitive,Debug,Clone,Copy,PartialEq,Eq,PartialOrd,Ord,Hash)] #[repr(u16)] pub enum TLVType { Unspec = 0, UUID = 1, CTransID = 2, Ino = 3, Size = 4, Mode = 5, UID = 6, GID = 7, RDev = 8, CTime = 9, MTime = 10, ATime = 11, OTime = 12, XAttrName = 13, XAttrData = 14, Path = 15, PathTo = 16, PathLink = 17, FileOffset = 18, Data = 19, CloneUUID = 20, CloneCTransID = 21, ClonePath = 22, CloneOffset = 23, CloneLen = 24, FAllocateMode = 25, FileAttr = 26, UnencodedFileLen = 27, UnencodedLen = 28, UnencodedOffset = 29, Compression = 30, Encryption = 31, } /// `u128` wrapper for UUIDs /// /// This mostly just overrides the `Display` trait to use the typical UUID format `XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX` #[derive(Clone)] pub struct UUID(pub u128); impl Display for UUID { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { let x = self.0.to_le_bytes(); write!(f, "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", x[0], x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9], x[10], x[11], x[12], x[13], x[14], x[15], ) } } impl Debug for UUID { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { let x = self.0.to_le_bytes(); write!(f, "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", x[0], x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9], x[10], x[11], x[12], x[13], x[14], x[15], ) } } /// time format for BTRFS stream /// /// Consists of a 64 bit UNIX timestamp and a 32 bit nanoseconds field. #[derive(Debug,Clone)] pub struct Time { pub secs: i64, pub nanos: u32, } impl Display for Time { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { write!(f, "{}", DateTime::from_timestamp(self.secs, self.nanos).ok_or(std::fmt::Error)?) } } /// `writeln`, but behaving more gracefully on shutdown macro_rules! out { ($($arg:tt)*) => { let result = writeln!(std::io::stdout(), $($arg)*); if let Some(e) = result.err().filter(|e| e.kind() != ErrorKind::BrokenPipe) { panic!("I/O error: {}", &e); } }; } #[derive(Clone,Copy,PartialEq,Eq,PartialOrd,Ord,Debug)] enum FileChange { Unchanged, Created, Deleted, Modified, ModifiedOnlyAttributes, } #[derive(Clone,Copy,PartialEq,Eq,Debug)] enum FileOp { Create, Delete, Modify, ModifyAttributes, } fn state_change(old: FileChange, op: FileOp) -> Option { let new = match (old, op) { (FileChange::Unchanged, FileOp::Create) => FileChange::Created, (FileChange::Unchanged, FileOp::Delete) => FileChange::Deleted, (FileChange::Unchanged, FileOp::Modify) => FileChange::Modified, (FileChange::Unchanged, FileOp::ModifyAttributes) => FileChange::ModifiedOnlyAttributes, (FileChange::Created, FileOp::Delete) => FileChange::Unchanged, (FileChange::Created, FileOp::Modify) => FileChange::Created, (FileChange::Created, FileOp::ModifyAttributes) => FileChange::Created, (FileChange::Deleted, FileOp::Create) => FileChange::Modified, (FileChange::Modified, FileOp::Delete) => FileChange::Deleted, (FileChange::Modified, FileOp::Modify) => FileChange::Modified, (FileChange::Modified, FileOp::ModifyAttributes) => FileChange::Modified, (FileChange::ModifiedOnlyAttributes, FileOp::Delete) => FileChange::Deleted, (FileChange::ModifiedOnlyAttributes, FileOp::Modify) => FileChange::Modified, (FileChange::ModifiedOnlyAttributes, FileOp::ModifyAttributes) => FileChange::ModifiedOnlyAttributes, _ => return None, }; Some(new) } fn main() -> Result<(), Box> { let mut contents: Vec = Vec::new(); File::open("../sync_diff_20250525_20251130")?.read_to_end(&mut contents)?; out!("Magic: {}", &String::from_utf8(contents[..12].to_vec())?); out!("Version: {}", u32::from_le_bytes(contents[13..17].try_into()?)); out!(""); let mut files: BTreeMap = BTreeMap::new(); let mut offset: usize = 17; fn do_op(files: &mut BTreeMap, path: &String, op: FileOp) -> Result<(), String> { let old = *files.get(path).unwrap_or(&FileChange::Unchanged); let new = state_change(old, op) .ok_or(format!("Invalid operation: {:?} on {:?}, path = {}", op, old, path))?; if new == FileChange::Unchanged { files.remove(path); } else { files.insert(path.clone(), new); } Ok(()) } while offset < contents.len() { let (cmd, new_offset) = Command::parse(&contents[offset..])?; offset += new_offset; if cmd.ty() == CommandType::Snapshot { out!("{:#?}", &cmd); } match &cmd { Command::MkFile { path, inode: _ } | Command::MkDir { path, inode: _ } | Command::Symlink { path, inode: _, link: _ } | Command::Link { path, link: _ } => { do_op(&mut files, path, FileOp::Create)?; }, Command::Rename { from, to } if from != to => { do_op(&mut files, from, FileOp::Delete)?; do_op(&mut files, to, FileOp::Create)?; }, Command::Unlink { path } | Command::RmDir { path } => { do_op(&mut files, path, FileOp::Delete)?; }, Command::Write { path, offset: _, data: _ } | Command::Truncate { path, size: _ } => { do_op(&mut files, path, FileOp::Modify)?; }, Command::Chmod { path, mode: _ } | Command::Chown { path, uid: _, gid: _ } => { do_op(&mut files, path, FileOp::ModifyAttributes)?; }, _ => (), } } let mut last: Option = None; for (file, &change) in &files { if let Some(last_) = &last && file.starts_with(last_) { // ignore } else if change == FileChange::Created { last = Some(file.clone()); out!("{file}"); } } Ok(()) } pub trait TlvData: Sized { fn parse_tlv_data(data: &[u8]) -> Result; } impl TlvData for u64 { fn parse_tlv_data(data: &[u8]) -> Result { Ok(u64::from_le_bytes(data.try_into().unwrap())) } } impl TlvData for u32 { fn parse_tlv_data(data: &[u8]) -> Result { Ok(u32::from_le_bytes(data.try_into().unwrap())) } } impl TlvData for String { fn parse_tlv_data(data: &[u8]) -> Result { String::from_utf8(data.to_vec()).map_err(Into::into) } } impl TlvData for UUID { fn parse_tlv_data(data: &[u8]) -> Result { Ok(UUID(u128::from_le_bytes(data.try_into().unwrap()))) } } impl TlvData for Time { fn parse_tlv_data(data: &[u8]) -> Result { let secs = i64::from_le_bytes(data[0..8].try_into().unwrap()); let nanos = u32::from_le_bytes(data[8..12].try_into().unwrap()); Ok(Time {secs, nanos}) } } impl Command { pub fn ty(&self) -> CommandType { match self { Command::Subvol { path: _, uuid: _, transid: _ } => CommandType::Subvol, Command::Snapshot { clone_transid: _, clone_uuid: _, uuid: _, path: _, transid: _ } => CommandType::Snapshot, Command::MkFile { path: _, inode: _ } => CommandType::MkFile, Command::MkDir { path: _, inode: _ } => CommandType::MkDir, Command::MkNod { path: _, mode: _, rdev: _ } => CommandType::MkNod, Command::MkFifo { path: _, inode: _ } => CommandType::MkFifo, Command::MkSock { path: _, inode: _ } => CommandType::MkSock, Command::Symlink { path: _, inode: _, link: _ } => CommandType::Symlink, Command::Rename { from: _, to: _ } => CommandType::Rename, Command::Link { path: _, link: _ } => CommandType::Link, Command::Unlink { path: _ } => CommandType::Unlink, Command::RmDir { path: _ } => CommandType::RmDir, Command::SetXAttr => CommandType::SetXAttr, Command::RemoveXAttr => CommandType::RemoveXAttr, Command::Write { path: _, offset: _, data: _ } => CommandType::Write, Command::Clone { path: _, offset: _, clone_path: _, clone_uuid: _, clone_len: _, clone_transid: _, clone_offset: _ } => CommandType::Clone, Command::Truncate { path: _, size: _ } => CommandType::Truncate, Command::Chmod { path: _, mode: _ } => CommandType::Chmod, Command::Chown { path: _, uid: _, gid: _ } => CommandType::Chown, Command::UTimes { path: _, atime: _, mtime: _, ctime: _ } => CommandType::UTimes, Command::End => CommandType::End, Command::UpdateExtent { path: _, offset: _, size: _ } => CommandType::UpdateExtent, Command::FAllocate { path: _, mode: _, offset: _, size: _ } => CommandType::FAllocate, Command::FileAttr => CommandType::FileAttr, Command::EncodedWrite => CommandType::EncodedWrite, _ => CommandType::Unspec, } } pub fn path(&self) -> Option<&str> { match self { Command::Symlink { path, inode: _, link: _ } | Command::Rename { from: path, to: _ } | Command::Link { path , link: _ } | Command::Unlink { path } | Command::RmDir { path } | Command::MkFile { path, inode: _ } | Command::MkDir { path, inode: _ } => Some(path), _ => None, } } pub fn parse(data: &[u8]) -> Result<(Self, usize), BtrfsStreamError> { if data.len() < 10 { return Err(BtrfsStreamError::IncompleteCommand); } let len = u32::from_le_bytes(data[0..4].try_into().unwrap()) as usize; let cmd = CommandType::try_from_primitive( u16::from_le_bytes(data[4..6].try_into().unwrap()) )?; let _checksum = u32::from_le_bytes(data[6..10].try_into().unwrap()); let mut tlvs: HashMap = HashMap::new(); let mut inner_offset = 0; while inner_offset < len { if data.len() < inner_offset + 14 { return Err(BtrfsStreamError::IncompleteCommand); } let tlvtype = TLVType::try_from_primitive( u16::from_le_bytes(data[inner_offset+10 .. inner_offset+12].try_into().unwrap()) )?; let tlvlen = u16::from_le_bytes(data[inner_offset+12 .. inner_offset+14].try_into().unwrap()) as usize; tlvs.insert(tlvtype, (inner_offset+14, inner_offset+14+tlvlen)); inner_offset += tlvlen + 4; } let tlv = |ty: TLVType| -> Result<&[u8], BtrfsStreamError> { let (s, e) = *tlvs.get(&ty).ok_or(BtrfsStreamError::MissingTLV(cmd, ty))?; Ok(&data[s..e]) }; let result = match cmd { CommandType::Subvol => Command::Subvol { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, uuid: UUID::parse_tlv_data(tlv(TLVType::UUID)?)?, transid: u64::parse_tlv_data(tlv(TLVType::CTransID)?)?, }, CommandType::Snapshot => Command::Snapshot { clone_transid: u64::parse_tlv_data(tlv(TLVType::CloneCTransID)?)?, clone_uuid: UUID::parse_tlv_data(tlv(TLVType::CloneUUID)?)?, uuid: UUID::parse_tlv_data(tlv(TLVType::UUID)?)?, path: String::parse_tlv_data(tlv(TLVType::Path)?)?, transid: u64::parse_tlv_data(tlv(TLVType::CTransID)?)?, }, CommandType::MkFile => Command::MkFile { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, inode: u64::parse_tlv_data(tlv(TLVType::Ino)?)?, }, CommandType::MkDir => Command::MkDir { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, inode: u64::parse_tlv_data(tlv(TLVType::Ino)?)?, }, CommandType::MkNod => Command::MkNod { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, mode: u64::parse_tlv_data(tlv(TLVType::Mode)?)?, rdev: u64::parse_tlv_data(tlv(TLVType::RDev)?)?, }, CommandType::MkFifo => Command::MkFifo { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, inode: u64::parse_tlv_data(tlv(TLVType::Ino)?)?, }, CommandType::MkSock => Command::MkSock { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, inode: u64::parse_tlv_data(tlv(TLVType::Ino)?)?, }, CommandType::Symlink => Command::Symlink { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, inode: u64::parse_tlv_data(tlv(TLVType::Ino)?)?, link: String::parse_tlv_data(tlv(TLVType::PathLink)?)?, }, CommandType::Rename => Command::Rename { from: String::parse_tlv_data(tlv(TLVType::Path)?)?, to: String::parse_tlv_data(tlv(TLVType::PathTo)?)?, }, CommandType::Link => Command::Link { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, link: String::parse_tlv_data(tlv(TLVType::PathLink)?)?, }, CommandType::Unlink => Command::Unlink { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, }, CommandType::RmDir => Command::RmDir { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, }, CommandType::Write => Command::Write { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, offset: u64::parse_tlv_data(tlv(TLVType::FileOffset)?)?, data: tlv(TLVType::Data)?.to_vec(), }, CommandType::Clone => Command::Clone { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, offset: u64::parse_tlv_data(tlv(TLVType::FileOffset)?)?, clone_path: String::parse_tlv_data(tlv(TLVType::ClonePath)?)?, clone_uuid: UUID::parse_tlv_data(tlv(TLVType::CloneUUID)?)?, clone_len: u64::parse_tlv_data(tlv(TLVType::CloneLen)?)?, clone_transid: u64::parse_tlv_data(tlv(TLVType::CloneCTransID)?)?, clone_offset: u64::parse_tlv_data(tlv(TLVType::CloneOffset)?)?, }, CommandType::Truncate => Command::Truncate { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, size: u64::parse_tlv_data(tlv(TLVType::Size)?)?, }, CommandType::Chmod => Command::Chmod { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, mode: u64::parse_tlv_data(tlv(TLVType::Mode)?)?, }, CommandType::Chown => Command::Chown { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, uid: u64::parse_tlv_data(tlv(TLVType::UID)?)?, gid: u64::parse_tlv_data(tlv(TLVType::GID)?)?, }, CommandType::UTimes => Command::UTimes { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, atime: Time::parse_tlv_data(tlv(TLVType::ATime)?)?, mtime: Time::parse_tlv_data(tlv(TLVType::MTime)?)?, ctime: Time::parse_tlv_data(tlv(TLVType::CTime)?)?, }, CommandType::End => Command::End, CommandType::UpdateExtent => Command::UpdateExtent { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, offset: u64::parse_tlv_data(tlv(TLVType::FileOffset)?)?, size: u64::parse_tlv_data(tlv(TLVType::Size)?)?, }, CommandType::FAllocate => Command::FAllocate { path: String::parse_tlv_data(tlv(TLVType::Path)?)?, mode: u32::parse_tlv_data(tlv(TLVType::FAllocateMode)?)?, offset: u64::parse_tlv_data(tlv(TLVType::FileOffset)?)?, size: u64::parse_tlv_data(tlv(TLVType::Size)?)?, }, // CommandType::SetXAttr => Command::SetXAttr, // CommandType::RemoveXAttr => Command::RemoveXAttr, // CommandType::FileAttr => Command::FileAttr, // CommandType::EncodedWrite => Command::EncodedWrite, _ => return Err(BtrfsStreamError::UnimplementedCommandType(cmd, tlvs)), }; Ok((result, len+10)) } } #[derive(Error, Debug)] pub enum BtrfsStreamError { #[error("missing TLV type {0:?} for command {1:?}")] MissingTLV(CommandType, TLVType), #[error("unimplemented command type {0:?} with TLVs: {1:?}")] UnimplementedCommandType(CommandType, HashMap), #[error("command ended prematurely")] IncompleteCommand, #[error("invalid command type: {}", .0.number)] InvalidCommandType(#[from] TryFromPrimitiveError), #[error("invalid TLV type: {}", .0.number)] InvalidTLVType(#[from] TryFromPrimitiveError), #[error("invalid UTF-8 string: {:?}", .0.as_bytes())] InvalidString(#[from] FromUtf8Error), }