add chunks functionality

btrfs_explorer/src/btrfs_lookup.rs

@@ -135,9 +135,9 @@ impl<'a> Tree<'a> {
 	/// ended iterator which iterates through the keys contained in the range, in ascending
 	/// or descending order.
     ///
-	/// The skip functions are ignored for now, but are intended as an optimization:
-	/// after a key `k` was returned by the iterator (or the reverse iterator), all keys
-	/// strictly lower than `forward_skip_fn(k)` are skipped (resp. all keys strictly above
+	/// The skip functions make it possible to efficiently iterate only through certain types of items.
+	/// After a key `k` was returned by the iterator (or the reverse iterator), all keys
+	/// lower or equal `forward_skip_fn(k)` are skipped (resp. all keys higher or equal
 	/// `backward_skip_fn(k)` are skipped.
     ///
 	/// If `forward_skip_fn` and `backward_skip_fn` are the identity, nothing is skipped

btrfs_explorer/src/http_chunk.rs

@@ -0,0 +1,327 @@
+use std::convert::identity;
+use maud::{Markup, html, DOCTYPE, PreEscaped};
+use rouille::{Request, Response};
+use crate::{
+	btrfs_lookup::Tree, btrfs_structs::{self, BlockRef, ChunkItem, ItemType, Key, TreeID, Value}, key, main_error::MainError, render_common::{render_page, size_name}
+};
+
+struct ChunkLineDisplay {
+	logical_address: Option<u64>,
+	link: bool,
+	physical_address: u64,
+	size: u64,
+	description: String,
+}
+
+struct ChunkResult {
+	pub offset: u64,
+	pub refs: Vec<Vec<u64>>,
+	pub color_special: bool,
+}
+
+pub fn http_allchunks(image: &[u8], _req: &Request) -> Result<Response, MainError> {
+	let tree = Tree::chunk(image)?;
+
+	let mut chunks: Vec<ChunkLineDisplay> = Vec::new();
+
+	for item in tree.iter() {
+		let Value::Chunk(chunk_item) = &item.value else { continue; };
+
+		for stripe in &chunk_item.stripes {
+			if stripe.devid != 1 {
+				println!("multiple devices not supported!");
+				continue;
+			}
+
+			let desc = match chunk_item.chunktype & 0x7 {
+				1 => format!("data chunk"),
+				2 => format!("system chunk"),
+				4 => format!("metadata chunk"),
+				_ => format!("(unknown chunk type)"),
+			};
+
+			chunks.push(ChunkLineDisplay {
+				logical_address: Some(item.key.key_offset),
+				link: chunk_item.chunktype & 0x7 != 1,
+				physical_address: stripe.offset,
+				size: chunk_item.size,
+				description: desc,
+			});
+		}
+	}
+
+	chunks.sort_by_key(|x|x.physical_address);
+
+	let mut chunks_filled: Vec<ChunkLineDisplay> = Vec::with_capacity(chunks.len());
+
+	let mut total_size: u64 = 0;
+	for x in chunks {
+		if total_size > x.physical_address {
+			// not so good
+		} else if total_size < x.physical_address {
+			chunks_filled.push(ChunkLineDisplay {
+				logical_address: None,
+				link: false,
+				physical_address: total_size,
+				size: x.physical_address - total_size,
+				description: format!("unassigned"),
+			});
+		}
+		total_size = x.physical_address + x.size;
+
+		chunks_filled.push(x);
+	}
+
+	Ok(Response::html(render_allchunks(chunks_filled)))
+}
+
+pub fn http_chunk(image: &[u8], offset: &str, _req: &Request) -> Result<Response, MainError> {
+	let logical_offset: u64 = u64::from_str_radix(offset, 16)?;
+
+	let tree = Tree::new(image, TreeID::Extent)?;
+
+	let start = key!(logical_offset, ItemType::BlockGroup, 0);
+	let end = key!(logical_offset, ItemType::BlockGroup, u64::MAX);
+	if let Some(bg) = tree.range(start..=end).next() {
+		//		let extent_list: Vec<(u64, u64, Vec<u64>)> = Vec::new();
+		let blockgroup_size = bg.key.key_offset;
+
+		// we'll just assume for now this is metadata, for every node we store the referencing trees
+		let nr_nodes = (blockgroup_size >> 14) as usize;
+		let mut node_list: Vec<Vec<u64>> = Vec::with_capacity(nr_nodes);
+
+		let start = key!(logical_offset, ItemType::Invalid, 0);
+		let end = key!(logical_offset + blockgroup_size, ItemType::Invalid, 0);
+
+		for item in tree.range(start..end) {
+			let Value::Extent(extent_item) = item.value else { continue };
+
+			if item.key.key_type == ItemType::Metadata {
+				let index = ((item.key.key_id - logical_offset) >> 14) as usize;
+
+				let process_ref = |rf: &BlockRef| {
+					match rf {
+						&BlockRef::Tree { id } => Some(id),
+						_ => None,
+					}
+				};
+				let refs: Vec<u64> = extent_item.block_refs.iter().filter_map(process_ref).collect();
+
+				while node_list.len() < index {
+					node_list.push(Vec::new());
+				}
+				node_list.push(refs);
+			}
+		}
+
+		while node_list.len() < nr_nodes {
+			node_list.push(Vec::new());
+		}
+
+		let chunk_data = ChunkResult {
+			offset: logical_offset,
+			refs: node_list,
+			color_special: true,
+		};
+		return Ok(Response::html(render_chunk(chunk_data)))
+	}
+
+	err!("No block group found at logical address {:x}", logical_offset)
+}
+
+const COLORS: &[&str] = &["lightgray", "#e6194b", "#3cb44b", "#ffe119", "#4363d8", "#f58231", "#911eb4", "#46f0f0", "#f032e6", "#bcf60c", "#fabebe", "#008080", "#e6beff", "#9a6324", "#fffac8", "#800000", "#aaffc3", "#808000", "#ffd8b1", "#000075", "#808080", "#000000"];
+
+fn render_chunk(data: ChunkResult) -> Markup {
+	let header = format!("Metadata nodes in chunk at address {:x}", data.offset);
+
+	let body = format!("{:?}", &data.refs);
+
+	let boxes: Vec<Vec<&str>> = data.refs
+		.chunks(64)
+		.map(|row|row.iter()
+			 .map(|noderefs| {
+				  if noderefs.len() == 0 {
+					  "lightgrey"
+				  } else if noderefs[0] == 1 { // root
+					  if data.color_special {"darkgreen"} else {"black"}
+				  } else if noderefs[0] == 2 { // extent
+					  if data.color_special {"orange"} else {"black"}
+				  } else if noderefs[0] == 3 { // extent
+					  if data.color_special {"brown"} else {"black"}
+				  } else if noderefs[0] == 4 { // device
+					  if data.color_special {"cyan"} else {"black"}
+				  } else if noderefs[0] == 7 { // checksum
+					  if data.color_special {"magenta"} else {"black"}
+				  } else if noderefs[0] == 9 { // uuid
+					  if data.color_special {"yellow"} else {"black"}
+				  } else if noderefs[0] == 10 { // free space
+					  if data.color_special {"#00ff00"} else {"black"}
+				  } else if noderefs[0] < 0x100 && noderefs[0] != 5 || noderefs[0] > u64::MAX - 0x100 {
+					  if data.color_special {"red"} else {"black"}
+				  } else if noderefs.len() == 1 {
+					  if noderefs[0] == 5 {
+						  if data.color_special {"black"} else {"darkgreen"}
+					  } else {
+						  if data.color_special {"black"} else {"blue"}
+					  }
+				  } else {
+					  if data.color_special {"black"} else {
+						  "conic-gradient(blue 0deg 45deg, darkgreen 45deg 225deg, blue 225deg 360deg)"
+					  }
+				  }
+			 })
+			 .collect())
+		.collect();
+
+	let content = html! {
+		h1 {
+			(header)
+		}
+
+		details open {
+			summary { "Explanation" }
+			(explanation_chunk())
+		}
+
+		br {}
+
+		table.blocks {
+			@for row in boxes {
+				tr {
+					@for cell in row {
+						td style={"background:" (cell) ";"} {}
+					}
+				}
+			}
+		}
+	};
+
+	render_page("", content)
+}
+
+fn render_allchunks(data: Vec<ChunkLineDisplay>) -> Markup {
+	let content = html! {
+		h1 {
+			"Physical disk layout"
+		}
+
+		details open {
+			summary { "Explanation" }
+			(explanation_allchunks())
+		}
+
+		br {}
+
+		@for bg in data {
+			div.item {
+				span.key.key_offset {
+					(format!("{:x}", bg.physical_address))
+				}
+				@if let Some(addr) = bg.logical_address {
+					span.key.key_offset {
+						(format!("{:x}", addr))
+					}
+				}
+				span.itemvalue {
+					@if bg.link {
+						a href=(format!("/chunk/{:x}", bg.logical_address.unwrap())) { (bg.description) } ", " (size_name(bg.size))
+
+					} @else {
+						(bg.description) ", " (size_name(bg.size))
+					}
+				}
+			}
+		}
+	};
+
+	render_page("", content)
+}
+
+fn explanation_allchunks() -> Markup {
+	html! {
+		p {
+			"This shows the on-disk format of a BTRFS file system on the most zoomed-out level. BTRFS includes a device mapper functionality where a single logical file system can be spread over multiple physical disks, and a logical block can have multiple copies of it stored on disk. Here we assume the file system has only one physical disk and there are no RAID features enabled."
+		}
+		p {
+			"This page shows the layout of the phyiscal disk. It is organized in \"chunks\", typically a few megabytes to a gigabyte in size, with possibly some unassigned space in between. There are three types of chunks:"
+		}
+
+		ul {
+			li {
+				b { "Data: " }
+				"A data chunk contains actual contents of files. The contents of one file do not have to be stored in the data chunk in a contiguous way, and can be spread out over multiple data chunks. The data chunk itself also contains no information about which files the data belongs to, all of that is stored in the metadata. Very small files (e.g. under 1 KiB) do not have their contents in here, but they are entirely stored in the metadata section."
+			}
+
+			li {
+				b { "Metadata: " }
+				"This contains all information about file names, directories, checksums, used and free space, devices etc. The data here is organized in the eponymous \"B-Trees\", which is essentially a type of key-value store. Click on a metadata chunk to find out what is stored inside it."
+			}
+
+			li {
+				b { "System: " }
+				"The system chunks are just additional metadata chunks, except that they are reserved for special kinds of metadata. Most importantly, the system chunks contain the mapping from logical to physical addresses, which is needed to find the other metadata chunks. The physical locations of the system chunks are stored in the superblock, thereby avoiding a chicken-and-egg problem while mounting the drive."
+			}
+		}
+
+		p {
+			"The first column in the following table shows the physical address of a chunk and the second column shows its logical address. You can click on the metadata or system chunks to find out how they are laid out."
+		}
+	}
+}
+
+fn explanation_chunk() -> Markup {
+	html! {
+		p { "The metadata of a BTRFS file system is organized in the form of multiple " b { "B-trees" } ", which consist of " b { "nodes" } ". Each node is 16 KiB in size." }
+
+		p { "This page shows the contents of a single metadata chunk. Every little box is one node of 16 KiB. Together they add up to the total size of the chunk. We show 64 nodes per row, so each row is 1 MiB." }
+
+		p { "The colors indicate which B-tree the node belongs to. Most of them belong to the filesystem trees. There is one filesystem tree for every subvolume, but we draw them all in the same color here. The other trees are:" }
+
+		table {
+			tr {
+				td { table.blocks { tr { td style="background: darkgreen;" {} } } }
+				td { "root tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: orange;" {} } } }
+				td { "extent tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: brown;" {} } } }
+				td { "chunk tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: cyan;" {} } } }
+				td { "device tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: magenta;" {} } } }
+				td { "checksum tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: yellow;" {} } } }
+				td { "uuid tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: #00ff00;" {} } } }
+				td { "free space tree" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: red;" {} } } }
+				td { "other trees" }
+			}
+
+			tr {
+				td { table.blocks { tr { td style="background: black;" {} } } }
+				td { "filesystem trees" }
+			}
+		}
+	}
+}

btrfs_explorer/src/lib.rs

@@ -8,6 +8,7 @@ pub mod http_tree;
 pub mod render_common;
 pub mod render_tree;
 pub mod main_error;
+pub mod http_chunk;
 
 #[cfg(test)]
 mod test;

btrfs_explorer/src/render_common.rs

@@ -1,6 +1,8 @@
-use maud::Render;
+use maud::{html, DOCTYPE, Markup, Render};
 use std::fmt::{Debug, UpperHex};
 
+pub const HTTP_PATH: &str = "";
+
 pub struct DebugRender<T>(pub T);
 
 impl<T: Debug> Render for DebugRender<T> {
@@ -36,3 +38,15 @@ pub fn size_name(x: u64) -> String {
 		format!("{} EiB", x / (1<<60))
 	}
 }
+
+pub fn render_page(title: &str, content: Markup) -> Markup {
+	html! {
+		(DOCTYPE)
+		head {
+			link rel="stylesheet" href={(HTTP_PATH) "/style.css"};
+		}
+		body {
+			(content)
+		}
+	}
+}

btrfs_explorer/src/render_tree.rs

@@ -1,5 +1,5 @@
 use crate::btrfs_structs::{Item, Key, ItemType, Value, ExtentDataBody};
-use crate::render_common::{Hex, size_name};
+use crate::render_common::{Hex, size_name, HTTP_PATH};
 use maud::{Markup, html, DOCTYPE, PreEscaped};
 use std::ffi::CStr;
 
@@ -13,11 +13,9 @@ pub struct TableResult<'a> {
 	pub last_key: Key,
 }
 
-const HTTP_PATH: &str = "/btrfs";
-
 pub fn render_table(table: TableResult) -> Markup {
 
-	let header: String = if let Some(desc) = table.tree_desc {
+	let header = if let Some(desc) = table.tree_desc {
 		format!("Tree {} ({})", table.tree_id, desc)
 	} else {
 		format!("Tree {}", table.tree_id)

btrfs_explorer_bin/src/main.rs

@@ -38,9 +38,11 @@ fn main() -> Result<(), MainError> {
 		router!(
 			request,
 			(GET) ["/"] =>
-				http_main_boxes(&image, request),
+				btrfs_explorer::http_chunk::http_allchunks(&image, request).unwrap(),
 			(GET) ["/root"] =>
 				btrfs_explorer::http_tree::http_root(&image, None, request),
+			(GET) ["/chunk/{offset}", offset: String] =>
+				btrfs_explorer::http_chunk::http_chunk(&image, &offset, request).unwrap(),
 			(GET) ["/tree/{tree}", tree: String] =>
 				btrfs_explorer::http_tree::http_tree(&image, &tree, None, request.get_param("key").as_deref(), request).unwrap(),
 			(GET) ["/tree/{tree}/{key}", tree: String, key: String] =>

style.css

@@ -52,7 +52,7 @@ a.nav {
     text-decoration: none;
 }
 
-details.item {
+.item {
     padding: 3px;
     background-color: #dde;
     border-radius: 4px;
@@ -76,7 +76,7 @@ details .details {
     border-radius: 4px;
 }
 
-details .itemvalue {
+.itemvalue {
     color: black;
     padding: 3px;
     margin: 1px 2px;
@@ -84,7 +84,7 @@ details .itemvalue {
     display: inline-block;
 }
 
-details .key {
+.key {
     color: white;
     background-color: #999;
     border-radius: 4px;
@@ -95,7 +95,7 @@ details .key {
     font-size: 12pt;
 }
 
-details .key a {
+.key a {
     color: white;
 }
 
@@ -166,3 +166,16 @@ span.nodeaddr {
 pre {
     white-space: pre-wrap;
 }
+
+table.blocks {
+    margin: 0 auto;
+    border-collapse: separate;
+    border-spacing: 2px;
+    width: 770px;
+}
+
+table.blocks td {
+    height: 10px;
+    width: 10px;
+    padding: 0;
+}