diff --git a/CLAUDE.md b/CLAUDE.md index f3fbf8a..fc9c72d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -105,6 +105,11 @@ by CI. - A `.hashdb` file is **immutable once published**; updates ship as new versioned files. - Files are **untrusted** (downloaded): the header + section bounds are validated on `open`, and every read bounds-checks its own extent. `verify()` (checksum + full scan) is opt-in. + Frames above `MAX_FRAME_SIZE` are rejected on open, which bounds what one lookup allocates. +- Published crates (`ltk_hashdb`, `ltk_mimir_cache`) keep public enums and structs with + public fields `#[non_exhaustive]` and enable the `missing_docs` and + `missing_debug_implementations` lints (CI fails on warnings). Don't expose dependency + types like `zeekstd::Error` in their APIs. - **A lookup miss must never touch the arena** - it's decided by binary search over the keys. There is a `decompressions` counter and a unit test asserting misses don't bump it; keep that invariant when changing the reader. diff --git a/Cargo.lock b/Cargo.lock index 1aa0cb4..ca67aa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -552,7 +552,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -561,18 +560,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" -[[package]] -name = "futures-io" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - [[package]] name = "futures-task" version = "0.3.32" @@ -586,10 +573,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", - "futures-io", - "futures-sink", "futures-task", - "memchr", "pin-project-lite", "slab", ] @@ -1745,9 +1729,7 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64", "bytes", - "futures-channel", "futures-core", - "futures-util", "http", "http-body", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index 1616cad..a06901e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,8 @@ members = ["crates/*"] [workspace.package] edition = "2021" +# std `File::try_lock` (the cache's update lock) was stabilized in 1.89. +rust-version = "1.89" license = "Apache-2.0" repository = "https://github.com/LeagueToolkit/mimir" authors = ["Crauzer <0xcrauzer@proton.me>", "League Toolkit contributors"] @@ -31,7 +33,7 @@ serde_json = "1" directories = "5" sha2 = "0.10" time = { version = "0.3", features = ["formatting", "parsing"] } -reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] } +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls"] } ureq = "2" # CLI diff --git a/README.md b/README.md index 4e18b8f..a8fb26d 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ let remote = UreqFetch::new(ReleaseSource::github("LeagueToolkit/mimir")); match store.update(&remote, UpdateOptions::default())? { UpdateOutcome::Completed(report) => println!("installed {:?}", report.installed), UpdateOutcome::Locked => println!("another process is already updating"), + _ => {} } ``` @@ -271,7 +272,7 @@ frame cache. `Send + Sync`. | `manifest` · `path_for` | what is installed, and where | | `check` · `check_async` | what an update *would* do - no download, no lock | | `update` · `update_async` | compare → download → verify → install → GC | -| `commit` · `gc` | publish versions, sweep superseded ones | +| `commit` · `gc` | publish versions, delete superseded ones (both take the update lock) | | `try_lock_update` · `lock_update_timeout` · `lock_holder` | take the update lock, wait for it, or ask who has it | **`Table`** - which logical table, and how it hashes. diff --git a/crates/ltk_hashdb/Cargo.toml b/crates/ltk_hashdb/Cargo.toml index 2854506..f69efcb 100644 --- a/crates/ltk_hashdb/Cargo.toml +++ b/crates/ltk_hashdb/Cargo.toml @@ -1,8 +1,12 @@ [package] name = "ltk_hashdb" -description = "The .hashdb binary format: an mmap-backed read-only integer-keyed string table (reader + streaming writer)" +description = "The .hashdb binary format: a read-only table mapping integer hashes to strings, with a reader and a writer" +readme = "README.md" +keywords = ["league-of-legends", "hash", "lookup-table", "mmap", "zstd"] +categories = ["data-structures", "encoding", "game-development"] version = "0.1.0" edition.workspace = true +rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true diff --git a/crates/ltk_hashdb/README.md b/crates/ltk_hashdb/README.md new file mode 100644 index 0000000..16d561d --- /dev/null +++ b/crates/ltk_hashdb/README.md @@ -0,0 +1,59 @@ +# ltk_hashdb + +The `.hashdb` binary format: a read-only table that maps integer hashes to strings. League +Toolkit uses it to turn WAD path hashes, bin hashes, and RST keys back into names. League +tables use the `.lhdb` extension. + +A file has a fixed 80-byte header, a sorted array of keys, per-entry offsets and lengths, +and a string arena that is either raw or compressed with the zstd seekable format. Strings +are stored in path order, so paths in the same directory compress together. + +- A lookup miss is decided by a binary search over the keys and never reads the arena. +- A hit on a compressed table decompresses only the frame that holds the string, and + caches it for later lookups. +- Files are treated as untrusted: opening validates the header and section bounds, one + lookup allocates at most a few `MAX_FRAME_SIZE` (1 MiB) frames, and `HashDb::verify` + runs a full checksum pass. + +The byte-level specification is +[`docs/FORMAT.md`](https://github.com/LeagueToolkit/mimir/blob/main/docs/FORMAT.md). + +## Example + +```rust +use std::io::Cursor; + +use ltk_hashdb::{Casing, Compression, HashDb, HashDbWriter, HashKind, KeyConfig, KeyWidth}; + +fn main() -> Result<(), Box> { + let config = KeyConfig::new(KeyWidth::U64, HashKind::Xxh64, Casing::AsciiInsensitive); + let path = "assets/characters/aatrox/aatrox.bin"; + + let mut writer = HashDbWriter::with_key_config(config, Compression::default()); + writer.insert(config.hash(path), path); + let mut file = Cursor::new(Vec::new()); + writer.build(&mut file)?; + + let db = HashDb::open_bytes(file.into_inner())?; + let hash = db.hash_path("ASSETS/Characters/Aatrox/Aatrox.bin"); + assert_eq!(db.get(hash).as_deref(), Some(path)); + Ok(()) +} +``` + +## Opening files + +`HashDb::open` memory-maps the file. The mapping is only valid while the file does not +change, so never rewrite or truncate a `.hashdb` in place while it is open: write a new +file and rename it over the old one. `HashDb::open_bytes` takes an in-memory image +instead. + +## Layering + +`LayeredHashDb` puts a writable in-memory overlay in front of one or more tables that +share a `KeyConfig`, for example to add mod paths at runtime or to search the `game` and +`lcu` tables with one lookup. + +## License + +Apache-2.0. diff --git a/crates/ltk_hashdb/src/error.rs b/crates/ltk_hashdb/src/error.rs index b8436ae..3996cc2 100644 --- a/crates/ltk_hashdb/src/error.rs +++ b/crates/ltk_hashdb/src/error.rs @@ -3,7 +3,7 @@ use thiserror::Error; -use crate::KeyConfig; +use crate::{HashKind, KeyConfig, KeyWidth}; /// Errors from opening a `.hashdb` file ([`HashDb::open`] / [`HashDb::open_bytes`]): /// I/O, or the untrusted header/section-bounds validation rejecting the file. @@ -11,67 +11,141 @@ use crate::KeyConfig; /// [`HashDb::open`]: crate::HashDb::open /// [`HashDb::open_bytes`]: crate::HashDb::open_bytes #[derive(Debug, Error)] +#[non_exhaustive] pub enum OpenError { + /// Reading or mapping the file failed. #[error("io error")] Io(#[from] std::io::Error), + /// The file does not start with [`MAGIC`](crate::MAGIC). #[error("bad magic: not a hashdb file")] BadMagic, + /// The header names a format version this build cannot read. #[error("unsupported format version {0}")] UnsupportedVersion(u16), + /// A header field is out of range or inconsistent. #[error("malformed header: {0}")] MalformedHeader(&'static str), + /// A section or the seek table is inconsistent with the header, including a + /// frame larger than [`MAX_FRAME_SIZE`](crate::MAX_FRAME_SIZE). #[error("malformed file: {0}")] Malformed(&'static str), - #[error("zstd seekable format error")] - Zeekstd(#[from] zeekstd::Error), + /// The compressed arena's seek table could not be parsed. + #[error(transparent)] + Compression(#[from] CompressionError), } -/// Errors from the opt-in full integrity check ([`HashDb::verify`]). +/// Errors from the opt-in integrity checks ([`HashDb::verify`], +/// [`HashDb::verify_index`]) and from [`HashDb::try_get`]. /// /// [`HashDb::verify`]: crate::HashDb::verify +/// [`HashDb::verify_index`]: crate::HashDb::verify_index +/// [`HashDb::try_get`]: crate::HashDb::try_get #[derive(Debug, Error)] +#[non_exhaustive] pub enum VerifyError { + /// The stored checksum does not match the file's sections. #[error("checksum mismatch")] ChecksumMismatch, + /// The sections describe an invalid table. #[error("malformed file: {0}")] Malformed(&'static str), + /// Decompressing a frame failed. #[error("io error")] Io(#[from] std::io::Error), - #[error("zstd seekable format error")] - Zeekstd(#[from] zeekstd::Error), + /// The compressed arena is inconsistent with its seek table. + #[error(transparent)] + Compression(#[from] CompressionError), } /// Errors from building a table ([`HashDbWriter::build`]): invalid input -/// entries, a bad compression config, or I/O while writing. +/// entries, a bad configuration, or I/O while writing. /// /// [`HashDbWriter::build`]: crate::HashDbWriter::build #[derive(Debug, Error)] +#[non_exhaustive] pub enum BuildError { + /// Writing the output failed. #[error("io error")] Io(#[from] std::io::Error), + /// One key was inserted with two different paths. #[error("duplicate key {key:#x} with conflicting paths")] - DuplicateKey { key: u64 }, + DuplicateKey { + /// The conflicting key. + key: u64, + }, + /// A key does not fit in a [`KeyWidth::U32`] table. #[error("key {key:#x} does not fit in a u32 table")] - KeyOutOfRange { key: u64 }, + KeyOutOfRange { + /// The out-of-range key. + key: u64, + }, + /// A path is longer than the format's `u16` length field allows. #[error("path for key {key:#x} is {len} bytes; lengths are u16 (max 65535)")] - PathTooLong { key: u64, len: usize }, + PathTooLong { + /// The key the path was inserted under. + key: u64, + + /// The path's length in bytes. + len: usize, + }, + + /// [`Compression::Zeekstd`](crate::Compression::Zeekstd) `frame_size` is + /// outside `1..=`[`MAX_FRAME_SIZE`](crate::MAX_FRAME_SIZE). + #[error( + "zeekstd frame_size {frame_size} is outside 1..={}", + crate::MAX_FRAME_SIZE + )] + InvalidFrameSize { + /// The rejected frame size. + frame_size: u32, + }, + + /// A 64-bit hash algorithm was recorded for a [`KeyWidth::U32`] table, so + /// [`HashDb::hash_path`](crate::HashDb::hash_path) could never produce one + /// of its keys. + #[error("hash kind {hash_kind} produces 64-bit keys but the table is {key_width}")] + HashKindTooWide { + /// The recorded algorithm. + hash_kind: HashKind, + + /// The table's key width. + key_width: KeyWidth, + }, + + /// Compressing the arena failed. + #[error(transparent)] + Compression(#[from] CompressionError), +} - #[error("zeekstd frame_size must be nonzero")] - ZeroFrameSize, +/// A failure inside the zstd seekable-format layer. +/// +/// Opaque so the compression library can change without a breaking release. +/// The underlying error is available through [`std::error::Error::source`]. +#[derive(Debug, Error)] +#[error("zstd seekable format error")] +pub struct CompressionError(#[source] zeekstd::Error); + +/// Converts zeekstd results without a public `From` impl, which +/// would put zeekstd in this crate's API. +pub(crate) trait ZeekstdResultExt { + fn zeek(self) -> Result; +} - #[error("zstd seekable format error")] - Zeekstd(#[from] zeekstd::Error), +impl ZeekstdResultExt for Result { + fn zeek(self) -> Result { + self.map_err(CompressionError) + } } /// A base rejected by [`LayeredHashDb`] because it does not hash its keys the way @@ -80,13 +154,15 @@ pub enum BuildError { /// [`LayeredHashDb`]: crate::LayeredHashDb #[derive(Debug, Clone, Copy, PartialEq, Eq, Error)] #[error( - "base {index} is keyed {found}, but the layer is keyed {expected}; a base that hashes differently can never be hit by a caller's precomputed probe" + "base {index} is keyed {found}, but the layer is keyed {expected}; a base that hashes \ + differently can never be hit by a caller's precomputed probe" )] +#[non_exhaustive] pub struct KeyConfigMismatch { /// Position of the rejected base, counting the ones already layered. pub index: usize, - /// What the layer hashes under - its first base's configuration. + /// What the layer hashes under: its first base's configuration. pub expected: KeyConfig, /// What the rejected base hashes under. diff --git a/crates/ltk_hashdb/src/hash.rs b/crates/ltk_hashdb/src/hash.rs index e8390c9..0c082b4 100644 --- a/crates/ltk_hashdb/src/hash.rs +++ b/crates/ltk_hashdb/src/hash.rs @@ -46,6 +46,7 @@ impl fmt::Display for Casing { /// recorded separately (see [`Casing`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] #[repr(u8)] +#[non_exhaustive] pub enum HashKind { /// Not recorded. [`HashKind::hash`] falls back on key width: /// u64 → [`HashKind::Xxh64`], u32 → [`HashKind::Fnv1a32`]. @@ -76,10 +77,9 @@ impl HashKind { /// Hash `path` with this algorithm under `casing`. `key_width` resolves the /// [`HashKind::Unspecified`] fallback. /// - /// Insensitive hashing is allocation-free for paths up to 512 bytes - every - /// path League ships - which lowercase into a stack buffer; longer ones pay - /// one allocation. This sits on the hunt engine's hot path, millions of - /// candidates per round, where the stack path measures ~2-3× faster. + /// Case-insensitive hashing does not allocate for paths up to 512 bytes, + /// which covers every path League ships: they are lowercased in a stack + /// buffer. Longer paths allocate once. pub fn hash(self, path: &str, casing: Casing, key_width: KeyWidth) -> u64 { let kind = match self { Self::Unspecified => match key_width { @@ -109,6 +109,12 @@ impl HashKind { } } + /// Whether this algorithm's output needs 64-bit keys, so it cannot key a + /// [`KeyWidth::U32`] table. + pub(crate) fn is_64_bit(self) -> bool { + matches!(self, Self::Xxh64 | Self::Xxh3) + } + /// `self` must be a concrete algorithm ([`HashKind::Unspecified`] already /// resolved by [`HashKind::hash`]). fn hash_bytes(self, bytes: &[u8]) -> u64 { @@ -155,14 +161,17 @@ impl KeyConfig { } } + /// Width of the keys. pub const fn key_width(self) -> KeyWidth { self.key_width } + /// The algorithm the keys are hashed with. pub const fn hash_kind(self) -> HashKind { self.hash_kind } + /// Whether paths are lowercased before hashing. pub const fn casing(self) -> Casing { self.casing } diff --git a/crates/ltk_hashdb/src/header.rs b/crates/ltk_hashdb/src/header.rs index 77d28a0..798dda6 100644 --- a/crates/ltk_hashdb/src/header.rs +++ b/crates/ltk_hashdb/src/header.rs @@ -41,8 +41,13 @@ use crate::{Casing, HashKind, KeyWidth, OpenError}; +/// The 8 bytes every `.hashdb` file starts with. pub const MAGIC: [u8; 8] = *b"HASHDB\0\0"; + +/// The format version this build reads and writes. pub const FORMAT_VERSION: u16 = 1; + +/// Size of the fixed file header, in bytes. pub const HEADER_SIZE: usize = 80; /// Header flag: the arena is a zeekstd seekable stream rather than raw bytes. @@ -195,6 +200,11 @@ impl Header { 8 => KeyWidth::U64, _ => return Err(OpenError::MalformedHeader("key_width must be 4 or 8")), }; + if key_width == KeyWidth::U32 && hash_kind.is_64_bit() { + return Err(OpenError::MalformedHeader( + "64-bit hash_kind in a u32-key table", + )); + } let offset_width = match buf[13] { 4 => OffsetWidth::U32, 8 => OffsetWidth::U64, diff --git a/crates/ltk_hashdb/src/lib.rs b/crates/ltk_hashdb/src/lib.rs index 583b3b9..a6947f4 100644 --- a/crates/ltk_hashdb/src/lib.rs +++ b/crates/ltk_hashdb/src/lib.rs @@ -1,12 +1,18 @@ -//! The `.hashdb` binary format: a read-only, mmap-backed table mapping integer -//! keys to string values (paths, in the League Toolkit case), laid out as: +//! The `.hashdb` binary format: a read-only table mapping integer keys to +//! string values (paths, in the League Toolkit case), laid out as: //! //! - a fixed 80-byte header //! - a sorted, binary-searchable array of keys //! - per-entry offset and length arrays //! - a string arena (raw or zeekstd-seekable), path-ordered so similar paths share frames //! -//! See `docs/FORMAT.md` for the byte-level spec. +//! See `docs/FORMAT.md` in the repository for the byte-level spec. +//! +//! Every opener validates the untrusted header and section bounds, and one +//! lookup allocates at most a few [`MAX_FRAME_SIZE`] frames whatever the file +//! claims. [`HashDb::verify`] runs the full checksum pass. + +#![warn(missing_docs, missing_debug_implementations)] mod cache; mod error; @@ -17,7 +23,7 @@ mod path; mod reader; mod writer; -pub use error::{BuildError, KeyConfigMismatch, OpenError, VerifyError}; +pub use error::{BuildError, CompressionError, KeyConfigMismatch, OpenError, VerifyError}; pub use hash::{Casing, HashKind, KeyConfig}; pub use header::{FORMAT_VERSION, HEADER_SIZE, MAGIC}; pub use layered::LayeredHashDb; @@ -25,6 +31,14 @@ pub use path::PathRef; pub use reader::{HashDb, HashDbOptions, WeakHashDb, DEFAULT_FRAME_CACHE_BYTES}; pub use writer::{BuildStats, HashDbWriter}; +/// Largest decompressed zeekstd frame, in bytes, that the writer produces and +/// the reader accepts (1 MiB). +/// +/// Lookups decompress whole frames, so this bounds what one lookup can allocate +/// on an untrusted file; the seekable format itself allows frames of up to +/// 1 GiB. Published tables use 16 KiB frames. +pub const MAX_FRAME_SIZE: u32 = 1 << 20; + /// Width of the integer keys in a table. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum KeyWidth { @@ -55,16 +69,20 @@ impl std::fmt::Display for KeyWidth { } /// Arena compression strategy. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[non_exhaustive] pub enum Compression { - /// Raw concatenated arena, borrowed directly from the mmap. + /// Raw concatenated arena, read directly from the file's bytes. None, /// Zstandard Seekable Format arena, one frame decompressed per hit. - /// - /// - `frame_size`: decompressed frame size in bytes - /// - `level`: zstd compression level (decompression speed is independent of it) - Zeekstd { frame_size: u32, level: i32 }, + Zeekstd { + /// Decompressed frame size in bytes, `1..=`[`MAX_FRAME_SIZE`]. + frame_size: u32, + + /// zstd compression level. Decompression speed does not depend on it. + level: i32, + }, } /// Whether a table carries the arena-order index in the file. @@ -83,7 +101,8 @@ pub enum Compression { /// [`HashDb::prefix`]: crate::HashDb::prefix /// [`HashDb::iter`]: crate::HashDb::iter /// [`HashDb::verify`]: crate::HashDb::verify -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +#[non_exhaustive] pub enum ArenaOrder { /// Leave it out; the reader rebuilds it the first time it is needed. /// @@ -105,7 +124,7 @@ pub enum ArenaOrder { } impl Default for Compression { - /// Publishing config: 16 KiB frames (the size/latency knee) at level 19. + /// The publishing config: 16 KiB frames at level 19 (see `docs/BENCHMARKS.md`). fn default() -> Self { Self::Zeekstd { frame_size: 16 << 10, @@ -113,3 +132,8 @@ impl Default for Compression { } } } + +/// Compiles the README's code blocks as doctests. +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeDoctests; diff --git a/crates/ltk_hashdb/src/reader.rs b/crates/ltk_hashdb/src/reader.rs index ef37fbf..5154742 100644 --- a/crates/ltk_hashdb/src/reader.rs +++ b/crates/ltk_hashdb/src/reader.rs @@ -1,4 +1,4 @@ -//! Read-only, mmap-backed `.hashdb` hash table. +//! Read-only `.hashdb` hash table over a memory map or an in-memory image. use std::borrow::Cow; use std::cell::RefCell; @@ -14,8 +14,11 @@ use xxhash_rust::xxh3::Xxh3; use zeekstd::SeekTable; use crate::cache::{Frame, FrameCache}; +use crate::error::ZeekstdResultExt; use crate::header::{arena_order_width, Header, ARENA_ORDER_CHECKSUM_SIZE}; -use crate::{Casing, HashKind, KeyConfig, KeyWidth, OpenError, PathRef, VerifyError}; +use crate::{ + Casing, HashKind, KeyConfig, KeyWidth, OpenError, PathRef, VerifyError, MAX_FRAME_SIZE, +}; /// Decompressed frame bytes a table caches by default: 4 MiB, i.e. 256 frames at the /// published 16 KiB frame size. @@ -47,6 +50,7 @@ impl Default for HashDbOptions { } impl HashDbOptions { + /// The default options: a [`DEFAULT_FRAME_CACHE_BYTES`] frame cache. pub fn new() -> Self { Self::default() } @@ -342,20 +346,20 @@ impl HashDb { // frame offsets can be trusted on later reads. let seek_table = if header.arena_compressed() { let mut cursor = std::io::Cursor::new(&data[arena.clone()]); - let st = SeekTable::from_seekable(&mut cursor)?; + let st = SeekTable::from_seekable(&mut cursor).zeek()?; let total = match st.num_frames() { 0 => 0, - n => st.frame_end_decomp(n - 1)?, + n => st.frame_end_decomp(n - 1).zeek()?, }; if total != header.arena_decompressed_size { return Err(OpenError::Malformed( "seek table decompressed size disagrees with header", )); } - if st.max_frame_size_decomp() as usize > zeekstd::SEEKABLE_MAX_FRAME_SIZE { - return Err(OpenError::Malformed( - "frame exceeds seekable-format maximum", - )); + // A lookup decompresses whole frames, so this bounds what one lookup + // allocates. The seek table is untrusted and may claim up to 1 GiB. + if st.max_frame_size_decomp() > u64::from(MAX_FRAME_SIZE) { + return Err(OpenError::Malformed("frame exceeds MAX_FRAME_SIZE")); } Some(st) } else { @@ -530,18 +534,22 @@ impl HashDb { } } + /// Number of entries. pub fn len(&self) -> usize { self.inner.len() } + /// Whether the table has no entries. pub fn is_empty(&self) -> bool { self.inner.header.entry_count == 0 } + /// Width of the stored keys. pub fn key_width(&self) -> KeyWidth { self.inner.header.key_width } + /// The algorithm recorded in the header (may be [`HashKind::Unspecified`]). pub fn hash_kind(&self) -> HashKind { self.inner.header.hash_kind } @@ -909,7 +917,7 @@ impl Inner { let first = seek_table.frame_index_decomp(start); let last = seek_table.frame_index_decomp(end - 1); let frame = self.frame(first)?; - let frame_start = seek_table.frame_start_decomp(first)?; + let frame_start = seek_table.frame_start_decomp(first).zeek()?; let offset = (start - frame_start) as usize; if first == last { @@ -936,7 +944,7 @@ impl Inner { for index in first + 1..=last { let frame = self.frame(index)?; - let frame_start = seek_table.frame_start_decomp(index)?; + let frame_start = seek_table.frame_start_decomp(index).zeek()?; let take = (end - frame_start).min(frame.bytes().len() as u64) as usize; let tail = frame.bytes().get(..take).ok_or(VerifyError::Malformed( "frame shorter than the seek table says", @@ -1069,15 +1077,15 @@ impl Inner { fn decompress(&self, index: u32) -> Result, VerifyError> { let seek_table = self.seek_table.as_ref().expect("compressed arena"); let arena = &self.backing.bytes()[self.arena.clone()]; - let start = seek_table.frame_start_comp(index)? as usize; - let end = seek_table.frame_end_comp(index)? as usize; - let size = seek_table.frame_size_decomp(index)? as usize; + let start = seek_table.frame_start_comp(index).zeek()? as usize; + let end = seek_table.frame_end_comp(index).zeek()? as usize; + let size = seek_table.frame_size_decomp(index).zeek()? as usize; let compressed = arena .get(start..end) .ok_or(VerifyError::Malformed("frame extent out of arena bounds"))?; - // Cap the capacity hint at the (header-pinned) arena size so a corrupt seek - // table can't force a huge allocation. + // `size` is at most MAX_FRAME_SIZE (checked on open), so this allocation + // stays small whatever the file claims. let capacity = size.min(self.header.arena_decompressed_size as usize); let mut buffer = self.cache.take_buffer(capacity); diff --git a/crates/ltk_hashdb/src/writer.rs b/crates/ltk_hashdb/src/writer.rs index 786ea81..26cff41 100644 --- a/crates/ltk_hashdb/src/writer.rs +++ b/crates/ltk_hashdb/src/writer.rs @@ -1,14 +1,17 @@ //! Streaming builder for `.hashdb` files. -use std::io::{Seek, Write}; +use std::io::Write; use xxhash_rust::xxh3::Xxh3; +use crate::error::ZeekstdResultExt; use crate::header::{ arena_order_width, ArenaOrderRef, Header, OffsetWidth, FLAG_ARENA_COMPRESSED, FLAG_CASE_INSENSITIVE, HEADER_SIZE, }; -use crate::{ArenaOrder, BuildError, Casing, Compression, HashKind, KeyConfig, KeyWidth}; +use crate::{ + ArenaOrder, BuildError, Casing, Compression, HashKind, KeyConfig, KeyWidth, MAX_FRAME_SIZE, +}; /// Collects `(key, path)` pairs, then [`HashDbWriter::build`] sorts by key, dedups, /// assigns arena offsets, and writes the file. @@ -21,18 +24,37 @@ pub struct HashDbWriter { entries: Vec<(u64, Box)>, } +impl std::fmt::Debug for HashDbWriter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HashDbWriter") + .field("key_width", &self.key_width) + .field("compression", &self.compression) + .field("hash_kind", &self.hash_kind) + .field("casing", &self.casing) + .field("arena_order", &self.arena_order) + .field("entries", &self.entries.len()) + .finish() + } +} + /// Sizes reported by a successful [`HashDbWriter::build`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub struct BuildStats { + /// Number of entries written, after duplicates were removed. pub entries: usize, + /// Total length of the stored paths, in bytes. pub arena_decompressed_size: u64, + /// Size of the arena on disk, in bytes. Equal to `arena_decompressed_size` + /// for [`Compression::None`]. pub arena_compressed_size: u64, /// Bytes spent on the arena-order section, `0` when it was omitted. pub arena_order_size: u64, + /// Size of the whole file, in bytes. pub file_len: u64, } @@ -62,7 +84,7 @@ impl HashDbWriter { } /// Record the algorithm the keys were hashed with, so readers can hash new - /// paths via `HashDb::hash_path`. + /// paths with [`HashDb::hash_path`](crate::HashDb::hash_path). pub fn hash_kind(mut self, kind: HashKind) -> Self { self.hash_kind = kind; self @@ -86,20 +108,45 @@ impl HashDbWriter { self } + /// Add one entry. pub fn insert(&mut self, key: u64, path: &str) { self.entries.push((key, path.into())); } + /// Add several entries. pub fn extend<'a>(&mut self, it: impl IntoIterator) { self.entries .extend(it.into_iter().map(|(k, p)| (k, Box::from(p)))); } - /// Sort by key, dedup, assign offsets, and write - /// header + keys + offsets + lengths + arena. + /// Sort by key, remove duplicates, assign offsets, and write the header, + /// keys, offsets, lengths, arena, and (optionally) arena-order section to + /// `out`. The whole file is assembled in memory before anything is written. + /// + /// # Errors /// - /// A key mapped to two different paths is a [`BuildError::DuplicateKey`]. - pub fn build(mut self, mut out: W) -> Result { + /// - [`BuildError::DuplicateKey`] if a key was inserted with two different paths + /// - [`BuildError::KeyOutOfRange`] if a key does not fit a [`KeyWidth::U32`] table + /// - [`BuildError::PathTooLong`] if a path exceeds 65535 bytes + /// - [`BuildError::InvalidFrameSize`] if the zeekstd frame size is outside + /// `1..=`[`MAX_FRAME_SIZE`] + /// - [`BuildError::HashKindTooWide`] if a 64-bit [`HashKind`] is recorded + /// for a [`KeyWidth::U32`] table + /// - [`BuildError::Io`] / [`BuildError::Compression`] if writing or + /// compressing fails + pub fn build(mut self, mut out: W) -> Result { + if self.key_width == KeyWidth::U32 && self.hash_kind.is_64_bit() { + return Err(BuildError::HashKindTooWide { + hash_kind: self.hash_kind, + key_width: self.key_width, + }); + } + if let Compression::Zeekstd { frame_size, .. } = self.compression { + if frame_size == 0 || frame_size > MAX_FRAME_SIZE { + return Err(BuildError::InvalidFrameSize { frame_size }); + } + } + self.entries.sort_unstable(); self.entries.dedup(); if let Some(w) = self.entries.windows(2).find(|w| w[0].0 == w[1].0) { @@ -170,16 +217,14 @@ impl HashDbWriter { let (stored_arena, mut flags) = match self.compression { Compression::None => (arena, 0), Compression::Zeekstd { frame_size, level } => { - if frame_size == 0 { - return Err(BuildError::ZeroFrameSize); - } let mut compressed = Vec::new(); let mut encoder = zeekstd::EncodeOptions::new() .compression_level(level) .frame_size_policy(zeekstd::FrameSizePolicy::Uncompressed(frame_size)) - .into_encoder(&mut compressed)?; + .into_encoder(&mut compressed) + .zeek()?; encoder.write_all(&arena)?; - encoder.finish()?; + encoder.finish().zeek()?; (compressed, FLAG_ARENA_COMPRESSED) } }; diff --git a/crates/ltk_hashdb/tests/proptests.rs b/crates/ltk_hashdb/tests/proptests.rs index bc7ae4f..2cd2588 100644 --- a/crates/ltk_hashdb/tests/proptests.rs +++ b/crates/ltk_hashdb/tests/proptests.rs @@ -82,21 +82,37 @@ proptest! { let _ = HashDb::open_bytes(bytes); } - /// Random corruption of a valid file: open either errors or yields a db whose - /// verify()/get() don't panic. + /// Random corruption of a valid file, raw or compressed: open either errors + /// or yields a db whose lookups (hits included), verify, and iteration + /// don't panic. #[test] fn corrupted_valid_file_never_panics( - entries in prop::collection::btree_map(any::(), ".{0,20}", 1..16), + entries in prop::collection::btree_map(any::(), ".{0,40}", 1..32), + compressed in any::(), + frame_size in 1u32..256, flips in prop::collection::vec((any::(), 1u8..=255), 1..8), probe in any::(), ) { - let mut bytes = build_bytes(KeyWidth::U64, &entries); + let compression = if compressed { + Compression::Zeekstd { frame_size, level: 3 } + } else { + Compression::None + }; + let mut bytes = build_bytes_with(KeyWidth::U64, compression, &entries); for (idx, mask) in flips { let i = idx.index(bytes.len()); bytes[i] ^= mask; } if let Ok(db) = HashDb::open_bytes(bytes) { - let _ = db.get(probe); + // Probe the real keys too: a random key almost always misses and + // never reaches the arena. + let mut probes: Vec = entries.keys().copied().collect(); + probes.push(probe); + for &k in &probes { + let _ = db.get(k); + let _ = db.try_get(k); + } + let _ = db.get_batch(&probes).count(); let _ = db.verify(); let _ = db.iter().count(); } diff --git a/crates/ltk_hashdb/tests/roundtrip.rs b/crates/ltk_hashdb/tests/roundtrip.rs index f2e6f61..00f9923 100644 --- a/crates/ltk_hashdb/tests/roundtrip.rs +++ b/crates/ltk_hashdb/tests/roundtrip.rs @@ -4,7 +4,7 @@ use std::io::Cursor; use ltk_hashdb::{ ArenaOrder, BuildError, Casing, Compression, HashDb, HashDbWriter, HashKind, KeyWidth, - LayeredHashDb, OpenError, VerifyError, + LayeredHashDb, OpenError, VerifyError, MAX_FRAME_SIZE, }; fn build_with( @@ -319,16 +319,64 @@ fn compressed_corruption_detected_by_verify() { } #[test] -fn zero_frame_size_rejected() { - let mut w = HashDbWriter::new( +fn out_of_range_frame_size_rejected() { + for frame_size in [0, MAX_FRAME_SIZE + 1] { + let mut w = HashDbWriter::new( + KeyWidth::U64, + Compression::Zeekstd { + frame_size, + level: 3, + }, + ); + w.insert(1, "a"); + let err = w.build(Cursor::new(Vec::new())).unwrap_err(); + assert!(matches!(err, BuildError::InvalidFrameSize { frame_size: f } if f == frame_size)); + } +} + +#[test] +fn wide_hash_kind_in_u32_table_rejected() { + let w = HashDbWriter::new(KeyWidth::U32, Compression::None).hash_kind(HashKind::Xxh64); + let err = w.build(Cursor::new(Vec::new())).unwrap_err(); + assert!(matches!(err, BuildError::HashKindTooWide { .. })); +} + +/// A seek table may claim frames of up to 1 GiB while the compressed data is +/// tiny. `open` must reject frames above `MAX_FRAME_SIZE`, or a single lookup +/// would allocate the claimed size. +#[test] +fn oversized_frame_claim_rejected_on_open() { + let mut bytes = build_with( KeyWidth::U64, + HashKind::Xxh64, Compression::Zeekstd { - frame_size: 0, + frame_size: 64, level: 3, }, + GAME_ENTRIES, ); - w.insert(1, "a"); - assert!(w.build(Cursor::new(Vec::new())).is_err()); + + // The seek table footer is the last 9 bytes: frame count (u32), descriptor + // (u8), magic (u32). The last entry sits right before it; its second u32 is + // the frame's decompressed size. + let len = bytes.len(); + assert_eq!(bytes[len - 4..], 0x8F92_EAB1u32.to_le_bytes()); + let entry_size = if bytes[len - 5] & 0x80 != 0 { 12 } else { 8 }; + let d_size_at = len - 9 - entry_size + 4; + let old = u32::from_le_bytes(bytes[d_size_at..d_size_at + 4].try_into().unwrap()); + let claimed = 0x4000_0000u32; + bytes[d_size_at..d_size_at + 4].copy_from_slice(&claimed.to_le_bytes()); + + // Keep the header's arena size consistent with the seek table, so only the + // frame-size check can reject the file. + let arena_size = u64::from_le_bytes(bytes[48..56].try_into().unwrap()); + let patched = arena_size - u64::from(old) + u64::from(claimed); + bytes[48..56].copy_from_slice(&patched.to_le_bytes()); + + assert!(matches!( + HashDb::open_bytes(bytes), + Err(OpenError::Malformed(_)) + )); } #[test] diff --git a/crates/ltk_mimir_cache/Cargo.toml b/crates/ltk_mimir_cache/Cargo.toml index 33b52d8..68209d8 100644 --- a/crates/ltk_mimir_cache/Cargo.toml +++ b/crates/ltk_mimir_cache/Cargo.toml @@ -1,9 +1,12 @@ [package] name = "ltk_mimir_cache" -description = "Shared cache directory, manifest, versioned publishing, update lock, and in-process updater for League Toolkit .lhdb hash tables" +description = "Shared, versioned on-disk cache and updater for League Toolkit .lhdb hash tables" readme = "README.md" +keywords = ["league-of-legends", "hash", "cache", "hashdb"] +categories = ["caching", "game-development"] version = "0.1.0" edition.workspace = true +rust-version.workspace = true license.workspace = true repository.workspace = true authors.workspace = true diff --git a/crates/ltk_mimir_cache/README.md b/crates/ltk_mimir_cache/README.md index a63e643..b2a0883 100644 --- a/crates/ltk_mimir_cache/README.md +++ b/crates/ltk_mimir_cache/README.md @@ -1,103 +1,139 @@ # ltk_mimir_cache -The shared, versioned, multi-process **cache** for League Toolkit hash tables - the I/O -policy layer that sits on top of the [`ltk_hashdb`](../ltk_hashdb) format. It decides -*where* tables live on disk, *which* version is active, and how new versions are published -so that many tools on one machine can read the same files while a single updater swaps in a -new release underneath them - without locks on the read path and without ever showing a -reader a half-written file. +A shared, versioned cache of League Toolkit hash tables that several processes on one +machine can use at once. It stores tables in the +[`ltk_hashdb`](https://crates.io/crates/ltk_hashdb) format and handles: -`ltk_hashdb` owns the byte format; this crate owns everything around it: directory -resolution, the `manifest.json` pointer, atomic versioned publishing, the single-updater -lock, and lazy garbage collection. +- where the cache directory is +- which version of each table is active (`manifest.json`) +- installing new versions atomically, so readers never see a partial file +- a lock so only one process updates the cache at a time +- removing versions that are no longer used +- checking for and downloading updates from a published release + +Readers take no locks. ## The cache directory -One directory holds every table plus a manifest and an update lock: +One directory holds every table, the manifest, and the update lock: -``` +```text hashes/ - game-2026-07-08.lhdb # versioned, immutable once written + game-2026-07-08.lhdb # versioned, never modified after it is written lcu-2026-07-08.lhdb binentries-2026-07-08.lhdb ... - manifest.json # pointer: active version + sha256 per table - .update.lock # cross-process single-updater lock + manifest.json # active version and sha256 per table + .update.lock # held by the process that is updating ``` -Its location is resolved (without being created) from the platform data directory by -default, which `MIMIR_DIR` overrides when set: +`HashStore::discover` finds the directory without creating it: -- `MIMIR_DIR` - points directly at the tables directory; overrides everything. -- Otherwise, the platform data dir: +- `MIMIR_DIR`, if set and non-empty +- otherwise the platform data directory: - Windows: `%LOCALAPPDATA%\LeagueToolkit\hashes\` - - Linux: `$XDG_DATA_HOME/LeagueToolkit/hashes` (fallback `~/.local/share/...`) + - Linux: `$XDG_DATA_HOME/LeagueToolkit/hashes` (default `~/.local/share/LeagueToolkit/hashes`) - macOS: `~/Library/Application Support/LeagueToolkit/hashes` -## Reading (lazy, lock-free) +Only this crate (or the `mimir` CLI) should change files in the directory. Tables are +memory-mapped, which relies on them never being modified. -Tables are immutable and the manifest is swapped atomically, so readers never coordinate: -read the manifest, `mmap` the active file, use it, drop it. +## Reading -```rust +```rust,no_run use ltk_mimir_cache::{HashStore, Table}; -let store = HashStore::discover()?; // MIMIR_DIR override, else platform dir -let db = store.open(Table::Game)?; // manifest → active file → mmap -if let Some(path) = db.get(0x1234_5678_9abc_def0) { - println!("{path}"); +fn main() -> Result<(), Box> { + let store = HashStore::discover()?; + + // One table. `open_shared` reuses a handle this process already has. + let db = store.open_shared(Table::Game)?; + if let Some(path) = db.get(0x1234_5678_9abc_def0) { + println!("{path}"); + } + + // WAD chunk hashes can be in either path table, so look in both; earlier tables win. + let (paths, unavailable) = store.open_layered(&[Table::Game, Table::Lcu])?; + for (table, error) in unavailable { + eprintln!("{table} unavailable: {error}"); + } + println!("{} base tables", paths.bases().len()); + Ok(()) +} +``` + +Opening checks the file's structure and that its key config matches the table. It does +not re-check the sha256, which was verified when the file was installed. Call +`HashDb::verify` for a full check. + +## Updating from a release + +`HashStore::update` compares the local manifest with a release, downloads the tables that +changed, checks their sha256, installs them, and removes old versions. The caller supplies +the download function; any closure `Fn(&str) -> Result, E>` works. With the `ureq` +feature, `UreqFetch::new(ReleaseSource::github("owner/repo"))` downloads from a GitHub +release. `HashStore::check` reports what an update would do without downloading anything. + +```rust,no_run +use std::path::Path; + +use ltk_mimir_cache::{HashStore, UpdateOptions, UpdateOutcome}; + +fn main() -> Result<(), Box> { + let store = HashStore::discover()?; + + // Read release files from a local directory. + let fetch = |filename: &str| std::fs::read(Path::new("release").join(filename)); + match store.update(&fetch, UpdateOptions::default())? { + UpdateOutcome::Completed(report) => println!("installed {:?}", report.installed), + UpdateOutcome::Locked => println!("another process is updating"), + _ => {} + } + Ok(()) } ``` -`open` validates structure only: it trusts the manifest's download-time sha256 and -stays cheap and lazy. Use [`HashDb::verify`](../ltk_hashdb) for a full checksum pass. +`update_async` does the same with an `AsyncFetch`, such as `ReqwestFetch` from the +`reqwest` feature. -## Committing (single updater, atomic) +## Committing tables you built -`commit` is the cache-side primitive: it installs one or more freshly built `.lhdb` files -under immutable `-.lhdb` names, then flips the manifest to point at them -**last**, so a reader mid-lookup sees either the whole old version or the whole new one. The -table copies and the manifest write are each a temp file + `fsync` + rename. +`commit` installs built `.lhdb` files under `
-.lhdb` names and then replaces +the manifest. Both `commit` and `gc` require the update lock: -```rust +```rust,no_run use ltk_mimir_cache::{CommitItem, HashStore, Table}; -let store = HashStore::discover()?; +fn main() -> Result<(), Box> { + let store = HashStore::discover()?; -// Serialize updaters (readers need no lock). `None` means another process is updating. -if let Some(_lock) = store.try_lock_update()? { - store.commit( - &[CommitItem::new(Table::Game, "2026-07-08", "build/game.lhdb")], - None, // optional provenance (source repo + commit / inputs sha256) - )?; - store.gc()?; // reclaim superseded versions no reader still maps + // `None` means another process is updating. + if let Some(lock) = store.try_lock_update()? { + let items = [CommitItem::new(Table::Game, "2026-07-08", "build/game.lhdb")]; + store.commit(&lock, &items, None)?; + store.gc(&lock)?; + } + Ok(()) } ``` -`gc` deletes versioned files no longer referenced by the manifest (plus stray `.tmp` -leftovers). A file the OS refuses to unlink - still mapped by a reader, the classic Windows -case - is left in place and reported in `GcReport::retained` to retry later, never surfaced -as an error. +`gc` deletes table files the manifest no longer references and leftover `.tmp` files. On +Windows a file that a reader still has mapped cannot be deleted; `gc` lists it in +`GcReport::retained` and tries again next time. -> The download-driven `mimir update` flow is built on exactly these primitives: fetch the -> release manifest, fingerprint-skip per table by sha256, download + verify what changed, -> then `commit` + `gc` under `try_lock_update`. +## Features -## API surface +| Feature | Adds | +|---------|------| +| `ureq` | `UreqFetch`, a blocking HTTP fetcher | +| `reqwest` | `ReqwestFetch`, an async HTTP fetcher | -| Item | Role | -|------|------| -| `HashStore::discover` / `at` | Resolve the cache dir, or use an explicit one | -| `HashStore::manifest` / `open` / `path_for` | Read the manifest; open / locate the active table | -| `HashStore::try_lock_update` → `UpdateLock` | Non-blocking cross-process single-updater lock | -| `HashStore::commit` | Install versioned files + atomically swap the manifest | -| `HashStore::gc` → `GcReport` | Reclaim unreferenced, unmapped versions | -| `HashStore::update` / `update_async` | The download-driven update loop (blocking or async) over a caller-supplied `Fetch` / `AsyncFetch` | -| `Table` | The eight logical tables (`id` / `from_id` / `ALL`) | -| `Manifest` / `Source` / `TableEntry` | The `manifest.json` schema (serde) | +Both use connect and read timeouts and refuse files larger than +`DEFAULT_MAX_ASSET_SIZE` (512 MiB) unless configured otherwise. -See [`docs/CONSUMERS.md`](../../docs/CONSUMERS.md) for the consumer-facing integration guide. +See the [consumer guide](https://github.com/LeagueToolkit/mimir/blob/main/docs/CONSUMERS.md) +for more detail. ## License -Apache-2.0. Copyright 2026 Crauzer <0xcrauzer@proton.me>. +Apache-2.0. diff --git a/crates/ltk_mimir_cache/src/error.rs b/crates/ltk_mimir_cache/src/error.rs index b3e973f..46557df 100644 --- a/crates/ltk_mimir_cache/src/error.rs +++ b/crates/ltk_mimir_cache/src/error.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; +use ltk_hashdb::KeyConfig; use thiserror::Error; use crate::{HashUniverse, Table}; @@ -50,40 +51,84 @@ impl std::error::Error for ParseTableError {} /// Errors from reading, parsing, or writing `manifest.json`. #[derive(Debug, Error)] +#[non_exhaustive] pub enum ManifestError { + /// Reading or writing the file failed. #[error(transparent)] Io(#[from] std::io::Error), + /// The file is not a valid manifest document. #[error("manifest json error")] Json(#[from] serde_json::Error), + /// There is no manifest at this path: nothing was ever committed to the cache. #[error("no manifest at {0}")] Missing(PathBuf), + /// The schema version is older than the first published one. #[error("manifest schema {0} predates the first published one")] UnsupportedSchema(u32), + /// The manifest needs a newer reader than this build. #[error( "this manifest requires a reader that understands schema {required}, and this build \ understands {supported}" )] - ReaderTooOld { required: u32, supported: u32 }, + ReaderTooOld { + /// The lowest schema that can read the manifest. + required: u32, + + /// The schema this build reads. + supported: u32, + }, } /// Errors from opening a cached table ([`HashStore::open`](crate::HashStore::open) / /// [`HashStore::path_for`](crate::HashStore::path_for)). #[derive(Debug, Error)] +#[non_exhaustive] pub enum OpenError { + /// The manifest could not be read. #[error(transparent)] Manifest(#[from] ManifestError), + /// The manifest has no entry for this table. #[error("table {0} is not in the manifest")] TableNotFound(Table), + /// The manifest entry's filename is not `
-.lhdb` with a + /// valid version label. + #[error("table {table}: invalid filename {file:?} in the manifest")] + InvalidFilename { + /// The table whose entry is invalid. + table: Table, + + /// The filename the manifest gave. + file: String, + }, + + /// The table file could not be opened or failed validation. #[error("opening the table file")] HashDb(#[from] ltk_hashdb::OpenError), - #[error("the table file does not hash its keys the way this table is defined to")] + /// The table file's key config is not the table's + /// [`key_config`](Table::key_config): the file is not the table the + /// manifest files it under. + #[error("table {table}: file is keyed {found}, but the table is keyed {expected}")] + WrongKeyConfig { + /// The table being opened. + table: Table, + + /// The table's key config. + expected: KeyConfig, + + /// The file's key config. + found: KeyConfig, + }, + + /// A table's key config does not match the tables layered before it in + /// [`open_layered`](crate::HashStore::open_layered). + #[error("the table file does not hash its keys the way the layer does")] KeyConfig(#[from] ltk_hashdb::KeyConfigMismatch), } @@ -91,11 +136,12 @@ pub enum OpenError { /// ([`HashStore::open_layered`](crate::HashStore::open_layered)). #[derive(Debug, Clone, Copy, Error)] #[error( - "cannot layer {table} ({found}) with {first} ({expected}): a hash means nothing outside \ - its own universe, so one table would answer the other with a confident, wrong path" + "cannot layer {table} ({found}) with {first} ({expected}): tables from different hash \ + universes would answer each other's hashes with unrelated paths" )] +#[non_exhaustive] pub struct UniverseMismatch { - /// The first table in the requested set - the universe the rest must match. + /// The first table in the requested set, whose universe the rest must match. pub first: Table, /// That table's universe. @@ -110,97 +156,161 @@ pub struct UniverseMismatch { /// Errors from installing tables ([`HashStore::commit`](crate::HashStore::commit)). #[derive(Debug, Error)] +#[non_exhaustive] pub enum CommitError { + /// Copying a file or creating the cache directory failed. #[error(transparent)] Io(#[from] std::io::Error), + /// The current manifest could not be read, or the new one written. #[error(transparent)] Manifest(#[from] ManifestError), + /// A source file is not a valid table. #[error("opening the built table file")] HashDb(#[from] ltk_hashdb::OpenError), - #[error("invalid version label {0:?}: must be non-empty and free of path separators")] + /// A source file's key config is not the table's [`key_config`](Table::key_config). + #[error("table {table}: built file is keyed {found}, but the table is keyed {expected}")] + WrongKeyConfig { + /// The table being installed. + table: Table, + + /// The table's key config. + expected: KeyConfig, + + /// The file's key config. + found: KeyConfig, + }, + + /// A version label is empty, too long, starts with `.`, or uses characters + /// outside `[A-Za-z0-9._-]`. + #[error( + "invalid version label {0:?}: must be 1-64 characters from [A-Za-z0-9._-] \ + and not start with '.'" + )] InvalidVersion(String), + /// The versioned file already exists with different content. #[error( "table {table:?}: version {version:?} is already published with different content; \ published versions are immutable" )] - VersionReused { table: Table, version: String }, + VersionReused { + /// The table being installed. + table: Table, + + /// The reused version label. + version: String, + }, } -/// Errors from sweeping unreferenced files ([`HashStore::gc`](crate::HashStore::gc)). +/// Errors from removing unreferenced files ([`HashStore::gc`](crate::HashStore::gc)). #[derive(Debug, Error)] +#[non_exhaustive] pub enum GcError { + /// Listing the cache directory failed. #[error(transparent)] Io(#[from] std::io::Error), + /// The manifest could not be read. #[error(transparent)] Manifest(#[from] ManifestError), } /// Why a streaming fetch stopped ([`Fetch::fetch_to`](crate::Fetch::fetch_to)). /// -/// The two halves have different owners: `Transport` is the fetcher's problem, -/// `Sink` is the caller's - a full disk, or a wrapping sink cancelling the -/// download by refusing the next chunk. +/// `Transport` is a failure in the fetcher. `Sink` is a failure writing the +/// bytes: a full disk, or a wrapping sink cancelling the download by refusing +/// the next chunk. #[derive(Debug, Error)] +#[non_exhaustive] pub enum FetchError { + /// The fetcher failed. #[error(transparent)] Transport(E), + /// The sink refused the bytes. #[error("writing the fetched bytes")] Sink(#[source] std::io::Error), } /// Errors from a lock-free comparison ([`HashStore::check`](crate::HashStore::check)). /// -/// Deliberately smaller than [`UpdateError`]: `check` installs nothing, so there -/// is no download to mis-checksum and no commit to fail. +/// Smaller than [`UpdateError`]: `check` installs nothing, so there is no +/// download to verify and no commit to fail. #[derive(Debug, Error)] +#[non_exhaustive] pub enum CheckError { + /// The local or remote manifest could not be read or parsed. #[error(transparent)] Manifest(#[from] ManifestError), + /// Fetching the remote manifest failed. #[error("fetching {file}")] Fetch { + /// The filename being fetched. file: String, + + /// Why the fetch failed. #[source] source: FetchError, }, } -/// Errors from an update run ([`HashStore::update`](crate::HashStore::update)). +/// Errors from an update run ([`HashStore::update`](crate::HashStore::update) / +/// [`HashStore::update_async`](crate::HashStore::update_async)). /// /// Generic over the fetcher's error type ([`Fetch::Error`](crate::Fetch::Error) / /// [`AsyncFetch::Error`](crate::AsyncFetch::Error)), so a failed download -/// surfaces the transport's concrete error instead of a boxed one. +/// returns the fetcher's own error type. #[derive(Debug, Error)] +#[non_exhaustive] pub enum UpdateError { + /// A local file operation failed. #[error(transparent)] Io(#[from] std::io::Error), + /// The local or remote manifest could not be read or parsed. #[error(transparent)] Manifest(#[from] ManifestError), + /// Downloading a file failed. #[error("fetching {file}")] Fetch { + /// The filename being fetched. file: String, + + /// Why the fetch failed. #[source] source: FetchError, }, + /// A downloaded file's sha256 does not match the remote manifest. #[error("{file}: sha256 mismatch (manifest {expected}, downloaded {actual})")] ChecksumMismatch { + /// The downloaded filename. file: String, + + /// The sha256 the remote manifest lists. expected: String, + + /// The sha256 of the downloaded bytes. actual: String, }, + /// A remote manifest entry's filename is not `
-.lhdb` with + /// a valid version label. #[error("table {id}: malformed filename {file:?} in the remote manifest")] - BadRemoteFilename { id: String, file: String }, + BadRemoteFilename { + /// The table id from the remote manifest. + id: String, + + /// The filename the remote manifest gave. + file: String, + }, + /// Installing the downloaded tables failed. Nothing was installed. #[error("installing the downloaded tables")] Commit(#[from] CommitError), } diff --git a/crates/ltk_mimir_cache/src/fetch.rs b/crates/ltk_mimir_cache/src/fetch.rs index dfc50b7..16caf50 100644 --- a/crates/ltk_mimir_cache/src/fetch.rs +++ b/crates/ltk_mimir_cache/src/fetch.rs @@ -10,18 +10,38 @@ //! plumbing the [`Fetch::Error`](crate::Fetch::Error) `Sized` bound would //! otherwise force. //! -//! - `ureq` feature → [`UreqFetch`], a blocking [`Fetch`](crate::Fetch). -//! - `reqwest` feature → [`ReqwestFetch`], an async +//! - `ureq` feature: [`UreqFetch`], a blocking [`Fetch`](crate::Fetch). +//! - `reqwest` feature: [`ReqwestFetch`], an async //! [`AsyncFetch`](crate::AsyncFetch). //! +//! Both use a 30 s connect timeout and a 60 s read timeout, so a stalled server +//! fails the update instead of holding the update lock indefinitely, and both +//! refuse a file larger than [`DEFAULT_MAX_ASSET_SIZE`] unless configured +//! otherwise. +//! //! Both fetchers stream into the caller's sink rather than buffering a whole //! table, and both are silent by design: per-file progress and cancellation stay //! with the caller, who wraps the fetcher and passes a sink of their own (see //! [`Fetch`](crate::Fetch)). +use std::time::Duration; + /// Default `User-Agent` for the bundled fetchers, matching the mimir CLI. const USER_AGENT: &str = concat!("mimir/", env!("CARGO_PKG_VERSION")); +/// Time allowed to establish a connection. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Time allowed between two reads of the response body. +const READ_TIMEOUT: Duration = Duration::from_secs(60); + +/// Default largest file the bundled fetchers download, in bytes (512 MiB). +/// +/// Downloads stream to disk, so this bounds the disk space a misbehaving server +/// or mirror can fill before the sha256 check rejects the file. Published tables +/// are far smaller. +pub const DEFAULT_MAX_ASSET_SIZE: u64 = 512 << 20; + /// Where release assets live: a GitHub repo's latest release, or an explicit /// base URL (a mirror). Shared by the blocking and async fetchers. #[derive(Debug, Clone)] @@ -53,21 +73,41 @@ impl ReleaseSource { } } +/// Errors from the bundled HTTP fetchers. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum HttpFetchError { - /// The request never produced an HTTP response (DNS, connect, TLS, or a - /// mid-body read failure). + /// The request did not complete: DNS, connect, TLS, timeout, or a failure + /// while reading the body. #[error("fetching {url}")] Transport { + /// The requested URL. url: String, + /// The HTTP client's error. #[source] source: Box, }, - /// The server answered, but with a non-success status. + /// The server answered with a non-success status. #[error("unexpected HTTP {status} for {url}")] - Status { status: u16, url: String }, + Status { + /// The HTTP status code. + status: u16, + + /// The requested URL. + url: String, + }, + + /// The file is larger than the fetcher's size limit. + #[error("{url} is larger than the {limit}-byte limit")] + TooLarge { + /// The requested URL. + url: String, + + /// The limit, in bytes. + limit: u64, + }, } /// The read buffer both fetchers pump through. Large enough that a 38 MiB table @@ -79,23 +119,53 @@ const CHUNK: usize = 64 * 1024; mod ureq_impl { use std::io::{Read, Write}; - use super::{HttpFetchError, ReleaseSource, CHUNK, USER_AGENT}; + use super::{ + HttpFetchError, ReleaseSource, CHUNK, CONNECT_TIMEOUT, DEFAULT_MAX_ASSET_SIZE, + READ_TIMEOUT, USER_AGENT, + }; use crate::{Fetch, FetchError}; - /// A blocking [`Fetch`] over `ureq`, pulling assets from a [`ReleaseSource`]. + /// A blocking [`Fetch`] over `ureq`, downloading from a [`ReleaseSource`]. + /// + /// Requires the `ureq` feature. + #[derive(Debug)] pub struct UreqFetch { source: ReleaseSource, agent: ureq::Agent, + + max_size: u64, } impl UreqFetch { - /// A fetcher for `source`, using a fresh agent with the mimir - /// `User-Agent`. + /// A fetcher for `source` with the mimir `User-Agent`, a 30 s connect + /// timeout, and a 60 s read timeout. pub fn new(source: ReleaseSource) -> Self { - let agent = ureq::AgentBuilder::new().user_agent(USER_AGENT).build(); + let agent = ureq::AgentBuilder::new() + .user_agent(USER_AGENT) + .timeout_connect(CONNECT_TIMEOUT) + .timeout_read(READ_TIMEOUT) + .build(); + + Self::with_agent(source, agent) + } - Self { source, agent } + /// A fetcher for `source` that sends requests through `agent`, for + /// custom proxies, TLS, or timeouts. + pub fn with_agent(source: ReleaseSource, agent: ureq::Agent) -> Self { + Self { + source, + agent, + max_size: DEFAULT_MAX_ASSET_SIZE, + } + } + + /// Set the largest file this fetcher downloads, in bytes. Defaults to + /// [`DEFAULT_MAX_ASSET_SIZE`]. + #[must_use] + pub fn max_size(mut self, bytes: u64) -> Self { + self.max_size = bytes; + self } } @@ -126,6 +196,16 @@ mod ureq_impl { } }; + let declared = response + .header("Content-Length") + .and_then(|len| len.parse::().ok()); + if declared.is_some_and(|len| len > self.max_size) { + return Err(FetchError::Transport(HttpFetchError::TooLarge { + url, + limit: self.max_size, + })); + } + // Hand-rolled rather than `io::copy`, which would fold a mid-body // read failure and the caller's sink refusing a chunk into one error. let mut reader = response.into_reader(); @@ -142,8 +222,15 @@ mod ureq_impl { return Ok(total); } - sink.write_all(&buf[..read]).map_err(FetchError::Sink)?; + // Content-Length can be absent or wrong, so count while reading. total += read as u64; + if total > self.max_size { + return Err(FetchError::Transport(HttpFetchError::TooLarge { + url, + limit: self.max_size, + })); + } + sink.write_all(&buf[..read]).map_err(FetchError::Sink)?; } } } @@ -157,24 +244,59 @@ mod reqwest_impl { use std::future::Future; use std::io::Write; - use super::{HttpFetchError, ReleaseSource, USER_AGENT}; + use super::{ + HttpFetchError, ReleaseSource, CONNECT_TIMEOUT, DEFAULT_MAX_ASSET_SIZE, READ_TIMEOUT, + USER_AGENT, + }; use crate::{AsyncFetch, FetchError}; + /// An async [`AsyncFetch`] over `reqwest`, downloading from a + /// [`ReleaseSource`]. + /// + /// Requires the `reqwest` feature. + #[derive(Debug, Clone)] pub struct ReqwestFetch { source: ReleaseSource, client: reqwest::Client, + + max_size: u64, } impl ReqwestFetch { - /// A fetcher for `source`, using a client with the mimir `User-Agent`. - pub fn new(source: ReleaseSource) -> Self { + /// A fetcher for `source` with the mimir `User-Agent`, a 30 s connect + /// timeout, and a 60 s read timeout. + /// + /// # Errors + /// + /// The `reqwest` error if the client cannot be built, for example when + /// the TLS backend fails to initialize. + pub fn new(source: ReleaseSource) -> Result { let client = reqwest::Client::builder() .user_agent(USER_AGENT) - .build() - .unwrap_or_default(); + .connect_timeout(CONNECT_TIMEOUT) + .read_timeout(READ_TIMEOUT) + .build()?; - Self { source, client } + Ok(Self::with_client(source, client)) + } + + /// A fetcher for `source` that sends requests through `client`, for + /// custom proxies, TLS, or timeouts. + pub fn with_client(source: ReleaseSource, client: reqwest::Client) -> Self { + Self { + source, + client, + max_size: DEFAULT_MAX_ASSET_SIZE, + } + } + + /// Set the largest file this fetcher downloads, in bytes. Defaults to + /// [`DEFAULT_MAX_ASSET_SIZE`]. + #[must_use] + pub fn max_size(mut self, bytes: u64) -> Self { + self.max_size = bytes; + self } } @@ -189,6 +311,7 @@ mod reqwest_impl { // Own what the request needs; only the sink is borrowed. let url = self.source.asset_url(filename); let client = self.client.clone(); + let limit = self.max_size; async move { let transport = |err: reqwest::Error, url: &str| { @@ -212,15 +335,29 @@ mod reqwest_impl { })); } + if response.content_length().is_some_and(|len| len > limit) { + return Err(FetchError::Transport(HttpFetchError::TooLarge { + url, + limit, + })); + } + // `chunk` rather than `bytes`, so the body never has to exist in // memory all at once. It also keeps the transport failure and the - // sink failure apart. + // sink failure apart. Content-Length can be absent or wrong, so + // the limit is also checked while reading. let mut total = 0; while let Some(chunk) = response.chunk().await.map_err(|err| transport(err, &url))? { - sink.write_all(&chunk).map_err(FetchError::Sink)?; total += chunk.len() as u64; + if total > limit { + return Err(FetchError::Transport(HttpFetchError::TooLarge { + url, + limit, + })); + } + sink.write_all(&chunk).map_err(FetchError::Sink)?; } Ok(total) @@ -291,7 +428,7 @@ mod tests { let payload = b"lhdb-bytes".to_vec(); let (base, server) = serve(payload.clone(), "/game-1.lhdb".to_string(), 2); - // Two fetchers → two connections, matching the server's count. + // Two fetchers, two connections, matching the server's count. // Straight into a caller's sink - no whole-table buffer anywhere. let ok = UreqFetch::new(ReleaseSource::base_url(&base)); let mut sink = Vec::new(); @@ -322,7 +459,7 @@ mod tests { .unwrap(); runtime.block_on(async { - let ok = ReqwestFetch::new(ReleaseSource::base_url(&base)); + let ok = ReqwestFetch::new(ReleaseSource::base_url(&base)).unwrap(); let mut sink = Vec::new(); assert_eq!( ok.fetch_to("game-1.lhdb", &mut sink).await.unwrap(), @@ -330,7 +467,7 @@ mod tests { ); assert_eq!(sink, payload); - let missing = ReqwestFetch::new(ReleaseSource::base_url(&base)); + let missing = ReqwestFetch::new(ReleaseSource::base_url(&base)).unwrap(); match missing.fetch("nope.lhdb").await { Err(FetchError::Transport(HttpFetchError::Status { status: 404, .. })) => {} other => panic!("expected a 404 status error, got {other:?}"), @@ -339,4 +476,46 @@ mod tests { server.join().unwrap(); } + + #[cfg(feature = "ureq")] + #[test] + fn ureq_fetch_enforces_max_size() { + let (base, server) = serve(vec![7u8; 32], "/game-1.lhdb".to_string(), 2); + + let at_limit = UreqFetch::new(ReleaseSource::base_url(&base)).max_size(32); + assert_eq!(at_limit.fetch("game-1.lhdb").unwrap().len(), 32); + + let below = UreqFetch::new(ReleaseSource::base_url(&base)).max_size(16); + match below.fetch("game-1.lhdb") { + Err(FetchError::Transport(HttpFetchError::TooLarge { limit: 16, .. })) => {} + other => panic!("expected a too-large error, got {other:?}"), + } + + server.join().unwrap(); + } + + #[cfg(feature = "reqwest")] + #[test] + fn reqwest_fetch_enforces_max_size() { + let (base, server) = serve(vec![7u8; 32], "/game-1.lhdb".to_string(), 2); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + let source = ReleaseSource::base_url(&base); + let at_limit = ReqwestFetch::new(source.clone()).unwrap().max_size(32); + assert_eq!(at_limit.fetch("game-1.lhdb").await.unwrap().len(), 32); + + let below = ReqwestFetch::new(source).unwrap().max_size(16); + match below.fetch("game-1.lhdb").await { + Err(FetchError::Transport(HttpFetchError::TooLarge { limit: 16, .. })) => {} + other => panic!("expected a too-large error, got {other:?}"), + } + }); + + server.join().unwrap(); + } } diff --git a/crates/ltk_mimir_cache/src/lib.rs b/crates/ltk_mimir_cache/src/lib.rs index f306dae..55f57a6 100644 --- a/crates/ltk_mimir_cache/src/lib.rs +++ b/crates/ltk_mimir_cache/src/lib.rs @@ -81,6 +81,7 @@ //! Some(holder) => println!("pid {} has been updating since {}", holder.pid, holder.since), //! None => println!("another process is updating"), //! }, +//! _ => {} // `UpdateOutcome` is non-exhaustive //! } //! //! Ok(()) @@ -108,20 +109,21 @@ //! [`commit`](HashStore::commit) is the write primitive underneath `update`, and //! what a builder calls directly: it installs files under immutable //! `
-.lhdb` names and flips the manifest last, each step a temp -//! file, an fsync and a rename. Hold the update lock across it, then -//! [`gc`](HashStore::gc) to unlink superseded versions nothing still has open: +//! file, an fsync and a rename. Both it and [`gc`](HashStore::gc), which +//! deletes superseded versions, take the update lock: //! //! ```no_run //! use ltk_mimir_cache::{CommitItem, HashStore, Table}; //! //! # fn main() -> Result<(), Box> { //! let store = HashStore::discover()?; -//! let Some(_lock) = store.try_lock_update()? else { +//! let Some(lock) = store.try_lock_update()? else { //! return Ok(()); // someone else is publishing //! }; //! -//! store.commit(&[CommitItem::new(Table::Game, "2026-08-24", "game.lhdb")], None)?; -//! println!("swept {} superseded files", store.gc()?.deleted.len()); +//! let items = [CommitItem::new(Table::Game, "2026-08-24", "game.lhdb")]; +//! store.commit(&lock, &items, None)?; +//! println!("removed {} superseded files", store.gc(&lock)?.deleted.len()); //! # Ok(()) //! # } //! ``` @@ -153,6 +155,8 @@ //! - **`reqwest`** - `ReqwestFetch`, an [`AsyncFetch`] for //! [`update_async`](HashStore::update_async). +#![warn(missing_docs, missing_debug_implementations)] + mod dir; mod error; #[cfg(any(feature = "ureq", feature = "reqwest"))] @@ -164,6 +168,8 @@ mod store; mod table; mod update; +pub use ltk_hashdb; + pub use error::{ CheckError, CommitError, FetchError, GcError, ManifestError, NoCacheDirError, OpenError, ParseTableError, UniverseMismatch, UpdateError, @@ -173,7 +179,7 @@ pub use fetch::ReqwestFetch; #[cfg(feature = "ureq")] pub use fetch::UreqFetch; #[cfg(any(feature = "ureq", feature = "reqwest"))] -pub use fetch::{HttpFetchError, ReleaseSource}; +pub use fetch::{HttpFetchError, ReleaseSource, DEFAULT_MAX_ASSET_SIZE}; pub use lock::{LockHolder, UpdateLock}; pub use manifest::{Manifest, Source, TableEntry, SCHEMA_VERSION}; pub use store::{CommitItem, GcReport, HashStore}; @@ -182,3 +188,8 @@ pub use update::{ AsyncFetch, CheckReport, Fetch, PlannedTable, TableDiff, TableStatus, UnsupportedTable, UpdateObserver, UpdateOptions, UpdateOutcome, UpdateReport, }; + +/// Compiles the README's code blocks as doctests. +#[cfg(doctest)] +#[doc = include_str!("../README.md")] +struct ReadmeDoctests; diff --git a/crates/ltk_mimir_cache/src/manifest.rs b/crates/ltk_mimir_cache/src/manifest.rs index b0d5390..f2bce0f 100644 --- a/crates/ltk_mimir_cache/src/manifest.rs +++ b/crates/ltk_mimir_cache/src/manifest.rs @@ -22,6 +22,7 @@ const OLDEST_SCHEMA: u32 = 1; /// The `manifest.json` document: schema version, generation timestamp, optional input /// provenance, and one [`TableEntry`] per published table keyed by [`Table::id`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] pub struct Manifest { /// The schema the writer used. Informational - see /// [`from_slice`](Manifest::from_slice) for what is actually gated. @@ -37,6 +38,7 @@ pub struct Manifest { #[serde(default, skip_serializing_if = "Option::is_none")] pub min_reader_schema: Option, + /// When the manifest was written, as an RFC 3339 UTC timestamp. pub generated_at: String, /// The inputs the most recent commit to this cache drew on. @@ -48,12 +50,22 @@ pub struct Manifest { #[serde(default, alias = "source", skip_serializing_if = "Option::is_none")] pub last_run: Option, + /// The active file per table, keyed by [`Table::id`]. Ids this build does + /// not know are kept as they are. #[serde(default)] pub tables: BTreeMap, } -/// Provenance of the inputs one table - or one publishing run - was built from. +/// Provenance of the inputs one table, or one publishing run, was built from. +/// +/// Build one from [`Source::default`] and set the fields: +/// +/// ``` +/// let mut source = ltk_mimir_cache::Source::default(); +/// source.repo = Some("CommunityDragon/Data".into()); +/// ``` #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] pub struct Source { /// Where the txt hash lists came from: a git URL or a GitHub `owner/repo` /// (canonically `CommunityDragon/Data`). @@ -72,10 +84,18 @@ pub struct Source { /// The active file for one table plus the metadata a reader/updater needs without /// opening it: download checksum, entry count, key width, and download size. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[non_exhaustive] pub struct TableEntry { + /// Filename in the cache directory, `
-.lhdb`. pub file: String, + + /// sha256 of the file, lowercase hex. pub sha256: String, + + /// Number of entries in the table. pub entries: u64, + + /// Key width in bytes (4 or 8). pub key_width: u8, /// The size of `file` in bytes, when the publisher recorded it. @@ -215,7 +235,8 @@ impl Manifest { } /// Read and parse the manifest at `path`. - pub fn read(path: &Path) -> Result { + pub fn read(path: impl AsRef) -> Result { + let path = path.as_ref(); match std::fs::read(path) { Ok(bytes) => Self::from_slice(&bytes), Err(e) if e.kind() == std::io::ErrorKind::NotFound => { @@ -226,7 +247,8 @@ impl Manifest { } /// Serialize (pretty, trailing newline) and atomically swap the manifest at `path`. - pub fn write_atomic(&self, path: &Path) -> Result<(), ManifestError> { + pub fn write_atomic(&self, path: impl AsRef) -> Result<(), ManifestError> { + let path = path.as_ref(); let mut json = serde_json::to_vec_pretty(self)?; json.push(b'\n'); fsutil::atomic_write(path, &json)?; diff --git a/crates/ltk_mimir_cache/src/store.rs b/crates/ltk_mimir_cache/src/store.rs index 9c00014..773b6d1 100644 --- a/crates/ltk_mimir_cache/src/store.rs +++ b/crates/ltk_mimir_cache/src/store.rs @@ -2,13 +2,14 @@ //! opening the active table, committing new immutable versions, and GC'ing old ones. use std::collections::HashMap; +use std::io::ErrorKind; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use ltk_hashdb::{HashDb, LayeredHashDb, WeakHashDb}; +use ltk_hashdb::{HashDb, KeyConfig, LayeredHashDb, WeakHashDb}; -use crate::manifest::{Manifest, Source, TableEntry}; +use crate::manifest::{version_of, Manifest, Source, TableEntry}; use crate::{ dir, fsutil, CommitError, GcError, LockHolder, ManifestError, NoCacheDirError, OpenError, Table, UniverseMismatch, UpdateLock, @@ -18,11 +19,25 @@ use crate::{ pub(crate) const MANIFEST_FILE: &str = "manifest.json"; /// Extension for published table files (League Toolkit convention). const TABLE_EXT: &str = "lhdb"; +/// Longest accepted version label, in bytes. +const MAX_VERSION_LEN: usize = 64; /// A shared, versioned, multi-process cache of hash tables rooted at one directory. /// /// Construction is cheap and does not touch the filesystem; the directory is created -/// lazily on the first [`commit`](HashStore::commit). +/// on the first [`try_lock_update`](HashStore::try_lock_update). +/// +/// # Directory contract +/// +/// Tables are memory-mapped, which is only sound while their bytes do not change. +/// This crate never modifies a table file after writing it: new versions get new +/// names, and [`gc`](HashStore::gc) only deletes. Other programs must not modify +/// `.lhdb` files in the cache directory either; change it through this crate or +/// the `mimir` CLI. +/// +/// Readers need no coordination. [`commit`](HashStore::commit) and +/// [`gc`](HashStore::gc) take the [`UpdateLock`], so only one process changes the +/// cache at a time. /// /// A store also remembers the tables it has opened through /// [`open_shared`](HashStore::open_shared), weakly - clones of a store share that @@ -39,12 +54,14 @@ pub struct HashStore { /// One table to install in a [`commit`](HashStore::commit) call. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct CommitItem { /// Which logical table this file is. pub table: Table, /// Version label used in the immutable filename (`
-.lhdb`), e.g. a - /// date or patch string. Must be non-empty and free of path separators. + /// date or patch string. 1-64 characters from `[A-Za-z0-9._-]`, not starting + /// with `.`. pub version: String, /// The freshly built `.lhdb` to install; copied into the cache under its @@ -106,11 +123,15 @@ impl CommitItem { } } -/// What [`HashStore::gc`] did: files removed vs. files the OS refused to delete -/// (still mmap'd by a reader - the common Windows case). +/// What [`HashStore::gc`] did. #[derive(Debug, Default, Clone)] +#[non_exhaustive] pub struct GcReport { + /// Files that were deleted. pub deleted: Vec, + + /// Files the OS refused to delete, usually because a reader still has them + /// mapped on Windows. A later `gc` retries them. pub retained: Vec, } @@ -142,15 +163,29 @@ impl HashStore { /// Read and parse the manifest. Errors with [`ManifestError::Missing`] if the cache /// has never been published to. pub fn manifest(&self) -> Result { - Manifest::read(&self.manifest_path()) + Manifest::read(self.manifest_path()) } /// The active on-disk path for `table`, per the manifest. + /// + /// # Errors + /// + /// [`OpenError::Manifest`] if the manifest cannot be read, + /// [`OpenError::TableNotFound`] if it has no entry for `table`, and + /// [`OpenError::InvalidFilename`] if the entry's filename is not + /// `
-.lhdb`, so a manifest cannot point outside the cache. pub fn path_for(&self, table: Table) -> Result { let manifest = self.manifest()?; let entry = manifest .entry(table) .ok_or(OpenError::TableNotFound(table))?; + if version_of(table, &entry.file).is_none() { + return Err(OpenError::InvalidFilename { + table, + file: entry.file.clone(), + }); + } + Ok(self.dir.join(&entry.file)) } @@ -162,8 +197,26 @@ impl HashStore { /// /// Every call maps the file afresh. Use [`open_shared`](HashStore::open_shared) /// when the same table may already be open in this process. + /// + /// # Errors + /// + /// As [`path_for`](HashStore::path_for), plus [`OpenError::HashDb`] if the + /// file cannot be opened or fails validation, and + /// [`OpenError::WrongKeyConfig`] if its key config is not + /// [`Table::key_config`]. pub fn open(&self, table: Table) -> Result { - Ok(HashDb::open(self.path_for(table)?)?) + retry_if_swept(|| { + let db = HashDb::open(self.path_for(table)?)?; + check_key_config(table, &db).map_err(|(expected, found)| { + OpenError::WrongKeyConfig { + table, + expected, + found, + } + })?; + + Ok(db) + }) } /// Open the active version of `table`, reusing a handle this store already has. @@ -182,6 +235,10 @@ impl HashStore { /// Fails like [`open`](HashStore::open): a missing manifest, a table the manifest /// does not carry, or a file that does not validate. pub fn open_shared(&self, table: Table) -> Result { + retry_if_swept(|| self.open_shared_once(table)) + } + + fn open_shared_once(&self, table: Table) -> Result { let path = self.path_for(table)?; if let Some(db) = self.registered(&path) { return Ok(db); @@ -191,6 +248,11 @@ impl HashStore { // handle and the loser's simply isn't the one registered, which is cheaper than // holding a lock across an mmap and a seek-table parse. let db = HashDb::open(&path)?; + check_key_config(table, &db).map_err(|(expected, found)| OpenError::WrongKeyConfig { + table, + expected, + found, + })?; let mut opened = self.opened.lock().unwrap_or_else(PoisonError::into_inner); opened.insert(path, db.downgrade()); @@ -304,14 +366,17 @@ impl HashStore { UpdateLock::holder(&self.dir) } - /// Install one or more freshly built tables and atomically flip the manifest to - /// point at them. + /// Install one or more freshly built tables and atomically switch the manifest + /// to them. /// - /// Each source is installed under an immutable `
-.lhdb` name - + /// Each source is installed under an immutable `
-.lhdb` name: /// copied, or renamed when the item was staged in this directory - /// ([`CommitItem::staged`]) - and the manifest is swapped only after every - /// file is durable, so a reader never sees a pointer to a partial table. Concurrent mutators should hold - /// [`try_lock_update`](HashStore::try_lock_update); readers need no coordination. + /// ([`CommitItem::staged`]). The manifest is replaced only after every file + /// is on disk, so a reader never sees a manifest pointing at a partial table. + /// `lock` must come from this store's + /// [`try_lock_update`](HashStore::try_lock_update) or + /// [`lock_update_timeout`](HashStore::lock_update_timeout); readers need no + /// coordination. /// /// `source` describes the run and lands in /// [`Manifest::last_run`](crate::Manifest::last_run); it is also the @@ -321,11 +386,25 @@ impl HashStore { /// not restamp the seven tables built from something else. /// /// Committing zero items still refreshes the timestamp and the run record. + /// + /// # Errors + /// + /// - [`CommitError::InvalidVersion`] for a bad version label + /// - [`CommitError::HashDb`] / [`CommitError::WrongKeyConfig`] if a file is not a + /// valid table with the table's [`key_config`](Table::key_config) + /// - [`CommitError::VersionReused`] if the versioned file already exists with + /// different content + /// - [`CommitError::Io`] / [`CommitError::Manifest`] for I/O failures + /// + /// On error the manifest is unchanged. pub fn commit( &self, + lock: &UpdateLock, items: &[CommitItem], source: Option, ) -> Result { + // Holding the lock is the requirement; there is nothing else to check. + let _ = lock; std::fs::create_dir_all(&self.dir)?; // Start from the current manifest so unpublished tables keep their pointers. @@ -347,6 +426,13 @@ impl HashStore { // Opening the built file validates it and yields entry count + key width. let (entries, key_width) = { let db = HashDb::open(&item.path)?; + check_key_config(item.table, &db).map_err(|(expected, found)| { + CommitError::WrongKeyConfig { + table: item.table, + expected, + found, + } + })?; (db.len() as u64, db.key_width().bytes() as u8) }; @@ -402,15 +488,29 @@ impl HashStore { ); } - manifest.write_atomic(&self.manifest_path())?; + manifest.write_atomic(self.manifest_path())?; Ok(manifest) } - /// Delete versioned `.lhdb` files the manifest no longer references, plus stray - /// `.tmp` leftovers. Files the OS refuses to delete (still mmap'd on Windows) are - /// reported in [`GcReport::retained`] and retried on a later run. A missing - /// manifest deletes nothing. - pub fn gc(&self) -> Result { + /// Delete versioned `.lhdb` files the manifest no longer references, and + /// leftover `.tmp` files from interrupted writes. + /// + /// `lock` must come from this store's + /// [`try_lock_update`](HashStore::try_lock_update) or + /// [`lock_update_timeout`](HashStore::lock_update_timeout). It guarantees + /// that no other process is writing a `.tmp` file this would delete. + /// + /// Files the OS refuses to delete (on Windows, files a reader still has + /// mapped) are listed in [`GcReport::retained`] and retried on a later run. + /// Without a manifest nothing is deleted. + /// + /// # Errors + /// + /// [`GcError::Manifest`] if the manifest cannot be read, [`GcError::Io`] if + /// the directory cannot be listed. + pub fn gc(&self, lock: &UpdateLock) -> Result { + // Holding the lock is the requirement; there is nothing else to check. + let _ = lock; let manifest = match self.manifest() { Ok(m) => m, Err(ManifestError::Missing(_)) => return Ok(GcReport::default()), @@ -445,11 +545,39 @@ impl HashStore { } } -/// A version label must be a single path component - non-empty, no separators - -/// so `
-.lhdb` can never escape the cache directory. +/// A version label is 1-64 characters from `[A-Za-z0-9._-]` and does not start +/// with `.`, so `
-.lhdb` is a plain filename on every platform: +/// no separators, no `:` (an NTFS alternate data stream), no control characters. pub(crate) fn is_valid_version(version: &str) -> bool { !version.is_empty() - && !version.contains('/') - && !version.contains('\\') - && !version.contains(std::path::MAIN_SEPARATOR) + && version.len() <= MAX_VERSION_LEN + && !version.starts_with('.') + && version + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')) +} + +/// Reject a table file whose header does not match `table`'s key config, such +/// as a bin table filed under `game`. Returns the expected and found configs. +fn check_key_config(table: Table, db: &HashDb) -> Result<(), (KeyConfig, KeyConfig)> { + let (expected, found) = (table.key_config(), db.key_config()); + if expected == found { + Ok(()) + } else { + Err((expected, found)) + } +} + +/// Run `open` once more if the file vanished: an update can replace the manifest +/// and delete the file it named between our manifest read and our open, and the +/// new manifest names a file that exists. +fn retry_if_swept( + mut open: impl FnMut() -> Result, +) -> Result { + match open() { + Err(OpenError::HashDb(ltk_hashdb::OpenError::Io(e))) if e.kind() == ErrorKind::NotFound => { + open() + } + res => res, + } } diff --git a/crates/ltk_mimir_cache/src/table.rs b/crates/ltk_mimir_cache/src/table.rs index fd98c61..45002d5 100644 --- a/crates/ltk_mimir_cache/src/table.rs +++ b/crates/ltk_mimir_cache/src/table.rs @@ -21,13 +21,28 @@ use crate::ParseTableError; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Table { + /// Game WAD paths (`hashes.game.txt`). Game, + + /// League client WAD paths (`hashes.lcu.txt`). Lcu, + + /// Bin entry names (`hashes.binentries.txt`). BinEntries, + + /// Bin class names (`hashes.bintypes.txt`). BinTypes, + + /// Bin field names (`hashes.binfields.txt`). BinFields, + + /// Bin hash values (`hashes.binhashes.txt`). BinHashes, + + /// RST string table keys, XXH64 (`hashes.rst.xxh64.txt`). Rst, + + /// RST string table keys, XXH3 (`hashes.rst.xxh3.txt`). RstXxh3, } diff --git a/crates/ltk_mimir_cache/src/update.rs b/crates/ltk_mimir_cache/src/update.rs index 936bdf9..a4ed61b 100644 --- a/crates/ltk_mimir_cache/src/update.rs +++ b/crates/ltk_mimir_cache/src/update.rs @@ -16,7 +16,7 @@ use crate::manifest::version_of; use crate::store::MANIFEST_FILE; use crate::{ fsutil, CheckError, CommitItem, FetchError, GcReport, HashStore, Manifest, ManifestError, - Source, Table, TableEntry, UpdateError, + Source, Table, TableEntry, UpdateError, UpdateLock, }; /// Fetch one release asset by filename (`manifest.json`, @@ -335,6 +335,7 @@ pub trait UpdateObserver: Sync { /// One table an update run is about to download. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub struct PlannedTable { /// The table that will be installed. pub table: Table, @@ -349,6 +350,7 @@ pub struct PlannedTable { /// What an update run did. #[derive(Debug)] +#[non_exhaustive] pub enum UpdateOutcome { /// Another process holds the update lock; nothing was done. Locked, @@ -359,6 +361,7 @@ pub enum UpdateOutcome { /// What an update would do to one table. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum TableStatus { /// The cache already holds exactly this file. Current, @@ -400,6 +403,7 @@ impl std::fmt::Display for TableStatus { /// One table as the release publishes it, next to what the cache holds. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub struct TableDiff { /// The table this describes. pub table: Table, @@ -419,6 +423,7 @@ pub struct TableDiff { /// Shaped like [`UpdateReport`] on purpose - the same run, described before it /// happens rather than after. #[derive(Debug, Clone, Default)] +#[non_exhaustive] pub struct CheckReport { /// One diff per remote table this build knows, in manifest order. pub tables: Vec, @@ -462,6 +467,7 @@ impl CheckReport { /// A remote table this build cannot read, and the format version it is in. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub struct UnsupportedTable { /// The table the release published. pub table: Table, @@ -472,6 +478,7 @@ pub struct UnsupportedTable { /// What a completed update run installed and cleaned up. #[derive(Debug, Default)] +#[non_exhaustive] pub struct UpdateReport { /// Tables that were downloaded, verified, and installed. pub installed: Vec
, @@ -548,7 +555,7 @@ impl HashStore { remote: &F, options: UpdateOptions<'_>, ) -> Result> { - let Some(_lock) = self.try_lock_update()? else { + let Some(lock) = self.try_lock_update()? else { return Ok(UpdateOutcome::Locked); }; @@ -572,7 +579,13 @@ impl HashStore { } } - self.finish(items, staged, remote_manifest.last_run.clone(), report) + self.finish( + &lock, + items, + staged, + remote_manifest.last_run.clone(), + report, + ) } /// Async twin of [`update`](HashStore::update): the same compare → @@ -594,7 +607,7 @@ impl HashStore { remote: &F, options: UpdateOptions<'_>, ) -> Result> { - let Some(_lock) = self.try_lock_update()? else { + let Some(lock) = self.try_lock_update()? else { return Ok(UpdateOutcome::Locked); }; @@ -616,7 +629,13 @@ impl HashStore { } } - self.finish(items, staged, remote_manifest.last_run.clone(), report) + self.finish( + &lock, + items, + staged, + remote_manifest.last_run.clone(), + report, + ) } /// Compare the cache against a published release without changing either. @@ -771,6 +790,7 @@ impl HashStore { /// [`update_async`](HashStore::update_async). fn finish( &self, + lock: &UpdateLock, items: Vec, staged: Staged, source: Option, @@ -778,7 +798,7 @@ impl HashStore { ) -> Result> { // Install atomically - table files first, manifest pointer last. if !items.is_empty() { - self.commit(&items, source)?; + self.commit(lock, &items, source)?; report.installed = items.iter().map(|item| item.table).collect(); } @@ -786,7 +806,9 @@ impl HashStore { // in-flight `.tmp` files. drop(staged); - report.gc = self.gc().unwrap_or_default(); + // A GC failure does not undo a successful install, so it leaves the + // report empty rather than failing the run. + report.gc = self.gc(lock).unwrap_or_default(); Ok(UpdateOutcome::Completed(report)) } diff --git a/crates/ltk_mimir_cache/tests/common/mod.rs b/crates/ltk_mimir_cache/tests/common/mod.rs index 27531a6..abe34b8 100644 --- a/crates/ltk_mimir_cache/tests/common/mod.rs +++ b/crates/ltk_mimir_cache/tests/common/mod.rs @@ -6,19 +6,19 @@ use std::fs::{self, File}; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; -use ltk_hashdb::{Compression, HashDbWriter, HashKind, KeyWidth}; +use ltk_hashdb::{Compression, HashDbWriter}; use ltk_mimir_cache::{ CommitItem, FetchError, HashStore, Manifest, Source, Table, UpdateOutcome, UpdateReport, }; -/// Build a tiny raw `.lhdb` and return its path. -fn build_table(dir: &Path, name: &str, entries: &[(u64, &str)]) -> PathBuf { - let mut writer = HashDbWriter::new(KeyWidth::U64, Compression::None).hash_kind(HashKind::Xxh64); +/// Build a tiny raw `.lhdb` with `table`'s key config and return its path. +fn build_table(dir: &Path, table: Table, entries: &[(u64, &str)]) -> PathBuf { + let mut writer = HashDbWriter::with_key_config(table.key_config(), Compression::None); for (hash, path) in entries { writer.insert(*hash, path); } - let path = dir.join(name); + let path = dir.join(format!("{}.lhdb", table.id())); writer.build(fs::File::create(&path).unwrap()).unwrap(); path } @@ -33,16 +33,16 @@ pub fn make_release(dir: &Path, version: &str, tables: &[(Table, &[(u64, &str)]) let items: Vec = tables .iter() .map(|(table, entries)| { - let built = build_table(&build, &format!("{}.lhdb", table.id()), entries); + let built = build_table(&build, *table, entries); CommitItem::new(*table, version, built) }) .collect(); - let source = Source { - repo: Some("test/data".into()), - commit: Some("deadbeef".into()), - inputs_sha256: None, - }; - HashStore::at(dir).commit(&items, Some(source)).unwrap(); + let mut source = Source::default(); + source.repo = Some("test/data".into()); + source.commit = Some("deadbeef".into()); + let store = HashStore::at(dir); + let lock = store.try_lock_update().unwrap().unwrap(); + store.commit(&lock, &items, Some(source)).unwrap(); fs::copy(dir.join("manifest.json"), dir.join(channel_asset())).unwrap(); fs::remove_dir_all(&build).unwrap(); @@ -68,7 +68,7 @@ pub fn edit_release_manifest(dir: &Path, edit: impl FnOnce(&mut Manifest)) { pub fn completed(outcome: UpdateOutcome) -> UpdateReport { match outcome { UpdateOutcome::Completed(report) => report, - UpdateOutcome::Locked => panic!("expected a completed run, got Locked"), + other => panic!("expected a completed run, got {other:?}"), } } diff --git a/crates/ltk_mimir_cache/tests/refresh.rs b/crates/ltk_mimir_cache/tests/refresh.rs index 3389676..a88626d 100644 --- a/crates/ltk_mimir_cache/tests/refresh.rs +++ b/crates/ltk_mimir_cache/tests/refresh.rs @@ -321,19 +321,14 @@ fn unknown_remote_table_is_skipped() { // A future mimir publishes a ninth table this build doesn't know. edit_release_manifest(&release, |manifest| { - manifest.tables.insert( - "shiny-new".into(), - ltk_mimir_cache::TableEntry { - file: "shiny-new-1.lhdb".into(), - sha256: "0".repeat(64), - entries: 0, - key_width: 8, - size_bytes: Some(4096), - version: "1".into(), - source: None, - format_version: ltk_hashdb::FORMAT_VERSION, - }, - ); + let mut entry = manifest.tables["game"].clone(); + entry.file = "shiny-new-1.lhdb".into(); + entry.sha256 = "0".repeat(64); + entry.entries = 0; + entry.size_bytes = Some(4096); + entry.version = "1".into(); + entry.source = None; + manifest.tables.insert("shiny-new".into(), entry); }); let store = HashStore::at(&cache); @@ -495,13 +490,9 @@ fn a_manifest_from_the_future_installs_what_it_can() { assert_eq!(report.installed, [Table::Game], "the readable table lands"); assert_eq!(report.unknown_tables, ["shiny-new"]); - assert_eq!( - report.unsupported_tables, - [ltk_mimir_cache::UnsupportedTable { - table: Table::Lcu, - format_version: 99, - }] - ); + assert_eq!(report.unsupported_tables.len(), 1); + assert_eq!(report.unsupported_tables[0].table, Table::Lcu); + assert_eq!(report.unsupported_tables[0].format_version, 99); assert!( store.open(Table::Lcu).is_err(), "a table this build cannot read is never installed" diff --git a/crates/ltk_mimir_cache/tests/store.rs b/crates/ltk_mimir_cache/tests/store.rs index 9ae211c..4b17e8d 100644 --- a/crates/ltk_mimir_cache/tests/store.rs +++ b/crates/ltk_mimir_cache/tests/store.rs @@ -6,15 +6,22 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use ltk_hashdb::{Compression, HashDbWriter, KeyWidth}; +use ltk_hashdb::{Compression, HashDbWriter}; use ltk_mimir_cache::{ CommitError, CommitItem, HashStore, HashUniverse, ManifestError, OpenError, Source, Table, + UpdateLock, }; use tempfile::tempdir; -/// Build a raw `.lhdb` at `path` from `entries`, returning the path. +/// Build a raw `.lhdb` at `path` from `entries` with the `game`/`lcu` key +/// config, returning the path. fn build_table(path: &Path, entries: &[(u64, &str)]) -> PathBuf { - let mut writer = HashDbWriter::new(KeyWidth::U64, Compression::None); + build_table_with(path, Table::Game, entries) +} + +/// Build a raw `.lhdb` at `path` with `table`'s key config. +fn build_table_with(path: &Path, table: Table, entries: &[(u64, &str)]) -> PathBuf { + let mut writer = HashDbWriter::with_key_config(table.key_config(), Compression::None); for &(hash, p) in entries { writer.insert(hash, p); } @@ -22,6 +29,19 @@ fn build_table(path: &Path, entries: &[(u64, &str)]) -> PathBuf { path.to_path_buf() } +/// Take the update lock, which `commit` and `gc` require. +fn lock(store: &HashStore) -> UpdateLock { + store.try_lock_update().unwrap().expect("no other updater") +} + +/// A `Source` naming a repo and commit (`Source` is non-exhaustive). +fn source(repo: Option, commit: Option) -> Source { + let mut source = Source::default(); + source.repo = repo; + source.commit = commit; + source +} + const ENTRIES: &[(u64, &str)] = &[ (0x1111, "assets/characters/ahri/skins/skin07/ahri.bin"), (0x2222, "data/characters/ahri/ahri.bin"), @@ -35,7 +55,11 @@ fn commit_open_roundtrip() { let src = build_table(&tmp.path().join("game-build.lhdb"), ENTRIES); let manifest = store - .commit(&[CommitItem::new(Table::Game, "2026-07-08", &src)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "2026-07-08", &src)], + None, + ) .unwrap(); // Manifest records the versioned filename + derived metadata. @@ -64,7 +88,11 @@ fn commit_new_version_supersedes_and_gc_reclaims() { let v1 = build_table(&tmp.path().join("g1.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v1", &v1)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &v1)], + None, + ) .unwrap(); let v1_file = tmp.path().join("game-v1.lhdb"); assert!(v1_file.exists()); @@ -72,7 +100,11 @@ fn commit_new_version_supersedes_and_gc_reclaims() { // A second commit flips the pointer to a new immutable filename. let v2 = build_table(&tmp.path().join("g2.lhdb"), ENTRIES); let manifest = store - .commit(&[CommitItem::new(Table::Game, "v2", &v2)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v2", &v2)], + None, + ) .unwrap(); assert_eq!(manifest.entry(Table::Game).unwrap().file, "game-v2.lhdb"); // Old version still on disk until GC, new one active. @@ -80,7 +112,7 @@ fn commit_new_version_supersedes_and_gc_reclaims() { assert!(tmp.path().join("game-v2.lhdb").exists()); // GC (no readers hold v1) reclaims the superseded file, keeps the active one. - let report = store.gc().unwrap(); + let report = store.gc(&lock(&store)).unwrap(); assert!(report.deleted.contains(&v1_file), "v1 should be collected"); assert!(!v1_file.exists()); assert!(tmp.path().join("game-v2.lhdb").exists()); @@ -93,7 +125,7 @@ fn commit_new_version_supersedes_and_gc_reclaims() { fn gc_without_manifest_is_a_noop() { let tmp = tempdir().unwrap(); let store = HashStore::at(tmp.path()); - let report = store.gc().unwrap(); + let report = store.gc(&lock(&store)).unwrap(); assert!(report.deleted.is_empty()); assert!(report.retained.is_empty()); } @@ -112,7 +144,11 @@ fn open_missing_manifest_and_table_error() { // Manifest exists but lacks the requested table. let src = build_table(&tmp.path().join("g.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v1", &src)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &src)], + None, + ) .unwrap(); assert!(matches!( store.open(Table::Lcu), @@ -130,6 +166,7 @@ fn open_layered_skips_missing_and_respects_order() { let lcu = &[(0xAAAA_u64, "lcu/shared"), (0x2222, "lcu/only")][..]; store .commit( + &lock(&store), &[ CommitItem::new( Table::Game, @@ -172,6 +209,7 @@ fn open_layered_reports_missing_table_but_stays_usable() { // Only Game is committed; Lcu is absent. store .commit( + &lock(&store), &[CommitItem::new( Table::Game, "v1", @@ -230,6 +268,7 @@ fn open_many_pairs_each_table_with_its_result() { let store = HashStore::at(tmp.path()); store .commit( + &lock(&store), &[CommitItem::new( Table::Game, "v1", @@ -277,9 +316,13 @@ fn invalid_version_is_rejected() { let store = HashStore::at(tmp.path()); let src = build_table(&tmp.path().join("g.lhdb"), ENTRIES); - for bad in ["", "a/b", "a\\b"] { + for bad in ["", "a/b", "a\\b", "x:y", ".hidden", "a b"] { assert!(matches!( - store.commit(&[CommitItem::new(Table::Game, bad, &src)], None), + store.commit( + &lock(&store), + &[CommitItem::new(Table::Game, bad, &src)], + None + ), Err(CommitError::InvalidVersion(_)) )); } @@ -294,14 +337,22 @@ fn reused_version_with_different_content_is_rejected() { let v1 = build_table(&tmp.path().join("a.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v1", &v1)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &v1)], + None, + ) .unwrap(); // The same version label with different bytes violates immutability: commit must // refuse rather than rename over the existing file (a reader may have it mmap'd). let different = build_table(&tmp.path().join("b.lhdb"), &ENTRIES[..2]); let err = store - .commit(&[CommitItem::new(Table::Game, "v1", &different)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &different)], + None, + ) .unwrap_err(); assert!( matches!( @@ -325,7 +376,11 @@ fn recommit_identical_version_is_idempotent() { let v1 = build_table(&tmp.path().join("a.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v1", &v1)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &v1)], + None, + ) .unwrap(); let dest = tmp.path().join("game-v1.lhdb"); let before = std::fs::metadata(&dest).unwrap().modified().unwrap(); @@ -334,7 +389,11 @@ fn recommit_identical_version_is_idempotent() { // succeed without rewriting the immutable file, so a reader's mapping is safe. let same = build_table(&tmp.path().join("c.lhdb"), ENTRIES); let manifest = store - .commit(&[CommitItem::new(Table::Game, "v1", &same)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &same)], + None, + ) .unwrap(); assert_eq!(manifest.entry(Table::Game).unwrap().file, "game-v1.lhdb"); let after = std::fs::metadata(&dest).unwrap().modified().unwrap(); @@ -349,19 +408,23 @@ fn commit_overwrites_the_stale_run_record() { let store = HashStore::at(tmp.path()); let src = build_table(&tmp.path().join("g1.lhdb"), ENTRIES); - let source = Source { - repo: Some("owner/repo".into()), - commit: Some("abc123".into()), - inputs_sha256: None, - }; + let source = source(Some("owner/repo".into()), Some("abc123".into())); store - .commit(&[CommitItem::new(Table::Game, "v1", &src)], Some(source)) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &src)], + Some(source), + ) .unwrap(); assert!(store.manifest().unwrap().last_run.is_some()); let src2 = build_table(&tmp.path().join("g2.lhdb"), ENTRIES); let manifest = store - .commit(&[CommitItem::new(Table::Game, "v2", &src2)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v2", &src2)], + None, + ) .unwrap(); assert!( manifest.last_run.is_none(), @@ -377,14 +440,11 @@ fn a_partial_commit_leaves_other_tables_provenance_alone() { let store = HashStore::at(tmp.path()); let src = build_table(&tmp.path().join("t.lhdb"), ENTRIES); - let from = |commit: &str| Source { - repo: Some("owner/repo".into()), - commit: Some(commit.into()), - inputs_sha256: None, - }; + let from = |commit: &str| source(Some("owner/repo".into()), Some(commit.into())); store .commit( + &lock(&store), &[ CommitItem::new(Table::Game, "v1", &src), CommitItem::new(Table::Lcu, "v1", &src), @@ -396,6 +456,7 @@ fn a_partial_commit_leaves_other_tables_provenance_alone() { // A second run rebuilds only `game`, from a different upstream commit. let manifest = store .commit( + &lock(&store), &[CommitItem::new(Table::Game, "v2", &src)], Some(from("bbbb")), ) @@ -428,19 +489,12 @@ fn an_item_can_override_the_run_wide_source() { let store = HashStore::at(tmp.path()); let src = build_table(&tmp.path().join("t.lhdb"), ENTRIES); - let run = Source { - repo: Some("owner/repo".into()), - commit: Some("aaaa".into()), - inputs_sha256: None, - }; - let mirror = Source { - repo: Some("mirror/repo".into()), - commit: Some("bbbb".into()), - inputs_sha256: None, - }; + let run = source(Some("owner/repo".into()), Some("aaaa".into())); + let mirror = source(Some("mirror/repo".into()), Some("bbbb".into())); let manifest = store .commit( + &lock(&store), &[ CommitItem::new(Table::Game, "v1", &src), CommitItem::new(Table::Lcu, "v1", &src).with_source(mirror), @@ -473,7 +527,11 @@ fn readers_never_break_during_commit() { // Seed an initial version so readers have something to open immediately. let seed = build_table(&tmp.path().join("seed.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v0", &seed)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v0", &seed)], + None, + ) .unwrap(); let stop = Arc::new(AtomicBool::new(false)); @@ -502,7 +560,11 @@ fn readers_never_break_during_commit() { for i in 1..=40 { let src = build_table(&tmp.path().join(format!("pub{i}.lhdb")), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, format!("v{i}"), &src)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, format!("v{i}"), &src)], + None, + ) .unwrap(); } @@ -525,7 +587,11 @@ fn gc_handles_mapped_superseded_file() { let v1 = build_table(&tmp.path().join("g1.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v1", &v1)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &v1)], + None, + ) .unwrap(); // Hold a mapping of the v1 file across the commit + GC. @@ -534,11 +600,15 @@ fn gc_handles_mapped_superseded_file() { let v2 = build_table(&tmp.path().join("g2.lhdb"), ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v2", &v2)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v2", &v2)], + None, + ) .unwrap(); let v1_file = tmp.path().join("game-v1.lhdb"); - let report = store.gc().unwrap(); + let report = store.gc(&lock(&store)).unwrap(); // Every v1 outcome is one of the two graceful paths, and they agree with the fs. let deleted = report.deleted.contains(&v1_file); @@ -565,7 +635,11 @@ fn open_shared_reuses_handles_until_the_version_changes() { let one = build_table(&tmp.path().join("game-1.build"), &[(1, "assets/one.bin")]); store - .commit(&[CommitItem::new(Table::Game, "1", &one)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "1", &one)], + None, + ) .unwrap(); let first = store.open_shared(Table::Game).unwrap(); @@ -580,7 +654,11 @@ fn open_shared_reuses_handles_until_the_version_changes() { &[(1, "assets/one.bin"), (2, "assets/two.bin")], ); store - .commit(&[CommitItem::new(Table::Game, "2", &two)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "2", &two)], + None, + ) .unwrap(); let after = store.open_shared(Table::Game).unwrap(); @@ -603,6 +681,7 @@ fn commit_moves_a_staged_file_instead_of_copying_it() { let manifest = store .commit( + &lock(&store), &[CommitItem::staged(Table::Game, "v1", &staged, &sha256)], None, ) @@ -634,7 +713,11 @@ fn commit_copies_a_file_it_was_not_handed() { build_table(&built, ENTRIES); store - .commit(&[CommitItem::new(Table::Game, "v1", &built)], None) + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &built)], + None, + ) .unwrap(); assert!(built.is_file(), "the caller's file is left where it was"); @@ -738,3 +821,72 @@ fn a_bounded_wait_takes_a_lock_that_frees_up() { releaser.join().unwrap(); } + +#[test] +fn commit_rejects_wrong_key_config() { + let tmp = tempdir().unwrap(); + let store = HashStore::at(tmp.path()); + + // A u32 bin table offered as `game`. + let src = build_table_with(&tmp.path().join("b.lhdb"), Table::BinEntries, &[(1, "x")]); + let err = store + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &src)], + None, + ) + .unwrap_err(); + assert!( + matches!( + err, + CommitError::WrongKeyConfig { + table: Table::Game, + .. + } + ), + "{err}" + ); + assert!(store.manifest().is_err(), "nothing was committed"); +} + +#[test] +fn open_rejects_wrong_key_config_and_bad_filenames() { + let tmp = tempdir().unwrap(); + let store = HashStore::at(tmp.path()); + let src = build_table(&tmp.path().join("g.lhdb"), ENTRIES); + store + .commit( + &lock(&store), + &[CommitItem::new(Table::Game, "v1", &src)], + None, + ) + .unwrap(); + + // Point the manifest at a u32 table written outside `commit`. + build_table_with( + &tmp.path().join("game-v9.lhdb"), + Table::BinEntries, + &[(1, "x")], + ); + let mut manifest = store.manifest().unwrap(); + manifest.tables.get_mut("game").unwrap().file = "game-v9.lhdb".into(); + manifest.write_atomic(store.manifest_path()).unwrap(); + assert!(matches!( + store.open(Table::Game), + Err(OpenError::WrongKeyConfig { + table: Table::Game, + .. + }) + )); + + // A filename that is not `game-.lhdb` is refused before any open. + manifest.tables.get_mut("game").unwrap().file = "../game-v1.lhdb".into(); + manifest.write_atomic(store.manifest_path()).unwrap(); + assert!(matches!( + store.path_for(Table::Game), + Err(OpenError::InvalidFilename { + table: Table::Game, + .. + }) + )); +} diff --git a/crates/ltk_mimir_cli/src/bundle.rs b/crates/ltk_mimir_cli/src/bundle.rs index 98779af..c35529d 100644 --- a/crates/ltk_mimir_cli/src/bundle.rs +++ b/crates/ltk_mimir_cli/src/bundle.rs @@ -134,12 +134,15 @@ pub fn run(opts: &Options) -> Result<()> { items.push(CommitItem::new(spec.table, &version, path)); } - let source = Source { - repo: Some(opts.source_repo.clone()), - commit: opts.source_commit.clone(), - inputs_sha256: Some(inputs_sha256), - }; - let manifest = HashStore::at(&opts.out).commit(&items, Some(source))?; + let mut source = Source::default(); + source.repo = Some(opts.source_repo.clone()); + source.commit = opts.source_commit.clone(); + source.inputs_sha256 = Some(inputs_sha256); + let store = HashStore::at(&opts.out); + let lock = store + .try_lock_update()? + .with_context(|| format!("{} is locked by another process", opts.out.display()))?; + let manifest = store.commit(&lock, &items, Some(source))?; fs::remove_dir_all(&build_dir).with_context(|| format!("removing {}", build_dir.display()))?; // The same document under a format-specific name. A build asks for its own diff --git a/crates/ltk_mimir_cli/src/update.rs b/crates/ltk_mimir_cli/src/update.rs index 636601c..b449efe 100644 --- a/crates/ltk_mimir_cli/src/update.rs +++ b/crates/ltk_mimir_cli/src/update.rs @@ -68,6 +68,7 @@ pub fn run(opts: &Options) -> Result<()> { return Ok(()); } UpdateOutcome::Completed(report) => report, + other => anyhow::bail!("unexpected update outcome: {other:?}"), }; for id in &report.unknown_tables { diff --git a/docs/CONSUMERS.md b/docs/CONSUMERS.md index 806c980..3b82b11 100644 --- a/docs/CONSUMERS.md +++ b/docs/CONSUMERS.md @@ -271,11 +271,12 @@ let fetch = |filename: &str| -> Result, MyClientError> { }; match store.update(&fetch, UpdateOptions::default())? { - UpdateOutcome::Locked => {} // another process is updating; leave it to them + UpdateOutcome::Locked => {} // another process is updating UpdateOutcome::Completed(report) => { if report.is_up_to_date() { /* nothing changed */ } for table in &report.installed { /* log the refresh */ } } + _ => {} // `UpdateOutcome` is non-exhaustive } ``` @@ -391,6 +392,7 @@ let fetch = |filename: &str| { match store.update_async(&fetch, UpdateOptions::default()).await? { UpdateOutcome::Locked => {} UpdateOutcome::Completed(report) => { /* as above */ } + _ => {} } ``` @@ -469,8 +471,9 @@ use ltk_mimir_cache::{CommitItem, HashStore, Source, Table}; let store = HashStore::discover()?; -// Become the single updater, or leave it to whoever already is. -let Some(_lock) = store.try_lock_update()? else { return Ok(()) }; +// Take the update lock, or stop if another process holds it. +// `commit` and `gc` require it. +let Some(lock) = store.try_lock_update()? else { return Ok(()) }; // Install atomically: files are copied durable first, the manifest pointer // swaps last, so a concurrent reader never sees a half-written table. @@ -479,14 +482,17 @@ let Some(_lock) = store.try_lock_update()? else { return Ok(()) }; // `last_run`. Tables the call does not mention keep the provenance they were // installed with; pass `CommitItem::with_source` when one table came from // somewhere else. +let mut source = Source::default(); // `Source` is non-exhaustive +source.repo = Some("CommunityDragon/Data".into()); store.commit( + &lock, &[CommitItem::new(Table::Game, "2026-07-10", built_game_path)], - Some(Source { repo: Some("CommunityDragon/Data".into()), commit, inputs_sha256 }), + Some(source), )?; -// Clean up superseded versions. Files still mapped by a reader are skipped -// (reported in `retained`) and retried on a later run - never an error. -let report = store.gc()?; +// Remove superseded versions. Files a reader still has mapped are skipped, +// listed in `retained`, and retried on a later run. +let report = store.gc(&lock)?; ``` ### Verifying a table diff --git a/docs/FORMAT.md b/docs/FORMAT.md index f7ea720..b23fb8e 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -205,11 +205,13 @@ On open, before trusting any offset: - magic, `version == 1`, `key_width ∈ {4,8}`, `offset_width ∈ {4,8}`, known `flags` bits only (`opt_flags` is *not* checked - see above), - known `hash_kind`; + known `hash_kind`, and no 64-bit `hash_kind` (XXH64, XXH3) with 4-byte keys; - for raw arenas, `arena_compressed_size == arena_decompressed_size`; - for compressed arenas: the trailing seek table parses, its total decompressed size - equals `arena_decompressed_size`, and no frame's decompressed size exceeds the - seekable-format maximum (1 GiB); + equals `arena_decompressed_size`, and no frame's decompressed size exceeds + `MAX_FRAME_SIZE` (1 MiB). A lookup decompresses whole frames, so this bounds what + one lookup allocates; the seekable format itself allows frames of up to 1 GiB. + Writers must use frame sizes of at most 1 MiB; - all section extents in bounds (overflow-checked); - if `arena_order_offset != 0`: `arena_order_width` is `1..=8` and wide enough for `entry_count`, and the section is in bounds. Its *contents* are not read at open -