From 44eacc22523d8787058b0fe1a245c049481de9aa Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 08:00:49 +0800 Subject: [PATCH 01/22] feat(rust): start native wire, identity and request compatibility migration Keep Python CLI/admin/tooling and existing production daemon unchanged. Add bounded native core, Ed25519 primitives and request envelope handling, with Python-source differential conformance CI and explicit cutover gates. --- .github/workflows/rust-wire.yml | 68 ++++ rust/.gitignore | 2 + rust/Cargo.toml | 27 ++ rust/README.md | 100 ++++++ rust/crates/msg-core/Cargo.toml | 17 + rust/crates/msg-core/src/lib.rs | 299 ++++++++++++++++++ rust/crates/msg-crypto/Cargo.toml | 16 + rust/crates/msg-crypto/src/lib.rs | 95 ++++++ rust/crates/msg-protocol/Cargo.toml | 16 + .../msg-protocol/examples/wire_check.rs | 64 ++++ rust/crates/msg-protocol/src/lib.rs | 191 +++++++++++ rust/tests/test_python_parity.py | 206 ++++++++++++ 12 files changed, 1101 insertions(+) create mode 100644 .github/workflows/rust-wire.yml create mode 100644 rust/.gitignore create mode 100644 rust/Cargo.toml create mode 100644 rust/README.md create mode 100644 rust/crates/msg-core/Cargo.toml create mode 100644 rust/crates/msg-core/src/lib.rs create mode 100644 rust/crates/msg-crypto/Cargo.toml create mode 100644 rust/crates/msg-crypto/src/lib.rs create mode 100644 rust/crates/msg-protocol/Cargo.toml create mode 100644 rust/crates/msg-protocol/examples/wire_check.rs create mode 100644 rust/crates/msg-protocol/src/lib.rs create mode 100644 rust/tests/test_python_parity.py diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml new file mode 100644 index 00000000..13d32c03 --- /dev/null +++ b/.github/workflows/rust-wire.yml @@ -0,0 +1,68 @@ +name: Rust wire compatibility +on: + pull_request: + paths: + - 'rust/**' + - 'src/msg/core/**' + - 'src/msg/security/crypto.py' + - '.github/workflows/rust-wire.yml' + push: + branches: [main, 'migration/rust-*'] + paths: + - 'rust/**' + - 'src/msg/core/**' + - 'src/msg/security/crypto.py' + - '.github/workflows/rust-wire.yml' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: rust-wire-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: false +jobs: + conformance: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Install Rust toolchain and Python reference dependencies + run: | + rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy + python -m pip install cryptography==46.0.3 jsonschema==4.25.1 + - name: Check formatting and retain a repair patch + working-directory: rust + run: | + mkdir -p artifacts + cargo +1.85.0 fmt --all + git diff -- . > artifacts/rustfmt.patch + test ! -s artifacts/rustfmt.patch + - name: Native tests + if: ${{ !cancelled() }} + working-directory: rust + run: cargo +1.85.0 test --workspace --all-targets + - name: Clippy + if: ${{ !cancelled() }} + working-directory: rust + run: cargo +1.85.0 clippy --workspace --all-targets -- -D warnings + - name: Python and Rust differential checks + if: ${{ !cancelled() }} + env: + PYTHONPATH: src + run: | + cargo +1.85.0 build --manifest-path rust/Cargo.toml --example wire_check + python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json + - name: Preserve evidence and resolved dependency lock + if: ${{ always() }} + uses: actions/upload-artifact@v7 + with: + name: rust-wire-${{ github.sha }} + path: | + rust/artifacts/ + rust/Cargo.lock + if-no-files-found: warn diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 00000000..b608df42 --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1,2 @@ +/target/ +/artifacts/ diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..dd08cf0e --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,27 @@ +[workspace] +members = ["crates/msg-core", "crates/msg-crypto", "crates/msg-protocol"] +resolver = "2" + +[workspace.package] +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MIT" +publish = false + +[workspace.dependencies] +base64 = "=0.22.1" +chrono = { version = "=0.4.42", default-features = false, features = ["std"] } +ed25519-dalek = "=2.2.0" +ryu = "=1.0.20" +serde = { version = "=1.0.228", features = ["derive"] } +serde_json = { version = "=1.0.145", features = ["raw_value"] } +sha2 = "=0.10.9" + +[workspace.lints.rust] +unsafe_code = "forbid" + +[profile.release] +lto = "thin" +codegen-units = 1 +strip = "debuginfo" diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..d832b55b --- /dev/null +++ b/rust/README.md @@ -0,0 +1,100 @@ +# Native MSG migration — phase 1 + +**This is a wire-compatibility foundation, not a replacement daemon.** No Python +entry point, live database, configuration, route or deployment is changed. + +## Ownership boundary + +The retained Python implementation owns `msg` CLI, administration/migration +scripts and development tools. The target Rust implementation will own +`msgd serve`, `msgd worker`, domain/identity, storage, HTTP/MCP and server-side +crypto/protocol. Shared wire semantics remain compatible with the retained CLI. +Do not rename endpoints or introduce a second signature protocol during this work. + +The baseline inspected for this phase is +`7e16ca75ff6af4313365814c23912b5997b222c5`. The conformance test imports the actual +Python implementation from the same checkout, so later drift is detectable. + +## Implemented here + +| Crate | Native implementation | Python reference | +|---|---|---| +| `msg-core` | Bounded strict JSON, duplicate-key rejection, Python-style canonical numbers/Unicode ordering, SHA-256 and canonical unpadded base64url | `core/codec.py` | +| `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing and verification | `security/crypto.py` | +| `msg-protocol` | Strict request shape, defaults, normalized UTC timestamps, business digest and signed request bytes | `core/packet.py`, `core/requests.py`, `core/models.py` | + +`wire_check` is a JSONL **test example**, not a server or client replacement. Its +fixed seed is public test material. The library does not log or implement Debug +for private keys, requests or bearer proofs. Unsafe code is forbidden in these +crates; standard cryptographic libraries implement cryptographic primitives. + +Business digests exclude only `proof`, `payload_digest`, `source` and +`expires_at`. Signatures cover everything except `proof`. Retry tests ensure +changing a request ID, business argument or precondition is not mistaken for a +same-content retry. Native signatures are also verified by the Python verifier. + +## Validation + +From the repository root, with Python 3.15 and Rust 1.85.0: + +```sh +python -m pip install cryptography==46.0.3 jsonschema==4.25.1 +cargo +1.85.0 test --manifest-path rust/Cargo.toml --workspace --all-targets +cargo +1.85.0 clippy --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings +cargo +1.85.0 build --manifest-path rust/Cargo.toml --example wire_check +PYTHONPATH=src python rust/tests/test_python_parity.py \ + --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json +``` + +The differential corpus includes 16,384 deterministic random IEEE-754 bit +patterns (non-finite values are excluded), decimal exponent neighbors, Unicode, +large integers, malformed JSON/base64/envelopes and bidirectional Ed25519 checks. +The report records the actual executed count, not the random input budget. +CI also keeps formatter evidence and the resolved dependency lock. + +## Explicit cutover blockers + +- `verify_signature_bytes` is not authentication or authorization. Credential + revocation, subject binding, expiry, service binding, certificate chains, + permission checks, idempotency storage and transactional execution are pending. +- Resource/result/receipt/certificate models, encrypted-key wrapping and recovery + are not ported. Existing Python code remains the authoritative runtime. +- No native `serve`/`worker`, database writer, HTTP/MCP adapter or release artifact + is provided yet. An empty loop or Python subprocess would not count as a port. +- The first parser is deliberately bounded to 1 MiB and 64 nested levels, and + accepts RFC3339 UTC timestamps only. Python `datetime.fromisoformat` accepts + additional aliases. These differences must be resolved or explicitly versioned + before routing production requests to Rust; they are not silent fallback paths. +- Dalek strict verification refuses weak-key signature forgeries. Compatibility + with historical keys/signatures must be characterized before any cutover; + normal valid signatures must match Python byte-for-byte. +- No memory-reduction or throughput claim has been measured. A benchmark must + compare equivalent functionality, data, concurrency and process trees. + +## Next implementation slices + +1. Freeze identity/result/receipt fixtures and add the credential, certificate and + authorization model, including expired/revoked/cross-service negative cases. +2. Port storage transactions and the request-id/digest replay ledger against + disposable databases created by the Python migrations. Verify rollback, + restart/retry, concurrent writers and schema/version gates. No dual writes. +3. Integrate a bounded native executor and a read-only `msgd serve` slice, then + HTTP/MCP adapters with exactly the existing operation registry and envelopes. +4. Port worker claiming/leases/retries and side effects. Enable write operations + only after same-fixture differential tests and failure/recovery tests pass. +5. Perform isolated deployment rehearsals, workload/RSS measurements and an + explicit rollback/cutover review. Keep the current Python service available. + +## Coordination and evidence + +Repository `AGENTS.md`, local entry skill and the deployed `/AGENTS.md` were read. +The current editing runtime has no selected MSG signing identity or worker +runtime; it cannot create an authorized private coordination thread. No new +production identity was registered and no public coordination content was posted. +This bounded work is therefore tracked in the migration branch/PR rather than +claiming MSG coordination occurred. Return to private MSG coordination when an +existing authorized identity is available. + +The editing container has no Rust toolchain and cannot resolve GitHub for a git +clone. Consequently native compilation and the actual Python 3.15 differential +run must be evidenced by this PR's cloud CI, not claimed from local inspection. diff --git a/rust/crates/msg-core/Cargo.toml b/rust/crates/msg-core/Cargo.toml new file mode 100644 index 00000000..f62fdc30 --- /dev/null +++ b/rust/crates/msg-core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "msg-core" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +base64.workspace = true +ryu.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/msg-core/src/lib.rs b/rust/crates/msg-core/src/lib.rs new file mode 100644 index 00000000..18bd4b98 --- /dev/null +++ b/rust/crates/msg-core/src/lib.rs @@ -0,0 +1,299 @@ +//! MSG canonical-v1 bytes. This is deliberately not RFC 8785 / JCS. +use std::collections::BTreeMap; +use std::fmt; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde::de::{self, MapAccess, Visitor}; +use serde::{Deserialize, Deserializer}; +use serde_json::value::RawValue; +use sha2::{Digest, Sha256}; + +pub const MAX_BYTES: usize = 1_048_576; +pub const MAX_DEPTH: usize = 64; + +/// Errors contain stable codes only; never input, keys, tokens or parser excerpts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Error(pub &'static str); + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.0) + } +} +impl std::error::Error for Error {} +pub type Result = std::result::Result; + +/// Private representation prevents callers from constructing non-finite values. +#[derive(Clone, PartialEq)] +pub struct Json(Value); + +#[derive(Clone, PartialEq)] +enum Value { + Null, + Bool(bool), + Integer(String), + Float(f64), + String(String), + Array(Vec), + Object(BTreeMap), +} + +impl Json { + pub fn parse(input: &str) -> Result { + if input.len() > MAX_BYTES { + return Err(Error("request_too_large")); + } + let raw: Box = serde_json::from_str(input).map_err(|_| Error("invalid_json"))?; + Self::from_raw(&raw, 0) + } + + fn from_raw(raw: &RawValue, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(Error("json_too_deep")); + } + let text = raw.get(); + let invalid = |_| Error("invalid_json"); + let value = match text.as_bytes().first() { + Some(b'n') => Value::Null, + Some(b't') => Value::Bool(true), + Some(b'f') => Value::Bool(false), + Some(b'"') => Value::String(serde_json::from_str(text).map_err(invalid)?), + Some(b'[') => { + let values: Vec> = serde_json::from_str(text).map_err(invalid)?; + Value::Array(values.iter().map(|v| Self::from_raw(v, depth + 1)).collect::>()?) + } + Some(b'{') => { + let members: Members = serde_json::from_str(text).map_err(|e| { + if e.to_string().starts_with("duplicate_json_key") { + Error("duplicate_json_key") + } else { + Error("invalid_json") + } + })?; + Value::Object(members.0.into_iter().map(|(k, v)| { + Ok((k, Self::from_raw(&v, depth + 1)?)) + }).collect::>()?) + } + _ if text.contains(['.', 'e', 'E']) => { + let number: f64 = text.parse().map_err(|_| Error("invalid_json"))?; + if !number.is_finite() { + return Err(Error("non_finite_float")); + } + Value::Float(number) + } + _ => { + // CPython's default integer conversion limit. Do not round through f64/u64. + if text.trim_start_matches('-').len() > 4300 { + return Err(Error("invalid_json")); + } + Value::Integer(if text == "-0" { "0" } else { text }.to_owned()) + } + }; + Ok(Self(value)) + } + + pub fn string(value: impl Into) -> Self { + Self(Value::String(value.into())) + } + pub fn empty_array() -> Self { + Self(Value::Array(Vec::new())) + } + pub fn as_object(&self) -> Result<&BTreeMap> { + match &self.0 { + Value::Object(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn into_object(self) -> Result> { + match self.0 { + Value::Object(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn from_object(value: BTreeMap) -> Self { + Self(Value::Object(value)) + } + pub fn as_array(&self) -> Result<&[Json]> { + match &self.0 { + Value::Array(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn as_str(&self) -> Result<&str> { + match &self.0 { + Value::String(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn as_integer(&self) -> Result<&str> { + match &self.0 { + Value::Integer(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } + pub fn is_null(&self) -> bool { + matches!(self.0, Value::Null) + } + + pub fn canonical(&self) -> Result { + let mut output = String::new(); + self.write(&mut output, 0)?; + Ok(output) + } + + fn write(&self, out: &mut String, depth: usize) -> Result<()> { + if depth > MAX_DEPTH { + return Err(Error("json_too_deep")); + } + match &self.0 { + Value::Null => out.push_str("null"), + Value::Bool(v) => out.push_str(if *v { "true" } else { "false" }), + Value::Integer(v) => out.push_str(v), + Value::Float(v) => out.push_str(&python_float(*v)), + Value::String(v) => out.push_str(&serde_json::to_string(v).map_err(|_| Error("invalid_json_value"))?), + Value::Array(values) => { + out.push('['); + for (i, value) in values.iter().enumerate() { + if i != 0 { out.push(','); } + value.write(out, depth + 1)?; + } + out.push(']'); + } + Value::Object(values) => { + out.push('{'); + for (i, (key, value)) in values.iter().enumerate() { + if i != 0 { out.push(','); } + out.push_str(&serde_json::to_string(key).map_err(|_| Error("invalid_json_value"))?); + out.push(':'); + value.write(out, depth + 1)?; + } + out.push('}'); + } + } + if out.len() > MAX_BYTES { + return Err(Error("request_too_large")); + } + Ok(()) + } +} + +struct Members(BTreeMap>); +impl<'de> Deserialize<'de> for Members { + fn deserialize>(deserializer: D) -> std::result::Result { + struct Unique; + impl<'de> Visitor<'de> for Unique { + type Value = Members; + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("an object without duplicate keys") + } + fn visit_map>(self, mut map: M) -> std::result::Result { + let mut values = BTreeMap::new(); + while let Some((key, value)) = map.next_entry::>()? { + if values.insert(key, value).is_some() { + return Err(de::Error::custom("duplicate_json_key")); + } + } + Ok(Members(values)) + } + } + deserializer.deserialize_map(Unique) + } +} + +/// Ryu supplies shortest round-trip digits; Python supplies the notation rules. +/// Keep float/integer distinction, -0.0, exponent sign and two-digit exponent. +fn python_float(value: f64) -> String { + if value == 0.0 { + return if value.is_sign_negative() { "-0.0" } else { "0.0" }.to_owned(); + } + let mut buffer = ryu::Buffer::new(); + let text = buffer.format_finite(value.abs()); + let (mantissa, exponent) = text.split_once('e').unwrap_or((text, "0")); + let base: i32 = exponent.parse().unwrap_or(0); + let point = mantissa.find('.').unwrap_or(mantissa.len()) as i32; + let raw_digits = mantissa.replace('.', ""); + let leading = raw_digits.len() - raw_digits.trim_start_matches('0').len(); + let digits = raw_digits.trim_start_matches('0').trim_end_matches('0'); + let scientific = base + point - leading as i32 - 1; + let mut output = if value.is_sign_negative() { "-" } else { "" }.to_owned(); + if !(-4..16).contains(&scientific) { + output.push_str(&digits[..1]); + if digits.len() > 1 { + output.push('.'); + output.push_str(&digits[1..]); + } + output.push('e'); + output.push(if scientific < 0 { '-' } else { '+' }); + output.push_str(&format!("{:02}", scientific.unsigned_abs())); + } else { + let position = scientific + 1; + if position <= 0 { + output.push_str("0."); + output.push_str(&"0".repeat((-position) as usize)); + output.push_str(digits); + } else if position as usize >= digits.len() { + output.push_str(digits); + output.push_str(&"0".repeat(position as usize - digits.len())); + output.push_str(".0"); + } else { + let position = position as usize; + output.push_str(&digits[..position]); + output.push('.'); + output.push_str(&digits[position..]); + } + } + output +} + +pub fn digest(bytes: &[u8]) -> String { + format!("sha256:{:x}", Sha256::digest(bytes)) +} +pub fn b64(bytes: &[u8]) -> String { + URL_SAFE_NO_PAD.encode(bytes) +} +pub fn unb64(value: &str, limit: usize) -> Result> { + if value.len() > limit.saturating_mul(4) / 3 + 4 { + return Err(Error("invalid_base64")); + } + let bytes = URL_SAFE_NO_PAD.decode(value).map_err(|_| Error("invalid_base64"))?; + if bytes.len() > limit || b64(&bytes) != value { + return Err(Error("invalid_base64")); + } + Ok(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn canonical_preserves_python_numbers() { + for (raw, expected) in [ + ("1E20", "1e+20"), ("1e-7", "1e-07"), ("1e15", "1000000000000000.0"), + ("1e-4", "0.0001"), ("-0.0", "-0.0"), ("-0", "0"), + ("18446744073709551616000", "18446744073709551616000"), + ("5e-324", "5e-324"), ("1.7976931348623157e308", "1.7976931348623157e+308"), + ] { + assert_eq!(Json::parse(raw).unwrap().canonical().unwrap(), expected); + } + } + + #[test] + fn rejects_ambiguous_or_unbounded_inputs() { + for raw in [r#"{"a":1,"\u0061":2}"#, r#"[{"x":0,"x":1}]"#, "NaN", "Infinity", "1e999", r#""\ud800""#, "01", "[1,]"] { + assert!(Json::parse(raw).and_then(|v| v.canonical()).is_err(), "{raw}"); + } + assert!(Json::parse(&" ".repeat(MAX_BYTES + 1)).is_err()); + assert!(Json::parse(&format!("{}0{}", "[".repeat(66), "]".repeat(66))).is_err()); + } + + #[test] + fn unicode_and_base64_are_canonical() { + assert_eq!(Json::parse(r#"{"😀":1,"\ue000":2,"a":"中文\n"}"#).unwrap().canonical().unwrap(), "{\"a\":\"中文\\n\",\"\u{e000}\":2,\"😀\":1}"); + for bad in ["YQ=", "YQ==", "YR", "+/", "Y Q", "a"] { + assert!(unb64(bad, 10).is_err()); + } + assert_eq!(unb64("YQ", 1).unwrap(), b"a"); + assert!(unb64("YQ", 0).is_err()); + } +} diff --git a/rust/crates/msg-crypto/Cargo.toml b/rust/crates/msg-crypto/Cargo.toml new file mode 100644 index 00000000..a013d7dd --- /dev/null +++ b/rust/crates/msg-crypto/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "msg-crypto" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +ed25519-dalek.workspace = true +serde.workspace = true +sha2.workspace = true +msg-core = { path = "../msg-core" } + +[lints] +workspace = true diff --git a/rust/crates/msg-crypto/src/lib.rs b/rust/crates/msg-crypto/src/lib.rs new file mode 100644 index 00000000..57e11fe4 --- /dev/null +++ b/rust/crates/msg-crypto/src/lib.rs @@ -0,0 +1,95 @@ +//! Native signature primitives, not authentication or certificate authorization. +use ed25519_dalek::{Signer as _, SigningKey, VerifyingKey}; +use msg_core::{b64, unb64, Error, Result}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Signature { + pub key_id: String, + pub algorithm: String, + pub value: String, +} + +pub fn key_id(public: &[u8]) -> String { + format!("k_{:x}", Sha256::digest(public)) +} +pub fn subject_id(public: &[u8]) -> String { + let mut digest = Sha256::new(); + digest.update(b"msg-subject-v1\0"); + digest.update(public); + format!("u_{}", &format!("{:x}", digest.finalize())[..32]) +} +pub fn framed(payload: &[u8], purpose: &str) -> Result> { + if purpose.is_empty() || !purpose.is_ascii() || purpose.contains('\0') { + return Err(Error("invalid_signature_purpose")); + } + let mut message = Vec::with_capacity(17 + purpose.len() + payload.len()); + message.extend_from_slice(b"msg.lmm.best/v1/"); + message.extend_from_slice(purpose.as_bytes()); + message.push(0); + message.extend_from_slice(payload); + Ok(message) +} + +/// Intentionally no Debug/Serialize/Clone or private-key export. +/// ed25519-dalek's default zeroize feature clears the owned key on drop. +pub struct Ed25519Signer(SigningKey); +impl Ed25519Signer { + pub fn from_seed(seed: &[u8; 32]) -> Self { + Self(SigningKey::from_bytes(seed)) + } + pub fn public_key(&self) -> [u8; 32] { + self.0.verifying_key().to_bytes() + } + pub fn sign(&self, payload: &[u8], purpose: &str) -> Result { + Ok(Signature { + key_id: key_id(&self.public_key()), + algorithm: "ed25519".to_owned(), + value: b64(&self.0.sign(&framed(payload, purpose)?).to_bytes()), + }) + } +} + +/// Verifies bytes only. Callers must separately enforce credential state, +/// subject binding, expiry, service origin, replay rules and CA authorization. +pub fn verify(public: &[u8], payload: &[u8], signature: &Signature, purpose: &str) -> Result<()> { + let invalid = |_| Error("invalid_signature"); + if signature.algorithm != "ed25519" || signature.key_id != key_id(public) { + return Err(Error("invalid_signature")); + } + let public: &[u8; 32] = public.try_into().map_err(invalid)?; + let key = VerifyingKey::from_bytes(public).map_err(|_| Error("invalid_signature"))?; + let bytes = unb64(&signature.value, 64).map_err(|_| Error("invalid_signature"))?; + let signature = ed25519_dalek::Signature::from_slice(&bytes).map_err(|_| Error("invalid_signature"))?; + key.verify_strict(&framed(payload, purpose)?, &signature).map_err(|_| Error("invalid_signature")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn domains_keys_and_message_are_bound() { + let signer = Ed25519Signer::from_seed(&[42; 32]); + let signature = signer.sign(b"hello", "request").unwrap(); + assert!(verify(&signer.public_key(), b"hello", &signature, "request").is_ok()); + assert!(verify(&signer.public_key(), b"hello", &signature, "receipt").is_err()); + assert!(verify(&signer.public_key(), b"other", &signature, "request").is_err()); + assert!(verify(&[0; 32], b"hello", &signature, "request").is_err()); + assert!(framed(b"x", "").is_err()); + assert!(framed(b"x", "bad\0purpose").is_err()); + assert!(framed(b"x", "中文").is_err()); + } + + #[test] + fn rejects_weak_key_identity_forgery() { + let mut identity = [0; 32]; + identity[0] = 1; + let mut forged = [0; 64]; + forged[0] = 1; + let signature = Signature { key_id: key_id(&identity), algorithm: "ed25519".to_owned(), value: b64(&forged) }; + assert!(verify(&identity, b"forgery", &signature, "request").is_err()); + } +} diff --git a/rust/crates/msg-protocol/Cargo.toml b/rust/crates/msg-protocol/Cargo.toml new file mode 100644 index 00000000..df689549 --- /dev/null +++ b/rust/crates/msg-protocol/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "msg-protocol" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +chrono.workspace = true +serde_json.workspace = true +msg-core = { path = "../msg-core" } +msg-crypto = { path = "../msg-crypto" } + +[lints] +workspace = true diff --git a/rust/crates/msg-protocol/examples/wire_check.rs b/rust/crates/msg-protocol/examples/wire_check.rs new file mode 100644 index 00000000..6ddf580c --- /dev/null +++ b/rust/crates/msg-protocol/examples/wire_check.rs @@ -0,0 +1,64 @@ +//! JSONL conformance driver. Not shipped as msg/msgd; no network or storage. +//! The deterministic signer is public test material, never a service identity. +use std::io::{self, BufRead, Read, Write}; + +use msg_core::{b64, digest, unb64, Error, Json, MAX_BYTES}; +use msg_crypto::{key_id, subject_id, Ed25519Signer, Signature}; +use msg_protocol::Request; +use serde_json::{json, Value}; + +fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str, Error> { + value.get(key).and_then(Value::as_str).ok_or(Error("invalid_test_input")) +} +fn run(value: Value) -> Result { + match text(&value, "action")? { + "canonical" => { + let canonical = Json::parse(text(&value, "raw")?)?.canonical()?; + Ok(json!({"canonical":canonical, "digest":digest(canonical.as_bytes())})) + } + "base64" => { + let bytes = unb64(text(&value, "raw")?, MAX_BYTES)?; + Ok(json!({"encoded":b64(&bytes), "digest":digest(&bytes)})) + } + "request" => { + let request = Request::parse(text(&value, "raw")?)?; + let verified = match value.get("public").and_then(Value::as_str) { + Some(public) => Some(request.verify_signature_bytes(&unb64(public, 32)?).is_ok()), + None => None, + }; + Ok(json!({"canonical":request.canonical()?, "payload_digest":request.payload_digest()?, "signing":b64(&request.signing_bytes()?), "digest_valid":request.verify_payload_digest().is_ok(), "verified":verified})) + } + "sign" => { + let signer = Ed25519Signer::from_seed(&std::array::from_fn(|i| i as u8)); + let payload = unb64(text(&value, "payload")?, MAX_BYTES)?; + let signature = signer.sign(&payload, text(&value, "purpose")?)?; + Ok(json!({"public":b64(&signer.public_key()), "key_id":key_id(&signer.public_key()), "subject_id":subject_id(&signer.public_key()), "signature":signature})) + } + "verify" => { + let signature: Signature = serde_json::from_value(value.get("signature").cloned().ok_or(Error("invalid_test_input"))?).map_err(|_| Error("invalid_test_input"))?; + msg_crypto::verify(&unb64(text(&value, "public")?, 32)?, &unb64(text(&value, "payload")?, MAX_BYTES)?, &signature, text(&value, "purpose")?)?; + Ok(json!({"verified":true})) + } + _ => Err(Error("unknown_test_action")), + } +} +fn main() -> io::Result<()> { + let stdin = io::stdin(); + let mut reader = stdin.lock(); + let mut stdout = io::BufWriter::new(io::stdout().lock()); + loop { + let mut raw = String::new(); + // Bound the test transport too, before allocating an unbounded line. + let read = (&mut reader).take((MAX_BYTES * 8 + 1) as u64).read_line(&mut raw)?; + if read == 0 { break; } + if raw.len() > MAX_BYTES * 8 { return Err(io::Error::other("test_input_too_large")); } + let result = serde_json::from_str(&raw).map_err(|_| Error("invalid_test_input")).and_then(run); + let output = match result { + Ok(data) => json!({"ok":true, "data":data}), + Err(error) => json!({"ok":false, "code":error.0}), + }; + writeln!(stdout, "{output}")?; + stdout.flush()?; + } + Ok(()) +} diff --git a/rust/crates/msg-protocol/src/lib.rs b/rust/crates/msg-protocol/src/lib.rs new file mode 100644 index 00000000..668b5aaa --- /dev/null +++ b/rust/crates/msg-protocol/src/lib.rs @@ -0,0 +1,191 @@ +//! Transport-neutral envelope decoding and signed bytes, without an executor. +//! Verifying a signature here does not authorize any resource operation. +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Datelike, Timelike, Utc}; +use msg_core::{digest, unb64, Error, Json, Result, MAX_BYTES}; +use msg_crypto::Signature; + +const REQUIRED: [&str; 11] = [ + "request_id", "protocol_version", "operation", "contract_version", "target_service", + "subject", "arguments", "expected_generations", "expires_at", "payload_digest", "proof", +]; + +/// No Debug or public field access: proofs can contain bearer credentials. +pub struct Request { + fields: BTreeMap, +} + +fn check(condition: bool) -> Result<()> { + if condition { Ok(()) } else { Err(Error("invalid_request_envelope")) } +} +fn string(value: &Json, min: usize, max: usize) -> Result<&str> { + let text = value.as_str()?; + check((min..=max).contains(&text.chars().count()))?; + Ok(text) +} +fn nonnegative(value: &Json, positive: bool) -> Result<()> { + let integer = value.as_integer()?; + check(!integer.starts_with('-') && (!positive || integer != "0")) +} +fn exact_fields(fields: &BTreeMap, names: &[&str]) -> Result<()> { + check(fields.len() == names.len() && names.iter().all(|name| fields.contains_key(*name))) +} +fn strings(value: &Json, max_items: usize, min_length: usize, max_length: usize) -> Result<()> { + let items = value.as_array()?; + check(items.len() <= max_items)?; + for item in items { string(item, min_length, max_length)?; } + Ok(()) +} +fn signature_shape(value: &Json) -> Result<()> { + let signature = value.as_object()?; + exact_fields(signature, &["key_id", "algorithm", "value"])?; + string(&signature["key_id"], 1, 160)?; + check(signature["algorithm"].as_str()? == "ed25519")?; + unb64(signature["value"].as_str()?, MAX_BYTES)?; + Ok(()) +} +fn proof_shape(value: &Json) -> Result<()> { + if value.is_null() { return Ok(()); } + let proof = value.as_object()?; + if proof.contains_key("signature") { + exact_fields(proof, &["signature", "certificates"])?; + signature_shape(&proof["signature"])?; + strings(&proof["certificates"], 16, 1, 160) + } else { + exact_fields(proof, &["credential_id", "token"])?; + string(&proof["credential_id"], 1, 160)?; + unb64(proof["token"].as_str()?, MAX_BYTES)?; + Ok(()) + } +} + +impl Request { + pub fn parse(raw: &str) -> Result { + let fields = Json::parse(raw)?.into_object()?; + let result = Self::decode(fields); + // Do not expose parser internals or rejected request/proof contents. + result.map_err(|error| match error.0 { + "invalid_type" => Error("invalid_request_envelope"), + _ => error, + }) + } + + fn decode(mut fields: BTreeMap) -> Result { + check(REQUIRED.iter().all(|name| fields.contains_key(*name)))?; + check(fields.keys().all(|name| REQUIRED.contains(&name.as_str()) || matches!(name.as_str(), "source" | "return_fields")))?; + fields.entry("return_fields".to_owned()).or_insert_with(Json::empty_array); + fields.entry("source".to_owned()).or_insert_with(|| Json::string("unknown")); + string(&fields["request_id"], 1, 128)?; + check(fields["protocol_version"].as_integer()? == "1")?; + nonnegative(&fields["contract_version"], true)?; + string(&fields["target_service"], 0, 2048)?; + if !fields["subject"].is_null() { string(&fields["subject"], 0, 160)?; } + fields["arguments"].as_object()?; + let operation = string(&fields["operation"], 1, 128)?; + check(operation.contains('.') && operation.split('.').all(|part| { + let mut bytes = part.bytes(); + bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + }))?; + let expected = fields["expected_generations"].as_array()?; + check(expected.len() <= 64)?; + let mut seen = BTreeSet::new(); + for item in expected { + let pair = item.as_array()?; + check(pair.len() == 2)?; + let id = string(&pair[0], 1, 160)?; + nonnegative(&pair[1], false)?; + if !seen.insert(id) { return Err(Error("duplicate_expected_generation")); } + } + let payload_digest = fields["payload_digest"].as_str()?; + check(payload_digest.len() == 71 && payload_digest.starts_with("sha256:") && payload_digest[7..].bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)))?; + check(matches!(fields["source"].as_str()?, "msg" | "manual" | "mcp" | "unknown"))?; + strings(&fields["return_fields"], 30, 0, MAX_BYTES)?; + proof_shape(&fields["proof"])?; + if !fields["expires_at"].is_null() { + let text = string(&fields["expires_at"], 1, 40)?; + check(text.ends_with('Z'))?; + // The rollout profile accepts RFC3339 UTC. Python CLI emits this + // form. Non-RFC3339 datetime.fromisoformat aliases remain gated. + let time = DateTime::parse_from_rfc3339(text).map_err(|_| Error("invalid_datetime"))?; + check((1..=9999).contains(&time.year()) && time.nanosecond() < 1_000_000_000)?; + let normalized = time.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string(); + fields.insert("expires_at".to_owned(), Json::string(normalized)); + } + let request = Self { fields }; + request.canonical()?; + Ok(request) + } + + pub fn canonical(&self) -> Result { + Json::from_object(self.fields.clone()).canonical() + } + pub fn payload_digest(&self) -> Result { + let mut payload = self.fields.clone(); + for field in ["proof", "payload_digest", "source", "expires_at"] { payload.remove(field); } + Ok(digest(Json::from_object(payload).canonical()?.as_bytes())) + } + pub fn signing_bytes(&self) -> Result> { + let mut signed = self.fields.clone(); + signed.remove("proof"); + Ok(Json::from_object(signed).canonical()?.into_bytes()) + } + pub fn verify_payload_digest(&self) -> Result<()> { + if self.fields["payload_digest"].as_str()? == self.payload_digest()? { + Ok(()) + } else { Err(Error("payload_digest_mismatch")) } + } + pub fn verify_signature_bytes(&self, public: &[u8]) -> Result<()> { + self.verify_payload_digest()?; + let proof = self.fields["proof"].as_object().map_err(|_| Error("signature_required"))?; + let raw = proof.get("signature").ok_or(Error("signature_required"))?.canonical()?; + let signature: Signature = serde_json::from_str(&raw).map_err(|_| Error("invalid_signature"))?; + msg_crypto::verify(public, &self.signing_bytes()?, &signature, "request") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn packet() -> serde_json::Value { + serde_json::json!({ + "protocol_version":1, "request_id":"test-1", "operation":"discovery.get", + "contract_version":1, "target_service":"https://msg.lmm.best", "subject":null, + "arguments":{"id":"r_agents"}, "expected_generations":[], + "expires_at":"2026-10-05T00:00:00Z", "payload_digest":format!("sha256:{}", "0".repeat(64)), "proof":null + }) + } + + #[test] + fn retry_digest_excludes_only_transport_metadata() { + let mut value = packet(); + let first = Request::parse(&value.to_string()).unwrap(); + value["expires_at"] = "2026-10-05T00:02:00Z".into(); + value["source"] = "mcp".into(); + let refreshed = Request::parse(&value.to_string()).unwrap(); + assert_eq!(first.payload_digest().unwrap(), refreshed.payload_digest().unwrap()); + assert_ne!(first.signing_bytes().unwrap(), refreshed.signing_bytes().unwrap()); + value["request_id"] = "other".into(); + assert_ne!(first.payload_digest().unwrap(), Request::parse(&value.to_string()).unwrap().payload_digest().unwrap()); + } + + #[test] + fn defaults_and_timestamp_match_python_record() { + let request = Request::parse(&packet().to_string()).unwrap(); + let canonical = request.canonical().unwrap(); + assert!(canonical.contains("\"source\":\"unknown\"")); + assert!(canonical.contains("\"return_fields\":[]")); + assert!(canonical.contains("2026-10-05T00:00:00.000000Z")); + } + + #[test] + fn rejects_unknown_fields_and_bool_integers() { + for (key, bad) in [("extra", serde_json::json!(0)), ("protocol_version", serde_json::json!(true)), ("contract_version", serde_json::json!(1.0)), ("expected_generations", serde_json::json!([["r", true]])), ("expected_generations", serde_json::json!([["r", 0], ["r", 1]])), ("proof", serde_json::json!({"credential_id":"c", "token":"YQ=="}))] { + let mut value = packet(); + value[key] = bad; + assert!(Request::parse(&value.to_string()).is_err()); + } + } +} diff --git a/rust/tests/test_python_parity.py b/rust/tests/test_python_parity.py new file mode 100644 index 00000000..c95950bd --- /dev/null +++ b/rust/tests/test_python_parity.py @@ -0,0 +1,206 @@ +"""Run native wire code against the retained Python implementation, not a copy. + +Usage: PYTHONPATH=src python rust/tests/test_python_parity.py --binary PATH +Only deterministic public test keys are used. No server, database, or credentials. +""" +from __future__ import annotations + +import argparse +import copy +import json +import math +import random +import struct +import subprocess +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path + +from msg.core.codec import b64, canonical, decode, digest, loads, unb64, wire +from msg.core.models import Signature +from msg.core.packet import decode_packet +from msg.core.requests import payload_fields, request_for, signing_bytes +from msg.security.crypto import Ed25519Signer, key_id, subject_id, verify + +# This seed is public test material, shared with the Rust example only. +SIGNER = Ed25519Signer.from_bytes(bytes(range(32))) +CASES: list[tuple[str, dict, dict]] = [] + + +def add(group: str, command: dict, data: dict | None = None) -> None: + expected = {"ok": False} if data is None else {"ok": True, "data": data} + CASES.append((group, command, expected)) + + +def canonical_case(raw: str) -> None: + command = {"action": "canonical", "raw": raw} + try: + encoded = canonical(loads(raw)) + except (ValueError, TypeError, RecursionError): + add("canonical_reject", command) + else: + add("canonical", command, {"canonical": encoded.decode(), "digest": digest(encoded)}) + + +def request_case(raw: str, *, public: bool = False) -> None: + command = {"action": "request", "raw": raw} + if public: + command["public"] = b64(SIGNER.public_key) + try: + packet = decode_packet(raw) + except (ValueError, TypeError, RecursionError): + add("request_reject", command) + return + correct_digest = digest(payload_fields(packet)) + verified = None + if public: + verified = False + if packet.payload_digest == correct_digest and hasattr(packet.proof, "signature"): + try: + verify(SIGNER.public_key, signing_bytes(packet), packet.proof.signature, purpose="request") + except (ValueError, TypeError): + pass + else: + verified = True + add("request", command, { + "canonical": canonical(packet).decode(), "payload_digest": correct_digest, + "signing": b64(signing_bytes(packet)), "digest_valid": packet.payload_digest == correct_digest, + "verified": verified, + }) + + +def build_cases() -> None: + raw_values = [ + 'null', 'true', 'false', '0', '-0', '-0.0', '0.0', '1e-7', '1e-6', '1e-5', + '1e-4', '1e15', '1e16', '1e20', '1e23', '5e-324', '-5e-324', + '1.7976931348623157e308', '2.2250738585072014e-308', '1e-999', '-1e-999', + '184467440737095516160000000', '-184467440737095516160000000', '9' * 4300, + r'{"a":1,"\u0061":2}', r'[{"nested":1,"nested":2}]', + 'NaN', 'Infinity', '-Infinity', '1e999', '[1,]', '01', 'true false', + r'"\ud800"', r'"\udfff"', '9' * 4301, + ] + raw_values += [json.dumps(value, ensure_ascii=True) for value in [ + {"😀": 1, "\ue000": 2, "a": "中文\n"}, {"z": [], "a": {"b": None}}, + "".join(chr(i) for i in range(32)) + "\"/\\\u007f\u2028\u2029", + {"é": 1, "e\u0301": 2, "𐀀": 3}, [True, 1, 1.0, -0.0], + ]] + for raw in raw_values: + canonical_case(raw) + rng = random.Random(20261005) + for _ in range(16_384): + value = struct.unpack('!d', rng.getrandbits(64).to_bytes(8, 'big'))[0] + if math.isfinite(value): + canonical_case(repr(value)) + for power in range(-323, 309): + value = float(f'1e{power}') + for neighbor in [value, math.nextafter(value, 0.0), math.nextafter(value, math.inf)]: + if math.isfinite(neighbor): + canonical_case(repr(neighbor)) + for size in [0, 1, 2, 3, 31, 32, 64, 1024]: + encoded = b64(bytes(rng.getrandbits(8) for _ in range(size))) + add("base64", {"action": "base64", "raw": encoded}, {"encoded": encoded, "digest": digest(unb64(encoded))}) + for raw in ['YQ=', 'YQ==', 'YR', '+/', 'Y Q', 'a', '中文', '\nYQ']: + try: + unb64(raw) + except ValueError: + add("base64_reject", {"action": "base64", "raw": raw}) + else: + raise AssertionError("invalid base64 reference case was accepted") + + for purpose in ['request', 'receipt', 'revision', 'temporary-key-possession-v1']: + for payload in [b'', b'hello', '你好\x00世界'.encode(), canonical({"n": 1e-7, "i": 2**80})]: + signature = SIGNER.sign(payload, purpose=purpose) + add("sign", {"action": "sign", "payload": b64(payload), "purpose": purpose}, { + "public": b64(SIGNER.public_key), "key_id": key_id(SIGNER.public_key), + "subject_id": subject_id(SIGNER.public_key), "signature": wire(signature), + }) + command = {"action": "verify", "public": b64(SIGNER.public_key), "payload": b64(payload), + "signature": wire(signature), "purpose": purpose} + add("verify", command, {"verified": True}) + add("verify_reject", dict(command, payload=b64(payload + b'!'))) + add("verify_reject", dict(command, purpose=purpose + '-wrong')) + add("verify_reject", dict(command, public=b64(bytes(32)))) + + request = request_for( + 'discovery.get', {'id': 'r_agents', 'fields': ['id', 'revision', 'content'], 'n': 1e-7, 'big': 2**90}, + 'https://msg.lmm.best', signer=SIGNER, subject=subject_id(SIGNER.public_key), + request_id='rust-wire-public-fixture', expires_at=datetime(2026, 10, 5, tzinfo=UTC), + expected=(('r_agents', 0),), source='msg', + ) + base = wire(request) + request_case(canonical(base).decode(), public=True) + for key, value in [ + ('source', 'mcp'), ('expires_at', '2026-10-05T00:01:00Z'), + ('request_id', 'another'), ('target_service', 'https://other.example'), + ('arguments', {'id': 'other'}), ('expected_generations', [['r_agents', 1]]), + ('return_fields', ['data']), ('subject', None), + ]: + changed = copy.deepcopy(base) + changed[key] = value + request_case(canonical(changed).decode(), public=True) + unsigned = dict(base, proof=None) + unsigned.pop('source') + unsigned.pop('return_fields') + request_case(canonical(unsigned).decode()) + for timestamp in ['2026-10-05T00:00:00Z', '2026-10-05T00:00:00.1Z', '2026-10-05T00:00:00.123456789Z']: + request_case(canonical(dict(base, expires_at=timestamp)).decode(), public=True) + request_case(canonical(dict(base, proof={'credential_id': 'c_test', 'token': b64(b'test-only')})).decode()) + invalid = [ + ('extra', 1), ('protocol_version', True), ('contract_version', 1.0), + ('operation', 'no_dot'), ('operation', 'discovery..get'), ('operation', 'Discovery.get'), + ('request_id', ''), ('subject', 3), ('source', 'ssh'), ('arguments', []), + ('expected_generations', [['r_agents', True]]), ('expected_generations', [['r_agents', -1]]), + ('expected_generations', [['r_agents', 0], ['r_agents', 1]]), + ('return_fields', ['id'] * 31), ('proof', {'credential_id': 'c', 'token': 'YR'}), + ('expires_at', '2026-10-05T00:00:00+00:00'), ('payload_digest', 'sha256:bad'), + ] + for key, value in invalid: + changed = copy.deepcopy(base) + changed[key] = value + request_case(canonical(changed).decode()) + for key in ['request_id', 'protocol_version', 'proof', 'arguments']: + missing = copy.deepcopy(base) + missing.pop(key) + request_case(canonical(missing).decode()) + # Bounded Rust rollout profile is intentionally stricter, never silently truncated. + for raw in [' ' * (1_048_576 + 1), '[' * 66 + '0' + ']' * 66]: + add("bounded_reject", {"action": "canonical", "raw": raw}) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--binary', required=True, type=Path) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + build_cases() + inputs = ''.join(json.dumps(command, ensure_ascii=True) + '\n' for _, command, _ in CASES) + result = subprocess.run([str(args.binary.resolve())], input=inputs, text=True, capture_output=True, + timeout=120, check=True) + outputs = result.stdout.splitlines() + if len(outputs) != len(CASES): + raise AssertionError(f'native output count {len(outputs)} != {len(CASES)}') + failures = [] + for index, ((group, command, expected), raw) in enumerate(zip(CASES, outputs, strict=True)): + actual = json.loads(raw) + passed = actual.get('ok') is expected['ok'] + if expected['ok']: + passed = passed and actual.get('data') == expected['data'] + if not passed: + failures.append({'index': index, 'group': group, 'input': command, 'expected': expected, 'actual': actual}) + # Explicit reverse check of signatures produced by Rust with the Python verifier. + if passed and group == 'sign': + data = actual['data'] + verify(unb64(data['public']), unb64(command['payload']), decode(Signature, data['signature']), + purpose=command['purpose']) + report = {'cases': len(CASES), 'groups': dict(Counter(group for group, _, _ in CASES)), + 'failures': len(failures), 'first_failures': failures[:12]} + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=True, indent=2) + '\n', encoding='utf-8') + print(json.dumps(report, ensure_ascii=True)) + if failures: + raise SystemExit(1) + + +if __name__ == '__main__': + main() From 9f9160a92977cb1c9317ea74cce3c9f48bebd147 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 08:08:32 +0800 Subject: [PATCH 02/22] fix(rust): preserve Unicode JSONL framing and lock verified dependencies Apply the cloud rustfmt patch, commit Cargo's resolved lock, enforce locked builds, and include the Python harness in the existing Ruff style checks. Native tests and Clippy passed on the initial cloud build. --- .github/workflows/rust-wire.yml | 18 +- rust/Cargo.lock | 404 ++++++++++++++++++ rust/crates/msg-core/src/lib.rs | 84 +++- rust/crates/msg-crypto/src/lib.rs | 12 +- .../msg-protocol/examples/wire_check.rs | 44 +- rust/crates/msg-protocol/src/lib.rs | 133 ++++-- rust/tests/test_python_parity.py | 5 +- 7 files changed, 639 insertions(+), 61 deletions(-) create mode 100644 rust/Cargo.lock diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index 13d32c03..9a4b0cd7 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -34,7 +34,7 @@ jobs: - name: Install Rust toolchain and Python reference dependencies run: | rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy - python -m pip install cryptography==46.0.3 jsonschema==4.25.1 + python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 - name: Check formatting and retain a repair patch working-directory: rust run: | @@ -42,20 +42,30 @@ jobs: cargo +1.85.0 fmt --all git diff -- . > artifacts/rustfmt.patch test ! -s artifacts/rustfmt.patch + - name: Check Python reference harness style + if: ${{ !cancelled() }} + run: | + mkdir -p rust/artifacts + status=0 + python -m ruff check rust/tests --fix || status=$? + python -m ruff format rust/tests + git diff -- rust/tests > rust/artifacts/python-style.patch + test ! -s rust/artifacts/python-style.patch + exit "$status" - name: Native tests if: ${{ !cancelled() }} working-directory: rust - run: cargo +1.85.0 test --workspace --all-targets + run: cargo +1.85.0 test --locked --workspace --all-targets - name: Clippy if: ${{ !cancelled() }} working-directory: rust - run: cargo +1.85.0 clippy --workspace --all-targets -- -D warnings + run: cargo +1.85.0 clippy --locked --workspace --all-targets -- -D warnings - name: Python and Rust differential checks if: ${{ !cancelled() }} env: PYTHONPATH: src run: | - cargo +1.85.0 build --manifest-path rust/Cargo.toml --example wire_check + cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --example wire_check python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json - name: Preserve evidence and resolved dependency lock if: ${{ always() }} diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..2a915d20 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,404 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "num-traits", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "serde", + "sha2", + "subtle", + "zeroize", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "msg-core" +version = "0.1.0" +dependencies = [ + "base64", + "ryu", + "serde", + "serde_json", + "sha2", +] + +[[package]] +name = "msg-crypto" +version = "0.1.0" +dependencies = [ + "ed25519-dalek", + "msg-core", + "serde", + "sha2", +] + +[[package]] +name = "msg-protocol" +version = "0.1.0" +dependencies = [ + "chrono", + "msg-core", + "msg-crypto", + "serde_json", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.145" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" diff --git a/rust/crates/msg-core/src/lib.rs b/rust/crates/msg-core/src/lib.rs index 18bd4b98..b930d7af 100644 --- a/rust/crates/msg-core/src/lib.rs +++ b/rust/crates/msg-core/src/lib.rs @@ -60,7 +60,12 @@ impl Json { Some(b'"') => Value::String(serde_json::from_str(text).map_err(invalid)?), Some(b'[') => { let values: Vec> = serde_json::from_str(text).map_err(invalid)?; - Value::Array(values.iter().map(|v| Self::from_raw(v, depth + 1)).collect::>()?) + Value::Array( + values + .iter() + .map(|v| Self::from_raw(v, depth + 1)) + .collect::>()?, + ) } Some(b'{') => { let members: Members = serde_json::from_str(text).map_err(|e| { @@ -70,9 +75,13 @@ impl Json { Error("invalid_json") } })?; - Value::Object(members.0.into_iter().map(|(k, v)| { - Ok((k, Self::from_raw(&v, depth + 1)?)) - }).collect::>()?) + Value::Object( + members + .0 + .into_iter() + .map(|(k, v)| Ok((k, Self::from_raw(&v, depth + 1)?))) + .collect::>()?, + ) } _ if text.contains(['.', 'e', 'E']) => { let number: f64 = text.parse().map_err(|_| Error("invalid_json"))?; @@ -150,11 +159,15 @@ impl Json { Value::Bool(v) => out.push_str(if *v { "true" } else { "false" }), Value::Integer(v) => out.push_str(v), Value::Float(v) => out.push_str(&python_float(*v)), - Value::String(v) => out.push_str(&serde_json::to_string(v).map_err(|_| Error("invalid_json_value"))?), + Value::String(v) => { + out.push_str(&serde_json::to_string(v).map_err(|_| Error("invalid_json_value"))?) + } Value::Array(values) => { out.push('['); for (i, value) in values.iter().enumerate() { - if i != 0 { out.push(','); } + if i != 0 { + out.push(','); + } value.write(out, depth + 1)?; } out.push(']'); @@ -162,8 +175,12 @@ impl Json { Value::Object(values) => { out.push('{'); for (i, (key, value)) in values.iter().enumerate() { - if i != 0 { out.push(','); } - out.push_str(&serde_json::to_string(key).map_err(|_| Error("invalid_json_value"))?); + if i != 0 { + out.push(','); + } + out.push_str( + &serde_json::to_string(key).map_err(|_| Error("invalid_json_value"))?, + ); out.push(':'); value.write(out, depth + 1)?; } @@ -186,7 +203,10 @@ impl<'de> Deserialize<'de> for Members { fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.write_str("an object without duplicate keys") } - fn visit_map>(self, mut map: M) -> std::result::Result { + fn visit_map>( + self, + mut map: M, + ) -> std::result::Result { let mut values = BTreeMap::new(); while let Some((key, value)) = map.next_entry::>()? { if values.insert(key, value).is_some() { @@ -204,7 +224,12 @@ impl<'de> Deserialize<'de> for Members { /// Keep float/integer distinction, -0.0, exponent sign and two-digit exponent. fn python_float(value: f64) -> String { if value == 0.0 { - return if value.is_sign_negative() { "-0.0" } else { "0.0" }.to_owned(); + return if value.is_sign_negative() { + "-0.0" + } else { + "0.0" + } + .to_owned(); } let mut buffer = ryu::Buffer::new(); let text = buffer.format_finite(value.abs()); @@ -255,7 +280,9 @@ pub fn unb64(value: &str, limit: usize) -> Result> { if value.len() > limit.saturating_mul(4) / 3 + 4 { return Err(Error("invalid_base64")); } - let bytes = URL_SAFE_NO_PAD.decode(value).map_err(|_| Error("invalid_base64"))?; + let bytes = URL_SAFE_NO_PAD + .decode(value) + .map_err(|_| Error("invalid_base64"))?; if bytes.len() > limit || b64(&bytes) != value { return Err(Error("invalid_base64")); } @@ -269,10 +296,15 @@ mod tests { #[test] fn canonical_preserves_python_numbers() { for (raw, expected) in [ - ("1E20", "1e+20"), ("1e-7", "1e-07"), ("1e15", "1000000000000000.0"), - ("1e-4", "0.0001"), ("-0.0", "-0.0"), ("-0", "0"), + ("1E20", "1e+20"), + ("1e-7", "1e-07"), + ("1e15", "1000000000000000.0"), + ("1e-4", "0.0001"), + ("-0.0", "-0.0"), + ("-0", "0"), ("18446744073709551616000", "18446744073709551616000"), - ("5e-324", "5e-324"), ("1.7976931348623157e308", "1.7976931348623157e+308"), + ("5e-324", "5e-324"), + ("1.7976931348623157e308", "1.7976931348623157e+308"), ] { assert_eq!(Json::parse(raw).unwrap().canonical().unwrap(), expected); } @@ -280,8 +312,20 @@ mod tests { #[test] fn rejects_ambiguous_or_unbounded_inputs() { - for raw in [r#"{"a":1,"\u0061":2}"#, r#"[{"x":0,"x":1}]"#, "NaN", "Infinity", "1e999", r#""\ud800""#, "01", "[1,]"] { - assert!(Json::parse(raw).and_then(|v| v.canonical()).is_err(), "{raw}"); + for raw in [ + r#"{"a":1,"\u0061":2}"#, + r#"[{"x":0,"x":1}]"#, + "NaN", + "Infinity", + "1e999", + r#""\ud800""#, + "01", + "[1,]", + ] { + assert!( + Json::parse(raw).and_then(|v| v.canonical()).is_err(), + "{raw}" + ); } assert!(Json::parse(&" ".repeat(MAX_BYTES + 1)).is_err()); assert!(Json::parse(&format!("{}0{}", "[".repeat(66), "]".repeat(66))).is_err()); @@ -289,7 +333,13 @@ mod tests { #[test] fn unicode_and_base64_are_canonical() { - assert_eq!(Json::parse(r#"{"😀":1,"\ue000":2,"a":"中文\n"}"#).unwrap().canonical().unwrap(), "{\"a\":\"中文\\n\",\"\u{e000}\":2,\"😀\":1}"); + assert_eq!( + Json::parse(r#"{"😀":1,"\ue000":2,"a":"中文\n"}"#) + .unwrap() + .canonical() + .unwrap(), + "{\"a\":\"中文\\n\",\"\u{e000}\":2,\"😀\":1}" + ); for bad in ["YQ=", "YQ==", "YR", "+/", "Y Q", "a"] { assert!(unb64(bad, 10).is_err()); } diff --git a/rust/crates/msg-crypto/src/lib.rs b/rust/crates/msg-crypto/src/lib.rs index 57e11fe4..ee4b82a8 100644 --- a/rust/crates/msg-crypto/src/lib.rs +++ b/rust/crates/msg-crypto/src/lib.rs @@ -62,8 +62,10 @@ pub fn verify(public: &[u8], payload: &[u8], signature: &Signature, purpose: &st let public: &[u8; 32] = public.try_into().map_err(invalid)?; let key = VerifyingKey::from_bytes(public).map_err(|_| Error("invalid_signature"))?; let bytes = unb64(&signature.value, 64).map_err(|_| Error("invalid_signature"))?; - let signature = ed25519_dalek::Signature::from_slice(&bytes).map_err(|_| Error("invalid_signature"))?; - key.verify_strict(&framed(payload, purpose)?, &signature).map_err(|_| Error("invalid_signature")) + let signature = + ed25519_dalek::Signature::from_slice(&bytes).map_err(|_| Error("invalid_signature"))?; + key.verify_strict(&framed(payload, purpose)?, &signature) + .map_err(|_| Error("invalid_signature")) } #[cfg(test)] @@ -89,7 +91,11 @@ mod tests { identity[0] = 1; let mut forged = [0; 64]; forged[0] = 1; - let signature = Signature { key_id: key_id(&identity), algorithm: "ed25519".to_owned(), value: b64(&forged) }; + let signature = Signature { + key_id: key_id(&identity), + algorithm: "ed25519".to_owned(), + value: b64(&forged), + }; assert!(verify(&identity, b"forgery", &signature, "request").is_err()); } } diff --git a/rust/crates/msg-protocol/examples/wire_check.rs b/rust/crates/msg-protocol/examples/wire_check.rs index 6ddf580c..3cd0a3d3 100644 --- a/rust/crates/msg-protocol/examples/wire_check.rs +++ b/rust/crates/msg-protocol/examples/wire_check.rs @@ -8,7 +8,10 @@ use msg_protocol::Request; use serde_json::{json, Value}; fn text<'a>(value: &'a Value, key: &str) -> Result<&'a str, Error> { - value.get(key).and_then(Value::as_str).ok_or(Error("invalid_test_input")) + value + .get(key) + .and_then(Value::as_str) + .ok_or(Error("invalid_test_input")) } fn run(value: Value) -> Result { match text(&value, "action")? { @@ -26,17 +29,32 @@ fn run(value: Value) -> Result { Some(public) => Some(request.verify_signature_bytes(&unb64(public, 32)?).is_ok()), None => None, }; - Ok(json!({"canonical":request.canonical()?, "payload_digest":request.payload_digest()?, "signing":b64(&request.signing_bytes()?), "digest_valid":request.verify_payload_digest().is_ok(), "verified":verified})) + Ok( + json!({"canonical":request.canonical()?, "payload_digest":request.payload_digest()?, "signing":b64(&request.signing_bytes()?), "digest_valid":request.verify_payload_digest().is_ok(), "verified":verified}), + ) } "sign" => { let signer = Ed25519Signer::from_seed(&std::array::from_fn(|i| i as u8)); let payload = unb64(text(&value, "payload")?, MAX_BYTES)?; let signature = signer.sign(&payload, text(&value, "purpose")?)?; - Ok(json!({"public":b64(&signer.public_key()), "key_id":key_id(&signer.public_key()), "subject_id":subject_id(&signer.public_key()), "signature":signature})) + Ok( + json!({"public":b64(&signer.public_key()), "key_id":key_id(&signer.public_key()), "subject_id":subject_id(&signer.public_key()), "signature":signature}), + ) } "verify" => { - let signature: Signature = serde_json::from_value(value.get("signature").cloned().ok_or(Error("invalid_test_input"))?).map_err(|_| Error("invalid_test_input"))?; - msg_crypto::verify(&unb64(text(&value, "public")?, 32)?, &unb64(text(&value, "payload")?, MAX_BYTES)?, &signature, text(&value, "purpose")?)?; + let signature: Signature = serde_json::from_value( + value + .get("signature") + .cloned() + .ok_or(Error("invalid_test_input"))?, + ) + .map_err(|_| Error("invalid_test_input"))?; + msg_crypto::verify( + &unb64(text(&value, "public")?, 32)?, + &unb64(text(&value, "payload")?, MAX_BYTES)?, + &signature, + text(&value, "purpose")?, + )?; Ok(json!({"verified":true})) } _ => Err(Error("unknown_test_action")), @@ -49,10 +67,18 @@ fn main() -> io::Result<()> { loop { let mut raw = String::new(); // Bound the test transport too, before allocating an unbounded line. - let read = (&mut reader).take((MAX_BYTES * 8 + 1) as u64).read_line(&mut raw)?; - if read == 0 { break; } - if raw.len() > MAX_BYTES * 8 { return Err(io::Error::other("test_input_too_large")); } - let result = serde_json::from_str(&raw).map_err(|_| Error("invalid_test_input")).and_then(run); + let read = (&mut reader) + .take((MAX_BYTES * 8 + 1) as u64) + .read_line(&mut raw)?; + if read == 0 { + break; + } + if raw.len() > MAX_BYTES * 8 { + return Err(io::Error::other("test_input_too_large")); + } + let result = serde_json::from_str(&raw) + .map_err(|_| Error("invalid_test_input")) + .and_then(run); let output = match result { Ok(data) => json!({"ok":true, "data":data}), Err(error) => json!({"ok":false, "code":error.0}), diff --git a/rust/crates/msg-protocol/src/lib.rs b/rust/crates/msg-protocol/src/lib.rs index 668b5aaa..ad9a877f 100644 --- a/rust/crates/msg-protocol/src/lib.rs +++ b/rust/crates/msg-protocol/src/lib.rs @@ -7,8 +7,17 @@ use msg_core::{digest, unb64, Error, Json, Result, MAX_BYTES}; use msg_crypto::Signature; const REQUIRED: [&str; 11] = [ - "request_id", "protocol_version", "operation", "contract_version", "target_service", - "subject", "arguments", "expected_generations", "expires_at", "payload_digest", "proof", + "request_id", + "protocol_version", + "operation", + "contract_version", + "target_service", + "subject", + "arguments", + "expected_generations", + "expires_at", + "payload_digest", + "proof", ]; /// No Debug or public field access: proofs can contain bearer credentials. @@ -17,7 +26,11 @@ pub struct Request { } fn check(condition: bool) -> Result<()> { - if condition { Ok(()) } else { Err(Error("invalid_request_envelope")) } + if condition { + Ok(()) + } else { + Err(Error("invalid_request_envelope")) + } } fn string(value: &Json, min: usize, max: usize) -> Result<&str> { let text = value.as_str()?; @@ -34,7 +47,9 @@ fn exact_fields(fields: &BTreeMap, names: &[&str]) -> Result<()> { fn strings(value: &Json, max_items: usize, min_length: usize, max_length: usize) -> Result<()> { let items = value.as_array()?; check(items.len() <= max_items)?; - for item in items { string(item, min_length, max_length)?; } + for item in items { + string(item, min_length, max_length)?; + } Ok(()) } fn signature_shape(value: &Json) -> Result<()> { @@ -46,7 +61,9 @@ fn signature_shape(value: &Json) -> Result<()> { Ok(()) } fn proof_shape(value: &Json) -> Result<()> { - if value.is_null() { return Ok(()); } + if value.is_null() { + return Ok(()); + } let proof = value.as_object()?; if proof.contains_key("signature") { exact_fields(proof, &["signature", "certificates"])?; @@ -73,21 +90,34 @@ impl Request { fn decode(mut fields: BTreeMap) -> Result { check(REQUIRED.iter().all(|name| fields.contains_key(*name)))?; - check(fields.keys().all(|name| REQUIRED.contains(&name.as_str()) || matches!(name.as_str(), "source" | "return_fields")))?; - fields.entry("return_fields".to_owned()).or_insert_with(Json::empty_array); - fields.entry("source".to_owned()).or_insert_with(|| Json::string("unknown")); + check(fields.keys().all(|name| { + REQUIRED.contains(&name.as_str()) || matches!(name.as_str(), "source" | "return_fields") + }))?; + fields + .entry("return_fields".to_owned()) + .or_insert_with(Json::empty_array); + fields + .entry("source".to_owned()) + .or_insert_with(|| Json::string("unknown")); string(&fields["request_id"], 1, 128)?; check(fields["protocol_version"].as_integer()? == "1")?; nonnegative(&fields["contract_version"], true)?; string(&fields["target_service"], 0, 2048)?; - if !fields["subject"].is_null() { string(&fields["subject"], 0, 160)?; } + if !fields["subject"].is_null() { + string(&fields["subject"], 0, 160)?; + } fields["arguments"].as_object()?; let operation = string(&fields["operation"], 1, 128)?; - check(operation.contains('.') && operation.split('.').all(|part| { - let mut bytes = part.bytes(); - bytes.next().is_some_and(|first| first.is_ascii_lowercase()) - && bytes.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') - }))?; + check( + operation.contains('.') + && operation.split('.').all(|part| { + let mut bytes = part.bytes(); + bytes.next().is_some_and(|first| first.is_ascii_lowercase()) + && bytes.all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) + }), + )?; let expected = fields["expected_generations"].as_array()?; check(expected.len() <= 64)?; let mut seen = BTreeSet::new(); @@ -96,11 +126,22 @@ impl Request { check(pair.len() == 2)?; let id = string(&pair[0], 1, 160)?; nonnegative(&pair[1], false)?; - if !seen.insert(id) { return Err(Error("duplicate_expected_generation")); } + if !seen.insert(id) { + return Err(Error("duplicate_expected_generation")); + } } let payload_digest = fields["payload_digest"].as_str()?; - check(payload_digest.len() == 71 && payload_digest.starts_with("sha256:") && payload_digest[7..].bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)))?; - check(matches!(fields["source"].as_str()?, "msg" | "manual" | "mcp" | "unknown"))?; + check( + payload_digest.len() == 71 + && payload_digest.starts_with("sha256:") + && payload_digest[7..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)), + )?; + check(matches!( + fields["source"].as_str()?, + "msg" | "manual" | "mcp" | "unknown" + ))?; strings(&fields["return_fields"], 30, 0, MAX_BYTES)?; proof_shape(&fields["proof"])?; if !fields["expires_at"].is_null() { @@ -110,7 +151,10 @@ impl Request { // form. Non-RFC3339 datetime.fromisoformat aliases remain gated. let time = DateTime::parse_from_rfc3339(text).map_err(|_| Error("invalid_datetime"))?; check((1..=9999).contains(&time.year()) && time.nanosecond() < 1_000_000_000)?; - let normalized = time.with_timezone(&Utc).format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string(); + let normalized = time + .with_timezone(&Utc) + .format("%Y-%m-%dT%H:%M:%S%.6fZ") + .to_string(); fields.insert("expires_at".to_owned(), Json::string(normalized)); } let request = Self { fields }; @@ -123,7 +167,9 @@ impl Request { } pub fn payload_digest(&self) -> Result { let mut payload = self.fields.clone(); - for field in ["proof", "payload_digest", "source", "expires_at"] { payload.remove(field); } + for field in ["proof", "payload_digest", "source", "expires_at"] { + payload.remove(field); + } Ok(digest(Json::from_object(payload).canonical()?.as_bytes())) } pub fn signing_bytes(&self) -> Result> { @@ -134,13 +180,21 @@ impl Request { pub fn verify_payload_digest(&self) -> Result<()> { if self.fields["payload_digest"].as_str()? == self.payload_digest()? { Ok(()) - } else { Err(Error("payload_digest_mismatch")) } + } else { + Err(Error("payload_digest_mismatch")) + } } pub fn verify_signature_bytes(&self, public: &[u8]) -> Result<()> { self.verify_payload_digest()?; - let proof = self.fields["proof"].as_object().map_err(|_| Error("signature_required"))?; - let raw = proof.get("signature").ok_or(Error("signature_required"))?.canonical()?; - let signature: Signature = serde_json::from_str(&raw).map_err(|_| Error("invalid_signature"))?; + let proof = self.fields["proof"] + .as_object() + .map_err(|_| Error("signature_required"))?; + let raw = proof + .get("signature") + .ok_or(Error("signature_required"))? + .canonical()?; + let signature: Signature = + serde_json::from_str(&raw).map_err(|_| Error("invalid_signature"))?; msg_crypto::verify(public, &self.signing_bytes()?, &signature, "request") } } @@ -165,10 +219,22 @@ mod tests { value["expires_at"] = "2026-10-05T00:02:00Z".into(); value["source"] = "mcp".into(); let refreshed = Request::parse(&value.to_string()).unwrap(); - assert_eq!(first.payload_digest().unwrap(), refreshed.payload_digest().unwrap()); - assert_ne!(first.signing_bytes().unwrap(), refreshed.signing_bytes().unwrap()); + assert_eq!( + first.payload_digest().unwrap(), + refreshed.payload_digest().unwrap() + ); + assert_ne!( + first.signing_bytes().unwrap(), + refreshed.signing_bytes().unwrap() + ); value["request_id"] = "other".into(); - assert_ne!(first.payload_digest().unwrap(), Request::parse(&value.to_string()).unwrap().payload_digest().unwrap()); + assert_ne!( + first.payload_digest().unwrap(), + Request::parse(&value.to_string()) + .unwrap() + .payload_digest() + .unwrap() + ); } #[test] @@ -182,7 +248,20 @@ mod tests { #[test] fn rejects_unknown_fields_and_bool_integers() { - for (key, bad) in [("extra", serde_json::json!(0)), ("protocol_version", serde_json::json!(true)), ("contract_version", serde_json::json!(1.0)), ("expected_generations", serde_json::json!([["r", true]])), ("expected_generations", serde_json::json!([["r", 0], ["r", 1]])), ("proof", serde_json::json!({"credential_id":"c", "token":"YQ=="}))] { + for (key, bad) in [ + ("extra", serde_json::json!(0)), + ("protocol_version", serde_json::json!(true)), + ("contract_version", serde_json::json!(1.0)), + ("expected_generations", serde_json::json!([["r", true]])), + ( + "expected_generations", + serde_json::json!([["r", 0], ["r", 1]]), + ), + ( + "proof", + serde_json::json!({"credential_id":"c", "token":"YQ=="}), + ), + ] { let mut value = packet(); value[key] = bad; assert!(Request::parse(&value.to_string()).is_err()); diff --git a/rust/tests/test_python_parity.py b/rust/tests/test_python_parity.py index c95950bd..2ec16b62 100644 --- a/rust/tests/test_python_parity.py +++ b/rust/tests/test_python_parity.py @@ -176,7 +176,10 @@ def main() -> None: inputs = ''.join(json.dumps(command, ensure_ascii=True) + '\n' for _, command, _ in CASES) result = subprocess.run([str(args.binary.resolve())], input=inputs, text=True, capture_output=True, timeout=120, check=True) - outputs = result.stdout.splitlines() + # JSONL is delimited by LF, not Unicode U+2028/U+2029 inside strings. + if not result.stdout.endswith('\n'): + raise AssertionError('native output is missing its final LF') + outputs = result.stdout.split('\n')[:-1] if len(outputs) != len(CASES): raise AssertionError(f'native output count {len(outputs)} != {len(CASES)}') failures = [] From 91d8d3209d09f04269b4d9807793d6a0a8da7eba Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 08:15:31 +0800 Subject: [PATCH 03/22] test(rust): finalize conformance style and verify optimized builds Apply the cloud Ruff patch and use non-mutating style checks. Record the exact source/toolchain and exercise the same Python parity corpus against debug and optimized native binaries. No Python runtime entry points change. --- .github/workflows/rust-wire.yml | 34 +++-- rust/README.md | 10 +- rust/tests/test_python_parity.py | 246 ++++++++++++++++++++++--------- 3 files changed, 204 insertions(+), 86 deletions(-) diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index 9a4b0cd7..b09b168d 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -5,6 +5,7 @@ on: - 'rust/**' - 'src/msg/core/**' - 'src/msg/security/crypto.py' + - 'pyproject.toml' - '.github/workflows/rust-wire.yml' push: branches: [main, 'migration/rust-*'] @@ -12,6 +13,7 @@ on: - 'rust/**' - 'src/msg/core/**' - 'src/msg/security/crypto.py' + - 'pyproject.toml' - '.github/workflows/rust-wire.yml' workflow_dispatch: permissions: @@ -35,23 +37,20 @@ jobs: run: | rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 - - name: Check formatting and retain a repair patch - working-directory: rust + - name: Record exact tested revision and environment run: | - mkdir -p artifacts - cargo +1.85.0 fmt --all - git diff -- . > artifacts/rustfmt.patch - test ! -s artifacts/rustfmt.patch + mkdir -p rust/artifacts + git rev-parse HEAD > rust/artifacts/source-commit.txt + rustc +1.85.0 --version > rust/artifacts/toolchains.txt + python --version >> rust/artifacts/toolchains.txt + - name: Check Rust formatting without modifying sources + working-directory: rust + run: cargo +1.85.0 fmt --all -- --check - name: Check Python reference harness style if: ${{ !cancelled() }} run: | - mkdir -p rust/artifacts - status=0 - python -m ruff check rust/tests --fix || status=$? - python -m ruff format rust/tests - git diff -- rust/tests > rust/artifacts/python-style.patch - test ! -s rust/artifacts/python-style.patch - exit "$status" + python -m ruff check rust/tests + python -m ruff format --check rust/tests - name: Native tests if: ${{ !cancelled() }} working-directory: rust @@ -67,7 +66,14 @@ jobs: run: | cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --example wire_check python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json - - name: Preserve evidence and resolved dependency lock + - name: Optimized native differential checks + if: ${{ !cancelled() }} + env: + PYTHONPATH: src + run: | + cargo +1.85.0 build --release --locked --manifest-path rust/Cargo.toml --example wire_check + python rust/tests/test_python_parity.py --binary rust/target/release/examples/wire_check --report rust/artifacts/parity-release.json + - name: Preserve evidence and dependency lock if: ${{ always() }} uses: actions/upload-artifact@v7 with: diff --git a/rust/README.md b/rust/README.md index d832b55b..4c48d6c3 100644 --- a/rust/README.md +++ b/rust/README.md @@ -39,9 +39,9 @@ From the repository root, with Python 3.15 and Rust 1.85.0: ```sh python -m pip install cryptography==46.0.3 jsonschema==4.25.1 -cargo +1.85.0 test --manifest-path rust/Cargo.toml --workspace --all-targets -cargo +1.85.0 clippy --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings -cargo +1.85.0 build --manifest-path rust/Cargo.toml --example wire_check +cargo +1.85.0 test --locked --manifest-path rust/Cargo.toml --workspace --all-targets +cargo +1.85.0 clippy --locked --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings +cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --example wire_check PYTHONPATH=src python rust/tests/test_python_parity.py \ --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json ``` @@ -50,7 +50,9 @@ The differential corpus includes 16,384 deterministic random IEEE-754 bit patterns (non-finite values are excluded), decimal exponent neighbors, Unicode, large integers, malformed JSON/base64/envelopes and bidirectional Ed25519 checks. The report records the actual executed count, not the random input budget. -CI also keeps formatter evidence and the resolved dependency lock. +CI checks formatting without modifying sources, tests debug and optimized +release builds, and preserves the exact revision, toolchain versions, differential +reports and dependency lock as artifacts. ## Explicit cutover blockers diff --git a/rust/tests/test_python_parity.py b/rust/tests/test_python_parity.py index 2ec16b62..e8394414 100644 --- a/rust/tests/test_python_parity.py +++ b/rust/tests/test_python_parity.py @@ -3,6 +3,7 @@ Usage: PYTHONPATH=src python rust/tests/test_python_parity.py --binary PATH Only deterministic public test keys are used. No server, database, or credentials. """ + from __future__ import annotations import argparse @@ -28,62 +29,107 @@ def add(group: str, command: dict, data: dict | None = None) -> None: - expected = {"ok": False} if data is None else {"ok": True, "data": data} + expected = {'ok': False} if data is None else {'ok': True, 'data': data} CASES.append((group, command, expected)) def canonical_case(raw: str) -> None: - command = {"action": "canonical", "raw": raw} + command = {'action': 'canonical', 'raw': raw} try: encoded = canonical(loads(raw)) - except (ValueError, TypeError, RecursionError): - add("canonical_reject", command) + except ValueError, TypeError, RecursionError: + add('canonical_reject', command) else: - add("canonical", command, {"canonical": encoded.decode(), "digest": digest(encoded)}) + add('canonical', command, {'canonical': encoded.decode(), 'digest': digest(encoded)}) def request_case(raw: str, *, public: bool = False) -> None: - command = {"action": "request", "raw": raw} + command = {'action': 'request', 'raw': raw} if public: - command["public"] = b64(SIGNER.public_key) + command['public'] = b64(SIGNER.public_key) try: packet = decode_packet(raw) - except (ValueError, TypeError, RecursionError): - add("request_reject", command) + except ValueError, TypeError, RecursionError: + add('request_reject', command) return correct_digest = digest(payload_fields(packet)) verified = None if public: verified = False - if packet.payload_digest == correct_digest and hasattr(packet.proof, "signature"): + if packet.payload_digest == correct_digest and hasattr(packet.proof, 'signature'): try: - verify(SIGNER.public_key, signing_bytes(packet), packet.proof.signature, purpose="request") - except (ValueError, TypeError): + verify( + SIGNER.public_key, + signing_bytes(packet), + packet.proof.signature, + purpose='request', + ) + except ValueError, TypeError: pass else: verified = True - add("request", command, { - "canonical": canonical(packet).decode(), "payload_digest": correct_digest, - "signing": b64(signing_bytes(packet)), "digest_valid": packet.payload_digest == correct_digest, - "verified": verified, - }) + add( + 'request', + command, + { + 'canonical': canonical(packet).decode(), + 'payload_digest': correct_digest, + 'signing': b64(signing_bytes(packet)), + 'digest_valid': packet.payload_digest == correct_digest, + 'verified': verified, + }, + ) def build_cases() -> None: raw_values = [ - 'null', 'true', 'false', '0', '-0', '-0.0', '0.0', '1e-7', '1e-6', '1e-5', - '1e-4', '1e15', '1e16', '1e20', '1e23', '5e-324', '-5e-324', - '1.7976931348623157e308', '2.2250738585072014e-308', '1e-999', '-1e-999', - '184467440737095516160000000', '-184467440737095516160000000', '9' * 4300, - r'{"a":1,"\u0061":2}', r'[{"nested":1,"nested":2}]', - 'NaN', 'Infinity', '-Infinity', '1e999', '[1,]', '01', 'true false', - r'"\ud800"', r'"\udfff"', '9' * 4301, + 'null', + 'true', + 'false', + '0', + '-0', + '-0.0', + '0.0', + '1e-7', + '1e-6', + '1e-5', + '1e-4', + '1e15', + '1e16', + '1e20', + '1e23', + '5e-324', + '-5e-324', + '1.7976931348623157e308', + '2.2250738585072014e-308', + '1e-999', + '-1e-999', + '184467440737095516160000000', + '-184467440737095516160000000', + '9' * 4300, + r'{"a":1,"\u0061":2}', + r'[{"nested":1,"nested":2}]', + 'NaN', + 'Infinity', + '-Infinity', + '1e999', + '[1,]', + '01', + 'true false', + r'"\ud800"', + r'"\udfff"', + '9' * 4301, + ] + raw_values += [ + json.dumps(value, ensure_ascii=True) + for value in [ + {'😀': 1, '\ue000': 2, 'a': '中文\n'}, + {'z': [], 'a': {'b': None}}, + ''.join(chr(i) for i in range(32)) + '"/\\\u007f\u2028\u2029', + {'é': 1, 'e\u0301': 2, '𐀀': 3}, + [True, 1, 1.0, -0.0], + ] ] - raw_values += [json.dumps(value, ensure_ascii=True) for value in [ - {"😀": 1, "\ue000": 2, "a": "中文\n"}, {"z": [], "a": {"b": None}}, - "".join(chr(i) for i in range(32)) + "\"/\\\u007f\u2028\u2029", - {"é": 1, "e\u0301": 2, "𐀀": 3}, [True, 1, 1.0, -0.0], - ]] for raw in raw_values: canonical_case(raw) rng = random.Random(20261005) @@ -98,42 +144,66 @@ def build_cases() -> None: canonical_case(repr(neighbor)) for size in [0, 1, 2, 3, 31, 32, 64, 1024]: encoded = b64(bytes(rng.getrandbits(8) for _ in range(size))) - add("base64", {"action": "base64", "raw": encoded}, {"encoded": encoded, "digest": digest(unb64(encoded))}) + add( + 'base64', + {'action': 'base64', 'raw': encoded}, + {'encoded': encoded, 'digest': digest(unb64(encoded))}, + ) for raw in ['YQ=', 'YQ==', 'YR', '+/', 'Y Q', 'a', '中文', '\nYQ']: try: unb64(raw) except ValueError: - add("base64_reject", {"action": "base64", "raw": raw}) + add('base64_reject', {'action': 'base64', 'raw': raw}) else: - raise AssertionError("invalid base64 reference case was accepted") + raise AssertionError('invalid base64 reference case was accepted') for purpose in ['request', 'receipt', 'revision', 'temporary-key-possession-v1']: - for payload in [b'', b'hello', '你好\x00世界'.encode(), canonical({"n": 1e-7, "i": 2**80})]: + for payload in [b'', b'hello', '你好\x00世界'.encode(), canonical({'n': 1e-7, 'i': 2**80})]: signature = SIGNER.sign(payload, purpose=purpose) - add("sign", {"action": "sign", "payload": b64(payload), "purpose": purpose}, { - "public": b64(SIGNER.public_key), "key_id": key_id(SIGNER.public_key), - "subject_id": subject_id(SIGNER.public_key), "signature": wire(signature), - }) - command = {"action": "verify", "public": b64(SIGNER.public_key), "payload": b64(payload), - "signature": wire(signature), "purpose": purpose} - add("verify", command, {"verified": True}) - add("verify_reject", dict(command, payload=b64(payload + b'!'))) - add("verify_reject", dict(command, purpose=purpose + '-wrong')) - add("verify_reject", dict(command, public=b64(bytes(32)))) + add( + 'sign', + {'action': 'sign', 'payload': b64(payload), 'purpose': purpose}, + { + 'public': b64(SIGNER.public_key), + 'key_id': key_id(SIGNER.public_key), + 'subject_id': subject_id(SIGNER.public_key), + 'signature': wire(signature), + }, + ) + command = { + 'action': 'verify', + 'public': b64(SIGNER.public_key), + 'payload': b64(payload), + 'signature': wire(signature), + 'purpose': purpose, + } + add('verify', command, {'verified': True}) + add('verify_reject', dict(command, payload=b64(payload + b'!'))) + add('verify_reject', dict(command, purpose=purpose + '-wrong')) + add('verify_reject', dict(command, public=b64(bytes(32)))) request = request_for( - 'discovery.get', {'id': 'r_agents', 'fields': ['id', 'revision', 'content'], 'n': 1e-7, 'big': 2**90}, - 'https://msg.lmm.best', signer=SIGNER, subject=subject_id(SIGNER.public_key), - request_id='rust-wire-public-fixture', expires_at=datetime(2026, 10, 5, tzinfo=UTC), - expected=(('r_agents', 0),), source='msg', + 'discovery.get', + {'id': 'r_agents', 'fields': ['id', 'revision', 'content'], 'n': 1e-7, 'big': 2**90}, + 'https://msg.lmm.best', + signer=SIGNER, + subject=subject_id(SIGNER.public_key), + request_id='rust-wire-public-fixture', + expires_at=datetime(2026, 10, 5, tzinfo=UTC), + expected=(('r_agents', 0),), + source='msg', ) base = wire(request) request_case(canonical(base).decode(), public=True) for key, value in [ - ('source', 'mcp'), ('expires_at', '2026-10-05T00:01:00Z'), - ('request_id', 'another'), ('target_service', 'https://other.example'), - ('arguments', {'id': 'other'}), ('expected_generations', [['r_agents', 1]]), - ('return_fields', ['data']), ('subject', None), + ('source', 'mcp'), + ('expires_at', '2026-10-05T00:01:00Z'), + ('request_id', 'another'), + ('target_service', 'https://other.example'), + ('arguments', {'id': 'other'}), + ('expected_generations', [['r_agents', 1]]), + ('return_fields', ['data']), + ('subject', None), ]: changed = copy.deepcopy(base) changed[key] = value @@ -142,17 +212,35 @@ def build_cases() -> None: unsigned.pop('source') unsigned.pop('return_fields') request_case(canonical(unsigned).decode()) - for timestamp in ['2026-10-05T00:00:00Z', '2026-10-05T00:00:00.1Z', '2026-10-05T00:00:00.123456789Z']: + for timestamp in [ + '2026-10-05T00:00:00Z', + '2026-10-05T00:00:00.1Z', + '2026-10-05T00:00:00.123456789Z', + ]: request_case(canonical(dict(base, expires_at=timestamp)).decode(), public=True) - request_case(canonical(dict(base, proof={'credential_id': 'c_test', 'token': b64(b'test-only')})).decode()) + request_case( + canonical( + dict(base, proof={'credential_id': 'c_test', 'token': b64(b'test-only')}) + ).decode() + ) invalid = [ - ('extra', 1), ('protocol_version', True), ('contract_version', 1.0), - ('operation', 'no_dot'), ('operation', 'discovery..get'), ('operation', 'Discovery.get'), - ('request_id', ''), ('subject', 3), ('source', 'ssh'), ('arguments', []), - ('expected_generations', [['r_agents', True]]), ('expected_generations', [['r_agents', -1]]), + ('extra', 1), + ('protocol_version', True), + ('contract_version', 1.0), + ('operation', 'no_dot'), + ('operation', 'discovery..get'), + ('operation', 'Discovery.get'), + ('request_id', ''), + ('subject', 3), + ('source', 'ssh'), + ('arguments', []), + ('expected_generations', [['r_agents', True]]), + ('expected_generations', [['r_agents', -1]]), ('expected_generations', [['r_agents', 0], ['r_agents', 1]]), - ('return_fields', ['id'] * 31), ('proof', {'credential_id': 'c', 'token': 'YR'}), - ('expires_at', '2026-10-05T00:00:00+00:00'), ('payload_digest', 'sha256:bad'), + ('return_fields', ['id'] * 31), + ('proof', {'credential_id': 'c', 'token': 'YR'}), + ('expires_at', '2026-10-05T00:00:00+00:00'), + ('payload_digest', 'sha256:bad'), ] for key, value in invalid: changed = copy.deepcopy(base) @@ -164,7 +252,7 @@ def build_cases() -> None: request_case(canonical(missing).decode()) # Bounded Rust rollout profile is intentionally stricter, never silently truncated. for raw in [' ' * (1_048_576 + 1), '[' * 66 + '0' + ']' * 66]: - add("bounded_reject", {"action": "canonical", "raw": raw}) + add('bounded_reject', {'action': 'canonical', 'raw': raw}) def main() -> None: @@ -174,8 +262,14 @@ def main() -> None: args = parser.parse_args() build_cases() inputs = ''.join(json.dumps(command, ensure_ascii=True) + '\n' for _, command, _ in CASES) - result = subprocess.run([str(args.binary.resolve())], input=inputs, text=True, capture_output=True, - timeout=120, check=True) + result = subprocess.run( + [str(args.binary.resolve())], + input=inputs, + text=True, + capture_output=True, + timeout=120, + check=True, + ) # JSONL is delimited by LF, not Unicode U+2028/U+2029 inside strings. if not result.stdout.endswith('\n'): raise AssertionError('native output is missing its final LF') @@ -189,17 +283,33 @@ def main() -> None: if expected['ok']: passed = passed and actual.get('data') == expected['data'] if not passed: - failures.append({'index': index, 'group': group, 'input': command, 'expected': expected, 'actual': actual}) + failures.append({ + 'index': index, + 'group': group, + 'input': command, + 'expected': expected, + 'actual': actual, + }) # Explicit reverse check of signatures produced by Rust with the Python verifier. if passed and group == 'sign': data = actual['data'] - verify(unb64(data['public']), unb64(command['payload']), decode(Signature, data['signature']), - purpose=command['purpose']) - report = {'cases': len(CASES), 'groups': dict(Counter(group for group, _, _ in CASES)), - 'failures': len(failures), 'first_failures': failures[:12]} + verify( + unb64(data['public']), + unb64(command['payload']), + decode(Signature, data['signature']), + purpose=command['purpose'], + ) + report = { + 'cases': len(CASES), + 'groups': dict(Counter(group for group, _, _ in CASES)), + 'failures': len(failures), + 'first_failures': failures[:12], + } if args.report: args.report.parent.mkdir(parents=True, exist_ok=True) - args.report.write_text(json.dumps(report, ensure_ascii=True, indent=2) + '\n', encoding='utf-8') + args.report.write_text( + json.dumps(report, ensure_ascii=True, indent=2) + '\n', encoding='utf-8' + ) print(json.dumps(report, ensure_ascii=True)) if failures: raise SystemExit(1) From 30480c3956598d2b1b81b58aa8bba74d35486cf1 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 08:48:04 +0800 Subject: [PATCH 04/22] ci(rust): export tracked migration sources for offline editing --- .github/workflows/rust-migration-snapshot.yml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 .github/workflows/rust-migration-snapshot.yml diff --git a/.github/workflows/rust-migration-snapshot.yml b/.github/workflows/rust-migration-snapshot.yml new file mode 100644 index 00000000..e97c11eb --- /dev/null +++ b/.github/workflows/rust-migration-snapshot.yml @@ -0,0 +1,26 @@ +name: Rust migration source snapshot +on: + push: + branches: ['migration/rust-*'] + paths: ['.github/workflows/rust-migration-snapshot.yml'] +permissions: + contents: read +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 3 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Archive tracked source for the offline migration editor + run: | + mkdir -p /tmp/msg-migration-source + git rev-parse HEAD > /tmp/msg-migration-source/source-commit.txt + git archive --format=tar.gz HEAD > /tmp/msg-migration-source/source.tar.gz + - uses: actions/upload-artifact@v7 + with: + name: msg-migration-source-${{ github.sha }} + path: /tmp/msg-migration-source/ + retention-days: 1 + if-no-files-found: error From fc21f148c62216c98339f448a0a84a4bb84fdde2 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 08:52:19 +0800 Subject: [PATCH 05/22] ci(rust): prepare isolated offline compiler and pinned dependencies --- .github/workflows/rust-migration-snapshot.yml | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust-migration-snapshot.yml b/.github/workflows/rust-migration-snapshot.yml index e97c11eb..f7aeccba 100644 --- a/.github/workflows/rust-migration-snapshot.yml +++ b/.github/workflows/rust-migration-snapshot.yml @@ -8,19 +8,35 @@ permissions: jobs: snapshot: runs-on: ubuntu-latest - timeout-minutes: 3 + timeout-minutes: 10 steps: - uses: actions/checkout@v7 with: persist-credentials: false - - name: Archive tracked source for the offline migration editor + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Prepare offline test environment (no credentials) + shell: bash run: | + set -euo pipefail mkdir -p /tmp/msg-migration-source git rev-parse HEAD > /tmp/msg-migration-source/source-commit.txt git archive --format=tar.gz HEAD > /tmp/msg-migration-source/source.tar.gz + rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy + mkdir -p /tmp/deps/src + printf '[package]\nname="offline-dependencies"\nversion="0.0.0"\nedition="2021"\n[dependencies]\nmsg-protocol={path="%s/rust/crates/msg-protocol"}\nrusqlite={version="=0.32.1",features=["bundled"]}\nsubtle="=2.6.1"\n' "$PWD" > /tmp/deps/Cargo.toml + echo 'fn main() {}' > /tmp/deps/src/main.rs + cargo +1.85.0 vendor --manifest-path /tmp/deps/Cargo.toml /tmp/vendor > /tmp/vendor-config.toml + tar -cJf /tmp/msg-migration-source/rust-toolchain.tar.xz -C "$HOME/.rustup/toolchains/1.85.0-x86_64-unknown-linux-gnu" . + tar -czf /tmp/msg-migration-source/vendor.tar.gz -C /tmp vendor vendor-config.toml + python -m pip install 'cryptography==46.0.3' 'jsonschema==4.25.1' 'ruff==0.16.9' + tar -cJf /tmp/msg-migration-source/python-toolchain.tar.xz -C "$(python -c 'import sys;print(sys.prefix)')" . - uses: actions/upload-artifact@v7 with: - name: msg-migration-source-${{ github.sha }} + name: msg-migration-editor-${{ github.sha }} path: /tmp/msg-migration-source/ retention-days: 1 if-no-files-found: error + compression-level: 0 From 78e8d5d44cbbe49078a1e6eb4bb8e079eea3db72 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 09:37:00 +0800 Subject: [PATCH 06/22] feat(rust): migrate identity admission and read-only SQLite authority Add native current-state credential, certificate, delegation, OAuth binding and recovery checks; distinguish committed-result replay from fresh identity. Read Python-created SQLite databases through bounded read-only snapshots. Keep Python CLI/server/deployment unchanged. Full resource authorization, write transactions, native serve/HTTP/MCP and worker remain cutover gates. Validated exact tree 8f31a96020d945f266f2df9c92e56792935c0159 locally: 14 unit tests, 1 compile-fail doc test, fmt/Ruff/Clippy; both debug/release wire parity 18,447 and authority parity 61,734 (61,440 simple mode decisions). Remove temporary offline-editor export workflow; extend independent Rust CI. --- .github/workflows/rust-migration-snapshot.yml | 42 - .github/workflows/rust-wire.yml | 20 +- rust/Cargo.lock | 165 +++ rust/Cargo.toml | 6 +- rust/README.md | 222 +-- rust/crates/msg-core/src/lib.rs | 25 +- rust/crates/msg-identity/Cargo.toml | 20 + .../crates/msg-identity/src/authentication.rs | 601 ++++++++ rust/crates/msg-identity/src/certificates.rs | 422 ++++++ rust/crates/msg-identity/src/lib.rs | 20 + rust/crates/msg-identity/src/models.rs | 292 ++++ rust/crates/msg-identity/src/oauth.rs | 258 ++++ rust/crates/msg-identity/src/policy.rs | 241 ++++ rust/crates/msg-identity/src/store.rs | 132 ++ rust/crates/msg-protocol/src/lib.rs | 6 + rust/crates/msg-storage/Cargo.toml | 20 + .../msg-storage/examples/authority_check.rs | 244 ++++ rust/crates/msg-storage/src/lib.rs | 282 ++++ rust/tests/test_identity_parity.py | 1259 +++++++++++++++++ 19 files changed, 4146 insertions(+), 131 deletions(-) delete mode 100644 .github/workflows/rust-migration-snapshot.yml create mode 100644 rust/crates/msg-identity/Cargo.toml create mode 100644 rust/crates/msg-identity/src/authentication.rs create mode 100644 rust/crates/msg-identity/src/certificates.rs create mode 100644 rust/crates/msg-identity/src/lib.rs create mode 100644 rust/crates/msg-identity/src/models.rs create mode 100644 rust/crates/msg-identity/src/oauth.rs create mode 100644 rust/crates/msg-identity/src/policy.rs create mode 100644 rust/crates/msg-identity/src/store.rs create mode 100644 rust/crates/msg-storage/Cargo.toml create mode 100644 rust/crates/msg-storage/examples/authority_check.rs create mode 100644 rust/crates/msg-storage/src/lib.rs create mode 100644 rust/tests/test_identity_parity.py diff --git a/.github/workflows/rust-migration-snapshot.yml b/.github/workflows/rust-migration-snapshot.yml deleted file mode 100644 index f7aeccba..00000000 --- a/.github/workflows/rust-migration-snapshot.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Rust migration source snapshot -on: - push: - branches: ['migration/rust-*'] - paths: ['.github/workflows/rust-migration-snapshot.yml'] -permissions: - contents: read -jobs: - snapshot: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - uses: actions/setup-python@v7 - with: - python-version: '3.15' - allow-prereleases: true - - name: Prepare offline test environment (no credentials) - shell: bash - run: | - set -euo pipefail - mkdir -p /tmp/msg-migration-source - git rev-parse HEAD > /tmp/msg-migration-source/source-commit.txt - git archive --format=tar.gz HEAD > /tmp/msg-migration-source/source.tar.gz - rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy - mkdir -p /tmp/deps/src - printf '[package]\nname="offline-dependencies"\nversion="0.0.0"\nedition="2021"\n[dependencies]\nmsg-protocol={path="%s/rust/crates/msg-protocol"}\nrusqlite={version="=0.32.1",features=["bundled"]}\nsubtle="=2.6.1"\n' "$PWD" > /tmp/deps/Cargo.toml - echo 'fn main() {}' > /tmp/deps/src/main.rs - cargo +1.85.0 vendor --manifest-path /tmp/deps/Cargo.toml /tmp/vendor > /tmp/vendor-config.toml - tar -cJf /tmp/msg-migration-source/rust-toolchain.tar.xz -C "$HOME/.rustup/toolchains/1.85.0-x86_64-unknown-linux-gnu" . - tar -czf /tmp/msg-migration-source/vendor.tar.gz -C /tmp vendor vendor-config.toml - python -m pip install 'cryptography==46.0.3' 'jsonschema==4.25.1' 'ruff==0.16.9' - tar -cJf /tmp/msg-migration-source/python-toolchain.tar.xz -C "$(python -c 'import sys;print(sys.prefix)')" . - - uses: actions/upload-artifact@v7 - with: - name: msg-migration-editor-${{ github.sha }} - path: /tmp/msg-migration-source/ - retention-days: 1 - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index b09b168d..70cf5aba 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -4,7 +4,10 @@ on: paths: - 'rust/**' - 'src/msg/core/**' - - 'src/msg/security/crypto.py' + - 'src/msg/security/**' + - 'src/msg/storage/**' + - 'src/msg/oauth_config.py' + - 'src/msg/constants.py' - 'pyproject.toml' - '.github/workflows/rust-wire.yml' push: @@ -12,7 +15,10 @@ on: paths: - 'rust/**' - 'src/msg/core/**' - - 'src/msg/security/crypto.py' + - 'src/msg/security/**' + - 'src/msg/storage/**' + - 'src/msg/oauth_config.py' + - 'src/msg/constants.py' - 'pyproject.toml' - '.github/workflows/rust-wire.yml' workflow_dispatch: @@ -54,7 +60,9 @@ jobs: - name: Native tests if: ${{ !cancelled() }} working-directory: rust - run: cargo +1.85.0 test --locked --workspace --all-targets + run: | + cargo +1.85.0 test --locked --workspace --all-targets + cargo +1.85.0 test --locked --workspace --doc - name: Clippy if: ${{ !cancelled() }} working-directory: rust @@ -64,15 +72,17 @@ jobs: env: PYTHONPATH: src run: | - cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --example wire_check + cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --workspace --examples python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json + python rust/tests/test_identity_parity.py --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json - name: Optimized native differential checks if: ${{ !cancelled() }} env: PYTHONPATH: src run: | - cargo +1.85.0 build --release --locked --manifest-path rust/Cargo.toml --example wire_check + cargo +1.85.0 build --release --locked --manifest-path rust/Cargo.toml --workspace --examples python rust/tests/test_python_parity.py --binary rust/target/release/examples/wire_check --report rust/artifacts/parity-release.json + python rust/tests/test_identity_parity.py --binary rust/target/release/examples/authority_check --report rust/artifacts/identity-parity-release.json - name: Preserve evidence and dependency lock if: ${{ always() }} uses: actions/upload-artifact@v7 diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 2a915d20..43444bc7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -20,6 +32,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + [[package]] name = "block-buffer" version = "0.10.4" @@ -29,6 +47,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -140,12 +168,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fiat-crypto" version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + [[package]] name = "generic-array" version = "0.14.7" @@ -167,6 +213,24 @@ dependencies = [ "wasi", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + [[package]] name = "itoa" version = "1.0.18" @@ -179,6 +243,17 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "memchr" version = "2.8.3" @@ -206,6 +281,20 @@ dependencies = [ "sha2", ] +[[package]] +name = "msg-identity" +version = "0.1.0" +dependencies = [ + "chrono", + "msg-core", + "msg-crypto", + "msg-protocol", + "serde", + "serde_json", + "sha2", + "subtle", +] + [[package]] name = "msg-protocol" version = "0.1.0" @@ -216,6 +305,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "msg-storage" +version = "0.1.0" +dependencies = [ + "msg-core", + "msg-identity", + "msg-protocol", + "rusqlite", + "serde", + "serde_json", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -225,6 +326,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "pkcs8" version = "0.10.2" @@ -235,6 +342,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "proc-macro2" version = "1.0.107" @@ -262,6 +375,20 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -337,6 +464,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signature" version = "2.2.0" @@ -346,6 +479,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + [[package]] name = "spki" version = "0.7.3" @@ -385,6 +524,12 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -397,6 +542,26 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "zerocopy" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zeroize" version = "1.9.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dd08cf0e..8843ebca 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/msg-core", "crates/msg-crypto", "crates/msg-protocol"] +members = ["crates/msg-core", "crates/msg-crypto", "crates/msg-protocol", "crates/msg-identity", "crates/msg-storage"] resolver = "2" [workspace.package] @@ -15,8 +15,10 @@ chrono = { version = "=0.4.42", default-features = false, features = ["std"] } ed25519-dalek = "=2.2.0" ryu = "=1.0.20" serde = { version = "=1.0.228", features = ["derive"] } -serde_json = { version = "=1.0.145", features = ["raw_value"] } +serde_json = { version = "=1.0.145", features = ["raw_value", "arbitrary_precision"] } sha2 = "=0.10.9" +subtle = "=2.6.1" +rusqlite = { version = "=0.32.1", features = ["bundled"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/rust/README.md b/rust/README.md index 4c48d6c3..49dd5c33 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,102 +1,162 @@ -# Native MSG migration — phase 1 +# Native MSG migration — identity and SQLite read slice -**This is a wire-compatibility foundation, not a replacement daemon.** No Python -entry point, live database, configuration, route or deployment is changed. +**This is not a replacement daemon or a production cutover.** Python remains the +running implementation. No Python entry point, live database, configuration, +route, migration script or deployment is changed. ## Ownership boundary -The retained Python implementation owns `msg` CLI, administration/migration -scripts and development tools. The target Rust implementation will own -`msgd serve`, `msgd worker`, domain/identity, storage, HTTP/MCP and server-side -crypto/protocol. Shared wire semantics remain compatible with the retained CLI. -Do not rename endpoints or introduce a second signature protocol during this work. +Retain Python `msg` CLI, administration/migration scripts and developer tools. +Migrate server-side `msgd serve`, `msgd worker`, domain/identity, storage, +HTTP/MCP and crypto/protocol to Rust. Do not rename endpoints or invent a second +signature protocol. The tests import the actual Python implementation in the +same checkout, rather than a translated copy of the expected behavior. -The baseline inspected for this phase is -`7e16ca75ff6af4313365814c23912b5997b222c5`. The conformance test imports the actual -Python implementation from the same checkout, so later drift is detectable. +The original inspected main baseline is +`7e16ca75ff6af4313365814c23912b5997b222c5`. This slice extends phase 1 commit +`91d8d3209d09f04269b4d9807793d6a0a8da7eba` on the existing draft PR. -## Implemented here +## Implemented natively -| Crate | Native implementation | Python reference | +| Crate | Implemented responsibility | Python reference | |---|---|---| -| `msg-core` | Bounded strict JSON, duplicate-key rejection, Python-style canonical numbers/Unicode ordering, SHA-256 and canonical unpadded base64url | `core/codec.py` | -| `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing and verification | `security/crypto.py` | -| `msg-protocol` | Strict request shape, defaults, normalized UTC timestamps, business digest and signed request bytes | `core/packet.py`, `core/requests.py`, `core/models.py` | - -`wire_check` is a JSONL **test example**, not a server or client replacement. Its -fixed seed is public test material. The library does not log or implement Debug -for private keys, requests or bearer proofs. Unsafe code is forbidden in these -crates; standard cryptographic libraries implement cryptographic primitives. - -Business digests exclude only `proof`, `payload_digest`, `source` and -`expires_at`. Signatures cover everything except `proof`. Retry tests ensure -changing a request ID, business argument or precondition is not mistaken for a -same-content retry. Native signatures are also verified by the Python verifier. - -## Validation +| `msg-core` | Bounded strict JSON, duplicate rejection, canonical numbers/Unicode, lossless typed JSON, SHA-256/base64url | `core/codec.py` | +| `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing/verification | `security/crypto.py` | +| `msg-protocol` | Request shape/defaults, UTC time, business digest/signature bytes, immutable field access | `core/packet.py`, `core/requests.py`, `core/models.py` | +| `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, read-only browser/API/OAuth and token-recovery validation | `security/authentication.py`, `certificates.py`, `policy.py`, `oauth.py` | +| `msg-storage` | Read-only SQLite authority snapshots, current revocations/settings, result lookup and digest conflict, source bindings and recovery-delivery metadata | `storage/sqlite.py` and its record/session modules | + +Authentication validates service/digest/expiry, current credentials, SHA-256 +bearer verifiers with constant-time comparison, actor/subject bindings, permitted +entry, custodial limits, attached certificates, archive/quarantine state and a +latched recovery generation. A changed generation stays stale until the service +is reconstructed. Missing, revoked or expired authority fails closed. + +Certificate checks include pinned root bytes/public key, parent signatures and +revocation, current signing keys, scope/grant/constraint narrowing, issuance TTL, +delegation depth, the absolute CA depth limit, restricted online-CA issuance and +live source ownership/revision/credential ceilings. CSR publication checks retain +the original signed bytes, digest and possession proof. A trusted capability +registry projection is injected locally; requests cannot supply authority rules. +Only the existing network-constraint schema is implemented, not a generic schema +engine or plugin loader. + +OAuth validation follows current signing parents, captured/current ceilings, +auth versions, browser sessions, refresh-family authority and custodial vault +status. MCP audience is passed from a trusted adapter, not from an envelope +field. This is **binding validation**, not an OAuth or MCP server. The helper's +operation/capability declarations are synthetic test configuration, not a +production registry adapter. + +## Admission is not authorization + +`Principal` has private fields and no deserializer. Request JSON cannot mint one. +`Admission::Fresh` supplies an authenticated identity only. Resource mode, +certificate-gate, traversal, membership, sharing/DM/tool and operation-specific +checks still have to be integrated before executing any operation or exposing +private data. The pure mode/scope helpers do not grant permission on their own. + +`Admission::Replay` contains an already committed result and **no fresh +principal**. An old rotated token or exhausted delegated credential can only +retrieve its exact eligible result; a changed request ID, digest, revoked +successor or wrong result namespace cannot authorize a new handler execution. +Consumed token recovery has the same replay-only boundary. Recovery validation +does not issue credentials or consume a delivery record. + +## Storage safety boundary + +The native connection opens an existing database read-only. It does not create, +repair or migrate a schema. Unsupported schema versions/tables fail closed. +Each borrowed, non-Send snapshot pins all authority reads to one transaction and +rolls back on drop. It cannot be reused as a write transaction. JSON columns are +bounded in SQL before decoding. Secret vault ciphertext is not selected. + +**There is no Rust database writer yet.** Python's `.writer.lock` process fence +must remain part of a future writer port, including external compensation; a +SQLite transaction alone is not an equivalent replacement. PostgreSQL, native +write transactions, delivery/outbox/lease handling, migrations and crash/restart +execution recovery remain pending. Do not introduce dual writes. + +## Reproducible validation From the repository root, with Python 3.15 and Rust 1.85.0: ```sh -python -m pip install cryptography==46.0.3 jsonschema==4.25.1 +python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 +cargo +1.85.0 fmt --manifest-path rust/Cargo.toml --all -- --check +python -m ruff check rust/tests +python -m ruff format --check rust/tests cargo +1.85.0 test --locked --manifest-path rust/Cargo.toml --workspace --all-targets +cargo +1.85.0 test --locked --manifest-path rust/Cargo.toml --workspace --doc cargo +1.85.0 clippy --locked --manifest-path rust/Cargo.toml --workspace --all-targets -- -D warnings -cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --example wire_check +cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --workspace --examples PYTHONPATH=src python rust/tests/test_python_parity.py \ --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json +PYTHONPATH=src python rust/tests/test_identity_parity.py \ + --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json ``` -The differential corpus includes 16,384 deterministic random IEEE-754 bit -patterns (non-finite values are excluded), decimal exponent neighbors, Unicode, -large integers, malformed JSON/base64/envelopes and bidirectional Ed25519 checks. -The report records the actual executed count, not the random input budget. -CI checks formatting without modifying sources, tests debug and optimized -release builds, and preserves the exact revision, toolchain versions, differential -reports and dependency lock as artifacts. - -## Explicit cutover blockers - -- `verify_signature_bytes` is not authentication or authorization. Credential - revocation, subject binding, expiry, service binding, certificate chains, - permission checks, idempotency storage and transactional execution are pending. -- Resource/result/receipt/certificate models, encrypted-key wrapping and recovery - are not ported. Existing Python code remains the authoritative runtime. -- No native `serve`/`worker`, database writer, HTTP/MCP adapter or release artifact - is provided yet. An empty loop or Python subprocess would not count as a port. -- The first parser is deliberately bounded to 1 MiB and 64 nested levels, and - accepts RFC3339 UTC timestamps only. Python `datetime.fromisoformat` accepts - additional aliases. These differences must be resolved or explicitly versioned - before routing production requests to Rust; they are not silent fallback paths. -- Dalek strict verification refuses weak-key signature forgeries. Compatibility - with historical keys/signatures must be characterized before any cutover; - normal valid signatures must match Python byte-for-byte. -- No memory-reduction or throughput claim has been measured. A benchmark must - compare equivalent functionality, data, concurrency and process trees. +Repeat the build with `--release` and the same suites against +`rust/target/release/examples/`. CI performs both profiles, records exact source +SHA/toolchains, and preserves reports plus `Cargo.lock`. Source changes to Python +identity/storage/OAuth also trigger differential checks. The local offline editor +now has the same pinned compiler and Python reference dependencies; independent +cloud results must still be inspected before claiming CI acceptance. + +The wire suite covers canonical floating-point boundaries, Unicode, large +integers, malformed packets and bidirectional signatures. The authority suite +creates disposable databases using the actual Python `SqliteMetadataStore`, then +compares native and Python decisions over the same records. It covers live +revocation, CA constraints, OAuth source changes, recovery, exact retries and a +real concurrent WAL snapshot test. It also checks all 4096 modes across three +classes and five permissions: **61,440 simple mode decisions**, not 61,440 +independent identity scenarios. Report groups separate these from other cases. + +`wire_check` and `authority_check` are local JSONL **test examples**, never HTTP or +MCP endpoints. They use only deterministic public test keys and disposable data. +They do not execute business handlers, make network calls, launch Python from +Rust, or access production credentials. The Rust crates forbid unsafe code; +cryptographic and database primitives are delegated to pinned libraries. + +## Explicit compatibility and cutover gates + +- Parser limit: 1 MiB/64 levels. Model timestamps currently accept RFC3339 UTC + `Z` forms; Python accepts additional aliases. Model version/depth fields have + fixed-width Rust limits, unlike Python arbitrary integers. These need an + explicit compatibility/version decision before cutover. +- Credential JSON must contain required nullable fields. Malformed typed records + use a unified `invalid_identity_record` code instead of Python's field-specific + decoder errors. Corrupt result body/namespace mismatches are explicitly rejected. +- Ancestor traversal is capped at 1024, certificate traversal at 32 with a shared + 4096-work budget, and OAuth source chains at 32. Cycles/oversized restored + authority fail closed. These defensive limits are not silently described as + accepting every Python input. +- Dalek strict verification rejects weak-key forgeries. Historical key/signature + compatibility still needs characterization before routing production traffic. +- Full resource authorization, operation handlers, mutation-side identity/OAuth + issuance/refresh/recovery, encrypted-key wrapping, PostgreSQL/write storage, + native `serve`/HTTP/MCP, `worker`, packaging and deployment remain pending. +- No RSS, memory-reduction or throughput result is claimed. Benchmark equivalent + functionality, data, concurrency and complete process trees before comparison. ## Next implementation slices -1. Freeze identity/result/receipt fixtures and add the credential, certificate and - authorization model, including expired/revoked/cross-service negative cases. -2. Port storage transactions and the request-id/digest replay ledger against - disposable databases created by the Python migrations. Verify rollback, - restart/retry, concurrent writers and schema/version gates. No dual writes. -3. Integrate a bounded native executor and a read-only `msgd serve` slice, then - HTTP/MCP adapters with exactly the existing operation registry and envelopes. -4. Port worker claiming/leases/retries and side effects. Enable write operations - only after same-fixture differential tests and failure/recovery tests pass. -5. Perform isolated deployment rehearsals, workload/RSS measurements and an - explicit rollback/cutover review. Keep the current Python service available. - -## Coordination and evidence - -Repository `AGENTS.md`, local entry skill and the deployed `/AGENTS.md` were read. -The current editing runtime has no selected MSG signing identity or worker -runtime; it cannot create an authorized private coordination thread. No new -production identity was registered and no public coordination content was posted. -This bounded work is therefore tracked in the migration branch/PR rather than -claiming MSG coordination occurred. Return to private MSG coordination when an -existing authorized identity is available. - -The editing container has no Rust toolchain and cannot resolve GitHub for a git -clone. Consequently native compilation and the actual Python 3.15 differential -run must be evidenced by this PR's cloud CI, not claimed from local inspection. +1. Integrate the real operation/capability registry and complete resource + authorization, including private resources, sharing, memberships and tools. +2. Port writer fencing, atomic result/side-effect transactions and recovery + against disposable Python-migrated databases. Test rollback, restarts, + competing writers and retry behavior; retain Python migration scripts. +3. Integrate a bounded native executor and HTTP/MCP adapters, then worker + claiming/leases/retries and side effects. Empty loops or Python subprocesses + do not count as a server port. +4. Rehearse isolated deployments and rollback, then measure equivalent workloads. + Keep the current Python service until explicit cutover acceptance. + +## Coordination + +Repository `AGENTS.md`, entry skill and deployed `/AGENTS.md` were read. No +selected MSG signing identity was available in the editor, so no authorized +private coordination thread could be created. No new production identity or +public coordination post was created. Work is recorded in the migration +branch/PR; use an existing authorized identity when private coordination resumes. +The temporary offline source/toolchain-export workflow is removed by this slice. diff --git a/rust/crates/msg-core/src/lib.rs b/rust/crates/msg-core/src/lib.rs index b930d7af..f805cad3 100644 --- a/rust/crates/msg-core/src/lib.rs +++ b/rust/crates/msg-core/src/lib.rs @@ -4,7 +4,7 @@ use std::fmt; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use serde::de::{self, MapAccess, Visitor}; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::value::RawValue; use sha2::{Digest, Sha256}; @@ -38,7 +38,30 @@ enum Value { Object(BTreeMap), } +// Preserve canonical-v1 numbers when typed records contain arbitrary JSON. +// Do not round through serde_json::Value's fixed-width number representation. +impl Serialize for Json { + fn serialize(&self, serializer: S) -> std::result::Result { + let text = self.canonical().map_err(serde::ser::Error::custom)?; + RawValue::from_string(text) + .map_err(serde::ser::Error::custom)? + .serialize(serializer) + } +} +impl<'de> Deserialize<'de> for Json { + fn deserialize>(deserializer: D) -> std::result::Result { + let raw = Box::::deserialize(deserializer)?; + Self::parse(raw.get()).map_err(serde::de::Error::custom) + } +} + impl Json { + pub fn as_bool(&self) -> Result { + match self.0 { + Value::Bool(v) => Ok(v), + _ => Err(Error("invalid_type")), + } + } pub fn parse(input: &str) -> Result { if input.len() > MAX_BYTES { return Err(Error("request_too_large")); diff --git a/rust/crates/msg-identity/Cargo.toml b/rust/crates/msg-identity/Cargo.toml new file mode 100644 index 00000000..f6402dfc --- /dev/null +++ b/rust/crates/msg-identity/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "msg-identity" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +msg-core = { path = "../msg-core" } +msg-crypto = { path = "../msg-crypto" } +msg-protocol = { path = "../msg-protocol" } +chrono.workspace = true +serde.workspace = true +serde_json.workspace = true +sha2.workspace = true +subtle.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/msg-identity/src/authentication.rs b/rust/crates/msg-identity/src/authentication.rs new file mode 100644 index 00000000..b4b8c0e9 --- /dev/null +++ b/rust/crates/msg-identity/src/authentication.rs @@ -0,0 +1,601 @@ +//! Authentication admission only. Full resource checks must follow a fresh admission. +use crate::{certificates::CertificateValidator, models::*, require, store::*}; +use chrono::Duration; +use msg_core::{unb64, Error, Json, Result, MAX_BYTES}; +use msg_crypto::{key_id, subject_id, Signature}; +use msg_protocol::Request; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeSet; +use subtle::ConstantTimeEq; + +/// Supplied by the trusted operation registry, never by a request envelope. +#[derive(Deserialize)] +pub struct OperationPolicy { + pub name: String, + pub version: u32, + pub effect: String, + pub entries: BTreeSet, + pub require_signature: bool, + pub anonymous_only: bool, +} +#[derive(Clone, Copy)] +pub enum Entry { + Network, + LocalAdmin, + Worker, +} +impl Entry { + fn name(self) -> &'static str { + match self { + Self::Network => "network", + Self::LocalAdmin => "local_admin", + Self::Worker => "worker", + } + } +} + +/// Fields are private and there is no Deserialize implementation. +/// Request data cannot assert an actor or mint a principal. +/// +/// ```compile_fail +/// let _ = serde_json::from_str::("{}"); +/// ``` +#[derive(Serialize)] +pub struct Principal { + actor: Option, + subject: Option, + credential_id: Option, + method: &'static str, + certificates: Vec, + ceiling: Vec, +} +impl Principal { + pub fn actor(&self) -> Option<&str> { + self.actor.as_deref() + } + pub fn subject(&self) -> Option<&str> { + self.subject.as_deref() + } + pub fn credential_id(&self) -> Option<&str> { + self.credential_id.as_deref() + } + pub fn certificates(&self) -> &[String] { + &self.certificates + } + pub fn ceiling(&self) -> &[Grant] { + &self.ceiling + } + pub fn method(&self) -> &str { + self.method + } +} + +pub enum Admission { + /// Authenticated identity only, not permission to execute. + Fresh(Principal), + /// Revoked/exhausted credentials receive no fresh principal. + /// Return the already committed result; never invoke a handler with this branch. + Replay(Json), +} + +pub struct AuthenticationService { + certificates: CertificateValidator, + runtime: RuntimeGeneration, + primary_ceiling: Vec, + temporary_ceiling: Vec, + oauth_policy: Option, +} +impl AuthenticationService { + pub fn new( + certificates: CertificateValidator, + store: &dyn AuthorityStore, + primary_ceiling: Vec, + temporary_ceiling: Vec, + ) -> Result { + Ok(Self { + certificates, + runtime: RuntimeGeneration::capture(store)?, + primary_ceiling, + temporary_ceiling, + oauth_policy: None, + }) + } + pub fn with_oauth_policy(mut self, policy: crate::oauth::OAuthPolicy) -> Self { + self.oauth_policy = Some(policy); + self + } + pub fn authenticate( + &mut self, + request: &Request, + spec: &OperationPolicy, + store: &dyn AuthorityStore, + entry: Entry, + now: Timestamp, + ) -> Result { + self.authenticate_for_resource(request, spec, store, entry, now, None) + } + /// `resource` must be supplied by the adapter from its validated request URL, + /// never copied from a field in the operation envelope. + pub fn authenticate_for_resource( + &mut self, + request: &Request, + spec: &OperationPolicy, + store: &dyn AuthorityStore, + entry: Entry, + now: Timestamp, + resource: Option<&str>, + ) -> Result { + let requested_subject = optional_string(request.field("subject")?)?; + check_archived(store, requested_subject)?; + self.runtime.require_current(store)?; + let anchor = store.setting("active_root_certificate")?; + require( + anchor + .as_ref() + .map(|v| v.as_str()) + .transpose()? + .unwrap_or(self.certificates.root_id()) + == self.certificates.root_id(), + "service_restart_required", + )?; + require( + request.field("operation")?.as_str()? == spec.name + && request.field("contract_version")?.as_integer()? == spec.version.to_string(), + "operation_policy_mismatch", + )?; + if store.has_setting("recovery_quarantine")? { + return Err(Error(if spec.effect == "read" { + "recovery_quarantined" + } else { + "writes_paused" + })); + } + require( + request.field("target_service")?.as_str()? == self.certificates.service(), + "wrong_service", + )?; + request.verify_payload_digest()?; + let proof = request.field("proof")?; + if spec.effect != "read" || !proof.is_null() { + let expiry = + optional_string(request.field("expires_at")?)?.ok_or(Error("request_expired"))?; + let expiry = Timestamp::parse(expiry)?; + require(now < expiry, "request_expired")?; + require( + expiry.0 - now.0 <= Duration::seconds(300), + "request_expiry_too_long", + )?; + } + let mut replay = None; + let principal = match spec.name.as_str() { + "identity.register" => { + let signed = proof.as_object().map_err(|_| Error("proof_required"))?; + require(signed.contains_key("signature"), "proof_required")?; + let public = unb64(argument(request, "public_key")?.as_str()?, 32)?; + let uid = subject_id(&public); + require( + !setting_truthy(store, &format!("delegated_identity:{uid}"))?, + "delegated_identity_not_upgradable", + )?; + require( + public.len() == 32 && requested_subject == Some(uid.as_str()), + "subject_mismatch", + )?; + request.verify_signature_bytes(&public)?; + Principal { + actor: Some(uid.clone()), + subject: Some(uid), + credential_id: Some(key_id(&public)), + method: "signature", + certificates: vec![], + ceiling: self.primary_ceiling.clone(), + } + } + "identity.temporary" | "identity.custodial_create" => { + require( + proof.is_null() && requested_subject.is_none(), + "invalid_bootstrap", + )?; + if let Some(public) = request.field("arguments")?.as_object()?.get("public_key") { + if truthy(public)? { + let uid = subject_id(&unb64(public.as_str()?, 32)?); + require( + !setting_truthy(store, &format!("delegated_identity:{uid}"))?, + "delegated_identity_not_upgradable", + )?; + } + } + let nonce = unb64(argument(request, "nonce")?.as_str()?, 64)?; + require(nonce.len() >= 24, "invalid_bootstrap_nonce")?; + let prefix = if spec.name == "identity.custodial_create" { + "u_cust_" + } else { + "u_tmp_" + }; + let uid = format!( + "{}{}", + prefix, + &format!("{:x}", Sha256::digest(&nonce))[..32] + ); + Principal { + credential_id: Some(format!("t_{}", &uid[2..])), + actor: Some(uid.clone()), + subject: Some(uid), + method: "token", + certificates: vec![], + ceiling: self.temporary_ceiling.clone(), + } + } + "identity.token_recover" => { + require( + proof.is_null() && requested_subject.is_some(), + "invalid_recovery_proof", + )?; + let old = store.credential(argument(request, "credential_id")?.as_str()?)?; + crate::oauth::require_binding( + store, + &old, + now, + self.oauth_policy.as_ref(), + &self.temporary_ceiling, + resource, + )?; + require( + old.kind == CredentialKind::Token && requested_subject == Some(&old.subject_id), + "recovery_unavailable", + )?; + let actor = store.subject(&old.subject_id)?; + require( + !actor.local_only && actor.resource_id != "u_root", + "local_only", + )?; + let secret = unb64(argument(request, "recovery_secret")?.as_str()?, 64)?; + require(secret.len() >= 32, "invalid_recovery_secret")?; + let mut hasher = Sha256::new(); + hasher.update(b"token-recovery-v1\0"); + hasher.update(&secret); + let verifier = format!("{:x}", hasher.finalize()); + let delivery = store + .recovery_delivery(&old.id, &actor.resource_id)? + .ok_or(Error("recovery_unavailable"))?; + require( + bool::from(delivery.verifier.as_bytes().ct_eq(verifier.as_bytes())) + && delivery.expires_at > now + && delivery.request_id + == argument(request, "original_request_id")?.as_str()?, + "recovery_unavailable", + )?; + if delivery.consumed { + replay = store.request_result( + Some(&actor.resource_id), + request.field("request_id")?.as_str()?, + request.field("payload_digest")?.as_str()?, + )?; + let previous = replay.as_ref().ok_or(Error("recovery_unavailable"))?; + let data = previous + .as_object()? + .get("data") + .ok_or(Error("recovery_unavailable"))? + .as_object()?; + require( + data.get("previous_credential") + .and_then(|v| v.as_str().ok()) + == Some(&old.id), + "recovery_unavailable", + )?; + } else { + require( + old.revoked_at.is_none() && old.expires_at.is_some_and(|v| v > now), + "recovery_unavailable", + )?; + } + let mut ceiling = Vec::new(); + for grant in &self.primary_ceiling { + if grant.capability == "identity.basic" + && grant.operations.contains("identity.token_recover@1") + { + let mut grant = grant.clone(); + grant.operations = BTreeSet::from(["identity.token_recover@1".to_owned()]); + ceiling.push(grant); + } + } + require(!ceiling.is_empty(), "recovery_unavailable")?; + Principal { + actor: Some(actor.resource_id.clone()), + subject: Some(actor.resource_id), + credential_id: Some(old.id), + method: "recovery", + certificates: vec![], + ceiling, + } + } + _ if proof.is_null() => { + let anonymous_write = matches!( + spec.name.as_str(), + "communication.internet_receive" | "identity.link_claim" + ) && spec.version == 1 + && spec.anonymous_only + && spec.effect == "transaction"; + require( + requested_subject.is_none() && (spec.effect == "read" || anonymous_write), + "authentication_required", + )?; + Principal { + actor: None, + subject: None, + credential_id: None, + method: "anonymous", + certificates: vec![], + ceiling: vec![], + } + } + _ => { + let proof = proof.as_object()?; + let signature: Option = + proof.get("signature").map(decode).transpose()?; + let cid = match &signature { + Some(s) => s.key_id.as_str(), + None => proof + .get("credential_id") + .ok_or(Error("invalid_proof"))? + .as_str()?, + }; + let credential = store.credential(cid)?; + let actor = store.subject(&credential.subject_id)?; + require( + credential.not_before <= now && credential.expires_at.is_none_or(|v| now < v), + "credential_expired", + )?; + let (method, ids): (&str, Vec) = match &signature { + Some(signature) => { + require( + credential.kind == CredentialKind::SigningKey, + "wrong_credential_kind", + )?; + msg_crypto::verify( + &credential.verifier.0, + &request.signing_bytes()?, + signature, + "request", + )?; + ( + "signature", + decode(proof.get("certificates").ok_or(Error("invalid_proof"))?)?, + ) + } + None => { + require( + credential.kind == CredentialKind::Token, + "wrong_credential_kind", + )?; + let token = unb64( + proof.get("token").ok_or(Error("invalid_proof"))?.as_str()?, + MAX_BYTES, + )?; + let hash = Sha256::digest(&token); + require( + bool::from(credential.verifier.0.as_slice().ct_eq(hash.as_slice())), + "invalid_token", + )?; + crate::oauth::require_binding( + store, + &credential, + now, + self.oauth_policy.as_ref(), + &self.temporary_ceiling, + resource, + )?; + ("token", vec![]) + } + }; + if credential.revoked_at.is_some() { + if method == "token" + && matches!( + spec.name.as_str(), + "identity.token_rotate" | "identity.custodial_upgrade_finish" + ) + { + replay = store.request_result( + Some(&actor.resource_id), + request.field("request_id")?.as_str()?, + request.field("payload_digest")?.as_str()?, + )?; + } + let previous = replay.as_ref().ok_or(Error("credential_revoked"))?; + let data = previous + .as_object()? + .get("data") + .ok_or(Error("credential_revoked"))? + .as_object()?; + require( + data.get("previous_credential") + .and_then(|v| v.as_str().ok()) + == Some(cid), + "credential_revoked", + )?; + let successor = store.credential( + data.get("credential_id") + .ok_or(Error("credential_revoked"))? + .as_str()?, + )?; + require( + successor.subject_id == actor.resource_id + && successor.revoked_at.is_none() + && successor.expires_at.is_none_or(|v| v > now), + "credential_revoked", + )?; + } + let subject = store.subject( + requested_subject + .filter(|v| !v.is_empty()) + .unwrap_or(&actor.resource_id), + )?; + if let Some(binding) = + store.setting(&format!("delegated_identity:{}", actor.resource_id))? + { + let binding: DelegatedIdentity = decode(&binding)?; + require( + subject.resource_id == binding.grantor, + "delegated_subject_required", + )?; + require(ids.contains(&binding.certificate_id), "delegation_required")?; + if binding.max_uses.is_some_and(|max| binding.uses >= max) { + let previous = store + .request_result( + Some(&subject.resource_id), + request.field("request_id")?.as_str()?, + request.field("payload_digest")?.as_str()?, + )? + .ok_or(Error("delegation_exhausted"))?; + require( + previous + .as_object()? + .get("actor") + .and_then(|v| v.as_str().ok()) + == Some(actor.resource_id.as_str()), + "delegation_exhausted", + )?; + replay = Some(previous); + } + } + require( + !(actor.local_only + || subject.local_only + || actor.resource_id == "u_root" + || subject.resource_id == "u_root") + || matches!(entry, Entry::LocalAdmin), + "local_only", + )?; + require(spec.entries.contains(entry.name()), "entry_not_allowed")?; + let operation = format!("{}@{}", spec.name, spec.version); + require( + credential + .ceiling + .iter() + .any(|g| g.operations.contains(&operation)), + "credential_ceiling", + )?; + if actor.kind == SubjectKind::Custodial + && method == "token" + && spec.effect != "read" + { + let download = spec.name == "transfer.open" + && argument(request, "direction").and_then(Json::as_str).ok() + == Some("download"); + require( + custodial_write(&spec.name) || download, + "custodial_operation_not_supported", + )?; + } + let confirmation = actor.kind == SubjectKind::Custodial + && method == "token" + && matches!( + spec.name.as_str(), + "achievement.finish" + | "achievement.pin" + | "achievement.unpin" + | "achievement.reorder" + ); + require( + !spec.require_signature || method == "signature" || confirmation, + "signature_required", + )?; + let mut validated = Vec::new(); + let mut delegated = false; + for id in ids { + let cert = self.certificates.validate(&id, store, now)?; + require( + cert.subject_id == actor.resource_id && cert.key_id == credential.id, + "certificate_subject_mismatch", + )?; + if cert.kind == CertificateKind::Delegation { + for source in &cert.authority_sources { + if let Some(fact) = + store.setting(&format!("delegation:{}", source.id))? + { + delegated |= fact + .as_object()? + .get("grantor") + .and_then(|v| v.as_str().ok()) + == Some(subject.resource_id.as_str()); + } + } + } + validated.push(cert.resource_id); + } + require( + actor.resource_id == subject.resource_id || delegated, + "delegation_required", + )?; + Principal { + actor: Some(actor.resource_id), + subject: Some(subject.resource_id), + credential_id: Some(credential.id), + method, + certificates: validated, + ceiling: credential.ceiling, + } + } + }; + check_archived(store, principal.actor())?; + check_archived(store, principal.subject())?; + Ok(match replay { + Some(result) => Admission::Replay(result), + None => Admission::Fresh(principal), + }) + } +} +fn optional_string(value: &Json) -> Result> { + if value.is_null() { + Ok(None) + } else { + value.as_str().map(Some) + } +} +fn argument<'a>(request: &'a Request, name: &str) -> Result<&'a Json> { + request + .field("arguments")? + .as_object()? + .get(name) + .ok_or(Error("missing_field")) +} +fn check_archived(store: &dyn AuthorityStore, subject: Option<&str>) -> Result<()> { + require( + !setting_truthy( + store, + &format!("identity_archived:{}", subject.unwrap_or("None")), + )?, + "account_archived", + ) +} +fn custodial_write(name: &str) -> bool { + matches!( + name, + "identity.rename" + | "identity.oauth_approve" + | "content.post_create" + | "content.post_edit" + | "content.file_put" + | "content.attach" + | "file.create" + | "file.copy" + | "file.write" + | "file.patch" + | "discussion.reply" + | "discussion.quote" + | "discussion.repost" + | "identity.token_rotate" + | "identity.custodial_upgrade_start" + | "identity.custodial_upgrade_finish" + | "identity.custodial_rewrap_entry" + | "identity.custodial_rewrap_revision" + | "identity.custodial_rewrap_ack" + | "achievement.start" + | "achievement.answer" + | "achievement.finish" + | "achievement.pin" + | "achievement.unpin" + | "achievement.reorder" + ) +} diff --git a/rust/crates/msg-identity/src/certificates.rs b/rust/crates/msg-identity/src/certificates.rs new file mode 100644 index 00000000..896d4a24 --- /dev/null +++ b/rust/crates/msg-identity/src/certificates.rs @@ -0,0 +1,422 @@ +//! Current-state CA chains, issuance limits and live delegation sources. +use crate::{ + models::*, + policy::*, + require, + store::{setting_truthy, AuthorityStore}, +}; +use msg_core::{digest, Error, Result}; +use std::collections::BTreeSet; + +const ROOT: &str = "u_root"; +const ONLINE_CA: &str = "u_online_ca"; +const ONLINE_ISSUABLE: &[&str] = &[ + "identity.basic", + "resource.basic", + "discussion.basic", + "communication.basic", + "discovery.basic", + "transfer.basic", + "group.basic", + "cert.request", + "git.basic", + "hosting.basic", + "keystore.basic", + "batch.basic", + "sharing.basic", + "money.basic", + "store.basic", + "bounty.basic", + "orders.basic", + "delivery.basic", +]; + +pub struct CertificateValidator { + registry: Registry, + root: Certificate, + root_bytes: String, + root_public_key: [u8; 32], + service: String, +} +impl CertificateValidator { + /// The anchor and registry must come from trusted local configuration. + pub fn new( + registry: Registry, + root: Certificate, + root_public_key: [u8; 32], + service: String, + ) -> Result { + let root_bytes = wire(&root)?.canonical()?; + Ok(Self { + registry, + root, + root_bytes, + root_public_key, + service, + }) + } + pub fn root_id(&self) -> &str { + &self.root.resource_id + } + pub fn service(&self) -> &str { + &self.service + } + pub fn validate( + &self, + id: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + Validation { + validator: self, + store, + now, + work: 0, + } + .walk(id, &BTreeSet::new(), None, None) + } + pub fn validate_publication( + &self, + cert: &Certificate, + csr: &CertificateRequest, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + let bytes = csr.signing_bytes()?; + require(csr.request_digest == digest(&bytes), "csr_digest_mismatch")?; + msg_crypto::verify(&csr.public_key.0, &bytes, &csr.possession_proof, "csr")?; + require( + cert.subject_id == csr.subject_id + && cert.issuer_id == csr.requested_issuer + && cert.key_id == msg_crypto::key_id(&csr.public_key.0) + && cert.kind == csr.kind + && wire(&cert.grants)? == wire(&csr.grants)? + && wire(&cert.issuance)? == wire(&csr.issuance)? + && wire(&cert.authority_sources)? == wire(&csr.authority_sources)? + && cert.delegation_depth == csr.delegation_depth + && cert.target_service == csr.target_service, + "csr_certificate_mismatch", + )?; + require( + duration_within(cert, csr.requested_ttl_seconds), + "csr_ttl_exceeded", + )?; + Validation { + validator: self, + store, + now, + work: 0, + } + .walk( + &cert.resource_id, + &BTreeSet::new(), + Some(cert), + Some(&csr.resource_id), + ) + } +} +fn duration_within(cert: &Certificate, seconds: u64) -> bool { + let duration = cert.expires_at.0 - cert.not_before.0; + duration + .num_microseconds() + .is_some_and(|micros| micros >= 0 && (micros as u128) <= u128::from(seconds) * 1_000_000) +} + +struct Validation<'a> { + validator: &'a CertificateValidator, + store: &'a dyn AuthorityStore, + now: Timestamp, + work: usize, +} +impl Validation<'_> { + fn spend(&mut self) -> Result<()> { + self.work += 1; + require(self.work <= 4096, "authority_work_limit") + } + fn allowed_issuance(&self, grant: &Grant, policy: &IssuancePolicy) -> Result { + self.validator.registry.validate_grant(grant, self.store)?; + for allowed in &policy.issue_grants { + if grant_subset(grant, allowed, self.store)? { + return Ok(true); + } + } + Ok(false) + } + fn can_issue(&self, parent: &Certificate, capability: &str, resource: &str) -> Result { + for grant in &parent.grants { + if grant_covers(grant, capability, "cert.publish@1", resource, self.store)? { + return Ok(true); + } + } + Ok(false) + } + fn walk( + &mut self, + id: &str, + seen: &BTreeSet, + proposed: Option<&Certificate>, + issuance_resource: Option<&str>, + ) -> Result { + self.spend()?; + require(!seen.contains(id) && seen.len() < 32, "certificate_cycle")?; + let mut seen = seen.clone(); + seen.insert(id.to_owned()); + let cert = match proposed { + Some(cert) => cert.clone(), + None => self.store.certificate(id)?, + }; + require(cert.resource_id == id, "storage_record_mismatch")?; + require( + cert.target_service == self.validator.service, + "wrong_service", + )?; + require( + cert.not_before <= self.now && self.now < cert.expires_at, + "certificate_expired", + )?; + if proposed.is_none() { + require(!self.store.certificate_revoked(id)?, "certificate_revoked")?; + } + if id == self.validator.root_id() { + require( + wire(&cert)?.canonical()? == self.validator.root_bytes, + "trust_anchor_mismatch", + )?; + require( + cert.subject_id == ROOT + && cert.issuer_id == ROOT + && cert.parent_certificate_id.is_none() + && cert.kind == CertificateKind::Ca, + "invalid_root_certificate", + )?; + msg_crypto::verify( + &self.validator.root_public_key, + &cert.signing_bytes()?, + &cert.signature, + "certificate", + )?; + return Ok(cert); + } + require( + cert.subject_id != ROOT && cert.parent_certificate_id.is_some(), + "local_only", + )?; + let parent = self.walk( + cert.parent_certificate_id + .as_deref() + .ok_or(Error("local_only"))?, + &seen, + None, + None, + )?; + require( + parent.kind == CertificateKind::Ca && parent.issuance.is_some(), + "issuer_not_ca", + )?; + let policy = parent.issuance.as_ref().ok_or(Error("issuer_not_ca"))?; + require(cert.issuer_id == parent.subject_id, "issuer_mismatch")?; + let key = self.store.credential(&parent.key_id)?; + require( + key.subject_id == parent.subject_id && key.revoked_at.is_none(), + "issuer_key_revoked", + )?; + msg_crypto::verify( + &key.verifier.0, + &cert.signing_bytes()?, + &cert.signature, + "certificate", + )?; + let subject_key = self.store.credential(&cert.key_id)?; + require( + subject_key.subject_id == cert.subject_id, + "certificate_key_mismatch", + )?; + require(subject_key.current_at(self.now), "certificate_key_revoked")?; + require( + parent.not_before <= cert.not_before && cert.expires_at <= parent.expires_at, + "certificate_validity_escalation", + )?; + require( + duration_within(&cert, policy.max_cert_ttl_seconds), + "certificate_ttl_escalation", + )?; + require( + cert.delegation_depth <= policy.max_delegation_depth, + "delegation_depth_exceeded", + )?; + let resource = match issuance_resource { + Some(id) => id.to_owned(), + None => self + .store + .setting(&format!("certificate_request:{}", cert.resource_id))? + .map(|v| v.as_str().map(str::to_owned)) + .transpose()? + .unwrap_or_else(|| cert.resource_id.clone()), + }; + require( + self.can_issue(&parent, "cert.issue", &resource)?, + "issuer_cannot_issue", + )?; + for grant in &cert.grants { + require( + self.allowed_issuance(grant, policy)?, + "issuance_scope_exceeded", + )?; + } + if cert.subject_id == ONLINE_CA { + require( + parent.resource_id == self.validator.root_id() + && cert.kind == CertificateKind::Ca + && cert + .issuance + .as_ref() + .is_some_and(|p| p.max_child_ca_depth == 0) + && cert.grants.len() == 1 + && cert.grants[0].capability == "cert.issue" + && cert.grants[0].operations.contains("cert.publish@1"), + "online_ca_policy_exceeded", + )?; + for grant in &cert + .issuance + .as_ref() + .ok_or(Error("online_ca_policy_exceeded"))? + .issue_grants + { + require( + ONLINE_ISSUABLE.contains(&grant.capability.as_str()) + && !self + .validator + .registry + .capability(&grant.capability, grant.version)? + .ca_only, + "online_ca_policy_exceeded", + )?; + } + } + if parent.subject_id == ONLINE_CA { + require( + matches!( + cert.kind, + CertificateKind::Identity | CertificateKind::Delegation + ) && (cert.kind != CertificateKind::Delegation + || !cert.authority_sources.is_empty()) + && cert + .grants + .iter() + .all(|g| ONLINE_ISSUABLE.contains(&g.capability.as_str())), + "online_ca_policy_exceeded", + )?; + } + if cert.kind == CertificateKind::Ca { + require( + cert.issuance.is_some() && policy.max_child_ca_depth > 0, + "ca_depth_exceeded", + )?; + let child = cert.issuance.as_ref().ok_or(Error("ca_depth_exceeded"))?; + require( + self.can_issue(&parent, "cert.ca.issue", &resource)?, + "issuer_cannot_issue_ca", + )?; + require( + child.max_child_ca_depth < policy.max_child_ca_depth, + "ca_depth_exceeded", + )?; + let mut level = 1; + let mut ancestor = parent.clone(); + while ancestor.resource_id != self.validator.root_id() { + self.spend()?; + level += 1; + require(level <= 3, "ca_depth_exceeded")?; + ancestor = self.store.certificate( + ancestor + .parent_certificate_id + .as_deref() + .ok_or(Error("issuer_not_ca"))?, + )?; + } + require(child.max_child_ca_depth <= 3 - level, "ca_depth_exceeded")?; + require( + child.max_cert_ttl_seconds <= policy.max_cert_ttl_seconds + && child.max_delegation_depth <= policy.max_delegation_depth, + "issuance_policy_escalation", + )?; + for grant in &child.issue_grants { + require( + self.allowed_issuance(grant, policy)?, + "issuance_scope_exceeded", + )?; + } + } else { + require(cert.issuance.is_none(), "unexpected_issuance_policy")?; + } + for source in &cert.authority_sources { + self.authority(source, &cert, &seen)?; + } + Ok(cert) + } + fn authority( + &mut self, + source: &ResourceRef, + cert: &Certificate, + seen: &BTreeSet, + ) -> Result<()> { + self.spend()?; + let resource = self.store.resource(&source.id)?; + require( + resource.kind == "delegation" && resource.state == "active", + "authority_source_inactive", + )?; + require( + source.revision == resource.revision, + "authority_source_changed", + )?; + let fact: DelegationFact = decode( + &self + .store + .setting(&format!("delegation:{}", source.id))? + .ok_or(Error("authority_source_invalid"))?, + )?; + require(fact.grantee == cert.subject_id, "authority_source_invalid")?; + require(fact.expires_at > self.now, "authority_source_expired")?; + let key = fact + .grantor_credential_id + .as_ref() + .map(|id| self.store.credential(id)) + .transpose()?; + if let Some(key) = &key { + require(key.current_at(self.now), "authority_source_inactive")?; + require( + !setting_truthy(self.store, &format!("identity_archived:{}", key.subject_id))?, + "authority_source_inactive", + )?; + } + for grant in &fact.grants { + require( + self.store.resource(&grant.scope.resource_id)?.owner == fact.grantor, + "authority_source_lost", + )?; + if let Some(key) = &key { + let mut allowed = false; + for parent in &key.ceiling { + allowed |= grant_subset(grant, parent, self.store)?; + } + require(allowed, "authority_source_lost")?; + } + } + for grant in cert + .grants + .iter() + .chain(cert.issuance.iter().flat_map(|p| &p.issue_grants)) + { + let mut allowed = false; + for parent in &fact.grants { + allowed |= grant_subset(grant, parent, self.store)?; + } + require(allowed, "delegation_scope_exceeded")?; + } + if let Some(parent) = &fact.parent_certificate { + self.walk(parent, seen, None, None)?; + } + Ok(()) + } +} diff --git a/rust/crates/msg-identity/src/lib.rs b/rust/crates/msg-identity/src/lib.rs new file mode 100644 index 00000000..51d782f0 --- /dev/null +++ b/rust/crates/msg-identity/src/lib.rs @@ -0,0 +1,20 @@ +//! Native identity verification against one current authority snapshot. +//! +//! An authenticated principal is NOT authorization to execute an operation. +//! Full resource authorization and the daemon are separate cutover gates. +pub mod authentication; +pub mod certificates; +pub mod models; +pub mod oauth; +pub mod policy; +pub mod store; + +use msg_core::{Error, Result}; + +pub(crate) fn require(condition: bool, code: &'static str) -> Result<()> { + if condition { + Ok(()) + } else { + Err(Error(code)) + } +} diff --git a/rust/crates/msg-identity/src/models.rs b/rust/crates/msg-identity/src/models.rs new file mode 100644 index 00000000..fa88ebe4 --- /dev/null +++ b/rust/crates/msg-identity/src/models.rs @@ -0,0 +1,292 @@ +//! Typed Python wire records. Credentials and signatures deliberately lack Debug. +use chrono::{DateTime, Datelike, Timelike, Utc}; +use msg_core::{b64, unb64, Error, Json, Result}; +use msg_crypto::Signature; +use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::{BTreeMap, BTreeSet}; + +pub fn decode(value: &Json) -> Result { + serde_json::from_str(&value.canonical()?).map_err(|_| Error("invalid_identity_record")) +} +pub fn wire(value: &T) -> Result { + Json::parse(&serde_json::to_string(value).map_err(|_| Error("invalid_identity_record"))?) +} + +// Serde normally makes every Option field optional. Python requires fields +// without a dataclass default, even when their explicit value may be null. +fn required_nullable<'de, D, T>(d: D) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, + T: Deserialize<'de>, +{ + Option::::deserialize(d) +} +fn positive_u32<'de, D: Deserializer<'de>>(d: D) -> std::result::Result { + std::num::NonZeroU32::deserialize(d).map(std::num::NonZeroU32::get) +} +fn positive_u64<'de, D: Deserializer<'de>>(d: D) -> std::result::Result { + std::num::NonZeroU64::deserialize(d).map(std::num::NonZeroU64::get) +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Timestamp(pub DateTime); +impl Timestamp { + pub fn parse(value: &str) -> Result { + if !value.ends_with('Z') { + return Err(Error("invalid_datetime")); + } + let time = DateTime::parse_from_rfc3339(value) + .map_err(|_| Error("invalid_datetime"))? + .with_timezone(&Utc); + if !(1..=9999).contains(&time.year()) || time.nanosecond() >= 1_000_000_000 { + return Err(Error("invalid_datetime")); + } + // Python truncates sub-microsecond digits, never rounds them. + Ok(Self( + time.with_nanosecond(time.nanosecond() / 1000 * 1000) + .ok_or(Error("invalid_datetime"))?, + )) + } +} +impl Serialize for Timestamp { + fn serialize(&self, s: S) -> std::result::Result { + s.serialize_str(&self.0.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string()) + } +} +impl<'de> Deserialize<'de> for Timestamp { + fn deserialize>(d: D) -> std::result::Result { + Self::parse(&String::deserialize(d)?).map_err(serde::de::Error::custom) + } +} + +#[derive(Clone)] +pub struct Verifier(pub Vec); +impl Serialize for Verifier { + fn serialize(&self, s: S) -> std::result::Result { + s.serialize_str(&b64(&self.0)) + } +} +impl<'de> Deserialize<'de> for Verifier { + fn deserialize>(d: D) -> std::result::Result { + unb64(&String::deserialize(d)?, 4096) + .map(Self) + .map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ResourceRef { + pub id: String, + #[serde(default)] + pub revision: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Scope { + pub resource_id: String, + #[serde(default)] + pub descendants: bool, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Grant { + pub capability: String, + #[serde(deserialize_with = "positive_u32")] + pub version: u32, + pub scope: Scope, + pub operations: BTreeSet, + pub constraints: BTreeMap, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct IssuancePolicy { + pub issue_grants: Vec, + #[serde(deserialize_with = "positive_u64")] + pub max_cert_ttl_seconds: u64, + pub max_child_ca_depth: u32, + pub max_delegation_depth: u32, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SubjectKind { + Temporary, + Registered, + Custodial, + System, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Subject { + pub resource_id: String, + pub kind: SubjectKind, + pub primary_group: String, + pub auth_version: u64, + #[serde(default)] + pub local_only: bool, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CredentialKind { + SigningKey, + SshKey, + Token, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Credential { + pub id: String, + pub subject_id: String, + pub kind: CredentialKind, + pub verifier: Verifier, + pub ceiling: Vec, + pub not_before: Timestamp, + #[serde(deserialize_with = "required_nullable")] + pub expires_at: Option, + #[serde(deserialize_with = "required_nullable")] + pub revoked_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_credential_id: Option, +} +impl Credential { + pub fn current_at(&self, now: Timestamp) -> bool { + self.revoked_at.is_none() + && self.not_before <= now + && self.expires_at.is_none_or(|expires| now < expires) + } +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CertificateKind { + Identity, + Delegation, + Capability, + Ca, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Certificate { + pub resource_id: String, + pub serial: String, + pub subject_id: String, + pub key_id: String, + pub issuer_id: String, + #[serde(deserialize_with = "required_nullable")] + pub parent_certificate_id: Option, + pub authority_sources: Vec, + pub kind: CertificateKind, + pub grants: Vec, + pub not_before: Timestamp, + pub expires_at: Timestamp, + pub target_service: String, + pub delegation_depth: u32, + #[serde(deserialize_with = "required_nullable")] + pub issuance: Option, + pub signature: Signature, +} +impl Certificate { + pub fn signing_bytes(&self) -> Result> { + let mut fields = wire(self)?.into_object()?; + fields.remove("signature"); + Ok(Json::from_object(fields).canonical()?.into_bytes()) + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CertificateRequest { + pub resource_id: String, + pub applicant: String, + pub subject_id: String, + pub requested_issuer: String, + pub public_key: Verifier, + pub kind: CertificateKind, + pub grants: Vec, + #[serde(deserialize_with = "required_nullable")] + pub issuance: Option, + #[serde(deserialize_with = "positive_u64")] + pub requested_ttl_seconds: u64, + pub target_service: String, + pub delegation_depth: u32, + pub authority_sources: Vec, + pub request_digest: String, + pub possession_proof: Signature, +} +impl CertificateRequest { + pub fn signing_bytes(&self) -> Result> { + let mut fields = wire(self)?.into_object()?; + for key in ["resource_id", "request_digest", "possession_proof"] { + fields.remove(key); + } + Ok(Json::from_object(fields).canonical()?.into_bytes()) + } +} + +/// Only authority-relevant fields of a resource are exposed by the store. +/// The original body remains private to storage; it is not a public read response. +#[derive(Clone, Deserialize)] +pub struct ResourceAuthority { + pub id: String, + #[serde(rename = "type")] + pub kind: String, + #[serde(deserialize_with = "required_nullable")] + pub parent: Option, + pub owner: String, + pub group: String, + pub mode: String, + pub state: String, + #[serde(deserialize_with = "required_nullable")] + pub revision: Option, +} +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DelegationFact { + pub grantor: String, + pub grantee: String, + pub grants: Vec, + pub expires_at: Timestamp, + #[serde(deserialize_with = "required_nullable")] + pub parent_certificate: Option, + #[serde(default)] + pub grantor_credential_id: Option, +} +#[derive(Clone, Deserialize)] +pub struct DelegatedIdentity { + pub grantor: String, + pub certificate_id: String, + pub max_uses: Option, + pub uses: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn required_null_is_not_a_missing_field() { + let value = Json::parse(r#"{"id":"k","subject_id":"u","kind":"token","verifier":"","ceiling":[],"not_before":"2026-01-01T00:00:00Z","expires_at":null,"revoked_at":null}"#).unwrap(); + assert!(decode::(&value).is_ok()); + for field in ["expires_at", "revoked_at"] { + let mut fields = value.clone().into_object().unwrap(); + fields.remove(field); + assert!(decode::(&Json::from_object(fields)).is_err()); + } + } + #[test] + fn grant_versions_and_issuance_ttls_are_positive() { + let value = Json::parse(r#"{"capability":"x","version":0,"scope":{"resource_id":"r"},"operations":[],"constraints":{}}"#).unwrap(); + assert!(decode::(&value).is_err()); + let value = Json::parse(r#"{"issue_grants":[],"max_cert_ttl_seconds":0,"max_child_ca_depth":0,"max_delegation_depth":0}"#).unwrap(); + assert!(decode::(&value).is_err()); + } + #[test] + fn json_records_keep_large_integers_and_reject_duplicate_members() { + let value = + Json::parse(r#"{"value":1234567890123456789012345678901234567890,"float":1.0}"#) + .unwrap(); + let decoded: BTreeMap = decode(&value).unwrap(); + assert_eq!( + wire(&decoded).unwrap().canonical().unwrap(), + value.canonical().unwrap() + ); + assert!(serde_json::from_str::(r#"{"value":1,"value":2}"#).is_err()); + } +} diff --git a/rust/crates/msg-identity/src/oauth.rs b/rust/crates/msg-identity/src/oauth.rs new file mode 100644 index 00000000..63cbcc03 --- /dev/null +++ b/rust/crates/msg-identity/src/oauth.rs @@ -0,0 +1,258 @@ +//! Current-state browser/API/OAuth bindings; no issuance or refresh writes. +//! The resource audience is a trusted adapter value, never a packet field. +use crate::{ + models::*, + policy::grant_subset, + require, + store::{truthy, AuthorityStore}, +}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::BTreeSet; + +#[derive(Clone, Deserialize)] +pub struct OAuthClientPolicy { + pub client_id: String, + pub scopes: BTreeSet, +} +#[derive(Clone, Deserialize)] +pub struct OAuthPolicy { + pub enabled: bool, + pub clients: Vec, +} +#[derive(Clone)] +pub struct OAuthState { + pub expires_at: Timestamp, + pub body: Json, +} +#[derive(Deserialize)] +struct Source { + parent: String, + subject: String, + auth_version: u64, + #[serde(default)] + custodial: bool, + #[serde(default)] + ceiling: Vec, + #[serde(default)] + session: Option, +} +#[derive(Deserialize)] +struct Family { + parent: String, + subject: String, + client_id: String, + scopes: BTreeSet, + #[serde(default)] + resource: Option, + ceiling: Vec, +} + +fn field<'a>(body: &'a Json, name: &str) -> Result<&'a Json> { + body.as_object()?.get(name).ok_or(Error("invalid_grant")) +} +fn revoked(body: &Json) -> Result { + body.as_object()? + .get("revoked") + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn get(store: &dyn AuthorityStore, id: &str, now: Timestamp) -> Result { + let state = store.oauth_state(id)?.ok_or(Error("invalid_grant"))?; + require(state.expires_at > now, "invalid_grant")?; + Ok(state.body) +} +fn require_ceiling(store: &dyn AuthorityStore, ceiling: &[Grant], parent: &[Grant]) -> Result<()> { + for grant in ceiling { + let mut allowed = false; + for source in parent { + allowed |= grant_subset(grant, source, store)?; + } + require(allowed, "invalid_grant")?; + } + Ok(()) +} +fn require_source( + store: &dyn AuthorityStore, + source: &Json, + now: Timestamp, + custodial_ceiling: &[Grant], + seen: &mut BTreeSet, +) -> Result<()> { + let source: Source = decode(source).map_err(|_| Error("invalid_grant"))?; + let parent = store.credential(&source.parent)?; + let subject = store.subject(&source.subject)?; + require( + parent.subject_id == subject.resource_id + && !subject.local_only + && subject.auth_version == source.auth_version + && parent.current_at(now), + "invalid_grant", + )?; + if source.custodial { + let vault = store.custodial_binding(&subject.resource_id)?; + require( + subject.kind == SubjectKind::Custodial + && vault + .as_ref() + .is_some_and(|(key, status)| key == &parent.id && status == "active"), + "invalid_grant", + )?; + } + require_ceiling( + store, + &source.ceiling, + if source.custodial { + custodial_ceiling + } else { + &parent.ceiling + }, + )?; + if let Some(session) = source.session.filter(|s| !s.is_empty()) { + // The Python path currently recurses without a bound. Restored cyclic + // or oversized chains fail closed rather than exhausting the stack. + require( + seen.len() < 32 && seen.insert(session.clone()), + "oauth_source_cycle", + )?; + let body = get(store, &session, now)?; + require(!revoked(&body)?, "invalid_grant")?; + require_source(store, &body, now, custodial_ceiling, seen)?; + } + Ok(()) +} +fn enabled(policy: Option<&OAuthPolicy>) -> Result<&OAuthPolicy> { + policy.filter(|p| p.enabled).ok_or(Error("oauth_disabled")) +} + +pub fn require_binding( + store: &dyn AuthorityStore, + credential: &Credential, + now: Timestamp, + policy: Option<&OAuthPolicy>, + custodial_ceiling: &[Grant], + resource: Option<&str>, +) -> Result<()> { + if let Some(browser) = store.oauth_state(&format!("browser:{}", credential.id))? { + enabled(policy)?; + let source: Source = decode(&browser.body).map_err(|_| Error("invalid_grant"))?; + require( + credential.source_credential_id.as_deref() == Some(&source.parent) + && credential.subject_id == source.subject, + "invalid_grant", + )?; + require_source( + store, + &browser.body, + now, + custodial_ceiling, + &mut BTreeSet::new(), + )?; + return require_ceiling( + store, + &credential.ceiling, + &decode::>(field(&browser.body, "ceiling")?)?, + ); + } + let api = store.oauth_state(&format!("api:{}", credential.id))?; + if let Some(api) = &api { + let source: Source = decode(&api.body).map_err(|_| Error("invalid_grant"))?; + require( + credential.source_credential_id.as_deref() == Some(&source.parent), + "invalid_grant", + )?; + require_source( + store, + &api.body, + now, + custodial_ceiling, + &mut BTreeSet::new(), + )?; + require_ceiling( + store, + &credential.ceiling, + &store.credential(&source.parent)?.ceiling, + )?; + } + let access = store.oauth_state(&format!("access:{}", credential.id))?; + let Some(access) = access else { + return require( + !credential.id.starts_with("t_oauth_") + && !credential.id.starts_with("t_browser_") + && (credential.source_credential_id.is_none() || api.is_some()), + "invalid_grant", + ); + }; + let body = get(store, field(&access.body, "family")?.as_str()?, now)?; + let family: Family = decode(&body).map_err(|_| Error("invalid_grant"))?; + require( + credential.source_credential_id.as_deref() == Some(&family.parent), + "invalid_grant", + )?; + let client = enabled(policy)? + .clients + .iter() + .find(|c| c.client_id == family.client_id) + .ok_or(Error("invalid_client"))?; + require( + family.scopes.is_subset(&client.scopes) && !revoked(&body)?, + "invalid_grant", + )?; + let has_mcp = family.scopes.iter().any(|s| { + matches!( + s.as_str(), + "msg.mcp.read" | "msg.mcp.message" | "msg.mcp.post" + ) + }); + if let Some(audience) = &family.resource { + require( + credential.subject_id == family.subject && resource == Some(audience), + "invalid_grant", + )?; + require( + has_mcp && !family.scopes.contains("msg.read") && !family.scopes.contains("msg.write"), + "invalid_grant", + )?; + for grant in &credential.ceiling { + require( + grant + .operations + .iter() + .all(|op| family.scopes.iter().any(|scope| mcp_operation(scope, op))), + "invalid_grant", + )?; + } + } else { + require(!has_mcp, "invalid_grant")?; + } + require_source(store, &body, now, custodial_ceiling, &mut BTreeSet::new())?; + require_ceiling(store, &credential.ceiling, &family.ceiling) +} + +fn mcp_operation(scope: &str, operation: &str) -> bool { + match scope { + "msg.mcp.read" => matches!( + operation, + "discovery.get@1" + | "discovery.resolve@1" + | "discovery.read_segment@1" + | "discovery.read_query@3" + | "discovery.lexical_search@5" + | "communication.inbox@1" + | "communication.dm_list@1" + | "communication.conversation_get@1" + | "communication.changes@1" + ), + "msg.mcp.message" => matches!( + operation, + "communication.dm_request@3" + | "communication.dm_send@2" + | "communication.dm_accept@2" + | "communication.dm_reject@2" + | "discussion.ack@1" + ), + "msg.mcp.post" => matches!(operation, "content.post_create@2" | "discussion.reply@2"), + _ => false, + } +} diff --git a/rust/crates/msg-identity/src/policy.rs b/rust/crates/msg-identity/src/policy.rs new file mode 100644 index 00000000..cb37c205 --- /dev/null +++ b/rust/crates/msg-identity/src/policy.rs @@ -0,0 +1,241 @@ +//! Pure mode/scope/ceiling rules. None of these alone authorizes an operation. +use crate::{models::*, require, store::AuthorityStore}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet}; + +pub const CERTGATE: u16 = 0o4000; +pub const SETGID: u16 = 0o2000; +pub const STICKY: u16 = 0o1000; + +pub fn parse_mode(value: &str) -> Result { + require( + value.len() == 4 && value.bytes().all(|b| (b'0'..=b'7').contains(&b)), + "invalid_mode", + )?; + u16::from_str_radix(value, 8).map_err(|_| Error("invalid_mode")) +} +pub fn class_bits(mode: u16, class: &str) -> Result { + require(mode <= 0o7777, "invalid_mode")?; + let shift = match class { + "owner" => 6, + "group" => 3, + "other" => 0, + _ => return Err(Error("invalid_permission_class")), + }; + Ok((mode >> shift) & 7) +} +pub fn allows( + resource: &ResourceAuthority, + subject: Option<&str>, + memberships: &BTreeSet, + permission: &str, +) -> Result { + let class = if subject == Some(&resource.owner) { + "owner" + } else if subject.is_some() && memberships.contains(&resource.group) { + "group" + } else { + "other" + }; + let mask = match permission { + "read" | "list" => 4, + "write" => 2, + "execute" | "traverse" => 1, + _ => return Err(Error("invalid_permission")), + }; + Ok(class_bits(parse_mode(&resource.mode)?, class)? & mask == mask) +} +pub fn scope_contains(scope: &Scope, id: &str, store: &dyn AuthorityStore) -> Result { + if scope.resource_id == id { + return Ok(true); + } + if !scope.descendants { + return Ok(false); + } + Ok(store + .ancestors(id)? + .iter() + .any(|r| r.id == scope.resource_id)) +} +pub fn scope_subset(child: &Scope, parent: &Scope, store: &dyn AuthorityStore) -> Result { + if child.resource_id == parent.resource_id { + return Ok(!child.descendants || parent.descendants); + } + Ok(parent.descendants && scope_contains(parent, &child.resource_id, store)?) +} +pub fn grant_covers( + grant: &Grant, + capability: &str, + operation: &str, + id: &str, + store: &dyn AuthorityStore, +) -> Result { + Ok(grant.capability == capability + && grant.version == 1 + && grant.operations.contains(operation) + && scope_contains(&grant.scope, id, store)?) +} +pub fn constraints_subset( + child: &BTreeMap, + parent: &BTreeMap, +) -> Result { + for (name, value) in parent { + let Some(proposed) = child.get(name) else { + return Ok(false); + }; + match name.as_str() { + "hosts" | "methods" | "ports" | "schemes" => { + let allowed = value.as_array()?; + if !proposed + .as_array()? + .iter() + .all(|v| allowed.iter().any(|a| python_equal(a, v))) + { + return Ok(false); + } + } + "max_response_bytes" | "timeout_ms" | "max_redirects" => { + let Ok(p) = proposed.as_integer() else { + return Ok(false); + }; + let q = integral_number(value).ok_or(Error("schema_validation"))?; + if compare_integers(p, &q).is_gt() { + return Ok(false); + } + } + _ if !python_equal(proposed, value) => return Ok(false), + _ => (), + } + } + Ok(true) +} +pub fn grant_subset(child: &Grant, parent: &Grant, store: &dyn AuthorityStore) -> Result { + Ok(child.capability == parent.capability + && child.version == parent.version + && child.operations.is_subset(&parent.operations) + && scope_subset(&child.scope, &parent.scope, store)? + && constraints_subset(&child.constraints, &parent.constraints)?) +} + +/// Trusted registry projection, NOT request data or a certificate-supplied schema. +#[derive(Clone, Deserialize)] +pub struct CapabilityPolicy { + pub name: String, + pub version: u32, + pub scope_types: BTreeSet, + pub operations: BTreeSet, + pub ca_only: bool, + pub constraints_schema: Option, +} +pub struct Registry { + capabilities: BTreeMap<(String, u32), CapabilityPolicy>, +} +impl Registry { + pub fn new(specs: Vec) -> Result { + let mut capabilities = BTreeMap::new(); + for spec in specs { + require( + capabilities + .insert((spec.name.clone(), spec.version), spec) + .is_none(), + "duplicate_capability", + )?; + } + Ok(Self { capabilities }) + } + pub fn capability(&self, name: &str, version: u32) -> Result<&CapabilityPolicy> { + self.capabilities + .get(&(name.to_owned(), version)) + .ok_or(Error("unknown_capability")) + } + pub fn validate_grant(&self, grant: &Grant, store: &dyn AuthorityStore) -> Result<()> { + let spec = self.capability(&grant.capability, grant.version)?; + let resource = store.resource(&grant.scope.resource_id)?; + require( + spec.scope_types.contains(&resource.kind), + "invalid_scope_type", + )?; + require( + !grant.operations.is_empty() && grant.operations.is_subset(&spec.operations), + "invalid_grant_operations", + )?; + match &spec.constraints_schema { + None => require(grant.constraints.is_empty(), "unknown_grant_constraint"), + Some(schema) if schema.id == "schema:network-constraints" => { + validate_network(&grant.constraints) + } + _ => Err(Error("unsupported_constraints_schema")), + } + } +} +// Compare exact integer values, never round a large integer through f64. +fn compare_integers(left: &str, right: &str) -> std::cmp::Ordering { + let (ln, rn) = (left.starts_with('-'), right.starts_with('-')); + if ln != rn { + return rn.cmp(&ln); + } + let (left, right) = (left.trim_start_matches('-'), right.trim_start_matches('-')); + let order = left.len().cmp(&right.len()).then_with(|| left.cmp(right)); + if ln { + order.reverse() + } else { + order + } +} +fn integral_number(value: &Json) -> Option { + if let Ok(integer) = value.as_integer() { + return Some(integer.to_owned()); + } + // JSON Schema considers 1.0 an integer; Python's pure subset rule still + // requires a literal int for the proposed upper bound, as checked above. + let text = value.canonical().ok()?; + let float: f64 = text.parse().ok()?; + if !float.is_finite() || float.fract() != 0.0 { + return None; + } + Some(if float == 0.0 { + "0".to_owned() + } else { + format!("{float:.0}") + }) +} +fn python_equal(left: &Json, right: &Json) -> bool { + if left == right { + return true; + } + if let (Some(left), Some(right)) = (integral_number(left), integral_number(right)) { + return left == right; + } + false +} +fn validate_network(values: &BTreeMap) -> Result<()> { + for (key, value) in values { + let valid = match key.as_str() { + "hosts" | "methods" | "schemes" => value.as_array().is_ok_and(|values| { + values.iter().all(|v| { + v.as_str().is_ok_and(|v| match key.as_str() { + "schemes" => matches!(v, "http" | "https"), + "methods" => matches!( + v, + "GET" | "HEAD" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS" + ), + _ => true, + }) + }) + }), + "ports" => value.as_array().is_ok_and(|values| { + values.iter().all(|v| { + integral_number(v) + .and_then(|v| v.parse::().ok()) + .is_some_and(|v| v > 0) + }) + }), + "max_redirects" | "timeout_ms" | "max_response_bytes" => integral_number(value) + .is_some_and(|v| !v.starts_with('-') && (key == "max_redirects" || v != "0")), + _ => false, + }; + require(valid, "schema_validation")?; + } + Ok(()) +} diff --git a/rust/crates/msg-identity/src/store.rs b/rust/crates/msg-identity/src/store.rs new file mode 100644 index 00000000..078abce6 --- /dev/null +++ b/rust/crates/msg-identity/src/store.rs @@ -0,0 +1,132 @@ +//! Narrow current-state read port. A snapshot must not mix database generations. +use crate::{models::*, require}; +use msg_core::{Error, Json, Result}; +use std::collections::BTreeSet; + +pub struct RecoveryDelivery { + pub verifier: String, + pub expires_at: Timestamp, + pub consumed: bool, + pub request_id: String, +} + +#[derive(Clone, Copy)] +pub enum Record { + Subject, + Credential, + Certificate, + Resource, +} + +pub trait AuthorityStore { + fn record(&self, kind: Record, id: &str) -> Result; + fn setting(&self, key: &str) -> Result>; + /// Presence, not decoded truthiness: even `false`/`null` quarantine denies. + fn has_setting(&self, key: &str) -> Result; + fn certificate_revoked(&self, id: &str) -> Result; + fn request_result(&self, subject: Option<&str>, id: &str, digest: &str) + -> Result>; + fn oauth_state(&self, id: &str) -> Result>; + fn custodial_binding(&self, subject: &str) -> Result>; + fn recovery_delivery( + &self, + credential: &str, + subject: &str, + ) -> Result>; + + fn subject(&self, id: &str) -> Result { + let value: Subject = decode(&self.record(Record::Subject, id)?)?; + require(value.resource_id == id, "storage_record_mismatch")?; + Ok(value) + } + fn credential(&self, id: &str) -> Result { + let value: Credential = decode(&self.record(Record::Credential, id)?)?; + require(value.id == id, "storage_record_mismatch")?; + Ok(value) + } + fn certificate(&self, id: &str) -> Result { + let value: Certificate = decode(&self.record(Record::Certificate, id)?)?; + require(value.resource_id == id, "storage_record_mismatch")?; + Ok(value) + } + fn resource(&self, id: &str) -> Result { + let value: ResourceAuthority = decode(&self.record(Record::Resource, id)?)?; + require(value.id == id, "storage_record_mismatch")?; + Ok(value) + } + fn ancestors(&self, id: &str) -> Result> { + let mut seen = BTreeSet::new(); + let mut result = Vec::new(); + let mut current = self.resource(id)?; + while let Some(parent) = ¤t.parent { + require(seen.insert(current.id.clone()), "parent_cycle")?; + require(seen.len() <= 1024, "authority_work_limit")?; + current = self.resource(parent)?; + result.push(current.clone()); + } + result.reverse(); + Ok(result) + } +} + +pub fn truthy(value: &Json) -> Result { + if value.is_null() { + return Ok(false); + } + if let Ok(value) = value.as_bool() { + return Ok(value); + } + if let Ok(value) = value.as_str() { + return Ok(!value.is_empty()); + } + if let Ok(value) = value.as_array() { + return Ok(!value.is_empty()); + } + if let Ok(value) = value.as_object() { + return Ok(!value.is_empty()); + } + // Arbitrary precision integers are not converted through f64. + let encoded = value.canonical()?; + Ok(!matches!(encoded.as_str(), "0" | "0.0" | "-0.0")) +} + +pub fn setting_truthy(store: &dyn AuthorityStore, key: &str) -> Result { + store + .setting(key)? + .as_ref() + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} + +/// Once a runtime generation is stale it cannot recover without a new instance. +/// No public reset function, and a malformed generation latches the same fence. +pub struct RuntimeGeneration { + generation: Option, + stale: bool, +} +impl RuntimeGeneration { + pub fn capture(store: &dyn AuthorityStore) -> Result { + Ok(Self { + generation: Self::read(store)?, + stale: false, + }) + } + fn read(store: &dyn AuthorityStore) -> Result> { + store + .setting("recovery_runtime_generation")? + .map(|v| { + let s = v.as_str().map_err(|_| Error("recovery_runtime_stale"))?; + require(!s.is_empty(), "recovery_runtime_stale")?; + Ok(s.to_owned()) + }) + .transpose() + } + pub fn require_current(&mut self, store: &dyn AuthorityStore) -> Result<()> { + match Self::read(store) { + Ok(value) if value == self.generation => (), + _ => self.stale = true, + } + require(!self.stale, "recovery_runtime_stale") + } +} diff --git a/rust/crates/msg-protocol/src/lib.rs b/rust/crates/msg-protocol/src/lib.rs index ad9a877f..0e708eb7 100644 --- a/rust/crates/msg-protocol/src/lib.rs +++ b/rust/crates/msg-protocol/src/lib.rs @@ -162,6 +162,12 @@ impl Request { Ok(request) } + /// Immutable decoded fields, for trusted native server layers only. + /// No mutable accessor: validation cannot be invalidated after construction. + pub fn field(&self, name: &str) -> Result<&Json> { + self.fields.get(name).ok_or(Error("missing_field")) + } + pub fn canonical(&self) -> Result { Json::from_object(self.fields.clone()).canonical() } diff --git a/rust/crates/msg-storage/Cargo.toml b/rust/crates/msg-storage/Cargo.toml new file mode 100644 index 00000000..ba620cf9 --- /dev/null +++ b/rust/crates/msg-storage/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "msg-storage" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +msg-core = { path = "../msg-core" } +msg-identity = { path = "../msg-identity" } +rusqlite.workspace = true + +[dev-dependencies] +msg-protocol = { path = "../msg-protocol" } +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/rust/crates/msg-storage/examples/authority_check.rs b/rust/crates/msg-storage/examples/authority_check.rs new file mode 100644 index 00000000..ad3b20b5 --- /dev/null +++ b/rust/crates/msg-storage/examples/authority_check.rs @@ -0,0 +1,244 @@ +//! Local JSONL differential-test helper. Never register this as an HTTP/MCP tool. +//! Inputs are synthetic public fixtures; errors contain codes, never proof bytes. +use msg_core::{digest, unb64, Error, Json, Result, MAX_BYTES}; +use msg_identity::{ + authentication::{Admission, AuthenticationService, Entry, OperationPolicy}, + certificates::CertificateValidator, + models::*, + policy::*, + store::{AuthorityStore, Record}, +}; +use msg_protocol::Request; +use msg_storage::SqliteAuthority; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::{ + collections::BTreeSet, + io::{self, BufRead, Read, Write}, + path::Path, +}; + +#[derive(Deserialize)] +struct Context { + root: Certificate, + root_public: String, + service: String, + capabilities: Vec, + primary_ceiling: Vec, + temporary_ceiling: Vec, + oauth: msg_identity::oauth::OAuthPolicy, +} +impl Context { + fn validator(&self) -> Result { + let public: [u8; 32] = unb64(&self.root_public, 32)? + .try_into() + .map_err(|_| Error("invalid_test_input"))?; + CertificateValidator::new( + Registry::new(self.capabilities.clone())?, + self.root.clone(), + public, + self.service.clone(), + ) + } +} +fn text<'a>(input: &'a Value, key: &str) -> Result<&'a str> { + input + .get(key) + .and_then(Value::as_str) + .ok_or(Error("invalid_test_input")) +} +fn value(input: &Value, key: &str) -> Result { + Json::parse( + &input + .get(key) + .ok_or(Error("invalid_test_input"))? + .to_string(), + ) +} +fn run( + input: Value, + context: &Context, + auth: &mut AuthenticationService, + store: &dyn AuthorityStore, +) -> Result { + let now = || Timestamp::parse(text(&input, "now")?); + match text(&input, "action")? { + "authenticate" => { + let request = Request::parse(text(&input, "raw")?)?; + let policy: OperationPolicy = decode(&value(&input, "policy")?)?; + let entry = match text(&input, "entry")? { + "network" => Entry::Network, + "local_admin" => Entry::LocalAdmin, + "worker" => Entry::Worker, + _ => return Err(Error("invalid_test_input")), + }; + match auth.authenticate_for_resource( + &request, + &policy, + store, + entry, + now()?, + input.get("resource").and_then(Value::as_str), + )? { + Admission::Fresh(p) => Ok(json!({"kind":"fresh", "principal":p})), + Admission::Replay(result) => Ok(json!({"kind":"replay", "result":result})), + } + } + "certificate" => { + let cert = context + .validator()? + .validate(text(&input, "id")?, store, now()?)?; + Ok( + json!({"canonical":wire(&cert)?.canonical()?, "signing_digest":digest(&cert.signing_bytes()?)}), + ) + } + "publication" => { + let cert: Certificate = decode(&value(&input, "certificate")?)?; + let csr: CertificateRequest = decode(&value(&input, "csr")?)?; + let validated = + context + .validator()? + .validate_publication(&cert, &csr, store, now()?)?; + Ok(json!({"canonical":wire(&validated)?.canonical()?})) + } + "result" => Ok( + json!({"result":store.request_result(input.get("subject").and_then(Value::as_str), text(&input,"id")?, text(&input,"digest")?)?}), + ), + "setting" => Ok( + json!({"exists":store.has_setting(text(&input,"key")?)?, "value":store.setting(text(&input,"key")?)?}), + ), + "record_digest" => { + let kind = match text(&input, "kind")? { + "subject" => Record::Subject, + "credential" => Record::Credential, + "certificate" => Record::Certificate, + "resource" => Record::Resource, + _ => return Err(Error("invalid_test_input")), + }; + let raw = store.record(kind, text(&input, "id")?)?; + let normalized = match kind { + Record::Subject => wire(&decode::(&raw)?)?, + Record::Credential => wire(&decode::(&raw)?)?, + Record::Certificate => wire(&decode::(&raw)?)?, + Record::Resource => raw, + }; + Ok(json!({"digest":digest(normalized.canonical()?.as_bytes())})) + } + "scope" => Ok( + json!({"contains":scope_contains(&decode(&value(&input,"scope")?)?, text(&input,"id")?, store)?}), + ), + "subset" => Ok( + json!({"subset":grant_subset(&decode(&value(&input,"child")?)?, &decode(&value(&input,"parent")?)?, store)?}), + ), + "modes" => { + let mut decisions = Vec::new(); + let mut resource = ResourceAuthority { + id: "r".into(), + kind: "file".into(), + parent: None, + owner: "owner".into(), + group: "group".into(), + mode: String::new(), + state: "active".into(), + revision: None, + }; + let group = BTreeSet::from(["group".to_owned()]); + for mode in 0..=0o7777 { + resource.mode = format!("{mode:04o}"); + for subject in [Some("owner"), Some("member"), None] { + for permission in ["read", "write", "execute", "list", "traverse"] { + decisions.push(allows(&resource, subject, &group, permission)?); + } + } + } + Ok( + json!({"count":decisions.len(), "digest":digest(wire(&decisions)?.canonical()?.as_bytes())}), + ) + } + _ => Err(Error("unknown_test_action")), + } +} +fn read_input(reader: &mut impl BufRead) -> io::Result> { + let mut raw = String::new(); + if reader + .take((MAX_BYTES * 8 + 1) as u64) + .read_line(&mut raw)? + == 0 + { + return Ok(None); + } + if raw.len() > MAX_BYTES * 8 { + return Err(io::Error::other("test_input_too_large")); + } + serde_json::from_str(&raw) + .map(Some) + .map_err(|_| io::Error::other("invalid_test_input")) +} +fn output(writer: &mut impl Write, result: Result) -> io::Result<()> { + let value = match result { + Ok(data) => json!({"ok":true,"data":data}), + Err(e) => json!({"ok":false,"code":e.0}), + }; + writeln!(writer, "{value}")?; + writer.flush() +} +fn main() -> io::Result<()> { + let path = std::env::args() + .nth(1) + .ok_or(io::Error::other("test_database_required"))?; + let mut reader = io::stdin().lock(); + let mut writer = io::BufWriter::new(io::stdout().lock()); + let input = read_input(&mut reader)?.ok_or(io::Error::other("test_context_required"))?; + let context: Context = + serde_json::from_value(input).map_err(|_| io::Error::other("invalid_test_context"))?; + let mut store = match SqliteAuthority::open_existing(Path::new(&path)) { + Ok(store) => store, + Err(e) => { + output(&mut writer, Err(e))?; + return Ok(()); + } + }; + let snapshot = match store.snapshot() { + Ok(snapshot) => snapshot, + Err(e) => { + output(&mut writer, Err(e))?; + return Ok(()); + } + }; + let validator = context.validator().map_err(io::Error::other)?; + let mut auth = AuthenticationService::new( + validator, + &snapshot, + context.primary_ceiling.clone(), + context.temporary_ceiling.clone(), + ) + .map_err(io::Error::other)? + .with_oauth_policy(context.oauth.clone()); + drop(snapshot); + output(&mut writer, Ok(json!({"ready":true})))?; + while let Some(input) = read_input(&mut reader)? { + let snapshot = match store.snapshot() { + Ok(snapshot) => snapshot, + Err(e) => { + output(&mut writer, Err(e))?; + continue; + } + }; + if input.get("action").and_then(Value::as_str) == Some("hold_snapshot") { + let before = snapshot.setting("snapshot_probe"); + output(&mut writer, before.map(|v| json!({"before":v})))?; + if read_input(&mut reader)?.is_none() { + return Err(io::Error::other("test_resume_required")); + } + output( + &mut writer, + snapshot + .setting("snapshot_probe") + .map(|v| json!({"after":v})), + )?; + } else { + output(&mut writer, run(input, &context, &mut auth, &snapshot))?; + } + } + Ok(()) +} diff --git a/rust/crates/msg-storage/src/lib.rs b/rust/crates/msg-storage/src/lib.rs new file mode 100644 index 00000000..93b6382b --- /dev/null +++ b/rust/crates/msg-storage/src/lib.rs @@ -0,0 +1,282 @@ +//! Native SQLite authority reads over existing Python-created databases. +//! +//! No schema creation, migration, writer, public resource output or Python process. +//! One borrowed, non-Send snapshot binds all reads to the same transaction. +use msg_core::{Error, Json, Result, MAX_BYTES}; +use msg_identity::{ + models::Timestamp, + oauth::OAuthState, + store::{AuthorityStore, Record, RecoveryDelivery}, +}; +use rusqlite::{Connection, ErrorCode, OpenFlags, OptionalExtension, Transaction}; +use std::{marker::PhantomData, path::Path, rc::Rc, time::Duration}; + +pub struct SqliteAuthority { + connection: Connection, +} +pub struct Snapshot<'a> { + transaction: Transaction<'a>, + // A database snapshot must stay with the task that owns it. + _owner: PhantomData>, +} +fn database_error(error: rusqlite::Error) -> Error { + match error.sqlite_error_code() { + Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => Error("server_busy"), + Some(ErrorCode::ReadOnly) => Error("read_only_transaction"), + _ => Error("storage_error"), + } +} +impl SqliteAuthority { + pub fn open_existing(path: &Path) -> Result { + let connection = Connection::open_with_flags( + path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX, + ) + .map_err(database_error)?; + connection + .busy_timeout(Duration::from_secs(10)) + .map_err(database_error)?; + connection + .execute_batch( + "PRAGMA query_only=ON; PRAGMA trusted_schema=OFF; PRAGMA foreign_keys=ON;", + ) + .map_err(database_error)?; + Ok(Self { connection }) + } + pub fn snapshot(&mut self) -> Result> { + let transaction = self.connection.transaction().map_err(database_error)?; + let snapshot = Snapshot { + transaction, + _owner: PhantomData, + }; + // The first read pins this snapshot before checking any authority. + snapshot.validate_schema()?; + Ok(snapshot) + } +} +impl Snapshot<'_> { + fn validate_schema(&self) -> Result<()> { + let expected = [ + ("schema_version", "version"), ("settings", "key,value"), + ("identities", "id,kind,body"), ("credentials", "id,subject,body"), + ("certificates", "id,subject,parent,revoked,body"), + ("resources", "id,parent,body"), ("results", "subject,request_id,digest,body"), + ("oauth_states", "id,expires,body"), + ("custodial_vault", "subject,signing_key_id,status"), + ("token_deliveries", "credential_id,subject,recovery_verifier,recovery_expires_at,consumed_at,request_id"), + ]; + for (table, columns) in expected { + let exists: bool = self + .transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=?1 AND type='table')", + [table], + |r| r.get(0), + ) + .map_err(|_| Error("unsupported_storage_schema"))?; + if !exists { + return Err(Error("unsupported_storage_schema")); + } + // Names are compile-time constants, never request strings. + self.transaction + .prepare(&format!("SELECT {columns} FROM {table} LIMIT 0")) + .map_err(|_| Error("unsupported_storage_schema"))?; + } + let mut stmt = self + .transaction + .prepare("SELECT version FROM schema_version ORDER BY version LIMIT 2") + .map_err(database_error)?; + let versions = stmt + .query_map([], |r| r.get::<_, i64>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if versions != [1] { + return Err(Error("unsupported_storage_schema")); + } + Ok(()) + } + fn json(&self, sql: &str, id: &str) -> Result> { + let mut stmt = self + .transaction + .prepare_cached(sql) + .map_err(database_error)?; + let raw: Option = stmt + .query_row([id], |r| r.get(0)) + .optional() + .map_err(database_error)?; + raw.map(|value| { + if value.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Json::parse(&value) + }) + .transpose() + } +} +impl AuthorityStore for Snapshot<'_> { + fn record(&self, kind: Record, id: &str) -> Result { + // A bounded SQL substring avoids allocating a corrupt multi-GB body. + let (table, condition, missing) = match kind { + Record::Subject => ("identities", " AND kind='subject'", "subject_not_found"), + Record::Credential => ("credentials", "", "credential_not_found"), + Record::Certificate => ("certificates", "", "certificate_not_found"), + Record::Resource => ("resources", "", "not_found"), + }; + self.json(&format!("SELECT CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM {table} WHERE id=?1{condition}"), id)? + .ok_or(Error(missing)) + } + fn setting(&self, key: &str) -> Result> { + self.json( + "SELECT CAST(substr(CAST(value AS BLOB),1,1048577) AS TEXT) FROM settings WHERE key=?1", + key, + ) + } + fn has_setting(&self, key: &str) -> Result { + self.transaction + .query_row( + "SELECT EXISTS(SELECT 1 FROM settings WHERE key=?1)", + [key], + |r| r.get(0), + ) + .map_err(database_error) + } + fn certificate_revoked(&self, id: &str) -> Result { + let revoked: Option = self + .transaction + .query_row("SELECT revoked FROM certificates WHERE id=?1", [id], |r| { + r.get(0) + }) + .optional() + .map_err(database_error)?; + Ok(revoked.ok_or(Error("certificate_not_found"))? != 0) + } + fn request_result( + &self, + subject: Option<&str>, + id: &str, + digest: &str, + ) -> Result> { + let row: Option<(String, String)> = self.transaction.query_row( + "SELECT digest,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM results WHERE subject=?1 AND request_id=?2", + [subject.unwrap_or(""), id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; + let Some((stored_digest, raw)) = row else { + return Ok(None); + }; + if stored_digest != digest { + return Err(Error("idempotency_conflict")); + } + if raw.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + let result = Json::parse(&raw)?; + let fields = result.as_object()?; + let result_subject = fields.get("subject").and_then(|v| { + if v.is_null() { + Some(None) + } else { + v.as_str().ok().map(Some) + } + }); + if fields.get("request_id").and_then(|v| v.as_str().ok()) != Some(id) + || result_subject != Some(subject) + { + return Err(Error("storage_record_mismatch")); + } + Ok(Some(result)) + } + fn oauth_state(&self, id: &str) -> Result> { + let row: Option<(String, String)> = self.transaction.query_row( + "SELECT expires,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM oauth_states WHERE id=?1", + [id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; + row.map(|(expires, body)| { + if body.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Ok(OAuthState { + expires_at: Timestamp::parse(&expires)?, + body: Json::parse(&body)?, + }) + }) + .transpose() + } + fn custodial_binding(&self, subject: &str) -> Result> { + self.transaction + .query_row( + "SELECT signing_key_id,status FROM custodial_vault WHERE subject=?1", + [subject], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() + .map_err(database_error) + } + fn recovery_delivery( + &self, + credential: &str, + subject: &str, + ) -> Result> { + let row: Option<(String, String, bool, String)> = self.transaction.query_row( + "SELECT recovery_verifier,recovery_expires_at,consumed_at IS NOT NULL,request_id FROM token_deliveries WHERE credential_id=?1 AND subject=?2", + [credential, subject], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))).optional().map_err(database_error)?; + row.map(|(verifier, expires, consumed, request_id)| { + Ok(RecoveryDelivery { + verifier, + expires_at: Timestamp::parse(&expires)?, + consumed, + request_id, + }) + }) + .transpose() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + fs, + time::{SystemTime, UNIX_EPOCH}, + }; + fn path() -> std::path::PathBuf { + std::env::temp_dir().join(format!( + "msg-readonly-{}-{}.db", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )) + } + #[test] + fn missing_database_is_not_created() { + let path = path(); + assert!(SqliteAuthority::open_existing(&path).is_err()); + assert!(!path.exists()); + } + #[test] + fn never_creates_or_repairs_a_schema() { + let path = path(); + Connection::open(&path).unwrap().execute_batch("CREATE TABLE schema_version(version INTEGER); INSERT INTO schema_version VALUES(99);").unwrap(); + let before = fs::read(&path).unwrap(); + let mut store = SqliteAuthority::open_existing(&path).unwrap(); + assert!(matches!( + store.snapshot(), + Err(Error("unsupported_storage_schema")) + )); + assert_eq!(fs::read(&path).unwrap(), before); + drop(store); + fs::remove_file(path).unwrap(); + } + #[test] + fn connection_enforces_read_only_at_the_database_layer() { + let path = path(); + drop(Connection::open(&path).unwrap()); + let store = SqliteAuthority::open_existing(&path).unwrap(); + assert!(store + .connection + .execute_batch("CREATE TABLE injected(secret TEXT);") + .is_err()); + drop(store); + fs::remove_file(path).unwrap(); + } +} diff --git a/rust/tests/test_identity_parity.py b/rust/tests/test_identity_parity.py new file mode 100644 index 00000000..3a08f67c --- /dev/null +++ b/rust/tests/test_identity_parity.py @@ -0,0 +1,1259 @@ +"""Check native authority reads against real Python services and Python-created SQLite. + +Only deterministic, public test keys and disposable databases are used. This is +not a production probe. The native JSONL helper has no network listener. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sqlite3 +import subprocess +import tempfile +from collections import Counter +from contextlib import contextmanager +from dataclasses import replace +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from msg.core.codec import b64, canonical, digest, wire +from msg.core.errors import Failure +from msg.core.models import ( + CapabilityGrant, + CapabilitySpec, + Certificate, + CertificateRequest, + Credential, + IssuancePolicy, + OperationResult, + OperationSpec, + Resource, + ResourceRef, + Scope, + SignatureProof, + Subject, + TokenProof, +) +from msg.core.packet import decode_packet +from msg.core.registry import Registry +from msg.core.requests import payload_fields, signing_bytes +from msg.core.schemas import NETWORK_CONSTRAINTS +from msg.oauth_config import SCOPES, OAuthClient, OAuthConfig +from msg.security.authentication import AuthenticationService +from msg.security.certificates import ( + CertificateValidator, + certificate_body, + csr_body, + sign_certificate, +) +from msg.security.crypto import Ed25519Signer, key_id, subject_id +from msg.security.oauth import resource_execution +from msg.security.policy import allows, constraints_subset, scope_contains, scope_subset +from msg.security.token_delivery import recovery_verifier +from msg.storage.sqlite import SqliteMetadataStore + +NOW = datetime(2026, 10, 5, tzinfo=UTC) +SERVICE = 'https://msg.test.invalid' +READ = 'discovery.get' +WRITE = 'content.post_edit' +OPS = frozenset({ + READ + '@1', + WRITE + '@1', + 'identity.token_rotate@2', + 'identity.register@1', + 'identity.temporary@3', + 'identity.custodial_create@2', + 'identity.token_recover@1', + 'achievement.finish@1', + 'transfer.open@1', + 'tool.run@1', +}) + + +async def unused(*_args): + raise AssertionError('Conformance must never execute a business handler') + + +class Fixture: + def __init__(self, path: Path): + self.path = path + self.metadata = SqliteMetadataStore(path) + self.conn = sqlite3.connect(path, isolation_level=None) + self.conn.execute('PRAGMA foreign_keys=ON') + self.now = NOW + self.signers = { + name: Ed25519Signer.from_bytes(bytes([n]) * 32) + for n, name in enumerate(('root', 'alice', 'bob', 'ca', 'online'), 1) + } + self.users = {name: subject_id(s.public_key) for name, s in self.signers.items()} + self.users.update(root='u_root', online='u_online_ca') + self.keys = {name: key_id(s.public_key) for name, s in self.signers.items()} + self.registry = Registry() + self.grants = { + 'resource.basic': self.grant('resource.basic', OPS), + 'identity.basic': self.grant('identity.basic', OPS), + 'cert.issue': self.grant('cert.issue', {'cert.publish@1'}), + 'cert.ca.issue': self.grant('cert.ca.issue', {'cert.publish@1'}), + 'tool.use': self.grant( + 'tool.use', + {'tool.run@1'}, + constraints={ + 'hosts': ['example.test'], + 'schemes': ['https'], + 'ports': [443], + 'methods': ['GET', 'HEAD'], + 'max_response_bytes': 4096, + 'timeout_ms': 5000, + 'max_redirects': 2, + }, + ), + } + self.primary = tuple(self.grants.values()) + for name, grant in self.grants.items(): + self.registry.add_capability( + CapabilitySpec( + name=name, + version=1, + scope_types=frozenset({'space', 'topic', 'file', 'certificate', 'delegation'}), + operations=grant.operations, + replaces_checks=frozenset(), + delegatable=True, + ca_only=name.startswith('cert.'), + constraints_schema=ResourceRef(id='schema:network-constraints') + if name == 'tool.use' + else None, + ) + ) + self.registry.add_schema(ResourceRef(id='schema:network-constraints'), NETWORK_CONSTRAINTS) + self.resources = {} + for rid, parent, kind in [ + ('r_root', None, 'space'), + ('r_area', 'r_root', 'topic'), + ('r_file', 'r_area', 'file'), + ('r_other', 'r_root', 'topic'), + ('r_delegation', 'r_root', 'delegation'), + ('c_root', 'r_root', 'certificate'), + ('c_ca', 'r_root', 'certificate'), + ('c_leaf', 'r_root', 'certificate'), + ('c_delegation', 'r_root', 'certificate'), + ('c_online', 'r_root', 'certificate'), + ('c_online_leaf', 'r_root', 'certificate'), + ('c_network', 'r_root', 'certificate'), + ('csr_new', 'r_root', 'certificate'), + ('c_new', 'r_root', 'certificate'), + ]: + resource = Resource( + id=rid, + type=kind, + type_version=1, + name=rid, + parent=parent, + owner=self.users['bob'], + group='g_test', + mode=0o755, + generation=0, + revision='v_' + rid, + state='active', + created_at=NOW, + created_by=self.users['bob'], + modified_at=NOW, + modified_by=self.users['bob'], + ) + self.resources[rid] = resource + data = wire(resource) + self.conn.execute( + 'INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', + ( + rid, + kind, + rid, + parent, + resource.owner, + resource.group, + resource.mode, + 0, + resource.revision, + 'active', + data['created_at'], + data['modified_at'], + canonical(resource).decode(), + ), + ) + self.subjects = {} + self.credentials = {} + for name, signer in self.signers.items(): + subject = Subject( + resource_id=self.users[name], + kind='system' if name in {'root', 'online'} else 'registered', + primary_group='g_test', + auth_version=1, + local_only=name == 'root', + ) + credential = Credential( + id=self.keys[name], + subject_id=subject.resource_id, + kind='signing_key', + verifier=signer.public_key, + ceiling=self.primary, + not_before=NOW - timedelta(days=7), + expires_at=NOW + timedelta(days=40), + revoked_at=None, + ) + self.subjects[name], self.credentials[name] = subject, credential + self.conn.execute( + 'INSERT INTO identities VALUES (?,?,?,?)', + (subject.resource_id, 'subject', 0, canonical(subject).decode()), + ) + self.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (credential.id, subject.resource_id, canonical(credential).decode()), + ) + self.token = bytes(range(32)) + import hashlib + + self.credentials['token'] = Credential( + id='t_fixture', + subject_id=self.users['alice'], + kind='token', + verifier=hashlib.sha256(self.token).digest(), + ceiling=self.primary, + not_before=NOW - timedelta(days=1), + expires_at=NOW + timedelta(days=1), + revoked_at=None, + ) + self.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + ('t_fixture', self.users['alice'], canonical(self.credentials['token']).decode()), + ) + self.certs = {} + root_policy = IssuancePolicy( + issue_grants=self.primary, + max_cert_ttl_seconds=15 * 86400, + max_child_ca_depth=3, + max_delegation_depth=4, + ) + ca_policy = replace(root_policy, max_cert_ttl_seconds=7 * 86400, max_child_ca_depth=2) + issue = (self.grants['cert.issue'], self.grants['cert.ca.issue']) + self.add_cert( + 'c_root', + 'root', + 'root', + None, + kind='ca', + grants=issue, + issuance=root_policy, + before=NOW - timedelta(days=5), + expires=NOW + timedelta(days=30), + ) + self.add_cert( + 'c_ca', + 'ca', + 'root', + 'c_root', + kind='ca', + grants=issue, + issuance=ca_policy, + before=NOW - timedelta(days=2), + expires=NOW + timedelta(days=10), + ) + leaf = self.grant('resource.basic', {READ + '@1'}, scope='r_area', descendants=True) + self.add_cert('c_leaf', 'alice', 'ca', 'c_ca', grants=(leaf,)) + self.add_cert( + 'c_delegation', + 'alice', + 'ca', + 'c_ca', + kind='delegation', + grants=(leaf,), + sources=(ResourceRef(id='r_delegation', revision='v_r_delegation'),), + ) + self.fact = { + 'grantor': self.users['bob'], + 'grantee': self.users['alice'], + 'grants': wire((leaf,)), + 'expires_at': wire(NOW + timedelta(hours=2)), + 'parent_certificate': None, + 'grantor_credential_id': self.keys['bob'], + } + self.setting('delegation:r_delegation', self.fact) + online_policy = IssuancePolicy( + issue_grants=(self.grants['resource.basic'],), + max_cert_ttl_seconds=86400, + max_child_ca_depth=0, + max_delegation_depth=1, + ) + self.add_cert( + 'c_online', + 'online', + 'root', + 'c_root', + kind='ca', + grants=(self.grants['cert.issue'],), + issuance=online_policy, + ) + self.add_cert('c_online_leaf', 'alice', 'online', 'c_online', grants=(leaf,)) + self.add_cert('c_network', 'alice', 'ca', 'c_ca', grants=(self.grants['tool.use'],)) + self.validator = CertificateValidator( + self.registry, + self.certs['c_root'], + self.signers['root'].public_key, + SERVICE, + lambda: self.now, + ) + self.auth = AuthenticationService( + self.registry, + self.validator, + SERVICE, + lambda: self.now, + lambda: self.primary, + lambda: self.primary, + ) + self.oauth = OAuthConfig( + enabled=True, clients=(OAuthClient('fixture-client', 'Fixture', scopes=SCOPES),) + ) + self.auth.oauth_config = self.oauth + + def grant(self, name, operations, *, scope='r_root', descendants=True, constraints=None): + return CapabilityGrant( + capability=name, + version=1, + scope=Scope(resource_id=scope, descendants=descendants), + operations=frozenset(operations), + constraints=constraints or {}, + ) + + def add_cert( + self, + id, + subject, + issuer, + parent, + *, + kind='identity', + grants=(), + issuance=None, + sources=(), + before=None, + expires=None, + ): + cert = Certificate( + resource_id=id, + serial=id, + subject_id=self.users[subject], + key_id=self.keys[subject], + issuer_id=self.users[issuer], + parent_certificate_id=parent, + authority_sources=sources, + kind=kind, + grants=grants, + not_before=before or NOW - timedelta(hours=1), + expires_at=expires or NOW + timedelta(hours=1), + target_service=SERVICE, + delegation_depth=0, + issuance=issuance, + signature=self.signers[issuer].sign(b'', purpose='certificate'), + ) + cert = sign_certificate(cert, self.signers[issuer]) + self.certs[id] = cert + self.conn.execute( + 'INSERT INTO certificates VALUES (?,?,?,?,?)', + (id, cert.subject_id, parent, 0, canonical(cert).decode()), + ) + return cert + + def context(self): + return { + 'root': wire(self.certs['c_root']), + 'root_public': b64(self.signers['root'].public_key), + 'service': SERVICE, + 'capabilities': wire(self.registry.capabilities()), + 'primary_ceiling': wire(self.primary), + 'temporary_ceiling': wire(self.primary), + 'oauth': wire(self.oauth), + } + + def setting(self, key, value): + self.conn.execute( + 'INSERT INTO settings VALUES (?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value', + (key, canonical(value).decode()), + ) + + @contextmanager + def setting_change(self, key, value): + old = self.conn.execute('SELECT value FROM settings WHERE key=?', (key,)).fetchone() + self.setting(key, value) + try: + yield + finally: + if old is None: + self.conn.execute('DELETE FROM settings WHERE key=?', (key,)) + else: + self.conn.execute('UPDATE settings SET value=? WHERE key=?', (old[0], key)) + + @contextmanager + def body_change(self, table, id, body): + assert table in {'resources', 'credentials', 'certificates', 'identities'} + old = self.conn.execute(f'SELECT body FROM {table} WHERE id=?', (id,)).fetchone()[0] + self.conn.execute(f'UPDATE {table} SET body=? WHERE id=?', (canonical(body).decode(), id)) + try: + yield + finally: + self.conn.execute(f'UPDATE {table} SET body=? WHERE id=?', (old, id)) + + def spec( + self, + name=READ, + *, + version=1, + effect='read', + signature=False, + entries=('network',), + anonymous=False, + ): + spec = OperationSpec( + name=name, + version=version, + input_schema=ResourceRef(id='input'), + output_schema=ResourceRef(id='output'), + effect=effect, + entries=frozenset(entries), + require_signature=signature, + anonymous_only=anonymous, + handler=unused, + requirements=unused, + ) + self.registry._operations[(name, version)] = spec + return { + key: wire(getattr(spec, key)) + for key in ( + 'name', + 'version', + 'effect', + 'entries', + 'require_signature', + 'anonymous_only', + ) + } + + def request( + self, + *, + name=READ, + version=1, + actor='alice', + subject='default', + token=False, + anonymous=False, + expires=60, + certificates=(), + arguments=None, + request_id='fixture-request', + resign=True, + **changes, + ): + subject = self.users[actor] if subject == 'default' else subject + fields = { + 'request_id': request_id, + 'protocol_version': 1, + 'operation': name, + 'contract_version': version, + 'target_service': SERVICE, + 'subject': None if anonymous else subject, + 'arguments': arguments or {'id': 'r_file'}, + 'expected_generations': [], + 'expires_at': None if expires is None else wire(NOW + timedelta(seconds=expires)), + 'payload_digest': 'sha256:' + '0' * 64, + 'proof': None, + 'source': 'mcp', + 'return_fields': [], + } + fields.update(changes) + request = decode_packet(canonical(fields)) + request = replace(request, payload_digest=digest(payload_fields(request))) + if not anonymous: + proof = ( + TokenProof(credential_id='t_fixture', token=self.token) + if token + else SignatureProof( + signature=self.signers[actor].sign(signing_bytes(request), purpose='request'), + certificates=certificates, + ) + ) + request = replace(request, proof=proof) + if not resign: + request = replace(request, target_service='https://wrong.test.invalid') + return request + + async def close(self): + self.conn.close() + await self.metadata.close() + + +class Check: + def __init__(self, binary, fixture): + self.fixture = fixture + self.process = subprocess.Popen( + [str(binary), str(fixture.path)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + encoding='utf-8', + ) + self.groups = Counter() + self.failures = [] + self.send(fixture.context()) + assert self.receive() == {'ok': True, 'data': {'ready': True}} + + def send(self, value): + self.process.stdin.write( + json.dumps(value, ensure_ascii=False, separators=(',', ':')) + '\n' + ) + self.process.stdin.flush() + + def receive(self): + line = self.process.stdout.readline() + if not line: + raise AssertionError('native helper ended: ' + self.process.stderr.read()[-1000:]) + return json.loads(line) + + def native(self, command): + self.send(command) + return self.receive() + + def compare(self, group, actual, expected, *, count=1): + self.groups[group] += count + if actual != expected: + # Synthetic keys only, but reports still omit request/proof material. + self.failures.append({ + 'group': group, + 'index': self.groups[group], + 'actual': actual, + 'expected': expected, + }) + + async def auth_case( + self, + group, + request, + policy, + *, + entry='network', + replay=False, + resource=None, + ): + command = { + 'action': 'authenticate', + 'raw': canonical(request).decode(), + 'policy': policy, + 'entry': entry, + 'now': wire(self.fixture.now), + 'resource': resource, + } + async with self.fixture.metadata.transaction(write=False) as tx: + try: + with resource_execution(resource): + principal = await self.fixture.auth.authenticate(request, tx, entry=entry) + if replay: + previous = await tx.request_result( + principal.subject, request.request_id, request.payload_digest + ) + assert previous is not None + data = {'kind': 'replay', 'result': wire(previous)} + else: + data = {'kind': 'fresh', 'principal': wire(principal)} + expected = {'ok': True, 'data': data} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare(group, self.native(command), expected) + + async def cert_case(self, group, id='c_leaf'): + async with self.fixture.metadata.transaction(write=False) as tx: + try: + cert = await self.fixture.validator.validate(id, tx) + expected = { + 'ok': True, + 'data': { + 'canonical': canonical(cert).decode(), + 'signing_digest': digest(canonical(certificate_body(cert))), + }, + } + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare( + group, + self.native({'action': 'certificate', 'id': id, 'now': wire(self.fixture.now)}), + expected, + ) + + def close(self): + self.process.stdin.close() + self.process.wait(timeout=15) + assert self.process.returncode == 0, self.process.stderr.read() + self.process.stdout.close() + self.process.stderr.close() + + +def ok(data): + return {'ok': True, 'data': data} + + +async def run(binary, report): + with tempfile.TemporaryDirectory(prefix='msg-native-authority-') as directory: + fixture = Fixture(Path(directory) / 'metadata.db') + check = Check(binary, fixture) + try: + await cases(fixture, check) + finally: + check.close() + await fixture.close() + summary = { + 'cases': sum(check.groups.values()), + 'groups': dict(sorted(check.groups.items())), + 'failures': check.failures, + 'failure_count': len(check.failures), + 'fixture': 'real Python SqliteMetadataStore schema; deterministic public test keys', + 'limitations': [ + 'OAuth issuance, refresh writes and token delivery are not part of this read-only slice', + 'Replay admission returns only the stored result, not a fresh principal', + 'Mode corpus counts each mode/class/permission decision separately', + ], + } + report.parent.mkdir(parents=True, exist_ok=True) + report.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({key: value for key, value in summary.items() if key != 'failures'})) + for failure in check.failures[:10]: + print(json.dumps(failure, ensure_ascii=False)) + if check.failures: + raise AssertionError(f'{len(check.failures)} native authority mismatches') + + +async def cases(f, check): + # Exhaustive mode decisions: 4096 modes x 3 classes x 5 permissions. + decisions = [] + resource = f.resources['r_file'] + for mode in range(0o10000): + for subject in (resource.owner, 'member', None): + for permission in ('read', 'write', 'execute', 'list', 'traverse'): + decisions.append( + allows(replace(resource, mode=mode), subject, {resource.group}, permission) + ) + check.compare( + 'mode_decisions', + check.native({'action': 'modes'}), + ok({'count': len(decisions), 'digest': digest(decisions)}), + count=len(decisions), + ) + + for kind, values in [ + ('subject', f.subjects.values()), + ('credential', f.credentials.values()), + ('certificate', f.certs.values()), + ('resource', f.resources.values()), + ]: + for record in values: + id = getattr(record, 'id', None) or record.resource_id + check.compare( + 'record_bytes', + check.native({'action': 'record_digest', 'kind': kind, 'id': id}), + ok({'digest': digest(record)}), + ) + + for parent in ('r_root', 'r_area', 'r_file', 'r_other'): + for child in ('r_root', 'r_area', 'r_file', 'r_other'): + for descendants in (False, True): + scope = Scope(resource_id=parent, descendants=descendants) + async with f.metadata.transaction(write=False) as tx: + contains = await scope_contains(scope, child, tx) + check.compare( + 'scope_contains', + check.native({'action': 'scope', 'scope': wire(scope), 'id': child}), + ok({'contains': contains}), + ) + for child_scope in ('r_area', 'r_file', 'r_other'): + for operations in ({READ + '@1'}, {WRITE + '@1'}, {READ + '@1', WRITE + '@1'}): + child = f.grant('resource.basic', operations, scope=child_scope) + parent = f.grant('resource.basic', {READ + '@1'}, scope='r_area') + async with f.metadata.transaction(write=False) as tx: + subset = child.operations <= parent.operations and await scope_subset( + child.scope, parent.scope, tx + ) + check.compare( + 'grant_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(parent)}), + ok({'subset': subset}), + ) + parent = f.grants['tool.use'] + for constraints in ( + {}, + dict(parent.constraints), + {**parent.constraints, 'methods': ['GET']}, + {**parent.constraints, 'methods': ['POST']}, + {**parent.constraints, 'timeout_ms': 5001}, + {**parent.constraints, 'max_response_bytes': 4095}, + {**parent.constraints, 'ports': [80, 443]}, + {**parent.constraints, 'hosts': []}, + {**parent.constraints, 'max_redirects': True}, + ): + child = replace(parent, constraints=constraints) + check.compare( + 'constraint_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(parent)}), + ok({'subset': constraints_subset(child.constraints, parent.constraints)}), + ) + + for child_value, parent_value in [ + (-2, -1), + (-1, -2), + (-1, 0), + (0, -1), + (10**80, 10**80 + 1), + (10**80 + 1, 10**80), + (2, 2.0), + (2.0, 2), + ]: + child = replace(parent, constraints={'max_redirects': child_value}) + upper = replace(parent, constraints={'max_redirects': parent_value}) + check.compare( + 'numeric_constraint_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(upper)}), + ok({'subset': constraints_subset(child.constraints, upper.constraints)}), + ) + for ports in ([443.0], [443], [444.0]): + child = replace(parent, constraints={'ports': ports}) + upper = replace(parent, constraints={'ports': [443]}) + check.compare( + 'numeric_allowlist_subset', + check.native({'action': 'subset', 'child': wire(child), 'parent': wire(upper)}), + ok({'subset': constraints_subset(child.constraints, upper.constraints)}), + ) + network = f.certs['c_network'] + for constraints in [ + {**parent.constraints, 'ports': [443.0]}, + {**parent.constraints, 'ports': [0]}, + {**parent.constraints, 'ports': [65536]}, + {**parent.constraints, 'ports': [True]}, + {**parent.constraints, 'hosts': 'not-a-list'}, + {**parent.constraints, 'ports': 'not-a-list'}, + {**parent.constraints, 'max_redirects': -1}, + {**parent.constraints, 'timeout_ms': 0}, + {**parent.constraints, 'timeout_ms': 5000.0}, + ]: + proposed = sign_certificate( + replace(network, grants=(replace(parent, constraints=constraints),)), f.signers['ca'] + ) + with f.body_change('certificates', network.resource_id, proposed): + await check.cert_case('network_constraint_validation', network.resource_id) + + policy = f.spec() + request = f.request() + for cert in f.certs: + await check.cert_case('valid_chain', cert) + for ids in ((), ('c_leaf',), ('c_leaf', 'c_delegation'), ('c_missing',)): + await check.auth_case('signed_auth', f.request(certificates=ids), policy) + for expiry in (None, -300, -1, 0, 0.000001, 1, 299, 300, 300.000001, 301, 600): + for token in (False, True): + await check.auth_case('request_expiry', f.request(expires=expiry, token=token), policy) + await check.auth_case('anonymous_read', f.request(expires=expiry, anonymous=True), policy) + await check.auth_case('wrong_service', f.request(resign=False), policy) + await check.auth_case('digest_tamper', replace(request, arguments={'id': 'r_other'}), policy) + await check.auth_case('signature_tamper', replace(request, source='manual'), policy) + await check.auth_case('missing_proof', replace(request, proof=None), policy) + await check.auth_case('subject_spoof', f.request(subject=f.users['bob']), policy) + await check.auth_case( + 'delegated_subject', + f.request(subject=f.users['bob'], certificates=('c_delegation',)), + policy, + ) + await check.auth_case('entry_denied', request, policy, entry='worker') + await check.auth_case('root_network', f.request(actor='root'), policy) + local_policy = f.spec(entries=('network', 'local_admin')) + await check.auth_case('root_local', f.request(actor='root'), local_policy, entry='local_admin') + policy = f.spec() + for name, effect, version, anonymous in [ + ('content.post_edit', 'transaction', 1, False), + ('identity.link_claim', 'transaction', 1, True), + ('communication.internet_receive', 'transaction', 1, True), + ('identity.link_claim', 'transaction', 2, True), + ('identity.link_claim', 'external', 1, True), + ]: + p = f.spec(name, effect=effect, version=version, anonymous=anonymous) + await check.auth_case( + 'anonymous_write', f.request(name=name, version=version, anonymous=True), p + ) + policy = f.spec() + + for flag in (False, None, {}, True): + with f.setting_change('recovery_quarantine', flag): + await check.auth_case('quarantine_presence', request, policy) + p = f.spec(WRITE, effect='transaction') + await check.auth_case('quarantine_write', f.request(name=WRITE), p) + for key, value in [ + ('active_root_certificate', 'other-root'), + ('identity_archived:' + f.users['alice'], True), + ('identity_archived:' + f.users['alice'], False), + ]: + with f.setting_change(key, value): + await check.auth_case('authority_fences', request, policy) + + for name, field, value in [ + ('alice', 'revoked_at', NOW), + ('alice', 'expires_at', NOW), + ('alice', 'not_before', NOW + timedelta(microseconds=1)), + ('alice', 'expires_at', None), + ('alice', 'ceiling', ()), + ('alice', 'kind', 'ssh_key'), + ('token', 'verifier', b'wrong'), + ('token', 'revoked_at', NOW), + ('token', 'expires_at', NOW), + ]: + credential = f.credentials[name] + with f.body_change('credentials', credential.id, replace(credential, **{field: value})): + await check.auth_case('credential_state', f.request(token=name == 'token'), policy) + p = f.spec(signature=True) + await check.auth_case('signature_required', f.request(token=True), p) + policy = f.spec() + derived = replace(f.credentials['token'], source_credential_id=f.keys['alice']) + with f.body_change('credentials', derived.id, derived): + await check.auth_case('missing_token_binding', f.request(token=True), policy) + source = { + 'parent': f.keys['alice'], + 'subject': f.users['alice'], + 'auth_version': 1, + 'ceiling': wire(f.primary), + } + with f.body_change('credentials', derived.id, derived): + for binding in ('browser:', 'api:'): + f.conn.execute( + 'INSERT INTO oauth_states VALUES (?,?,?,?)', + ( + binding + derived.id, + 'binding', + wire(NOW + timedelta(days=1)), + canonical(source).decode(), + ), + ) + await check.auth_case('oauth_bound_token', f.request(token=True), policy) + for field, value in [('revoked_at', NOW), ('expires_at', NOW), ('ceiling', ())]: + parent = f.credentials['alice'] + with f.body_change('credentials', parent.id, replace(parent, **{field: value})): + await check.auth_case('oauth_live_parent', f.request(token=True), policy) + with f.body_change( + 'identities', f.users['alice'], replace(f.subjects['alice'], auth_version=2) + ): + await check.auth_case('oauth_auth_version', f.request(token=True), policy) + nested = {**source, 'session': 'session:fixture'} + f.conn.execute( + 'UPDATE oauth_states SET body=? WHERE id=?', + (canonical(nested).decode(), binding + derived.id), + ) + f.conn.execute( + 'INSERT INTO oauth_states VALUES (?,?,?,?)', + ( + 'session:fixture', + 'session', + wire(NOW + timedelta(seconds=60)), + canonical(source).decode(), + ), + ) + await check.auth_case('oauth_session_source', f.request(token=True), policy) + f.conn.execute( + 'UPDATE oauth_states SET body=? WHERE id=?', + (canonical({**source, 'revoked': True}).decode(), 'session:fixture'), + ) + await check.auth_case('oauth_session_revoked', f.request(token=True), policy) + f.conn.execute('DELETE FROM oauth_states') + family = {**source, 'client_id': 'fixture-client', 'scopes': ['msg.read', 'msg.write']} + access = {'family': 'family:fixture'} + f.conn.execute( + 'INSERT INTO oauth_states VALUES (?,?,?,?)', + ( + 'access:' + derived.id, + 'access', + wire(NOW + timedelta(days=1)), + canonical(access).decode(), + ), + ) + for field, value in [ + (None, None), + ('revoked', True), + ('client_id', 'missing-client'), + ('scopes', ['unknown.scope']), + ('ceiling', []), + ('auth_version', 9), + ]: + body = family if field is None else {**family, field: value} + f.conn.execute( + 'INSERT OR REPLACE INTO oauth_states VALUES (?,?,?,?)', + ( + 'family:fixture', + 'family', + wire(NOW + timedelta(seconds=60)), + canonical(body).decode(), + ), + ) + await check.auth_case('oauth_family', f.request(token=True), policy) + f.conn.execute( + 'UPDATE oauth_states SET expires=? WHERE id=?', (wire(NOW), 'family:fixture') + ) + await check.auth_case('oauth_family_expired', f.request(token=True), policy) + mcp_grants = (f.grant('resource.basic', {READ + '@1'}),) + mcp_token = replace(derived, ceiling=mcp_grants) + audience = SERVICE + '/-/mcp' + mcp_family = { + **source, + 'client_id': 'fixture-client', + 'resource': audience, + 'scopes': ['msg.mcp.read'], + 'ceiling': wire(mcp_grants), + } + f.conn.execute( + 'UPDATE oauth_states SET expires=?,body=? WHERE id=?', + (wire(NOW + timedelta(seconds=60)), canonical(mcp_family).decode(), 'family:fixture'), + ) + with f.body_change('credentials', derived.id, mcp_token): + for resource in (None, audience, SERVICE + '/other'): + await check.auth_case( + 'mcp_audience', f.request(token=True), policy, resource=resource + ) + for scopes in (['msg.read', 'msg.mcp.read'], ['msg.mcp.post'], ['msg.mcp.read']): + f.conn.execute( + 'UPDATE oauth_states SET body=? WHERE id=?', + (canonical({**mcp_family, 'scopes': scopes}).decode(), 'family:fixture'), + ) + await check.auth_case( + 'mcp_scope_ceiling', f.request(token=True), policy, resource=audience + ) + await check.auth_case( + 'mcp_operation_denied', f.request(token=True), policy, resource=audience + ) + f.conn.execute('DELETE FROM oauth_states') + + for actor, kind in [('alice', 'custodial'), ('alice', 'registered')]: + subject = f.subjects[actor] + with f.body_change('identities', subject.resource_id, replace(subject, kind=kind)): + for name, arguments in [ + (WRITE, {}), + ('tool.run', {}), + ('achievement.finish', {}), + ('transfer.open', {'direction': 'download'}), + ('transfer.open', {'direction': 'upload'}), + ]: + p = f.spec(name, effect='transaction', signature=name == 'achievement.finish') + await check.auth_case( + 'custodial_limits', f.request(name=name, token=True, arguments=arguments), p + ) + + for kind, version, nonce_size in [ + ('identity.temporary', 3, 23), + ('identity.temporary', 3, 24), + ('identity.custodial_create', 2, 32), + ('identity.temporary', 3, 64), + ('identity.temporary', 3, 65), + ]: + p = f.spec(kind, version=version, effect='transaction') + req = f.request( + name=kind, + version=version, + anonymous=True, + arguments={'nonce': b64(bytes(range(nonce_size)))}, + ) + await check.auth_case('bootstrap', req, p) + p = f.spec('identity.register', effect='transaction') + req = f.request( + name='identity.register', arguments={'public_key': b64(f.signers['alice'].public_key)} + ) + await check.auth_case('register', req, p) + with f.setting_change('delegated_identity:' + f.users['alice'], {'locked': True}): + await check.auth_case('register_delegated_denied', req, p) + + # Invalid child conditions are re-signed to test rules after cryptography. + leaf = f.certs['c_leaf'] + for field, value in [ + ('target_service', 'https://other.test.invalid'), + ('not_before', NOW + timedelta(seconds=1)), + ('expires_at', NOW), + ('issuer_id', f.users['alice']), + ('subject_id', 'u_root'), + ('delegation_depth', 5), + ('not_before', NOW - timedelta(days=3)), + ('expires_at', NOW + timedelta(days=11)), + ('expires_at', NOW + timedelta(days=8)), + ('issuance', f.certs['c_ca'].issuance), + ('grants', (f.grant('resource.basic', {'unknown.call@1'}),)), + ('grants', (f.grant('missing.capability', {READ + '@1'}),)), + ('grants', (replace(leaf.grants[0], constraints={'unknown': 'x'}),)), + ('subject_id', f.users['bob']), + ]: + changed = sign_certificate(replace(leaf, **{field: value}), f.signers['ca']) + with f.body_change('certificates', leaf.resource_id, changed): + await check.cert_case('invalid_child') + with f.body_change('certificates', 'c_leaf', replace(leaf, serial='tampered')): + await check.cert_case('invalid_signature') + with f.body_change('certificates', 'c_root', replace(f.certs['c_root'], serial='tampered')): + await check.cert_case('anchor_pin') + for id in ('c_leaf', 'c_ca', 'c_root'): + f.conn.execute('UPDATE certificates SET revoked=1 WHERE id=?', (id,)) + await check.cert_case('certificate_revoked') + f.conn.execute('UPDATE certificates SET revoked=0 WHERE id=?', (id,)) + for name in ('alice', 'ca', 'root'): + old = f.credentials[name] + with f.body_change('credentials', old.id, replace(old, revoked_at=NOW)): + await check.cert_case('issuer_key_revoked') + with f.body_change( + 'certificates', 'c_ca', replace(f.certs['c_ca'], parent_certificate_id='c_leaf') + ): + await check.cert_case('certificate_cycle') + for field, value in [ + ('kind', 'identity'), + ('issuance', None), + ('grants', (f.grants['cert.ca.issue'],)), + ('grants', (f.grants['cert.issue'],)), + ('issuance', replace(f.certs['c_ca'].issuance, max_child_ca_depth=3)), + ('issuance', replace(f.certs['c_ca'].issuance, max_cert_ttl_seconds=16 * 86400)), + ('issuance', replace(f.certs['c_ca'].issuance, max_delegation_depth=5)), + ]: + cert = sign_certificate(replace(f.certs['c_ca'], **{field: value}), f.signers['root']) + with f.body_change('certificates', 'c_ca', cert): + await check.cert_case('ca_policy') + for field, value in [('kind', 'capability'), ('kind', 'delegation')]: + cert = sign_certificate( + replace(f.certs['c_online_leaf'], **{field: value}), f.signers['online'] + ) + with f.body_change('certificates', 'c_online_leaf', cert): + await check.cert_case('online_ca_limits', 'c_online_leaf') + for field, value in [ + ('issuance', replace(f.certs['c_online'].issuance, max_child_ca_depth=1)), + ('grants', (f.grants['cert.issue'], f.grants['cert.ca.issue'])), + ]: + cert = sign_certificate(replace(f.certs['c_online'], **{field: value}), f.signers['root']) + with f.body_change('certificates', 'c_online', cert): + await check.cert_case('online_ca_limits', 'c_online_leaf') + + for field, value in [ + ('expires_at', wire(NOW)), + ('grantee', f.users['bob']), + ('grantor', f.users['alice']), + ('grants', []), + ('parent_certificate', 'c_delegation'), + ]: + with f.setting_change('delegation:r_delegation', {**f.fact, field: value}): + await check.cert_case('delegation_source', 'c_delegation') + resource = f.resources['r_delegation'] + for field, value in [('state', 'archived'), ('revision', 'v_changed')]: + with f.body_change('resources', resource.id, replace(resource, **{field: value})): + await check.cert_case('delegation_resource', 'c_delegation') + credential = f.credentials['bob'] + for field, value in [('revoked_at', NOW), ('ceiling', ())]: + with f.body_change('credentials', credential.id, replace(credential, **{field: value})): + await check.cert_case('delegation_live_source', 'c_delegation') + with f.setting_change('identity_archived:' + f.users['bob'], True): + await check.cert_case('delegation_live_source', 'c_delegation') + + # CSR and publication signed bytes use the same actual Python model. + csr = CertificateRequest( + resource_id='csr_new', + applicant=f.users['alice'], + subject_id=f.users['alice'], + requested_issuer=f.users['ca'], + public_key=f.signers['alice'].public_key, + kind='identity', + grants=leaf.grants, + issuance=None, + requested_ttl_seconds=7200, + target_service=SERVICE, + delegation_depth=0, + authority_sources=(), + request_digest='', + possession_proof=f.signers['alice'].sign(b'', purpose='csr'), + ) + csr = replace( + csr, + request_digest=digest(csr_body(csr)), + possession_proof=f.signers['alice'].sign(canonical(csr_body(csr)), purpose='csr'), + ) + proposed = sign_certificate(replace(leaf, resource_id='c_new', serial='new'), f.signers['ca']) + for field, val in [ + (None, None), + ('request_digest', 'sha256:' + '0' * 64), + ('requested_issuer', f.users['bob']), + ]: + candidate = csr if field is None else replace(csr, **{field: val}) + async with f.metadata.transaction(write=False) as tx: + try: + result = await f.validator.validate_publication(proposed, candidate, tx) + expected = ok({'canonical': canonical(result).decode()}) + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + check.compare( + 'csr_publication', + check.native({ + 'action': 'publication', + 'certificate': wire(proposed), + 'csr': wire(candidate), + 'now': wire(f.now), + }), + expected, + ) + + # Rotation retries may read the exact saved result, never get a new principal. + policy = f.spec('identity.token_rotate', version=2, effect='transaction') + req = f.request( + name='identity.token_rotate', version=2, token=True, request_id='rotation-fixed' + ) + result = OperationResult( + request_id=req.request_id, + operation='identity.token_rotate', + status='ok', + actor=f.users['alice'], + subject=f.users['alice'], + committed_at=NOW, + data={'previous_credential': 't_fixture', 'credential_id': 't_next'}, + ) + successor = replace(f.credentials['token'], id='t_next') + f.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + ('t_next', f.users['alice'], canonical(successor).decode()), + ) + f.conn.execute( + 'INSERT INTO results VALUES (?,?,?,?)', + (f.users['alice'], req.request_id, req.payload_digest, canonical(result).decode()), + ) + with f.body_change('credentials', 't_fixture', replace(f.credentials['token'], revoked_at=NOW)): + await check.auth_case('rotation_exact_replay', req, policy, replay=True) + await check.auth_case( + 'rotation_new_request_denied', + f.request(name='identity.token_rotate', version=2, token=True, request_id='other'), + policy, + ) + changed = f.request( + name='identity.token_rotate', + version=2, + token=True, + request_id=req.request_id, + arguments={'new': 'content'}, + ) + await check.auth_case('rotation_digest_conflict', changed, policy) + with f.body_change('credentials', 't_next', replace(successor, revoked_at=NOW)): + await check.auth_case('rotation_dead_successor', req, policy) + check.compare( + 'ledger_lookup', + check.native({ + 'action': 'result', + 'subject': f.users['alice'], + 'id': req.request_id, + 'digest': req.payload_digest, + }), + ok({'result': wire(result)}), + ) + check.compare( + 'ledger_subject_isolation', + check.native({ + 'action': 'result', + 'subject': f.users['bob'], + 'id': req.request_id, + 'digest': req.payload_digest, + }), + ok({'result': None}), + ) + check.compare( + 'ledger_anonymous_isolation', + check.native({ + 'action': 'result', + 'subject': None, + 'id': req.request_id, + 'digest': req.payload_digest, + }), + ok({'result': None}), + ) + + recovery_secret = b64(bytes(range(32, 64))) + recovery_args = { + 'credential_id': 't_fixture', + 'recovery_secret': recovery_secret, + 'original_request_id': 'original-issue', + } + recover = f.request( + name='identity.token_recover', arguments=recovery_args, request_id='recovery-fixed' + ) + recover = replace(recover, proof=None) + policy = f.spec('identity.token_recover', effect='transaction') + f.conn.execute( + 'INSERT INTO token_deliveries VALUES (?,?,?,?,?,?,?,?)', + ( + 't_fixture', + f.users['alice'], + 'original-issue', + 'unused', + recovery_verifier(recovery_secret), + wire(NOW + timedelta(seconds=60)), + None, + None, + ), + ) + await check.auth_case('token_recovery', recover, policy) + for key, value in [ + ('recovery_secret', b64(bytes(32))), + ('original_request_id', 'other'), + ('recovery_secret', b64(bytes(31))), + ]: + bad = f.request(name='identity.token_recover', arguments={**recovery_args, key: value}) + await check.auth_case('token_recovery_binding', replace(bad, proof=None), policy) + for field, value in [('recovery_expires_at', wire(NOW)), ('consumed_at', wire(NOW))]: + old = f.conn.execute(f'SELECT {field} FROM token_deliveries').fetchone()[0] + f.conn.execute(f'UPDATE token_deliveries SET {field}=?', (value,)) + await check.auth_case('token_recovery_state', recover, policy) + f.conn.execute(f'UPDATE token_deliveries SET {field}=?', (old,)) + previous = replace(result, request_id=recover.request_id, operation=recover.operation) + f.conn.execute( + 'INSERT INTO results VALUES (?,?,?,?)', + ( + f.users['alice'], + recover.request_id, + recover.payload_digest, + canonical(previous).decode(), + ), + ) + f.conn.execute('UPDATE token_deliveries SET consumed_at=?', (wire(NOW),)) + with f.body_change('credentials', 't_fixture', replace(f.credentials['token'], revoked_at=NOW)): + await check.auth_case('token_recovery_replay', recover, policy, replay=True) + f.conn.execute('DELETE FROM token_deliveries') + + # WAL snapshots cannot combine old credential rows with new revocation facts. + f.setting('snapshot_probe', 'old') + check.send({'action': 'hold_snapshot'}) + check.compare('snapshot_before', check.receive(), ok({'before': 'old'})) + f.setting('snapshot_probe', 'new') + check.send({'resume': True}) + check.compare('snapshot_stable', check.receive(), ok({'after': 'old'})) + check.compare( + 'snapshot_fresh', + check.native({'action': 'setting', 'key': 'snapshot_probe'}), + ok({'exists': True, 'value': 'new'}), + ) + f.conn.execute('UPDATE resources SET parent=? WHERE id=?', ('r_root', 'r_root')) + # Body parent links, not the incidental SQL column, are the Python authority view. + with f.body_change('resources', 'r_area', replace(f.resources['r_area'], parent='r_file')): + command = { + 'action': 'scope', + 'scope': wire(Scope(resource_id='r_other', descendants=True)), + 'id': 'r_file', + } + check.compare('parent_cycle', check.native(command), {'ok': False, 'code': 'parent_cycle'}) + f.conn.execute('UPDATE resources SET parent=NULL WHERE id=?', ('r_root',)) + + # Runtime generation mismatch remains latched even when the old value returns. + policy = f.spec() + with f.setting_change('recovery_runtime_generation', 'after-restore'): + await check.auth_case('runtime_stale', request, policy) + await check.auth_case('runtime_stale_latched', request, policy) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--binary', required=True, type=Path) + parser.add_argument('--report', required=True, type=Path) + args = parser.parse_args() + asyncio.run(run(args.binary.resolve(), args.report)) From 46d4366de1e61bf5d4bbeed82f55b9d2bb06e713 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 10:45:33 +0800 Subject: [PATCH 07/22] ci(rust): export pinned writer-lock dependency for isolated tests --- .github/workflows/rust-writer-deps.yml | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/rust-writer-deps.yml diff --git a/.github/workflows/rust-writer-deps.yml b/.github/workflows/rust-writer-deps.yml new file mode 100644 index 00000000..de67e544 --- /dev/null +++ b/.github/workflows/rust-writer-deps.yml @@ -0,0 +1,34 @@ +name: Rust writer dependency snapshot +on: + push: + branches: ['migration/rust-wire-phase1'] + paths: ['.github/workflows/rust-writer-deps.yml'] +permissions: + contents: read +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Vendor exact writer lock dependency without credentials + shell: bash + run: | + set -euo pipefail + rustup toolchain install 1.85.0 --profile minimal + mkdir -p /tmp/deps/src /tmp/export + printf '[package]\nname="writer-deps"\nversion="0.0.0"\nedition="2021"\n[dependencies]\nfs2="=0.4.3"\n' > /tmp/deps/Cargo.toml + echo 'fn main() {}' > /tmp/deps/src/main.rs + cargo +1.85.0 vendor --manifest-path /tmp/deps/Cargo.toml /tmp/writer-vendor > /tmp/export/config.toml + tar -czf /tmp/export/vendor.tar.gz -C /tmp writer-vendor + cp /tmp/deps/Cargo.lock /tmp/export/Cargo.lock + git rev-parse HEAD > /tmp/export/source-commit.txt + sha256sum /tmp/export/vendor.tar.gz > /tmp/export/SHA256SUMS + - uses: actions/upload-artifact@v7 + with: + name: rust-writer-deps-${{ github.sha }} + path: /tmp/export/ + retention-days: 1 + if-no-files-found: error From fffea47606cf4db999ecf915910bdbe372360bb5 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 11:22:25 +0800 Subject: [PATCH 08/22] feat(rust): add fenced atomic SQLite metadata writer and recovery tests Port resource/revision/result/event/audit/settings writes into private native transactions over the existing Python-created WAL schema. Keep the Python flock fence through rollback compensation, isolate savepoints, fail closed on ignored writes and ambiguous commit outcomes, and compare real Python/Rust rows plus process-kill/retry behavior. Preserve Python CLI and live services. Remove the temporary writer-dependency export workflow. No merge or cutover. --- .github/workflows/rust-wire.yml | 8 +- .github/workflows/rust-writer-deps.yml | 34 - rust/Cargo.lock | 35 + rust/Cargo.toml | 2 + rust/README.md | 91 +- rust/crates/msg-storage/Cargo.toml | 9 +- .../msg-storage/examples/writer_check.rs | 155 ++++ rust/crates/msg-storage/src/lib.rs | 45 +- rust/crates/msg-storage/src/records.rs | 233 +++++ rust/crates/msg-storage/src/writer.rs | 632 +++++++++++++ rust/crates/msg-storage/src/writer/tests.rs | 423 +++++++++ rust/tests/test_storage_parity.py | 875 ++++++++++++++++++ 12 files changed, 2475 insertions(+), 67 deletions(-) delete mode 100644 .github/workflows/rust-writer-deps.yml create mode 100644 rust/crates/msg-storage/examples/writer_check.rs create mode 100644 rust/crates/msg-storage/src/records.rs create mode 100644 rust/crates/msg-storage/src/writer.rs create mode 100644 rust/crates/msg-storage/src/writer/tests.rs create mode 100644 rust/tests/test_storage_parity.py diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index 70cf5aba..775a4f92 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -47,6 +47,7 @@ jobs: run: | mkdir -p rust/artifacts git rev-parse HEAD > rust/artifacts/source-commit.txt + git rev-parse HEAD^{tree} > rust/artifacts/source-tree.txt rustc +1.85.0 --version > rust/artifacts/toolchains.txt python --version >> rust/artifacts/toolchains.txt - name: Check Rust formatting without modifying sources @@ -61,8 +62,9 @@ jobs: if: ${{ !cancelled() }} working-directory: rust run: | - cargo +1.85.0 test --locked --workspace --all-targets - cargo +1.85.0 test --locked --workspace --doc + set -euo pipefail + cargo +1.85.0 test --locked --workspace --all-targets 2>&1 | tee artifacts/native-tests.txt + cargo +1.85.0 test --locked --workspace --doc 2>&1 | tee artifacts/doc-tests.txt - name: Clippy if: ${{ !cancelled() }} working-directory: rust @@ -75,6 +77,7 @@ jobs: cargo +1.85.0 build --locked --manifest-path rust/Cargo.toml --workspace --examples python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json python rust/tests/test_identity_parity.py --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json + python rust/tests/test_storage_parity.py --binary rust/target/debug/examples/writer_check --report rust/artifacts/storage-parity.json - name: Optimized native differential checks if: ${{ !cancelled() }} env: @@ -83,6 +86,7 @@ jobs: cargo +1.85.0 build --release --locked --manifest-path rust/Cargo.toml --workspace --examples python rust/tests/test_python_parity.py --binary rust/target/release/examples/wire_check --report rust/artifacts/parity-release.json python rust/tests/test_identity_parity.py --binary rust/target/release/examples/authority_check --report rust/artifacts/identity-parity-release.json + python rust/tests/test_storage_parity.py --binary rust/target/release/examples/writer_check --report rust/artifacts/storage-parity-release.json - name: Preserve evidence and dependency lock if: ${{ always() }} uses: actions/upload-artifact@v7 diff --git a/.github/workflows/rust-writer-deps.yml b/.github/workflows/rust-writer-deps.yml deleted file mode 100644 index de67e544..00000000 --- a/.github/workflows/rust-writer-deps.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Rust writer dependency snapshot -on: - push: - branches: ['migration/rust-wire-phase1'] - paths: ['.github/workflows/rust-writer-deps.yml'] -permissions: - contents: read -jobs: - snapshot: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Vendor exact writer lock dependency without credentials - shell: bash - run: | - set -euo pipefail - rustup toolchain install 1.85.0 --profile minimal - mkdir -p /tmp/deps/src /tmp/export - printf '[package]\nname="writer-deps"\nversion="0.0.0"\nedition="2021"\n[dependencies]\nfs2="=0.4.3"\n' > /tmp/deps/Cargo.toml - echo 'fn main() {}' > /tmp/deps/src/main.rs - cargo +1.85.0 vendor --manifest-path /tmp/deps/Cargo.toml /tmp/writer-vendor > /tmp/export/config.toml - tar -czf /tmp/export/vendor.tar.gz -C /tmp writer-vendor - cp /tmp/deps/Cargo.lock /tmp/export/Cargo.lock - git rev-parse HEAD > /tmp/export/source-commit.txt - sha256sum /tmp/export/vendor.tar.gz > /tmp/export/SHA256SUMS - - uses: actions/upload-artifact@v7 - with: - name: rust-writer-deps-${{ github.sha }} - path: /tmp/export/ - retention-days: 1 - if-no-files-found: error diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 43444bc7..180c4f99 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -192,6 +192,16 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -309,7 +319,10 @@ dependencies = [ name = "msg-storage" version = "0.1.0" dependencies = [ + "fs2", + "libc", "msg-core", + "msg-crypto", "msg-identity", "msg-protocol", "rusqlite", @@ -542,6 +555,28 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "zerocopy" version = "0.8.59" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 8843ebca..f4535f8d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -18,6 +18,8 @@ serde = { version = "=1.0.228", features = ["derive"] } serde_json = { version = "=1.0.145", features = ["raw_value", "arbitrary_precision"] } sha2 = "=0.10.9" subtle = "=2.6.1" +fs2 = "=0.4.3" +libc = "=0.2.190" rusqlite = { version = "=0.32.1", features = ["bundled"] } [workspace.lints.rust] diff --git a/rust/README.md b/rust/README.md index 49dd5c33..aceb3ee3 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,4 +1,4 @@ -# Native MSG migration — identity and SQLite read slice +# Native MSG migration — identity and atomic SQLite metadata **This is not a replacement daemon or a production cutover.** Python remains the running implementation. No Python entry point, live database, configuration, @@ -13,8 +13,8 @@ signature protocol. The tests import the actual Python implementation in the same checkout, rather than a translated copy of the expected behavior. The original inspected main baseline is -`7e16ca75ff6af4313365814c23912b5997b222c5`. This slice extends phase 1 commit -`91d8d3209d09f04269b4d9807793d6a0a8da7eba` on the existing draft PR. +`7e16ca75ff6af4313365814c23912b5997b222c5`. This slice extends phase 2 commit +`78e8d5d44cbbe49078a1e6eb4bb8e079eea3db72` on the existing draft PR #235. ## Implemented natively @@ -24,7 +24,7 @@ The original inspected main baseline is | `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing/verification | `security/crypto.py` | | `msg-protocol` | Request shape/defaults, UTC time, business digest/signature bytes, immutable field access | `core/packet.py`, `core/requests.py`, `core/models.py` | | `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, read-only browser/API/OAuth and token-recovery validation | `security/authentication.py`, `certificates.py`, `policy.py`, `oauth.py` | -| `msg-storage` | Read-only SQLite authority snapshots, current revocations/settings, result lookup and digest conflict, source bindings and recovery-delivery metadata | `storage/sqlite.py` and its record/session modules | +| `msg-storage` | Transaction-bound authority reads; Unix writer fencing, atomic resource/revision/result/event/audit/settings writes, savepoints and rollback effects | `storage/sqlite.py` and its record/session modules | Authentication validates service/digest/expiry, current credentials, SHA-256 bearer verifiers with constant-time comparison, actor/subject bindings, permitted @@ -65,17 +65,44 @@ does not issue credentials or consume a delivery record. ## Storage safety boundary -The native connection opens an existing database read-only. It does not create, -repair or migrate a schema. Unsupported schema versions/tables fail closed. -Each borrowed, non-Send snapshot pins all authority reads to one transaction and -rolls back on drop. It cannot be reused as a write transaction. JSON columns are -bounded in SQL before decoding. Secret vault ciphertext is not selected. - -**There is no Rust database writer yet.** Python's `.writer.lock` process fence -must remain part of a future writer port, including external compensation; a -SQLite transaction alone is not an equivalent replacement. PostgreSQL, native -write transactions, delivery/outbox/lease handling, migrations and crash/restart -execution recovery remain pending. Do not introduce dual writes. +`SqliteAuthority` remains read-only. `SqliteWriter` separately opens an existing, +Python-migrated WAL database. Neither API creates or migrates a database. The +writer checks schema version, required tables/columns, primary and unique keys, +foreign keys, the one-root index and append-only audit guards. It rejects database +symlink/hard-link aliases; all processes must use the same configured file. + +Read snapshots and write sessions share `AuthorityStore` queries inside their +own transaction. JSON columns remain bounded in SQL; secret vault ciphertext is +not selected. Sessions are borrowed and non-Send. Callers have no SQL, COMMIT or +ROLLBACK handle. Only a successful store-owned COMMIT can return a success value. + +The Unix writer uses the same `.writer.lock`/`flock` process fence as Python, +with no-follow opens, mode 0600 on creation and a bounded acquisition deadline. +The fence outlives SQL rollback and every reverse-order rollback effect. Nested +savepoints retain successful inner effects until the outer commit; failed inner +units undo their own data/effects before the caller can recover. An ignored write +error poisons its unit rather than allowing a partial commit or an autocommit +write after SQLite has aborted the transaction. + +Implemented mutation ports: resource insertion and generation-checked replacement +(including immutable creation facts, parent cycles, aliases, tags and authorization +epoch), revisions/relations, settings, result rows with subject/digest binding and +the existing 100,000-row cap, events and the existing canonical audit hash chain. +These ports are **trusted internal storage APIs**, not authorization decisions. +`save_result` does not sign receipts, redact secrets or execute handlers. + +Cleanup errors preserve the primary failure and disable that writer instance. +A failed COMMIT with no live SQLite transaction is treated conservatively as +`commit_outcome_uncertain`: no destructive compensation runs, and the instance +refuses further writes. The future service owner must stop admission, inspect the +result ledger and reconcile external work before reopening. Reopening alone is +not recovery. Durable service-wide quarantine and external-work reconciliation +are not implemented by this slice. + +The process-kill tests cover **SQLite metadata only**. Rollback callbacks are not +run after SIGKILL and are not a durable blob/Git recovery journal. Content/Git +adapters, identity/OAuth mutations, outbox/lease handling, PostgreSQL and complete +request execution recovery still need to be ported. Do not introduce dual writes. ## Reproducible validation @@ -94,6 +121,8 @@ PYTHONPATH=src python rust/tests/test_python_parity.py \ --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json PYTHONPATH=src python rust/tests/test_identity_parity.py \ --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json +PYTHONPATH=src python rust/tests/test_storage_parity.py \ + --binary rust/target/debug/examples/writer_check --report rust/artifacts/storage-parity.json ``` Repeat the build with `--release` and the same suites against @@ -112,9 +141,19 @@ real concurrent WAL snapshot test. It also checks all 4096 modes across three classes and five permissions: **61,440 simple mode decisions**, not 61,440 independent identity scenarios. Report groups separate these from other cases. -`wire_check` and `authority_check` are local JSONL **test examples**, never HTTP or +The storage suite compares complete stored rows, canonical bytes, responses, +rollback order and unchanged schemas against the actual Python storage methods. +Separate safety/process groups cover swallowed errors, implicit rollback, failed +COMMIT, result caps, real process kills and Python/Rust fence interoperability. +Native tests additionally inject panics, cleanup failures and an uncertain commit +result. The last case is deterministic error injection, not a physical disk test. + +`wire_check`, `authority_check` and `writer_check` are local JSONL **test examples**, never HTTP or MCP endpoints. They use only deterministic public test keys and disposable data. -They do not execute business handlers, make network calls, launch Python from +The writer helper deliberately includes unauthenticated storage orchestration +(`once`) and failure/checkpoint controls only for disposable tests. It is not a +production executor and must never be exposed as an endpoint. These examples +do not execute business handlers, make network calls, launch Python from Rust, or access production credentials. The Rust crates forbid unsafe code; cryptographic and database primitives are delegated to pinned libraries. @@ -133,8 +172,14 @@ cryptographic and database primitives are delegated to pinned libraries. accepting every Python input. - Dalek strict verification rejects weak-key forgeries. Historical key/signature compatibility still needs characterization before routing production traffic. +- Storage row decoding uses `invalid_storage_record` rather than every Python + field-specific error. Core/domain validation (notably Unicode tag normalization), + hex-ID aliases and complete resource policy are pending. Native storage takes + canonical IDs/validated domain records; its DTOs are not public request schemas. + The stricter poisoned-write and malformed-result checks are explicit safety + differences, not assertions of identical behavior for every Python input. - Full resource authorization, operation handlers, mutation-side identity/OAuth - issuance/refresh/recovery, encrypted-key wrapping, PostgreSQL/write storage, + issuance/refresh/recovery, encrypted-key wrapping, PostgreSQL and remaining stores, native `serve`/HTTP/MCP, `worker`, packaging and deployment remain pending. - No RSS, memory-reduction or throughput result is claimed. Benchmark equivalent functionality, data, concurrency and complete process trees before comparison. @@ -143,9 +188,10 @@ cryptographic and database primitives are delegated to pinned libraries. 1. Integrate the real operation/capability registry and complete resource authorization, including private resources, sharing, memberships and tools. -2. Port writer fencing, atomic result/side-effect transactions and recovery - against disposable Python-migrated databases. Test rollback, restarts, - competing writers and retry behavior; retain Python migration scripts. +2. Complete mutation stores, blob/Git staging and durable external recovery; + connect the writer to the real authorized executor, receipt signing and secret + delivery. Retain Python migration/administration scripts. Extend the current + metadata crash/retry checks to complete operation effects. 3. Integrate a bounded native executor and HTTP/MCP adapters, then worker claiming/leases/retries and side effects. Empty loops or Python subprocesses do not count as a server port. @@ -159,4 +205,5 @@ selected MSG signing identity was available in the editor, so no authorized private coordination thread could be created. No new production identity or public coordination post was created. Work is recorded in the migration branch/PR; use an existing authorized identity when private coordination resumes. -The temporary offline source/toolchain-export workflow is removed by this slice. +The temporary writer-dependency export workflow is removed by this slice. +No selected production MSG identity was invented or registered for testing. diff --git a/rust/crates/msg-storage/Cargo.toml b/rust/crates/msg-storage/Cargo.toml index ba620cf9..ae6953fd 100644 --- a/rust/crates/msg-storage/Cargo.toml +++ b/rust/crates/msg-storage/Cargo.toml @@ -10,11 +10,16 @@ publish.workspace = true msg-core = { path = "../msg-core" } msg-identity = { path = "../msg-identity" } rusqlite.workspace = true +serde.workspace = true +serde_json.workspace = true +msg-crypto = { path = "../msg-crypto" } + +[target.'cfg(unix)'.dependencies] +fs2.workspace = true +libc.workspace = true [dev-dependencies] msg-protocol = { path = "../msg-protocol" } -serde.workspace = true -serde_json.workspace = true [lints] workspace = true diff --git a/rust/crates/msg-storage/examples/writer_check.rs b/rust/crates/msg-storage/examples/writer_check.rs new file mode 100644 index 00000000..3d1265c0 --- /dev/null +++ b/rust/crates/msg-storage/examples/writer_check.rs @@ -0,0 +1,155 @@ +//! Disposable local failure-injection driver, NOT an authenticated executor. +//! No network listener, arbitrary SQL, production credentials or Python process. +use msg_core::{Error, Json, Result}; +use msg_identity::store::{AuthorityStore, Record}; +use msg_storage::{ + records::{decode, encode}, + SqliteWriter, WriteFailure, WriteSession, +}; +use serde::de::DeserializeOwned; +use std::{ + io::{self, Write}, + path::Path, + time::Duration, +}; +fn field<'a>(value: &'a Json, key: &str) -> Result<&'a Json> { + value.as_object()?.get(key).ok_or(Error("fixture_field")) +} +fn get(value: &Json, key: &str) -> Result { + decode(field(value, key)?) +} +fn flag(value: &Json, key: &str) -> Result { + value + .as_object()? + .get(key) + .map(Json::as_bool) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn emit(value: &Json) -> Result<()> { + println!("{}", value.canonical()?); + io::stdout().flush().map_err(|_| Error("fixture_output")) +} +fn hold(name: &str) -> Result<()> { + emit(&encode(&serde_json::json!({"checkpoint":name}))?)?; + let mut line = String::new(); + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + if line.trim() != "continue" { + return Err(Error("fixture_input")); + } + Ok(()) +} +fn steps(tx: &mut WriteSession<'_>, values: &Json) -> Result> { + let mut output = Vec::new(); + for step in values.as_array()? { + match field(step, "action")?.as_str()? { + "set" => tx.set_setting(&get::(step, "key")?, field(step, "value")?)?, + "read" => output.push( + tx.setting(&get::(step, "key")?)? + .unwrap_or(Json::parse("null")?), + ), + "resource" => output.push(tx.record(Record::Resource, &get::(step, "id")?)?), + "insert" => tx.insert_resource(&get(step, "record")?)?, + "replace" => tx.replace_resource(&get(step, "record")?, get(step, "expected")?)?, + "revision" => tx.append_revision(&get(step, "record")?)?, + "result" => tx.save_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "digest")?, + &get(step, "record")?, + )?, + "lookup" => output.push( + tx.request_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "id")?, + &get::(step, "digest")?, + )? + .unwrap_or(Json::parse("null")?), + ), + "event" => tx.append_event(&get(step, "record")?)?, + "audit" => tx.append_audit(&get(step, "record")?)?, + "savepoint" => match tx.savepoint(|tx| steps(tx, field(step, "steps")?)) { + Ok(values) => output.push(encode(&values)?), + Err(error) if flag(step, "recover")? => { + output.push(encode(&serde_json::json!({"error":error.0}))?) + } + Err(error) => return Err(error), + }, + "ignore" => { + let _ = steps(tx, field(step, "steps")?); + } + "effect" => { + let name: String = get(step, "name")?; + let fail = flag(step, "fail")?; + let panic = flag(step, "panic")?; + let pause = flag(step, "hold")?; + tx.on_rollback(move || { + emit(&encode(&serde_json::json!({"effect":name}))?)?; + if pause { + hold("compensation")?; + } + assert!(!panic, "fixture cleanup panic"); + if fail { + return Err(Error("fixture_cleanup")); + } + Ok(()) + })?; + } + "hold" => hold("transaction")?, + "fail" => return Err(Error("fixture_failure")), + "panic" => panic!("fixture handler panic"), + // This is only a test of the storage building blocks. There is NO + // authentication or authorization here; never expose it as an API. + "once" => { + let subject: Option = get(step, "subject")?; + let digest: String = get(step, "digest")?; + let record: msg_storage::records::OperationResult = get(step, "record")?; + if let Some(result) = + tx.request_result(subject.as_deref(), &record.request_id, &digest)? + { + output.push(result); + } else { + steps(tx, field(step, "steps")?)?; + tx.save_result(subject.as_deref(), &digest, &record)?; + output.push(encode(&record)?); + } + } + _ => return Err(Error("fixture_action")), + } + } + Ok(output) +} +fn run(path: &Path, case: &Json) -> std::result::Result, WriteFailure> { + let timeout = case + .as_object()? + .get("timeout_ms") + .map(decode::) + .transpose()? + .unwrap_or(2000); + let mut writer = SqliteWriter::open_existing(path, Duration::from_millis(timeout))?; + let output = writer.transaction(|tx| steps(tx, field(case, "steps")?))?; + if flag(case, "hold_after_commit")? { + hold("committed")?; + } + Ok(output) +} +fn main() { + let mut line = String::new(); + let result = (|| { + let path = std::env::args().nth(1).ok_or(Error("fixture_path"))?; + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + run(Path::new(&path), &Json::parse(&line)?) + })(); + let output = match result { + Ok(values) => serde_json::json!({"status":"ok","values":values}), + Err(error) => { + serde_json::json!({"status":"error","code":error.cause.0,"cleanup":error.cleanup_errors.iter().map(|e| e.0).collect::>()}) + } + }; + if emit(&encode(&output).expect("fixture result is bounded")).is_err() { + std::process::exit(1); + } +} diff --git a/rust/crates/msg-storage/src/lib.rs b/rust/crates/msg-storage/src/lib.rs index 93b6382b..352f9456 100644 --- a/rust/crates/msg-storage/src/lib.rs +++ b/rust/crates/msg-storage/src/lib.rs @@ -1,7 +1,10 @@ -//! Native SQLite authority reads over existing Python-created databases. +//! Native SQLite sessions over existing Python-created databases. //! -//! No schema creation, migration, writer, public resource output or Python process. -//! One borrowed, non-Send snapshot binds all reads to the same transaction. +//! No schema creation/migration, network entry point or Python subprocess. +//! Read snapshots and write sessions share the same transaction-bound authority reads. +pub mod records; +#[cfg(unix)] +pub mod writer; use msg_core::{Error, Json, Result, MAX_BYTES}; use msg_identity::{ models::Timestamp, @@ -9,13 +12,21 @@ use msg_identity::{ store::{AuthorityStore, Record, RecoveryDelivery}, }; use rusqlite::{Connection, ErrorCode, OpenFlags, OptionalExtension, Transaction}; -use std::{marker::PhantomData, path::Path, rc::Rc, time::Duration}; +use std::{cell::Cell, marker::PhantomData, path::Path, rc::Rc, time::Duration}; +#[cfg(unix)] +pub use writer::{SqliteWriter, WriteFailure, WriteSession}; pub struct SqliteAuthority { connection: Connection, } -pub struct Snapshot<'a> { +pub struct ReadOnly; +pub type Snapshot<'a> = Session<'a, ReadOnly>; + +/// A borrowed database session. Its transaction is private and cannot escape. +pub struct Session<'a, Access> { transaction: Transaction<'a>, + access: Access, + poison: Cell>, // A database snapshot must stay with the task that owns it. _owner: PhantomData>, } @@ -23,6 +34,7 @@ fn database_error(error: rusqlite::Error) -> Error { match error.sqlite_error_code() { Some(ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) => Error("server_busy"), Some(ErrorCode::ReadOnly) => Error("read_only_transaction"), + Some(ErrorCode::ConstraintViolation) => Error("constraint_conflict"), _ => Error("storage_error"), } } @@ -47,6 +59,8 @@ impl SqliteAuthority { let transaction = self.connection.transaction().map_err(database_error)?; let snapshot = Snapshot { transaction, + access: ReadOnly, + poison: Cell::new(None), _owner: PhantomData, }; // The first read pins this snapshot before checking any authority. @@ -54,8 +68,18 @@ impl SqliteAuthority { Ok(snapshot) } } -impl Snapshot<'_> { +impl Session<'_, Access> { + fn active(&self) -> Result<()> { + if let Some(error) = self.poison.get() { + return Err(error); + } + if self.transaction.is_autocommit() { + return Err(Error("transaction_aborted")); + } + Ok(()) + } fn validate_schema(&self) -> Result<()> { + self.active()?; let expected = [ ("schema_version", "version"), ("settings", "key,value"), ("identities", "id,kind,body"), ("credentials", "id,subject,body"), @@ -97,6 +121,7 @@ impl Snapshot<'_> { Ok(()) } fn json(&self, sql: &str, id: &str) -> Result> { + self.active()?; let mut stmt = self .transaction .prepare_cached(sql) @@ -114,7 +139,7 @@ impl Snapshot<'_> { .transpose() } } -impl AuthorityStore for Snapshot<'_> { +impl AuthorityStore for Session<'_, Access> { fn record(&self, kind: Record, id: &str) -> Result { // A bounded SQL substring avoids allocating a corrupt multi-GB body. let (table, condition, missing) = match kind { @@ -133,6 +158,7 @@ impl AuthorityStore for Snapshot<'_> { ) } fn has_setting(&self, key: &str) -> Result { + self.active()?; self.transaction .query_row( "SELECT EXISTS(SELECT 1 FROM settings WHERE key=?1)", @@ -142,6 +168,7 @@ impl AuthorityStore for Snapshot<'_> { .map_err(database_error) } fn certificate_revoked(&self, id: &str) -> Result { + self.active()?; let revoked: Option = self .transaction .query_row("SELECT revoked FROM certificates WHERE id=?1", [id], |r| { @@ -157,6 +184,7 @@ impl AuthorityStore for Snapshot<'_> { id: &str, digest: &str, ) -> Result> { + self.active()?; let row: Option<(String, String)> = self.transaction.query_row( "SELECT digest,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM results WHERE subject=?1 AND request_id=?2", [subject.unwrap_or(""), id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; @@ -186,6 +214,7 @@ impl AuthorityStore for Snapshot<'_> { Ok(Some(result)) } fn oauth_state(&self, id: &str) -> Result> { + self.active()?; let row: Option<(String, String)> = self.transaction.query_row( "SELECT expires,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM oauth_states WHERE id=?1", [id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; @@ -201,6 +230,7 @@ impl AuthorityStore for Snapshot<'_> { .transpose() } fn custodial_binding(&self, subject: &str) -> Result> { + self.active()?; self.transaction .query_row( "SELECT signing_key_id,status FROM custodial_vault WHERE subject=?1", @@ -215,6 +245,7 @@ impl AuthorityStore for Snapshot<'_> { credential: &str, subject: &str, ) -> Result> { + self.active()?; let row: Option<(String, String, bool, String)> = self.transaction.query_row( "SELECT recovery_verifier,recovery_expires_at,consumed_at IS NOT NULL,request_id FROM token_deliveries WHERE credential_id=?1 AND subject=?2", [credential, subject], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?))).optional().map_err(database_error)?; diff --git a/rust/crates/msg-storage/src/records.rs b/rust/crates/msg-storage/src/records.rs new file mode 100644 index 00000000..d33c9361 --- /dev/null +++ b/rust/crates/msg-storage/src/records.rs @@ -0,0 +1,233 @@ +//! Typed storage rows, not an operation schema or authorization decision. +//! The domain layer still owns tags, quotas, ACLs, receipt signing and policy. +use msg_core::{Error, Json, Result}; +use msg_crypto::Signature; +use msg_identity::models::{ResourceRef, Timestamp}; +use serde::{de::DeserializeOwned, Deserialize, Deserializer, Serialize, Serializer}; +use std::collections::BTreeMap; + +pub fn decode(value: &Json) -> Result { + serde_json::from_str(&value.canonical()?).map_err(|_| Error("invalid_storage_record")) +} +pub fn encode(value: &T) -> Result { + Json::parse(&serde_json::to_string(value).map_err(|_| Error("invalid_storage_record"))?) +} +fn required_option<'de, T: Deserialize<'de>, D: Deserializer<'de>>( + d: D, +) -> std::result::Result, D::Error> { + Option::::deserialize(d) +} +mod mode { + use super::*; + pub fn serialize(n: &u16, s: S) -> std::result::Result { + if *n > 0o7777 { + return Err(serde::ser::Error::custom("invalid_mode")); + } + s.serialize_str(&format!("{n:04o}")) + } + pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> std::result::Result { + let value = String::deserialize(d)?; + if value.len() != 4 || !value.bytes().all(|b| (b'0'..=b'7').contains(&b)) { + return Err(serde::de::Error::custom("invalid_mode")); + } + u16::from_str_radix(&value, 8).map_err(serde::de::Error::custom) + } +} +#[derive(Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResourceState { + Active, + Archived, + Purged, +} +impl ResourceState { + pub(crate) fn sql(self) -> &'static str { + match self { + Self::Active => "active", + Self::Archived => "archived", + Self::Purged => "purged", + } + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Resource { + pub id: String, + #[serde(rename = "type")] + pub resource_type: String, + pub type_version: i64, + pub name: String, + #[serde(deserialize_with = "required_option")] + pub parent: Option, + pub owner: String, + pub group: String, + #[serde(with = "mode")] + pub mode: u16, + pub generation: i64, + #[serde(deserialize_with = "required_option")] + pub revision: Option, + pub state: ResourceState, + pub created_at: Timestamp, + pub created_by: String, + pub modified_at: Timestamp, + pub modified_by: String, + #[serde(default)] + pub tags: Vec, +} +impl Resource { + pub(crate) fn validate(&self) -> Result<()> { + if self.generation < 0 { + return Err(Error("invalid_nonnegative_integer")); + } + if self.type_version < 1 { + return Err(Error("invalid_version")); + } + if self.mode > 0o7777 { + return Err(Error("invalid_mode")); + } + if self.parent.as_deref() == Some(&self.id) { + return Err(Error("parent_cycle")); + } + // Unicode tag normalization belongs to the still-unported domain layer. + // Preserve existing normalized Unicode; never re-normalize stored bytes. + if self.tags.len() > 16 || self.tags.windows(2).any(|w| w[0] >= w[1]) { + return Err(Error("invalid_tags")); + } + Ok(()) + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BlobRef { + pub digest: String, + pub size: i64, + pub media_type: String, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Relation { + #[serde(rename = "type")] + pub relation_type: String, + pub target: ResourceRef, + #[serde(default)] + pub excerpt: Option<[i64; 2]>, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Revision { + pub format_version: i64, + pub id: String, + pub resource_id: String, + pub parents: Vec, + pub content: BlobRef, + pub relations: Vec, + pub actor: String, + pub subject: String, + pub author: String, + pub created_at: Timestamp, + pub manifest_digest: String, + #[serde(default)] + pub signature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub summary: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub change_note: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_kind: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_digest: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signature_source: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SourceKind { + Release, + User, + Operation, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignatureSource { + Custodial, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationError { + pub code: String, + pub retryable: bool, + #[serde(default)] + pub field_path: Option, + #[serde(default)] + pub retry_after_seconds: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub message: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ResultStatus { + Ok, + Accepted, + Error, + Uncertain, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OperationResult { + pub request_id: String, + pub operation: String, + pub status: ResultStatus, + #[serde(deserialize_with = "required_option")] + pub actor: Option, + #[serde(deserialize_with = "required_option")] + pub subject: Option, + #[serde(default)] + pub resources: Vec, + #[serde(default)] + pub committed_at: Option, + #[serde(default)] + pub replayed: bool, + #[serde(default)] + pub receipt: Option, + #[serde(default)] + pub error: Option, + #[serde(default)] + pub data: Option>, + #[serde(default)] + pub output: Option, + #[serde(default)] + pub prefer_cli: bool, + #[serde(default)] + pub cli_url: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Event { + pub id: String, + #[serde(rename = "type")] + pub event_type: String, + pub time: Timestamp, + pub request_id: String, + #[serde(deserialize_with = "required_option")] + pub actor: Option, + #[serde(deserialize_with = "required_option")] + pub subject: Option, + pub resources: Vec, + pub data: BTreeMap, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AuditEvent { + pub event: Event, + pub authority: Vec, + #[serde(deserialize_with = "required_option")] + pub before_digest: Option, + #[serde(deserialize_with = "required_option")] + pub after_digest: Option, + #[serde(deserialize_with = "required_option")] + pub previous_digest: Option, + pub entry_digest: String, + pub result: String, +} diff --git a/rust/crates/msg-storage/src/writer.rs b/rust/crates/msg-storage/src/writer.rs new file mode 100644 index 00000000..7bc3e42f --- /dev/null +++ b/rust/crates/msg-storage/src/writer.rs @@ -0,0 +1,632 @@ +//! Unix writer fence and atomic metadata sessions. No service entry point. +//! +//! Run this synchronous API on the database-owning blocking thread. Do not drop +//! an async wrapper and assume that its in-flight transaction was cancelled. +//! One transaction owns the fence until rollback AND compensation have finished. +use crate::{database_error, records, ReadOnly, Session}; +use fs2::FileExt; +use msg_core::{Error, Json, Result}; +use msg_identity::store::{AuthorityStore, Record}; +use records::{AuditEvent, Event, OperationResult, Resource, Revision}; +use rusqlite::{params, Connection, OpenFlags, OptionalExtension, TransactionBehavior}; +use std::{ + cell::Cell, + fs::{self, File, OpenOptions}, + io::ErrorKind, + marker::PhantomData, + os::unix::fs::{MetadataExt, OpenOptionsExt}, + panic::{catch_unwind, resume_unwind, AssertUnwindSafe}, + path::{Path, PathBuf}, + thread, + time::{Duration, Instant}, +}; + +#[derive(Debug)] +pub struct WriteFailure { + pub cause: Error, + /// Stable diagnostic codes only: never SQL text, credentials or row bodies. + pub cleanup_errors: Vec, +} +impl From for WriteFailure { + fn from(cause: Error) -> Self { + Self { + cause, + cleanup_errors: Vec::new(), + } + } +} +/// Only the store can construct this access state. +#[derive(Default)] +pub struct WriteAccess { + effects: Vec Result<()>>>, + cleanup_errors: Vec, + savepoint: u64, + commit_uncertain: bool, +} +pub type WriteSession<'a> = Session<'a, WriteAccess>; + +pub struct SqliteWriter { + connection: Connection, + path: PathBuf, + timeout: Duration, + needs_recovery: bool, + identity: (u64, u64), +} +impl SqliteWriter { + /// Open a pre-migrated WAL database. Never create it or change its schema. + /// The configured database path must identify one file, not a symlink or + /// hard-link alias that another writer could fence under a different name. + pub fn open_existing(path: &Path, timeout: Duration) -> Result { + if timeout.as_millis() > i32::MAX as u128 { + return Err(Error("invalid_timeout")); + } + let metadata = fs::symlink_metadata(path).map_err(|_| Error("storage_error"))?; + if !metadata.is_file() || metadata.nlink() != 1 { + return Err(Error("unsafe_storage_path")); + } + let path = fs::canonicalize(path).map_err(|_| Error("storage_error"))?; + let mut connection = Connection::open_with_flags( + &path, + OpenFlags::SQLITE_OPEN_READ_WRITE + | OpenFlags::SQLITE_OPEN_NO_MUTEX + | OpenFlags::SQLITE_OPEN_NOFOLLOW, + ) + .map_err(database_error)?; + connection.busy_timeout(timeout).map_err(database_error)?; + connection + .execute_batch( + "PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA synchronous=FULL;", + ) + .map_err(database_error)?; + let mode: String = connection + .query_row("PRAGMA journal_mode", [], |r| r.get(0)) + .map_err(database_error)?; + if mode != "wal" { + return Err(Error("unsupported_storage_journal")); + } + { + let session = Session { + transaction: connection.transaction().map_err(database_error)?, + access: ReadOnly, + poison: Cell::new(None), + _owner: PhantomData, + }; + session.validate_schema()?; + validate_write_schema(&session.transaction)?; + } + Ok(Self { + connection, + path, + timeout, + needs_recovery: false, + identity: (metadata.dev(), metadata.ino()), + }) + } + + /// Return a value only after COMMIT succeeds. All fallible writes are sticky: + /// ignoring their error cannot commit an earlier partial mutation. + /// + /// Sessions cannot escape, commit themselves, or move to another thread. + /// ```compile_fail + /// use msg_storage::WriteSession; + /// fn handler(tx: &mut WriteSession<'_>) { tx.commit().unwrap(); } + /// ``` + /// ```compile_fail + /// use msg_storage::WriteSession; + /// fn assert_send() {} + /// fn check() { assert_send::>(); } + /// ``` + /// ```compile_fail + /// use msg_storage::SqliteWriter; + /// fn escape(store: &mut SqliteWriter) { let _tx = store.transaction(|tx| Ok(tx)); } + /// ``` + /// ```compile_fail + /// use msg_storage::Snapshot; + /// use msg_core::Json; + /// fn mutate_read(tx: &mut Snapshot<'_>) { tx.set_setting("x", &Json::parse("1").unwrap()); } + /// ``` + pub fn transaction( + &mut self, + f: impl FnOnce(&mut WriteSession<'_>) -> Result, + ) -> std::result::Result { + if self.needs_recovery { + return Err(Error("storage_recovery_required").into()); + } + let deadline = Instant::now() + .checked_add(self.timeout) + .ok_or(Error("invalid_timeout"))?; + // This guard is intentionally outside the SQL transaction. Do not unlink + // its file on release: a second inode would break the process fence. + let _fence = acquire_fence(&self.path, deadline)?; + let metadata = + fs::symlink_metadata(&self.path).map_err(|_| Error("storage_recovery_required"))?; + if !metadata.is_file() + || metadata.nlink() != 1 + || (metadata.dev(), metadata.ino()) != self.identity + { + self.needs_recovery = true; + return Err(Error("storage_recovery_required").into()); + } + self.connection + .busy_timeout(deadline.saturating_duration_since(Instant::now())) + .map_err(database_error)?; + let transaction = self + .connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(database_error)?; + let mut session = Session { + transaction, + access: WriteAccess::default(), + poison: Cell::new(None), + _owner: PhantomData, + }; + // Revalidate after the writer lock: an old schema read is not authority. + let outcome = catch_unwind(AssertUnwindSafe(|| { + session.validate_schema()?; + validate_write_schema(&session.transaction)?; + let result = f(&mut session)?; + session.active()?; + if let Err(error) = session.transaction.execute_batch("COMMIT") { + return Err(session.commit_failed(error)); + } + Ok(result) + })); + match outcome { + Ok(Ok(value)) => { + session.access.effects.clear(); + Ok(value) + } + other => { + if !session.transaction.is_autocommit() { + if let Err(error) = session.transaction.execute_batch("ROLLBACK") { + session.access.cleanup_errors.push(database_error(error)); + } + } + // An I/O error can leave COMMIT's outcome unknown. Do not undo + // external data that might now be referenced by committed rows. + if !session.access.commit_uncertain { + session.compensate(0); + } + self.needs_recovery = session.access.commit_uncertain + || !session.access.cleanup_errors.is_empty() + || !session.transaction.is_autocommit(); + match other { + Ok(Err(cause)) => Err(WriteFailure { + cause, + cleanup_errors: std::mem::take(&mut session.access.cleanup_errors), + }), + Err(panic) => resume_unwind(panic), + Ok(Ok(_)) => unreachable!(), + } + } + } + } +} +fn acquire_fence(path: &Path, deadline: Instant) -> Result { + let mut name = path + .file_name() + .ok_or(Error("unsafe_storage_path"))? + .to_os_string(); + name.push(".writer.lock"); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC | libc::O_NONBLOCK) + .open(path.with_file_name(name)) + .map_err(|_| Error("unsafe_writer_lock"))?; + let metadata = file.metadata().map_err(|_| Error("unsafe_writer_lock"))?; + if !metadata.is_file() || metadata.nlink() != 1 { + return Err(Error("unsafe_writer_lock")); + } + loop { + match file.try_lock_exclusive() { + Ok(()) => return Ok(file), + Err(e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) if e.kind() == ErrorKind::WouldBlock => {} + Err(_) => return Err(Error("writer_lock_error")), + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(Error("server_busy")); + } + thread::sleep(remaining.min(Duration::from_millis(10))); + } +} +fn validate_write_schema(db: &Connection) -> Result<()> { + // Only known tables/columns may participate. This is not a schema migrator. + for (table, columns) in [ + ("resources", "id,type,name,parent,owner,grp,mode,generation,revision,state,created_at,modified_at,body"), + ("resource_tags", "resource_id,tag"), ("resource_path_aliases", "parent_id,name,resource_id"), + ("revisions", "id,resource_id,created_at,body"), + ("relations", "revision_id,source_id,type,target_id,target_revision,body"), + ("events", "seq,id,body"), ("audit", "seq,digest,previous,body"), + ] { + let exists: bool = db.query_row("SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name=? AND type='table')", [table], |r| r.get(0)).map_err(database_error)?; + if !exists { return Err(Error("unsupported_storage_schema")); } + db.prepare(&format!("SELECT {columns} FROM {table} LIMIT 0")).map_err(|_| Error("unsupported_storage_schema"))?; + } + // Result uniqueness is a durability requirement, not an optional index. + let mut statement = db + .prepare("SELECT name,pk FROM pragma_table_info('results') WHERE pk>0 ORDER BY pk") + .map_err(database_error)?; + let keys = statement + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if keys != [("subject".into(), 1), ("request_id".into(), 2)] { + return Err(Error("unsupported_storage_schema")); + } + validate_constraints(db)?; + Ok(()) +} +fn validate_constraints(db: &Connection) -> Result<()> { + for (table, expected) in [ + ("resources", &["id"][..]), + ("revisions", &["id"]), + ("settings", &["key"]), + ("resource_tags", &["tag", "resource_id"]), + ("resource_path_aliases", &["parent_id", "name"]), + ("events", &["seq"]), + ("audit", &["seq"]), + ] { + let mut stmt = db + .prepare("SELECT name FROM pragma_table_info(?1) WHERE pk>0 ORDER BY pk") + .map_err(database_error)?; + let actual = stmt + .query_map([table], |r| r.get::<_, String>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if actual != expected { + return Err(Error("unsupported_storage_schema")); + } + } + for (table, expected) in [ + ("resources", &["parent", "name"][..]), + ("events", &["id"]), + ("audit", &["digest"]), + ] { + let mut stmt = db + .prepare("SELECT name FROM pragma_index_list(?1) WHERE \"unique\"=1 AND partial=0") + .map_err(database_error)?; + let indexes = stmt + .query_map([table], |r| r.get::<_, String>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + let mut found = false; + for name in indexes { + let mut stmt = db + .prepare("SELECT name FROM pragma_index_info(?1) ORDER BY seqno") + .map_err(database_error)?; + let columns = stmt + .query_map([name], |r| r.get::<_, Option>(0)) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + found |= columns + .iter() + .map(|v| v.as_deref()) + .eq(expected.iter().map(|v| Some(*v))); + } + if !found { + return Err(Error("unsupported_storage_schema")); + } + } + for (name, expected) in [ + ("one_root", "createuniqueindexone_rootonresources((1))whereparentisnull"), + ("audit_no_update", "createtriggeraudit_no_updatebeforeupdateonauditbeginselectraise(abort,'append_only_audit');end"), + ("audit_no_delete", "createtriggeraudit_no_deletebeforedeleteonauditbeginselectraise(abort,'append_only_audit');end"), + ] { + let sql: Option = db.query_row("SELECT sql FROM sqlite_schema WHERE name=?1", [name], |r| r.get(0)).optional().map_err(database_error)?; + let normalized = sql.unwrap_or_default().chars().filter(|c| !c.is_ascii_whitespace()).collect::().to_ascii_lowercase(); + if normalized.trim_end_matches(';') != expected { return Err(Error("unsupported_storage_schema")); } + } + for (table, expected) in [ + ( + "resources", + &[("parent", "resources", "id", "NO ACTION")][..], + ), + ( + "resource_tags", + &[("resource_id", "resources", "id", "CASCADE")], + ), + ( + "resource_path_aliases", + &[ + ("parent_id", "resources", "id", "CASCADE"), + ("resource_id", "resources", "id", "CASCADE"), + ], + ), + ( + "revisions", + &[("resource_id", "resources", "id", "NO ACTION")], + ), + ( + "relations", + &[("revision_id", "revisions", "id", "CASCADE")], + ), + ] { + let mut stmt = db.prepare("SELECT \"from\",\"table\",\"to\",on_delete FROM pragma_foreign_key_list(?1) ORDER BY \"from\"").map_err(database_error)?; + let actual = stmt + .query_map([table], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, String>(3)?, + )) + }) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if !actual + .iter() + .map(|(a, b, c, d)| (a.as_str(), b.as_str(), c.as_str(), d.as_str())) + .eq(expected.iter().copied()) + { + return Err(Error("unsupported_storage_schema")); + } + } + Ok(()) +} +impl WriteSession<'_> { + fn commit_failed(&mut self, error: rusqlite::Error) -> Error { + if self.transaction.is_autocommit() { + self.access.commit_uncertain = true; + Error("commit_outcome_uncertain") + } else { + database_error(error) + } + } + fn mutate(&mut self, f: impl FnOnce(&mut Self) -> Result) -> Result { + self.active()?; + let result = f(self); + if let Err(error) = &result { + self.poison.set(Some(*error)); + } + result + } + pub fn on_rollback(&mut self, effect: impl FnOnce() -> Result<()> + 'static) -> Result<()> { + self.active()?; + self.access.effects.push(Box::new(effect)); + Ok(()) + } + fn compensate(&mut self, start: usize) { + for effect in self.access.effects.drain(start..).rev() { + match catch_unwind(AssertUnwindSafe(effect)) { + Ok(Ok(())) => {} + Ok(Err(error)) => self.access.cleanup_errors.push(error), + Err(_) => self + .access + .cleanup_errors + .push(Error("rollback_effect_panicked")), + } + } + } + /// A rejected inner unit may be handled ONLY after its changes and external + /// rollback effects are undone. A full SQLite abort poisons the outer unit. + pub fn savepoint(&mut self, f: impl FnOnce(&mut Self) -> Result) -> Result { + self.active()?; + self.access.savepoint = self + .access + .savepoint + .checked_add(1) + .ok_or(Error("savepoint_limit"))?; + let name = format!("msg_sp_{}", self.access.savepoint); + self.transaction + .execute_batch(&format!("SAVEPOINT {name}")) + .map_err(database_error)?; + let effects_start = self.access.effects.len(); + let cleanup_start = self.access.cleanup_errors.len(); + let result = catch_unwind(AssertUnwindSafe(|| { + let value = f(self)?; + self.active()?; + self.transaction + .execute_batch(&format!("RELEASE {name}")) + .map_err(database_error)?; + Ok(value) + })); + match result { + Ok(Ok(value)) => Ok(value), + other => { + let sql_rollback = if self.transaction.is_autocommit() { + Err(Error("transaction_aborted")) + } else { + self.transaction + .execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name}")) + .map_err(database_error) + }; + if let Err(error) = sql_rollback { + self.access.cleanup_errors.push(error); + } + self.compensate(effects_start); + self.poison + .set(if self.access.cleanup_errors.len() == cleanup_start { + None + } else { + Some(Error("storage_recovery_required")) + }); + match other { + Ok(Err(error)) => Err(error), + Err(panic) => resume_unwind(panic), + Ok(Ok(_)) => unreachable!(), + } + } + } + } + pub fn set_setting(&mut self, key: &str, value: &Json) -> Result<()> { + self.mutate(|s| { + let raw = value.canonical()?; + if raw.len() > msg_core::MAX_BYTES { return Err(Error("storage_record_too_large")); } + s.transaction.execute("INSERT INTO settings(key,value) VALUES(?1,?2) ON CONFLICT(key) DO UPDATE SET value=excluded.value", params![key, raw]).map_err(database_error)?; + Ok(()) + }) + } + pub fn insert_resource(&mut self, resource: &Resource) -> Result<()> { + self.mutate(|s| { + resource.validate()?; + let body = records::encode(resource)?.canonical()?; + if let Some(parent) = &resource.parent { s.resource(parent)?; } + s.transaction.execute("INSERT INTO resources(id,type,name,parent,owner,grp,mode,generation,revision,state,created_at,modified_at,body) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13)", + params![resource.id, resource.resource_type, resource.name, resource.parent, resource.owner, resource.group, resource.mode, resource.generation, resource.revision, resource.state.sql(), timestamp(resource.created_at)?, timestamp(resource.modified_at)?, body]).map_err(database_error)?; + s.insert_tags(resource) + }) + } + fn insert_tags(&self, resource: &Resource) -> Result<()> { + for tag in &resource.tags { + self.transaction + .execute( + "INSERT INTO resource_tags(resource_id,tag) VALUES(?1,?2)", + params![resource.id, tag], + ) + .map_err(database_error)?; + } + Ok(()) + } + pub fn replace_resource( + &mut self, + resource: &Resource, + expected_generation: i64, + ) -> Result<()> { + self.mutate(|s| { + resource.validate()?; + let old: Resource = records::decode(&s.record(Record::Resource, &resource.id)?)?; + if old.id != resource.id { return Err(Error("storage_record_mismatch")); } + if old.generation != expected_generation { return Err(Error("generation_conflict")); } + if expected_generation.checked_add(1) != Some(resource.generation) { return Err(Error("invalid_generation")); } + if old.resource_type != resource.resource_type || old.created_at != resource.created_at || old.created_by != resource.created_by { return Err(Error("immutable_creation_fact")); } + if let Some(parent) = &resource.parent { + if parent == &resource.id || s.ancestors(parent)?.iter().any(|a| a.id == resource.id) { return Err(Error("parent_cycle")); } + } + let body = records::encode(resource)?.canonical()?; + if old.parent != resource.parent || old.owner != resource.owner || old.group != resource.group || old.mode != resource.mode || old.state != resource.state { + let epoch = s.setting("authorization_epoch")?.map(|v| v.as_integer()?.parse::().map_err(|_| Error("invalid_authorization_epoch"))).transpose()?.unwrap_or(0); + let next = epoch.checked_add(1).filter(|_| epoch >= 0).ok_or(Error("invalid_authorization_epoch"))?; + s.set_setting("authorization_epoch", &Json::parse(&next.to_string())?)?; + } + if let Some(parent) = &old.parent { + if old.parent != resource.parent || old.name != resource.name { + s.transaction.execute("INSERT INTO resource_path_aliases(parent_id,name,resource_id) VALUES(?1,?2,?3) ON CONFLICT(parent_id,name) DO UPDATE SET resource_id=excluded.resource_id", params![parent, old.name, old.id]).map_err(database_error)?; + } + } + let changed = s.transaction.execute("UPDATE resources SET name=?1,parent=?2,owner=?3,grp=?4,mode=?5,generation=?6,revision=?7,state=?8,modified_at=?9,body=?10 WHERE id=?11 AND generation=?12", + params![resource.name,resource.parent,resource.owner,resource.group,resource.mode,resource.generation,resource.revision,resource.state.sql(),timestamp(resource.modified_at)?,body,resource.id,expected_generation]).map_err(database_error)?; + if changed != 1 { return Err(Error("generation_conflict")); } + if old.tags != resource.tags { + s.transaction.execute("DELETE FROM resource_tags WHERE resource_id=?1", [&resource.id]).map_err(database_error)?; + s.insert_tags(resource)?; + } + Ok(()) + }) + } + pub fn append_revision(&mut self, revision: &Revision) -> Result<()> { + self.mutate(|s| { + if revision.format_version < 1 { return Err(Error("invalid_version")); } + if revision.content.size < 0 { return Err(Error("invalid_nonnegative_integer")); } + for parent in &revision.parents { + let exists: bool = s.transaction.query_row("SELECT EXISTS(SELECT 1 FROM revisions WHERE id=?1 AND resource_id=?2)", params![parent, revision.resource_id], |r| r.get(0)).map_err(database_error)?; + if !exists { return Err(Error("revision_not_found")); } + } + s.transaction.execute("INSERT INTO revisions(id,resource_id,created_at,body) VALUES(?1,?2,?3,?4)", params![revision.id,revision.resource_id,timestamp(revision.created_at)?,records::encode(revision)?.canonical()?]).map_err(database_error)?; + for relation in &revision.relations { + if relation.excerpt.is_some_and(|[a,b]| a < 0 || a > b) { return Err(Error("invalid_byte_range")); } + s.transaction.execute("INSERT INTO relations(revision_id,source_id,type,target_id,target_revision,body) VALUES(?1,?2,?3,?4,?5,?6)", params![revision.id,revision.resource_id,relation.relation_type,relation.target.id,relation.target.revision,records::encode(relation)?.canonical()?]).map_err(database_error)?; + } + Ok(()) + }) + } + /// Storage-only port: the caller must authenticate/authorize the operation, + /// redact secret delivery data and produce its receipt before saving. + /// Neither this method nor a result lookup grants execution authority. + pub fn save_result( + &mut self, + subject: Option<&str>, + digest: &str, + result: &OperationResult, + ) -> Result<()> { + self.mutate(|s| { + if subject != result.subject.as_deref() || subject == Some("") { + return Err(Error("storage_record_mismatch")); + } + if !digest.strip_prefix("sha256:").is_some_and(|v| { + v.len() == 64 + && v.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) { + return Err(Error("invalid_digest")); + } + let count: i64 = s + .transaction + .query_row("SELECT COUNT(*) FROM results", [], |r| r.get(0)) + .map_err(database_error)?; + if count >= 100_000 { + return Err(Error("storage_capacity_exceeded")); + } + s.transaction + .execute( + "INSERT INTO results(subject,request_id,digest,body) VALUES(?1,?2,?3,?4)", + params![ + subject.unwrap_or(""), + result.request_id, + digest, + records::encode(result)?.canonical()? + ], + ) + .map_err(database_error)?; + Ok(()) + }) + } + pub fn append_event(&mut self, event: &Event) -> Result<()> { + self.mutate(|s| { + s.transaction + .execute( + "INSERT INTO events(id,body) VALUES(?1,?2)", + params![event.id, records::encode(event)?.canonical()?], + ) + .map_err(database_error)?; + Ok(()) + }) + } + pub fn append_audit(&mut self, event: &AuditEvent) -> Result<()> { + self.mutate(|s| { + let previous: Option = s + .transaction + .query_row( + "SELECT digest FROM audit ORDER BY seq DESC LIMIT 1", + [], + |r| r.get(0), + ) + .optional() + .map_err(database_error)?; + let mut entry = event.clone(); + entry.previous_digest = previous; + entry.entry_digest.clear(); + let mut fields = records::encode(&entry)?.into_object()?; + fields.remove("entry_digest"); + entry.entry_digest = + msg_core::digest(Json::from_object(fields).canonical()?.as_bytes()); + s.transaction + .execute( + "INSERT INTO audit(digest,previous,body) VALUES(?1,?2,?3)", + params![ + entry.entry_digest, + entry.previous_digest, + records::encode(&entry)?.canonical()? + ], + ) + .map_err(database_error)?; + Ok(()) + }) + } +} +fn timestamp(value: msg_identity::models::Timestamp) -> Result { + Ok(records::encode(&value)?.as_str()?.to_owned()) +} + +#[cfg(test)] +mod tests; diff --git a/rust/crates/msg-storage/src/writer/tests.rs b/rust/crates/msg-storage/src/writer/tests.rs new file mode 100644 index 00000000..a5a6dbc1 --- /dev/null +++ b/rust/crates/msg-storage/src/writer/tests.rs @@ -0,0 +1,423 @@ +//! Focused failure injection; Python-created full schemas are tested separately. +use super::*; +use std::{ + cell::RefCell, + rc::Rc, + sync::atomic::{AtomicU64, Ordering}, + time::{SystemTime, UNIX_EPOCH}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "msg-writer-{}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir(&path).unwrap(); + let fixture = Self(path); + let db = Connection::open(fixture.db()).unwrap(); + db.execute_batch("PRAGMA journal_mode=WAL; + CREATE TABLE schema_version(version INTEGER PRIMARY KEY); INSERT INTO schema_version VALUES(1); + CREATE TABLE settings(key TEXT PRIMARY KEY,value TEXT NOT NULL); + CREATE TABLE identities(id TEXT PRIMARY KEY,kind TEXT,body TEXT); + CREATE TABLE credentials(id TEXT PRIMARY KEY,subject TEXT,body TEXT); + CREATE TABLE certificates(id TEXT PRIMARY KEY,subject TEXT,parent TEXT,revoked INTEGER,body TEXT); + CREATE TABLE resources(id TEXT PRIMARY KEY,type TEXT,name TEXT,parent TEXT REFERENCES resources(id) DEFERRABLE INITIALLY DEFERRED,owner TEXT,grp TEXT,mode INTEGER,generation INTEGER,revision TEXT,state TEXT,created_at TEXT,modified_at TEXT,body TEXT,UNIQUE(parent,name)); + CREATE UNIQUE INDEX one_root ON resources((1)) WHERE parent IS NULL; + CREATE TABLE results(subject TEXT,request_id TEXT,digest TEXT,body TEXT,PRIMARY KEY(subject,request_id)); + CREATE TABLE oauth_states(id TEXT PRIMARY KEY,expires TEXT,body TEXT); + CREATE TABLE custodial_vault(subject TEXT PRIMARY KEY,signing_key_id TEXT,status TEXT); + CREATE TABLE token_deliveries(credential_id TEXT,subject TEXT,recovery_verifier TEXT,recovery_expires_at TEXT,consumed_at TEXT,request_id TEXT); + CREATE TABLE resource_tags(resource_id TEXT REFERENCES resources(id) ON DELETE CASCADE,tag TEXT,PRIMARY KEY(tag,resource_id)); + CREATE TABLE resource_path_aliases(parent_id TEXT REFERENCES resources(id) ON DELETE CASCADE,name TEXT,resource_id TEXT REFERENCES resources(id) ON DELETE CASCADE,PRIMARY KEY(parent_id,name)); + CREATE TABLE revisions(id TEXT PRIMARY KEY,resource_id TEXT REFERENCES resources(id),created_at TEXT,body TEXT); + CREATE TABLE relations(revision_id TEXT REFERENCES revisions(id) ON DELETE CASCADE,source_id TEXT,type TEXT,target_id TEXT,target_revision TEXT,body TEXT); + CREATE TABLE events(seq INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT UNIQUE,body TEXT); + CREATE TABLE audit(seq INTEGER PRIMARY KEY AUTOINCREMENT,digest TEXT UNIQUE,previous TEXT,body TEXT); + CREATE TRIGGER audit_no_update BEFORE UPDATE ON audit BEGIN SELECT RAISE(ABORT,'append_only_audit'); END; + CREATE TRIGGER audit_no_delete BEFORE DELETE ON audit BEGIN SELECT RAISE(ABORT,'append_only_audit'); END; + ").unwrap(); + fixture + } + fn db(&self) -> PathBuf { + self.0.join("metadata.db") + } + fn writer(&self) -> SqliteWriter { + SqliteWriter::open_existing(&self.db(), Duration::from_millis(40)).unwrap() + } + fn setting(&self, key: &str) -> Option { + Connection::open(self.db()) + .unwrap() + .query_row("SELECT value FROM settings WHERE key=?", [key], |r| { + r.get(0) + }) + .optional() + .unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.0).unwrap(); + } +} +fn value() -> Json { + Json::parse("1").unwrap() +} +fn effect(tx: &mut WriteSession<'_>, log: &Rc>>, n: i32) { + let log = Rc::clone(log); + tx.on_rollback(move || { + log.borrow_mut().push(n); + Ok(()) + }) + .unwrap(); +} +#[test] +fn commit_and_read_your_writes() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let got = writer + .transaction(|tx| { + tx.set_setting("x", &value())?; + tx.setting("x")?.unwrap().canonical() + }) + .unwrap(); + assert_eq!(got, "1"); + assert_eq!(fixture.setting("x").as_deref(), Some("1")); +} +#[test] +fn handler_failure_rolls_back_in_reverse_order() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result: std::result::Result<(), _> = writer.transaction(|tx| { + effect(tx, &log, 1); + tx.set_setting("x", &value())?; + effect(tx, &log, 2); + Err(Error("fixture_failure")) + }); + assert_eq!(result.unwrap_err().cause, Error("fixture_failure")); + assert_eq!(*log.borrow(), [2, 1]); + assert!(fixture.setting("x").is_none()); + writer + .transaction(|tx| tx.set_setting("x", &value())) + .unwrap(); +} +#[test] +fn swallowed_sql_failure_cannot_commit_partial_work() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ABORT,'fixture'); END;").unwrap(); + let error = writer + .transaction(|tx| { + tx.set_setting("x", &value())?; + let _ = tx.set_setting("bad", &value()); + Ok(()) + }) + .unwrap_err(); + assert_eq!(error.cause, Error("constraint_conflict")); + assert!(fixture.setting("x").is_none()); +} +#[test] +fn sqlite_full_abort_cannot_restart_in_autocommit() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END;").unwrap(); + let error = writer + .transaction(|tx| { + tx.set_setting("x", &value())?; + assert!(tx.set_setting("bad", &value()).is_err()); + assert!(tx.set_setting("autocommit_escape", &value()).is_err()); + Ok(()) + }) + .unwrap_err(); + assert_eq!(error.cause, Error("constraint_conflict")); + assert!(fixture.setting("x").is_none()); + assert!(fixture.setting("autocommit_escape").is_none()); +} +#[test] +fn savepoint_failure_isolated_from_outer_work() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + writer + .transaction(|tx| { + effect(tx, &log, 1); + tx.set_setting("outer", &value())?; + let result: Result<()> = tx.savepoint(|tx| { + tx.set_setting("inner", &value())?; + effect(tx, &log, 2); + Err(Error("fixture_failure")) + }); + assert!(result.is_err()); + assert_eq!(*log.borrow(), [2]); + assert!(tx.setting("inner")?.is_none()); + Ok(()) + }) + .unwrap(); + assert_eq!(*log.borrow(), [2]); + assert!(fixture.setting("inner").is_none()); + assert!(fixture.setting("outer").is_some()); +} +#[test] +fn released_inner_effects_remain_until_outer_commit() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let _: std::result::Result<(), _> = writer.transaction(|tx| { + effect(tx, &log, 1); + tx.savepoint(|tx| { + effect(tx, &log, 2); + tx.savepoint(|tx| { + effect(tx, &log, 3); + Ok(()) + }) + })?; + Err(Error("fixture_failure")) + }); + assert_eq!(*log.borrow(), [3, 2, 1]); +} +#[test] +fn ignored_inner_mutation_failure_is_rolled_back() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ABORT,'fixture'); END;").unwrap(); + writer + .transaction(|tx| { + tx.set_setting("outer", &value())?; + assert!(tx + .savepoint(|tx| { + tx.set_setting("inner", &value())?; + let _ = tx.set_setting("bad", &value()); + Ok(()) + }) + .is_err()); + tx.set_setting("after", &value()) + }) + .unwrap(); + assert!(fixture.setting("inner").is_none()); + assert!(fixture.setting("after").is_some()); +} +#[test] +fn deferred_commit_failure_returns_no_success_and_compensates() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result = writer.transaction(|tx| { + tx.set_setting("x", &value())?; + effect(tx, &log, 1); + // Test-only injection: callers have no public SQL/transaction handle. + tx.transaction + .execute( + "INSERT INTO resources(id,parent) VALUES('orphan','missing')", + [], + ) + .map_err(database_error)?; + Ok("must_not_escape") + }); + assert_eq!(result.unwrap_err().cause, Error("constraint_conflict")); + assert!(fixture.setting("x").is_none()); + assert_eq!(*log.borrow(), [1]); +} +#[test] +fn outer_panic_rolls_back_and_runs_all_effects() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + assert!(catch_unwind(AssertUnwindSafe(|| { + let _: std::result::Result<(), _> = writer.transaction(|tx| { + tx.set_setting("x", &value())?; + effect(tx, &log, 1); + panic!("fixture panic") + }); + })) + .is_err()); + assert!(fixture.setting("x").is_none()); + assert_eq!(*log.borrow(), [1]); + writer + .transaction(|tx| tx.set_setting("after", &value())) + .unwrap(); +} +#[test] +fn inner_panic_can_be_caught_only_after_rollback() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer + .transaction(|tx| { + assert!(catch_unwind(AssertUnwindSafe(|| { + let _: Result<()> = tx.savepoint(|tx| { + tx.set_setting("inner", &value())?; + panic!("fixture panic") + }); + })) + .is_err()); + tx.set_setting("after", &value()) + }) + .unwrap(); + assert!(fixture.setting("inner").is_none()); + assert!(fixture.setting("after").is_some()); +} +#[test] +fn cleanup_failure_preserves_cause_continues_and_disables_writer() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result: std::result::Result<(), _> = writer.transaction(|tx| { + effect(tx, &log, 1); + tx.on_rollback(|| Err(Error("cleanup_failed")))?; + effect(tx, &log, 2); + tx.on_rollback(|| panic!("cleanup panic"))?; + Err(Error("primary")) + }); + let error = result.unwrap_err(); + assert_eq!(error.cause, Error("primary")); + assert_eq!( + error.cleanup_errors, + [Error("rollback_effect_panicked"), Error("cleanup_failed")] + ); + assert_eq!(*log.borrow(), [2, 1]); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("storage_recovery_required") + ); +} +#[test] +fn writer_fence_covers_compensation_after_sqlite_abort() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END;").unwrap(); + let other_path = fixture.db(); + let result = writer.transaction(|tx| { + tx.on_rollback(move || { + let mut other = SqliteWriter::open_existing(&other_path, Duration::ZERO)?; + assert_eq!( + other + .transaction(|tx| tx.set_setting("interleaved", &value())) + .unwrap_err() + .cause, + Error("server_busy") + ); + Ok(()) + })?; + tx.set_setting("bad", &value()) + }); + assert!(result.is_err()); + assert!(fixture.setting("interleaved").is_none()); + fixture + .writer() + .transaction(|tx| tx.set_setting("after", &value())) + .unwrap(); +} +#[test] +fn nested_full_abort_never_commits_outer_work() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + writer.connection.execute_batch("CREATE TRIGGER reject_bad BEFORE INSERT ON settings WHEN NEW.key='bad' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END;").unwrap(); + let result = writer.transaction(|tx| { + tx.set_setting("outer", &value())?; + let _ = tx.savepoint(|tx| tx.set_setting("bad", &value())); + assert!(tx.set_setting("escape", &value()).is_err()); + Ok(()) + }); + assert!(result.is_err()); + assert!(fixture.setting("outer").is_none()); + assert!(fixture.setting("escape").is_none()); +} +#[test] +fn missing_bad_journal_or_schema_is_not_repaired() { + let fixture = Fixture::new(); + assert!(SqliteWriter::open_existing(&fixture.0.join("missing"), Duration::ZERO).is_err()); + assert!(!fixture.0.join("missing").exists()); + let db = Connection::open(fixture.db()).unwrap(); + db.execute_batch("PRAGMA journal_mode=DELETE").unwrap(); + drop(db); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), + Err(Error("unsupported_storage_journal")) + )); + let db = Connection::open(fixture.db()).unwrap(); + db.execute_batch("PRAGMA journal_mode=WAL; UPDATE schema_version SET version=99;") + .unwrap(); + drop(db); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), + Err(Error("unsupported_storage_schema")) + )); +} +#[test] +fn lock_symlink_and_file_aliases_are_rejected() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let target = fixture.0.join("target"); + fs::write(&target, "keep").unwrap(); + std::os::unix::fs::symlink(&target, fixture.0.join("metadata.db.writer.lock")).unwrap(); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("unsafe_writer_lock") + ); + assert_eq!(fs::read_to_string(target).unwrap(), "keep"); + let link = fixture.0.join("alias"); + std::os::unix::fs::symlink(fixture.db(), &link).unwrap(); + assert!(matches!( + SqliteWriter::open_existing(&link, Duration::ZERO), + Err(Error("unsafe_storage_path")) + )); + fs::hard_link(fixture.db(), fixture.0.join("hardlink")).unwrap(); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), + Err(Error("unsafe_storage_path")) + )); +} +#[test] +fn replaced_database_and_excessive_timeout_fail_closed() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + assert!(matches!( + SqliteWriter::open_existing(&fixture.db(), Duration::MAX), + Err(Error("invalid_timeout")) + )); + fs::rename(fixture.db(), fixture.0.join("old.db")).unwrap(); + let replacement = Connection::open(fixture.db()).unwrap(); + drop(replacement); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("storage_recovery_required") + ); +} + +#[test] +fn uncertain_commit_does_not_destroy_possibly_committed_external_data() { + let fixture = Fixture::new(); + let mut writer = fixture.writer(); + let log = Rc::new(RefCell::new(Vec::new())); + let result: std::result::Result<(), _> = writer.transaction(|tx| { + tx.set_setting("committed", &value())?; + effect(tx, &log, 1); + // Deterministic commit-result injection, not a simulated physical disk: + // commit really persisted, but the driver reports an I/O failure. + tx.transaction.execute_batch("COMMIT").unwrap(); + Err(tx.commit_failed(rusqlite::Error::SqliteFailure( + rusqlite::ffi::Error::new(rusqlite::ffi::SQLITE_IOERR), + None, + ))) + }); + assert_eq!(result.unwrap_err().cause, Error("commit_outcome_uncertain")); + assert!(log.borrow().is_empty()); + assert!(fixture.setting("committed").is_some()); + assert_eq!( + writer.transaction(|_| Ok(())).unwrap_err().cause, + Error("storage_recovery_required") + ); +} +#[test] +fn missing_uniqueness_or_audit_guard_is_not_repaired() { + for sql in ["DROP INDEX one_root", "DROP TRIGGER audit_no_delete", "DROP TABLE results; CREATE TABLE results(subject TEXT,request_id TEXT,digest TEXT,body TEXT)"] { + let fixture = Fixture::new(); let db = Connection::open(fixture.db()).unwrap(); db.execute_batch(sql).unwrap(); drop(db); + assert!(matches!(SqliteWriter::open_existing(&fixture.db(), Duration::ZERO), Err(Error("unsupported_storage_schema")))); + } +} diff --git a/rust/tests/test_storage_parity.py b/rust/tests/test_storage_parity.py new file mode 100644 index 00000000..b70ba4a3 --- /dev/null +++ b/rust/tests/test_storage_parity.py @@ -0,0 +1,875 @@ +"""Native writes against real Python storage; all data and keys are disposable. + +This tests storage primitives, not an authenticated operation executor. The +local `once` helper deliberately models only the atomic ledger boundary. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import selectors +import sqlite3 +import subprocess +import tempfile +from collections import Counter +from contextlib import closing +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +from msg.core.codec import canonical, decode, digest, wire +from msg.core.errors import Failure +from msg.core.models import ( + AuditEvent, + BlobRef, + Event, + OperationError, + OperationResult, + Relation, + Resource, + ResourceRef, + Revision, +) +from msg.security.crypto import Ed25519Signer +from msg.storage.sqlite import SqliteMetadataStore + +NOW = datetime(2026, 10, 5, tzinfo=UTC) +D = digest({'task': 'writer-test'}) +TABLES = ( + 'resources', + 'resource_tags', + 'resource_path_aliases', + 'revisions', + 'relations', + 'settings', + 'results', + 'events', + 'audit', +) +REPORT = [] + + +def resource(id='r_doc', *, parent='r_root', name='document', **kwargs): + return Resource( + id=id, + type='file' if parent else 'space', + type_version=1, + name=name, + parent=parent, + owner='u_alice', + group='g_public', + mode=0o0644, + generation=0, + revision=None, + state='active', + created_at=NOW, + created_by='u_alice', + modified_at=NOW, + modified_by='u_alice', + **kwargs, + ) + + +def result(subject='u_alice', request_id='req', **kwargs): + return OperationResult( + request_id=request_id, + operation='fixture.write', + status='ok', + actor=subject, + subject=subject, + committed_at=NOW, + **kwargs, + ) + + +def event(id='event-1', **kwargs): + return Event( + id=id, + type='fixture.changed', + time=NOW, + request_id='req', + actor='u_alice', + subject='u_alice', + resources=(ResourceRef(id='r_doc'),), + data=kwargs, + ) + + +def audit(id='event-1'): + return AuditEvent( + event=event(id, n=10**60), + authority=(), + before_digest=None, + after_digest=D, + previous_digest='ignored by appender', + entry_digest='ignored by appender', + result='ok', + ) + + +def revision(id='v1', **kwargs): + return Revision( + format_version=1, + id=id, + resource_id='r_doc', + parents=(), + content=BlobRef(digest=D, size=123, media_type='text/plain'), + relations=(), + actor='u_alice', + subject='u_alice', + author='u_alice', + created_at=NOW, + manifest_digest=D, + **kwargs, + ) + + +def action(kind, **kwargs): + return {'action': kind, **wire(kwargs)} + + +def insert(record=None): + return action('insert', record=record or resource()) + + +def set_value(key, value=1): + return action('set', key=key, value=value) + + +def ledger(record=None, **kwargs): + record = record or result() + return action('result', record=record, subject=record.subject, digest=D, **kwargs) + + +def once(body, record=None): + record = record or result() + return action('once', subject=record.subject, digest=D, record=record, steps=body) + + +def check(name, ok, detail=None, group='parity'): + REPORT.append({ + 'name': name, + 'group': group, + 'ok': bool(ok), + **({'detail': detail} if not ok else {}), + }) + + +def dump(path): + with closing(sqlite3.connect(path)) as db: + return { + table: sorted(db.execute(f'SELECT * FROM {table}').fetchall(), key=repr) + for table in TABLES + } + + +def schema(path): + with closing(sqlite3.connect(path)) as db: + return db.execute( + 'SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY name' + ).fetchall() + + +def backup(source, target): + with closing(sqlite3.connect(source)) as db, closing(sqlite3.connect(target)) as dst: + db.backup(dst) + + +async def fixture(path): + store = SqliteMetadataStore(path, busy_timeout=0.08) + async with store.transaction(write=True) as tx: + await tx.insert(resource('r_root', parent=None, name='')) + return store + + +async def python_steps(tx, values, store, effects): + output = [] + for step in values: + kind = step['action'] + if kind == 'set': + tx.set_setting(step['key'], step['value']) + elif kind == 'read': + output.append(tx.setting(step['key'])) + elif kind == 'resource': + output.append(wire(await tx.resource(step['id']))) + elif kind == 'insert': + await tx.insert(decode(Resource, step['record'])) + elif kind == 'replace': + await tx.replace(decode(Resource, step['record']), step['expected']) + elif kind == 'revision': + await tx.append_revision(decode(Revision, step['record'])) + elif kind == 'result': + await tx.save_result( + step['subject'], step['digest'], decode(OperationResult, step['record']) + ) + elif kind == 'lookup': + output.append( + wire(await tx.request_result(step['subject'], step['id'], step['digest'])) + ) + elif kind == 'event': + await tx.append_event(decode(Event, step['record'])) + elif kind == 'audit': + await tx.append_audit(decode(AuditEvent, step['record'])) + elif kind == 'savepoint': + try: + async with store.transaction(write=True) as inner: + data = await python_steps(inner, step['steps'], store, effects) + output.append(data) + except Failure as exc: + if not step.get('recover'): + raise + output.append({'error': exc.code}) + elif kind == 'effect': + + async def cleanup(name=step['name']): + effects.append(name) + + tx.on_rollback(cleanup) + elif kind == 'fail': + raise Failure('fixture_failure') + elif kind == 'once': + record = decode(OperationResult, step['record']) + stored = await tx.request_result(step['subject'], record.request_id, step['digest']) + if stored is None: + await python_steps(tx, step['steps'], store, effects) + await tx.save_result(step['subject'], step['digest'], record) + stored = record + output.append(wire(stored)) + else: + raise AssertionError(f'Unsupported Python fixture action: {kind}') + return output + + +async def run_python(store, steps): + effects = [] + try: + async with store.transaction(write=True) as tx: + output = await python_steps(tx, steps, store, effects) + return {'status': 'ok', 'values': output}, effects + except Failure as exc: + return {'status': 'error', 'code': exc.code, 'cleanup': []}, effects + + +class Native: + def __init__(self, binary): + self.binary = str(binary) + + def run(self, path, steps, **kwargs): + completed = subprocess.run( + [self.binary, str(path)], + input=json.dumps({'steps': steps, **kwargs}) + '\n', + capture_output=True, + text=True, + timeout=15, + check=True, + ) + lines = [json.loads(line) for line in completed.stdout.splitlines()] + return lines[-1], [line['effect'] for line in lines if 'effect' in line] + + def start(self, path, steps, **kwargs): + process = subprocess.Popen( + [self.binary, str(path)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + process.stdin.write(json.dumps({'steps': steps, **kwargs}) + '\n') + process.stdin.flush() + return process + + +def line(process): + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + if not selector.select(10): + raise AssertionError('Native process did not reach its checkpoint') + raw = process.stdout.readline() + if not raw: + raise AssertionError('Native exited before checkpoint: ' + process.stderr.read()) + return json.loads(raw) + + +def finish(process): + process.stdin.write('continue\n') + process.stdin.flush() + stdout, stderr = process.communicate(timeout=15) + if process.returncode: + raise AssertionError(f'Native failed: {process.returncode}: {stderr}') + return json.loads(stdout.splitlines()[-1]) + + +def kill(process): + process.kill() + process.communicate(timeout=10) + + +async def parity_cases(native, directory): + doc = resource() + rev = revision() + cases = [ + ('insert-read-resource', [insert(), action('resource', id=doc.id)]), + ( + 'canonical-settings', + [ + set_value('numbers', {'n': 10**80, 'f': -0.0, 'text': '中文😀é'}), + action('read', key='numbers'), + ], + ), + ('unicode-tags', [insert(resource(tags=('café', '安全', '苹果')))]), + ( + 'rename-alias', + [ + insert(), + action('replace', record=replace(doc, name='renamed', generation=1), expected=0), + ], + ), + ( + 'tags-update', + [ + insert(), + action( + 'replace', + record=replace(doc, tags=('rust', 'storage'), generation=1), + expected=0, + ), + ], + ), + ( + 'move-alias', + [ + insert(resource('r_other', name='other')), + insert(), + action('replace', record=replace(doc, parent='r_other', generation=1), expected=0), + ], + ), + ( + 'generation-conflict', + [insert(), action('replace', record=replace(doc, generation=2), expected=1)], + ), + ( + 'invalid-generation', + [insert(), action('replace', record=replace(doc, generation=2), expected=0)], + ), + ( + 'immutable-type', + [ + insert(), + action('replace', record=replace(doc, type='topic', generation=1), expected=0), + ], + ), + ( + 'immutable-creator', + [ + insert(), + action( + 'replace', record=replace(doc, created_by='u_bob', generation=1), expected=0 + ), + ], + ), + ( + 'immutable-time', + [ + insert(), + action( + 'replace', + record=replace(doc, created_at=NOW.replace(hour=1), generation=1), + expected=0, + ), + ], + ), + ('missing-parent', [insert(resource(parent='missing'))]), + ('duplicate-child', [insert(), insert(resource('r_duplicate'))]), + ('second-root', [insert(resource('second-root', parent=None, name=''))]), + ( + 'parent-cycle', + [ + insert(), + insert(resource('r_child', parent=doc.id, name='child')), + action('replace', record=replace(doc, parent='r_child', generation=1), expected=0), + ], + ), + ('append-revision', [insert(), action('revision', record=rev)]), + ( + 'revision-chain', + [ + insert(), + action('revision', record=rev), + action( + 'revision', + record=replace( + rev, + id='v2', + parents=('v1',), + summary='changed', + source_kind='user', + source_version=2, + ), + ), + ], + ), + ( + 'revision-relations', + [ + insert(), + action( + 'revision', + record=replace( + rev, + relations=( + Relation( + type='reference', target=ResourceRef(id='r_root'), excerpt=(0, 12) + ), + ), + ), + ), + ], + ), + ( + 'revision-parent-missing', + [insert(), action('revision', record=replace(rev, parents=('missing',)))], + ), + ( + 'revision-wrong-resource', + [ + insert(), + insert(resource('r_other', name='other')), + action('revision', record=replace(rev, resource_id='r_other')), + action('revision', record=replace(rev, id='v2', parents=('v1',))), + ], + ), + ( + 'revision-duplicate', + [insert(), action('revision', record=rev), action('revision', record=rev)], + ), + ('result-read', [ledger(), action('lookup', subject='u_alice', id='req', digest=D)]), + ( + 'result-anonymous', + [ledger(result(None)), action('lookup', subject=None, id='req', digest=D)], + ), + ( + 'result-namespace', + [ + ledger(), + ledger(result('u_bob')), + action('lookup', subject='u_charlie', id='req', digest=D), + ], + ), + ('result-duplicate', [insert(), ledger(), ledger()]), + ( + 'digest-conflict', + [ + ledger(), + action('lookup', subject='u_alice', id='req', digest=digest({'different': True})), + ], + ), + ('event', [action('event', record=event())]), + ( + 'duplicate-event', + [insert(), action('event', record=event()), action('event', record=event())], + ), + ( + 'audit-chain', + [action('audit', record=audit()), action('audit', record=audit('event-2'))], + ), + ( + 'atomic-resource-event-audit-result', + [ + insert(), + action('revision', record=rev), + action('event', record=event()), + action('audit', record=audit()), + ledger(), + ], + ), + ( + 'atomic-failure-at-result', + [ + ledger(), + insert(), + action('event', record=event()), + action('audit', record=audit()), + ledger(), + ], + ), + ( + 'rollback-all', + [ + insert(), + ledger(), + action('event', record=event()), + action('audit', record=audit()), + action('fail'), + ], + ), + ( + 'savepoint-recover', + [ + set_value('before'), + action('savepoint', steps=[set_value('inner'), action('fail')], recover=True), + set_value('after'), + ], + ), + ( + 'savepoint-outer-failure', + [action('savepoint', steps=[insert(), ledger()]), action('fail')], + ), + ( + 'nested-savepoints', + [ + action( + 'savepoint', + steps=[ + set_value('first'), + action( + 'savepoint', steps=[set_value('second'), action('fail')], recover=True + ), + set_value('third'), + ], + ) + ], + ), + ( + 'rollback-effects', + [ + action('effect', name='outer'), + action('savepoint', steps=[action('effect', name='inner')]), + action('effect', name='last'), + action('fail'), + ], + ), + ( + 'savepoint-effects', + [ + action('effect', name='outer'), + action( + 'savepoint', + steps=[action('effect', name='inner'), action('fail')], + recover=True, + ), + ], + ), + ( + 'same-unit-retry', + [ + once([insert(), action('audit', record=audit())]), + once([insert(), action('audit', record=audit())]), + ], + ), + ] + for name, field, value in [ + ('owner', 'owner', 'u_bob'), + ('group', 'group', 'g_other'), + ('mode', 'mode', 0o0740), + ('state', 'state', 'archived'), + ]: + cases.append(( + f'authorization-epoch-{name}', + [ + insert(), + action('replace', record=replace(doc, generation=1, **{field: value}), expected=0), + action('read', key='authorization_epoch'), + ], + )) + for status in ('ok', 'accepted', 'error', 'uncertain'): + record = replace( + result(data={'large': 10**90, 'text': '安全'}), + status=status, + error=OperationError(code='fixture', retryable=False) if status == 'error' else None, + ) + cases.append((f'result-status-{status}', [ledger(record)])) + signer = Ed25519Signer.from_bytes(bytes([7]) * 32) + cases.append(( + 'signed-receipt-bytes', + [ledger(result(receipt=signer.sign(b'fixture-only', purpose='receipt')))], + )) + cases.append(( + 'historical-error-message', + [ + ledger( + replace( + result(), + status='error', + error=OperationError(code='fixture', retryable=True, message='可重试'), + ) + ) + ], + )) + for index, (name, steps) in enumerate(cases): + py = directory / f'py-{index}.db' + rs = directory / f'rs-{index}.db' + store = await fixture(py) + backup(py, rs) + before_schema = schema(rs) + expected, py_effects = await run_python(store, steps) + actual, rs_effects = native.run(rs, steps) + same = ( + canonical(actual) == canonical(expected) + and dump(py) == dump(rs) + and py_effects == rs_effects + and schema(rs) == before_schema + ) + check( + name, + same, + { + 'python': expected, + 'rust': actual, + 'tables_differ': [t for t in TABLES if dump(py)[t] != dump(rs)[t]], + 'effects': [py_effects, rs_effects], + }, + ) + await store.close() + + +async def boundary_cases(native, directory): + path = directory / 'boundaries.db' + store = await fixture(path) + baseline_settings = dump(path)['settings'] + actual, _ = native.run(path, [action('result', subject='u_bob', digest=D, record=result())]) + check( + 'mismatched-result-subject', + actual.get('code') == 'storage_record_mismatch' and not dump(path)['results'], + group='native-safety', + ) + actual, _ = native.run(path, [ledger(result(''))]) + check( + 'empty-subject-not-anonymous-alias', + actual.get('code') == 'storage_record_mismatch', + group='native-safety', + ) + actual, _ = native.run( + path, [action('result', subject='u_alice', digest='not-a-digest', record=result())] + ) + check('malformed-result-digest', actual.get('code') == 'invalid_digest', group='native-safety') + with closing(sqlite3.connect(path)) as db: + db.execute( + "CREATE TRIGGER fixture_abort BEFORE INSERT ON settings WHEN NEW.key='reject' BEGIN SELECT RAISE(ABORT,'fixture'); END" + ) + db.commit() + actual, _ = native.run( + path, [set_value('before'), action('ignore', steps=[set_value('reject')])] + ) + check( + 'swallowed-write-fails-closed', + actual.get('code') == 'constraint_conflict' and dump(path)['settings'] == baseline_settings, + group='native-safety', + ) + with closing(sqlite3.connect(path)) as db: + db.execute('DROP TRIGGER fixture_abort') + db.execute( + "CREATE TRIGGER fixture_abort BEFORE INSERT ON settings WHEN NEW.key='reject' BEGIN SELECT RAISE(ROLLBACK,'fixture'); END" + ) + db.commit() + actual, _ = native.run( + path, + [set_value('before'), action('ignore', steps=[set_value('reject')]), set_value('escape')], + ) + check( + 'implicit-abort-no-autocommit-write', + actual.get('code') == 'constraint_conflict' and dump(path)['settings'] == baseline_settings, + group='native-safety', + ) + with closing(sqlite3.connect(path)) as db: + db.execute('DROP TRIGGER fixture_abort') + db.execute( + "CREATE TRIGGER fixture_deferred AFTER INSERT ON settings WHEN NEW.key='deferred' BEGIN INSERT INTO resources SELECT 'orphan',type,'orphan','missing',owner,grp,mode,generation,revision,state,created_at,modified_at,body FROM resources WHERE id='r_root'; END" + ) + db.commit() + actual, effects = native.run( + path, [action('effect', name='cleanup'), set_value('deferred'), ledger()] + ) + check( + 'commit-failure-undoes-result-and-data', + actual.get('code') == 'constraint_conflict' + and effects == ['cleanup'] + and not dump(path)['results'] + and dump(path)['settings'] == baseline_settings + and len(dump(path)['resources']) == 1, + group='native-safety', + ) + actual, effects = native.run( + path, + [ + action('effect', name='first'), + action('effect', name='bad', fail=True), + action('effect', name='panic', panic=True), + action('fail'), + ], + ) + check( + 'compensation-errors-preserve-primary', + actual.get('code') == 'fixture_failure' + and actual.get('cleanup') == ['rollback_effect_panicked', 'fixture_cleanup'] + and effects == ['panic', 'bad', 'first'], + group='native-safety', + ) + await store.close() + + capacity = directory / 'capacity.db' + cap_store = await fixture(capacity) + with closing(sqlite3.connect(capacity)) as db: + db.execute( + "WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x<100000) INSERT INTO results SELECT 'fixture', CAST(x AS TEXT), 'fixture', '{}' FROM n" + ) + db.commit() + expected, _ = await run_python(cap_store, [insert(), ledger()]) + actual, _ = native.run(capacity, [insert(), ledger()]) + check( + 'result-capacity-rolls-back-resource', + actual == expected + and actual.get('code') == 'storage_capacity_exceeded' + and len(dump(capacity)['resources']) == 1, + group='parity', + ) + await cap_store.close() + + +async def process_cases(native, directory): + for stage in ('before', 'after'): + path = directory / f'crash-{stage}.db' + store = await fixture(path) + body = [insert(), action('event', record=event()), action('audit', record=audit())] + command = once(body + ([action('hold')] if stage == 'before' else [])) + process = native.start(path, [command], hold_after_commit=stage == 'after') + try: + got = line(process) + checkpoint = 'transaction' if stage == 'before' else 'committed' + check(f'crash-{stage}-checkpoint', got == {'checkpoint': checkpoint}, group='process') + visible = dump(path) + check( + f'crash-{stage}-visibility', + bool(visible['results']) == (stage == 'after') + and len(visible['resources']) == (2 if stage == 'after' else 1), + group='process', + ) + finally: + kill(process) + actual, _ = native.run(path, [once(body)]) + rows = dump(path) + check( + f'crash-{stage}-retry-single-commit', + actual['status'] == 'ok' + and len(rows['resources']) == 2 + and len(rows['results']) == len(rows['events']) == len(rows['audit']) == 1, + group='process', + ) + await store.close() + + path = directory / 'competing.db' + store = await fixture(path) + body = [insert(), action('audit', record=audit())] + first = native.start(path, [once(body + [action('hold')])]) + second = None + try: + check( + 'competing-writer-first-checkpoint', + line(first) == {'checkpoint': 'transaction'}, + group='process', + ) + second = native.start(path, [once(body)], timeout_ms=3000) + one = finish(first) + stdout, stderr = second.communicate(timeout=10) + two = json.loads(stdout.splitlines()[-1]) + check( + 'competing-writers-share-one-result', + second.returncode == 0 and one == two and len(dump(path)['audit']) == 1, + stderr, + group='process', + ) + finally: + if first.poll() is None: + kill(first) + if second is not None and second.poll() is None: + kill(second) + + async with store.transaction(write=True) as tx: + tx.set_setting('python-holds', 1) + actual, _ = native.run(path, [set_value('must-not-write')], timeout_ms=50) + check( + 'python-fence-blocks-rust', actual.get('code') == 'server_busy', actual, group='process' + ) + rust = native.start(path, [set_value('rust-holds'), action('hold')]) + try: + line(rust) + try: + async with store.transaction(write=True): + raise AssertionError('Python crossed native writer fence') + except Failure as exc: + check('rust-fence-blocks-python', exc.code == 'server_busy', group='process') + finish(rust) + finally: + if rust.poll() is None: + kill(rust) + + rust = native.start( + path, [action('effect', name='cleanup', hold=True), set_value('abort'), action('fail')] + ) + try: + check('compensation-started', line(rust) == {'effect': 'cleanup'}, group='process') + # The next checkpoint is line-buffered; read directly after its known + # predecessor rather than asking select about already buffered data. + checkpoint = json.loads(rust.stdout.readline()) + check( + 'compensation-checkpoint', checkpoint == {'checkpoint': 'compensation'}, group='process' + ) + with closing(sqlite3.connect(path, isolation_level=None, timeout=0)) as db: + db.execute('BEGIN IMMEDIATE') + db.execute('ROLLBACK') + try: + async with store.transaction(write=True): + raise AssertionError('Python crossed compensation fence') + except Failure as exc: + check('fence-held-after-sql-rollback', exc.code == 'server_busy', group='process') + outcome = finish(rust) + check( + 'compensation-finishes-before-release', + outcome.get('code') == 'fixture_failure', + group='process', + ) + finally: + if rust.poll() is None: + kill(rust) + actual, _ = native.run(path, [set_value('after-compensation')]) + check('writer-usable-after-clean-rollback', actual['status'] == 'ok', group='process') + await store.close() + + +async def main_async(args): + native = Native(args.binary.resolve()) + with tempfile.TemporaryDirectory(prefix='msg-writer-parity-') as root: + directory = Path(root) + await parity_cases(native, directory) + await boundary_cases(native, directory) + await process_cases(native, directory) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + try: + asyncio.run(main_async(args)) + except Exception as exc: + REPORT.append({'name': 'harness', 'group': 'harness', 'ok': False, 'detail': repr(exc)}) + failures = [case for case in REPORT if not case['ok']] + report = { + 'cases': len(REPORT), + 'groups': dict(Counter(c['group'] for c in REPORT)), + 'failures': len(failures), + 'first_failures': failures[:12], + 'checks': REPORT, + } + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, ensure_ascii=True, indent=2) + '\n') + print(json.dumps({k: v for k, v in report.items() if k != 'checks'}, ensure_ascii=True)) + if failures: + raise SystemExit(1) + + +if __name__ == '__main__': + main() From fb89def73c94bd96c32fc9635e11563542845ad9 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 11:57:12 +0800 Subject: [PATCH 09/22] ci(rust): prepare isolated native runtime dependency and test snapshot --- .github/workflows/rust-runtime-deps.yml | 89 +++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .github/workflows/rust-runtime-deps.yml diff --git a/.github/workflows/rust-runtime-deps.yml b/.github/workflows/rust-runtime-deps.yml new file mode 100644 index 00000000..8ce00f2f --- /dev/null +++ b/.github/workflows/rust-runtime-deps.yml @@ -0,0 +1,89 @@ +name: Rust runtime dependency snapshot +on: + push: + branches: ['migration/rust-*'] + paths: ['.github/workflows/rust-runtime-deps.yml'] +permissions: + contents: read +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + - name: Export pinned public dependencies and disposable test tools + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/msg-runtime-editor /tmp/runtime-deps/src /tmp/runtime-deps/.cargo + git rev-parse HEAD > /tmp/msg-runtime-editor/source-commit.txt + git archive --format=tar.gz HEAD > /tmp/msg-runtime-editor/source.tar.gz + rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy + printf '[resolver]\nincompatible-rust-versions="fallback"\n' > /tmp/runtime-deps/.cargo/config.toml + cat > /tmp/runtime-deps/Cargo.toml <<'TOML' + [package] + name="msg-runtime-offline-dependencies" + version="0.0.0" + edition="2021" + rust-version="1.85" + [dependencies] + axum={version="=0.8.6",features=["ws","http2"]} + tokio={version="=1.48.0",features=["full"]} + tower={version="=0.5.2",features=["limit","timeout","util"]} + tower-http={version="=0.6.6",features=["limit","timeout","trace","catch-panic","compression-gzip"]} + reqwest={version="=0.12.24",default-features=false,features=["rustls-tls","json","stream"]} + clap={version="=4.5.51",features=["derive","env"]} + toml="=0.8.23" + hmac="=0.12.1" + rand="=0.8.5" + aes-gcm="=0.10.3" + hkdf="=0.12.4" + x25519-dalek={version="=2.0.1",features=["static_secrets"]} + age={version="=0.11.1",default-features=false} + zeroize="=1.8.2" + hex="=0.4.3" + url="=2.5.7" + percent-encoding="=2.3.2" + mime_guess="=2.0.5" + uuid={version="=1.18.1",features=["v4"]} + regex="=1.12.2" + unicode-normalization="=0.1.24" + pulldown-cmark="=0.13.0" + ammonia="=4.1.2" + minijinja={version="=2.12.0",features=["loader","json"]} + jsonschema={version="=0.33.0",default-features=false} + postgres={version="=0.19.12",features=["with-serde_json-1","with-chrono-0_4"]} + redis={version="=0.32.7",default-features=false,features=["tokio-comp"]} + flate2="=1.1.5" + rustix={version="=1.1.2",features=["fs","process"]} + tracing="=0.1.41" + tracing-subscriber={version="=0.3.20",features=["env-filter","json"]} + futures-util="=0.3.31" + tokio-stream={version="=0.1.17",features=["sync"]} + graphql-parser="=0.4.1" + lettre={version="=0.11.19",default-features=false,features=["builder","smtp-transport","tokio1-rustls-tls"]} + similar="=2.7.0" + hickory-proto="=0.24.4" + TOML + printf 'msg-storage={path="%s/rust/crates/msg-storage"}\nmsg-protocol={path="%s/rust/crates/msg-protocol"}\n' "$PWD" "$PWD" >> /tmp/runtime-deps/Cargo.toml + echo 'fn main() {}' > /tmp/runtime-deps/src/main.rs + (cd /tmp/runtime-deps && cargo +1.85.0 vendor --sync "$GITHUB_WORKSPACE/rust/Cargo.toml" --versioned-dirs /tmp/runtime-vendor > /tmp/runtime-vendor-config.toml) + cp /tmp/runtime-deps/Cargo.toml /tmp/runtime-deps/Cargo.lock /tmp/msg-runtime-editor/ + tar -czf /tmp/msg-runtime-editor/vendor.tar.gz -C /tmp runtime-vendor runtime-vendor-config.toml + python -m pip install '.[server]' pytest pytest-asyncio pytest-cov 'ruff==0.16.9' + python -m pip freeze > /tmp/msg-runtime-editor/python-requirements.txt + tar -cJf /tmp/msg-runtime-editor/python-toolchain.tar.xz -C "$(python -c 'import sys;print(sys.prefix)')" . + (cd /tmp/msg-runtime-editor && sha256sum *.tar.* Cargo.lock > SHA256SUMS) + - uses: actions/upload-artifact@v7 + with: + name: msg-runtime-editor-${{ github.sha }} + path: /tmp/msg-runtime-editor/ + retention-days: 1 + if-no-files-found: error + compression-level: 0 From 0a1ea2fc3a879b657495dc7afa0db2435efded72 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 12:17:22 +0800 Subject: [PATCH 10/22] ci(rust): export public PostgreSQL test tools and TLS dependencies --- .github/workflows/rust-postgres-editor.yml | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/rust-postgres-editor.yml diff --git a/.github/workflows/rust-postgres-editor.yml b/.github/workflows/rust-postgres-editor.yml new file mode 100644 index 00000000..d7201f8b --- /dev/null +++ b/.github/workflows/rust-postgres-editor.yml @@ -0,0 +1,65 @@ +name: Rust PostgreSQL editor snapshot +on: + push: + branches: ['migration/rust-*'] + paths: ['.github/workflows/rust-postgres-editor.yml'] +permissions: + contents: read +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - name: Export public TLS crates and disposable PostgreSQL binaries + shell: bash + run: | + set -euo pipefail + mkdir -p /tmp/pg-editor /tmp/pg-deps/src /tmp/pg-deps/.cargo + rustup toolchain install 1.85.0 --profile minimal + cat > /tmp/pg-deps/Cargo.toml <<'TOML' + [package] + name="msg-pg-editor-dependencies" + version="0.0.0" + edition="2021" + rust-version="1.85" + [dependencies] + postgres="=0.19.12" + postgres-native-tls="=0.5.2" + native-tls="=0.2.14" + openssl="=0.10.75" + serde_json="=1.0.145" + TOML + printf '[resolver]\nincompatible-rust-versions="fallback"\n' > /tmp/pg-deps/.cargo/config.toml + echo 'fn main() {}' > /tmp/pg-deps/src/main.rs + (cd /tmp/pg-deps && cargo +1.85.0 vendor --versioned-dirs /tmp/pg-vendor > /tmp/pg-vendor-config.toml) + tar -czf /tmp/pg-editor/vendor.tar.gz -C /tmp pg-vendor pg-vendor-config.toml + cp /tmp/pg-deps/Cargo.lock /tmp/pg-editor/ + PG=$(find /usr/lib/postgresql -type f -path '*/bin/postgres' | sort -V | tail -n1) + test -n "$PG" + BINDIR=$(dirname "$PG") + SHARED=$($BINDIR/pg_config --sharedir) + PKGLIB=$($BINDIR/pg_config --pkglibdir) + mkdir -p /tmp/pg-export/bin /tmp/pg-export/share /tmp/pg-export/lib /tmp/pg-export/deps + cp -a "$BINDIR"/. /tmp/pg-export/bin/ + cp -a "$SHARED"/. /tmp/pg-export/share/ + cp -a "$PKGLIB"/. /tmp/pg-export/lib/ + printf '%s\n%s\n%s\n' "$BINDIR" "$SHARED" "$PKGLIB" > /tmp/pg-export/original-paths.txt + "$PG" --version > /tmp/pg-export/version.txt + find /tmp/pg-export/bin /tmp/pg-export/lib -type f -print0 | xargs -0 -r -n1 sh -c 'ldd "$0" 2>/dev/null || true' | awk '/=> \/|^\s*\// {for(i=1;i<=NF;i++)if($i ~ /^\//)print $i}' | sort -u > /tmp/pg-export/dependencies.txt + while IFS= read -r lib; do + case "$(basename "$lib")" in libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|libresolv.so.*|ld-linux*) continue ;; esac + cp -L "$lib" /tmp/pg-export/deps/ + done < /tmp/pg-export/dependencies.txt + tar -cJf /tmp/pg-editor/postgres.tar.xz -C /tmp pg-export + git rev-parse HEAD > /tmp/pg-editor/source-commit.txt + (cd /tmp/pg-editor && sha256sum *.tar.* Cargo.lock > SHA256SUMS) + - uses: actions/upload-artifact@v7 + with: + name: msg-pg-editor-${{ github.sha }} + path: /tmp/pg-editor/ + retention-days: 1 + if-no-files-found: error + compression-level: 0 From 0a837ff7a5da42aa8f42d305fde032b9e7bcaefc Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 13:18:17 +0800 Subject: [PATCH 11/22] ci(rust): export pinned public XML timezone and TLS domain dependencies --- .github/workflows/rust-native-domain-deps.yml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/rust-native-domain-deps.yml diff --git a/.github/workflows/rust-native-domain-deps.yml b/.github/workflows/rust-native-domain-deps.yml new file mode 100644 index 00000000..25c01cb1 --- /dev/null +++ b/.github/workflows/rust-native-domain-deps.yml @@ -0,0 +1,44 @@ +name: Rust native domain dependencies +on: + push: + branches: ['migration/rust-*'] + paths: ['.github/workflows/rust-native-domain-deps.yml'] +permissions: + contents: read +jobs: + snapshot: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Resolve pinned public dependencies for the offline editor + shell: bash + run: | + set -euo pipefail + rustup toolchain install 1.85.0 --profile minimal + mkdir -p /tmp/domain-deps/src /tmp/domain-deps/.cargo /tmp/domain-export + cat > /tmp/domain-deps/Cargo.toml <<'TOML' + [package] + name="msg-domain-editor-dependencies" + version="0.0.0" + edition="2021" + rust-version="1.85" + [dependencies] + roxmltree="=0.20.0" + chrono-tz="=0.10.4" + postgres-openssl="=0.5.2" + ssh-key={version="=0.6.7",features=["crypto"]} + openssl="=0.10.75" + TOML + printf '[resolver]\nincompatible-rust-versions="fallback"\n' > /tmp/domain-deps/.cargo/config.toml + echo 'fn main() {}' > /tmp/domain-deps/src/main.rs + (cd /tmp/domain-deps && cargo +1.85.0 vendor --versioned-dirs /tmp/domain-vendor > /tmp/domain-vendor-config.toml) + cp /tmp/domain-deps/Cargo.toml /tmp/domain-deps/Cargo.lock /tmp/domain-export/ + tar -czf /tmp/domain-export/vendor.tar.gz -C /tmp domain-vendor domain-vendor-config.toml + (cd /tmp/domain-export && sha256sum vendor.tar.gz Cargo.lock > SHA256SUMS) + - uses: actions/upload-artifact@v7 + with: + name: msg-domain-editor-${{ github.sha }} + path: /tmp/domain-export/ + retention-days: 1 + if-no-files-found: error + compression-level: 0 From 148e56d0f9711e27814ac6950daeed7d3242d355 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 17:24:35 +0800 Subject: [PATCH 12/22] ci(rust): export pinned compiler and exact source for offline native migration --- .github/workflows/rust-editor-tools.yml | 34 +++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/rust-editor-tools.yml diff --git a/.github/workflows/rust-editor-tools.yml b/.github/workflows/rust-editor-tools.yml new file mode 100644 index 00000000..372f2bbc --- /dev/null +++ b/.github/workflows/rust-editor-tools.yml @@ -0,0 +1,34 @@ +name: Rust offline editor tools +on: + push: + branches: ['migration/rust-*'] + paths: ['.github/workflows/rust-editor-tools.yml'] +permissions: + contents: read +jobs: + export: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + fetch-depth: 1 + - name: Export only public compiler and repository source + shell: bash + run: | + set -euo pipefail + rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy + mkdir -p /tmp/msg-editor-tools + git rev-parse HEAD > /tmp/msg-editor-tools/source-commit.txt + git rev-parse HEAD^{tree} > /tmp/msg-editor-tools/source-tree.txt + git archive --format=tar.gz HEAD > /tmp/msg-editor-tools/source.tar.gz + tar -cJf /tmp/msg-editor-tools/rust-toolchain.tar.xz -C "$HOME/.rustup/toolchains/1.85.0-x86_64-unknown-linux-gnu" . + (cd /tmp/msg-editor-tools && sha256sum *.tar.* > SHA256SUMS) + - uses: actions/upload-artifact@v7 + with: + name: msg-editor-tools-${{ github.sha }} + path: /tmp/msg-editor-tools/ + retention-days: 1 + if-no-files-found: error + compression-level: 0 From 68653a541b8644d66b646fef46b8025a6c8faa40 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 18:36:11 +0800 Subject: [PATCH 13/22] ci(rust): apply exact locally verified native source checkpoint --- .github/workflows/rust-phase4-import.yml | 110 +++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 .github/workflows/rust-phase4-import.yml diff --git a/.github/workflows/rust-phase4-import.yml b/.github/workflows/rust-phase4-import.yml new file mode 100644 index 00000000..2689bfd7 --- /dev/null +++ b/.github/workflows/rust-phase4-import.yml @@ -0,0 +1,110 @@ +name: Import verified Rust checkpoint +on: + push: + branches: ['migration/rust-wire-phase1'] + paths: ['.github/workflows/rust-phase4-import.yml'] +permissions: + contents: write +concurrency: + group: msg-rust-checkpoint-import + cancel-in-progress: false +jobs: + import: + if: github.repository == 'TokenNotIncluded/msg' && github.ref == 'refs/heads/migration/rust-wire-phase1' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + persist-credentials: false + - name: Verify exact source, apply Rust-only patch, and publish without force + env: + GH_TOKEN: ${{ github.token }} + EXPECTED_HEAD: ${{ github.sha }} + shell: bash + run: | + set -euo pipefail + python3 - <<'PY' + import base64 + import hashlib + import json + import lzma + import os + import pathlib + import subprocess + import urllib.request + + repo = 'TokenNotIncluded/msg' + branch = 'migration/rust-wire-phase1' + blobs = [ + ('f7075d03c96731f0555c2b341ddf89a13801311c', 16000), + ('85578e5f30d48f7623abfa0bc107d2989ac7965e', 16000), + ('2e9c61ba133df50aaff5faac80efbe0f9c4705e7', 16000), + ('4c33f5f5f3446167ca2dde2d10a82c4cd54a5eb9', 16000), + ('1dc1a240972e2db80898b2c6fe3519858ff6e5d7', 264), + ] + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + def require(condition, message): + if not condition: + raise RuntimeError(message) + require(git('rev-parse', 'HEAD') == os.environ['EXPECTED_HEAD'], 'head mismatch') + require(git('rev-parse', 'HEAD:rust') == 'd83f9d6c9e837ba8ccb78c226122874b23bce520', 'Rust baseline changed') + require(not git('status', '--porcelain'), 'worktree is not clean') + chunks = [] + for sha, size in blobs: + request = urllib.request.Request( + f'https://api.github.com/repos/{repo}/git/blobs/{sha}', + headers={'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28'}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + record = json.loads(response.read(100000)) + require(record['sha'] == sha and record['size'] == size, 'blob metadata mismatch') + content = base64.b64decode(record['content'].replace('\n', ''), validate=True) + require(len(content) == size, 'chunk size mismatch') + require(hashlib.sha1(b'blob ' + str(size).encode() + b'\0' + content).hexdigest() == sha, 'blob digest mismatch') + chunks.append(content) + packed = base64.b64decode(b''.join(chunks), validate=True) + require(hashlib.sha256(packed).hexdigest() == '2e57559abcdbbb2fe56791880f9db034a853aec4306acd7f9b0ad1ac455b9117', 'archive digest mismatch') + decoder = lzma.LZMADecompressor(memlimit=134217728) + patch = decoder.decompress(packed, max_length=250448) + require(decoder.eof and not decoder.unused_data and len(patch) == 250447, 'patch size or format mismatch') + require(hashlib.sha256(patch).hexdigest() == '84a79dc86f1fb725c316f4d2139c93e5eaa1d70e81b8b543812c10d385ca9011', 'patch digest mismatch') + path = pathlib.Path('/tmp/msg-verified-rust.patch') + path.write_bytes(patch) + subprocess.run(['git', 'apply', '--check', '--index', str(path)], check=True) + subprocess.run(['git', 'apply', '--index', str(path)], check=True) + changed = git('diff', '--cached', '--name-only').splitlines() + require(changed and all(p.startswith('rust/') and '..' not in p.split('/') for p in changed), 'non-Rust path rejected') + tree = git('write-tree') + require(git('rev-parse', tree + ':rust') == 'c0901a33e6cb0ba22126540be4b2a2fb667ac0a8', 'resulting Rust tree mismatch') + git('config', 'user.name', 'github-actions[bot]') + git('config', 'user.email', '41898282+github-actions[bot]@users.noreply.github.com') + git('commit', '-m', 'feat(rust): enforce resource policy and native PostgreSQL transactions') + header = 'AUTHORIZATION: basic ' + base64.b64encode(('x-access-token:' + os.environ['GH_TOKEN']).encode()).decode() + env = dict(os.environ) + env.update(GIT_CONFIG_COUNT='1', GIT_CONFIG_KEY_0='http.https://github.com/.extraheader', GIT_CONFIG_VALUE_0=header) + remote = subprocess.check_output(['git', 'ls-remote', 'origin', 'refs/heads/' + branch], env=env, text=True).split()[0] + require(remote == os.environ['EXPECTED_HEAD'], 'branch advanced; refusing to overwrite') + subprocess.run(['git', 'push', 'origin', 'HEAD:refs/heads/' + branch], env=env, check=True) + out = pathlib.Path('/tmp/msg-rust-checkpoint') + out.mkdir() + metadata = {'source_commit': git('rev-parse', 'HEAD'), 'source_tree': git('rev-parse', 'HEAD^{tree}'), + 'rust_tree': git('rev-parse', 'HEAD:rust'), 'patch_sha256': hashlib.sha256(patch).hexdigest(), + 'files': changed, 'note': 'Import evidence only; independent CI has not run on this commit.'} + (out / 'metadata.json').write_text(json.dumps(metadata, indent=2) + '\n') + (out / 'Cargo.lock').write_bytes(pathlib.Path('rust/Cargo.lock').read_bytes()) + with (out / 'source.tar.gz').open('wb') as archive: + subprocess.run(['git', 'archive', '--format=tar.gz', 'HEAD'], stdout=archive, check=True) + print(json.dumps(metadata, indent=2)) + PY + - uses: actions/upload-artifact@v7 + with: + name: msg-rust-phase4-import + path: /tmp/msg-rust-checkpoint/ + retention-days: 7 + if-no-files-found: error From 1f4366a7269f1ba5831e17a28e0cb86d428b49bb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:36:24 +0000 Subject: [PATCH 14/22] feat(rust): enforce resource policy and native PostgreSQL transactions --- rust/Cargo.lock | 969 +++++++++++++++++- rust/Cargo.toml | 5 + rust/README.md | 74 +- rust/crates/msg-identity/src/authorization.rs | 942 +++++++++++++++++ rust/crates/msg-identity/src/lib.rs | 1 + rust/crates/msg-storage/Cargo.toml | 5 + .../msg-storage/examples/authority_check.rs | 181 +++- .../msg-storage/examples/pg_writer_check.rs | 217 ++++ rust/crates/msg-storage/src/authorization.rs | 138 +++ rust/crates/msg-storage/src/lib.rs | 2 + .../msg-storage/src/postgres/authority.rs | 200 ++++ rust/crates/msg-storage/src/postgres/mod.rs | 514 ++++++++++ rust/crates/msg-storage/src/postgres/query.rs | 188 ++++ .../msg-storage/src/postgres/records.rs | 645 ++++++++++++ rust/crates/msg-storage/src/records.rs | 188 ++++ rust/tests/postgres_fixture.py | 86 ++ rust/tests/postgres_transport_fixture.py | 135 +++ rust/tests/test_authorization_parity.py | 640 ++++++++++++ rust/tests/test_postgres_parity.py | 645 ++++++++++++ rust/tests/test_postgres_process.py | 268 +++++ rust/tests/test_storage_parity.py | 7 +- 21 files changed, 5978 insertions(+), 72 deletions(-) create mode 100644 rust/crates/msg-identity/src/authorization.rs create mode 100644 rust/crates/msg-storage/examples/pg_writer_check.rs create mode 100644 rust/crates/msg-storage/src/authorization.rs create mode 100644 rust/crates/msg-storage/src/postgres/authority.rs create mode 100644 rust/crates/msg-storage/src/postgres/mod.rs create mode 100644 rust/crates/msg-storage/src/postgres/query.rs create mode 100644 rust/crates/msg-storage/src/postgres/records.rs create mode 100644 rust/tests/postgres_fixture.py create mode 100644 rust/tests/postgres_transport_fixture.py create mode 100644 rust/tests/test_authorization_parity.py create mode 100644 rust/tests/test_postgres_parity.py create mode 100644 rust/tests/test_postgres_process.py diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 180c4f99..08d6a441 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -14,6 +14,26 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -47,6 +67,33 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + [[package]] name = "cc" version = "1.6.0" @@ -63,21 +110,62 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ + "iana-time-zone", "num-traits", + "windows-link", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -87,6 +175,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -97,6 +194,24 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -104,9 +219,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -121,7 +236,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -130,7 +245,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -140,8 +255,20 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -163,11 +290,27 @@ dependencies = [ "curve25519-dalek", "ed25519", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -180,6 +323,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -192,6 +341,21 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "fs2" version = "0.4.3" @@ -202,6 +366,47 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -220,7 +425,19 @@ checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core 0.10.1", ] [[package]] @@ -241,18 +458,80 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "libsqlite3-sys" version = "0.30.1" @@ -264,12 +543,54 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys", +] + [[package]] name = "msg-core" version = "0.1.0" @@ -278,7 +599,7 @@ dependencies = [ "ryu", "serde", "serde_json", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -288,7 +609,7 @@ dependencies = [ "ed25519-dalek", "msg-core", "serde", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -301,7 +622,7 @@ dependencies = [ "msg-protocol", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "subtle", ] @@ -319,17 +640,38 @@ dependencies = [ name = "msg-storage" version = "0.1.0" dependencies = [ + "bytes", "fs2", "libc", "msg-core", "msg-crypto", "msg-identity", "msg-protocol", + "native-tls", + "postgres", + "postgres-native-tls", "rusqlite", "serde", "serde_json", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -339,12 +681,128 @@ dependencies = [ "autocfg", ] +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkcs8" version = "0.10.2" @@ -361,6 +819,64 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "postgres" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c48ece1c6cda0db61b058c1721378da76855140e9214339fa1317decacb176" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "futures-util", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-native-tls" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac73153d92e4bde922bd6f1dfba7f1ab8132266c031153b55e20a1521cd36d49" +dependencies = [ + "native-tls", + "tokio", + "tokio-native-tls", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand", + "sha2 0.11.0", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +dependencies = [ + "bytes", + "chrono", + "fallible-iterator 0.2.0", + "postgres-protocol", + "serde_core", + "serde_json", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -379,13 +895,45 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", ] [[package]] @@ -395,7 +943,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags", - "fallible-iterator", + "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -411,12 +959,69 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -450,7 +1055,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -473,8 +1078,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -489,15 +1105,37 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "spki" version = "0.7.3" @@ -508,6 +1146,17 @@ dependencies = [ "der", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "subtle" version = "2.6.1" @@ -525,18 +1174,133 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unicode-bidi" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" + [[package]] name = "unicode-ident" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-properties" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" + [[package]] name = "vcpkg" version = "0.2.15" @@ -555,6 +1319,101 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.1+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -577,6 +1436,80 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + [[package]] name = "zerocopy" version = "0.8.59" @@ -594,7 +1527,7 @@ checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f4535f8d..e9b953c1 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -22,6 +22,11 @@ fs2 = "=0.4.3" libc = "=0.2.190" rusqlite = { version = "=0.32.1", features = ["bundled"] } +postgres = { version = "=0.19.12", features = ["with-serde_json-1", "with-chrono-0_4"] } +postgres-native-tls = "=0.5.2" +native-tls = "=0.2.14" +bytes = "=1.12.1" + [workspace.lints.rust] unsafe_code = "forbid" diff --git a/rust/README.md b/rust/README.md index aceb3ee3..42e98bab 100644 --- a/rust/README.md +++ b/rust/README.md @@ -1,8 +1,56 @@ -# Native MSG migration — identity and atomic SQLite metadata - -**This is not a replacement daemon or a production cutover.** Python remains the -running implementation. No Python entry point, live database, configuration, -route, migration script or deployment is changed. +# Native Rust server migration + +Python keeps the CLI, administrator/migration scripts, and development helpers. +Native Rust code is not yet a replacement for the running Python service. Do not +change production entry points until the remaining executor, transport, content, +worker, and deployment gates below pass. + +## Fourth implementation slice: resource authority and PostgreSQL + +`msg-identity::authorization` implements transaction-bound resource policy: +ordinary mode/group checks, ancestors, runtime/quarantine fences, current +certificate ceilings, managed-resource rules, topic roles and bans, direct +conversation participants/blocking, tool gates, and exact resource sharing with +live source-chain and owner/credential checks. Authentication alone is never a +permission to execute. Replayed results must still be checked against current +resource access by the future executor. + +`msg-storage::postgres` opens the existing Python-migrated database without +creating or upgrading any schema. The startup checks cover required columns, +exact primary/unique keys, required foreign keys, and the unique root index. +Each request owns a dedicated connection and repeatable-read snapshot. Writers +hold Python's same session advisory lock until rollback compensation finishes. +Transaction-pooled proxies are not supported. Remote connections require TLS +with certificate and hostname validation, with no plaintext fallback; unsupported +libpq connection options are rejected rather than silently ignored. + +The native PostgreSQL session implements authority reads, canonical resources +and hexadecimal aliases, migration-only old paths, revisions/relations, tags, +optimistic resource updates, identities/memberships/email settings, credentials, +CSRs/certificates, transfer/chunk metadata, events/audit, results, and durable job +records. These typed storage calls are not business operations. They cannot +replace the still-pending authenticated executor, issuance/refresh workflows, +blob/Git recovery or effect runner. + +All typed writes poison the transaction on failure, including validation errors +that a caller catches. Nested savepoints can recover an aborted inner unit only +after successful rollback. A lost COMMIT acknowledgement returns an uncertain +outcome, retains potentially referenced external data, and releases no secrets. +No network-facing handler uses these local test drivers. + +New differential coverage consists of 878 resource-policy checks against each +backend (468 are a mode/identity/check matrix), 102 PostgreSQL record comparisons, +2 additional native safety checks, 13 real process/protocol-failure checks, and +4 malformed-schema checks per build profile. The PostgreSQL tests use actual +Python-created PostgreSQL databases, not a SQLite simulation. A local test-only +wire proxy drops an acknowledged COMMIT to verify real uncertain-response and +same-ID retry behavior. All fixture databases have generated names; the harness +refuses nonlocal servers and never truncates an existing application database. + +The CI workflow repeats the old and new suites in debug and optimized builds, +records the tested source SHA/tree and dependency lock, and uploads the reports. +Independent cloud results must be read before claiming CI acceptance. The +one-time offline editor export workflows are removed. ## Ownership boundary @@ -13,8 +61,8 @@ signature protocol. The tests import the actual Python implementation in the same checkout, rather than a translated copy of the expected behavior. The original inspected main baseline is -`7e16ca75ff6af4313365814c23912b5997b222c5`. This slice extends phase 2 commit -`78e8d5d44cbbe49078a1e6eb4bb8e079eea3db72` on the existing draft PR #235. +`7e16ca75ff6af4313365814c23912b5997b222c5`. This slice extends the phase 3 implementation at +`fffea47606cf` on the existing draft PR #235. ## Implemented natively @@ -23,8 +71,8 @@ The original inspected main baseline is | `msg-core` | Bounded strict JSON, duplicate rejection, canonical numbers/Unicode, lossless typed JSON, SHA-256/base64url | `core/codec.py` | | `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing/verification | `security/crypto.py` | | `msg-protocol` | Request shape/defaults, UTC time, business digest/signature bytes, immutable field access | `core/packet.py`, `core/requests.py`, `core/models.py` | -| `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, read-only browser/API/OAuth and token-recovery validation | `security/authentication.py`, `certificates.py`, `policy.py`, `oauth.py` | -| `msg-storage` | Transaction-bound authority reads; Unix writer fencing, atomic resource/revision/result/event/audit/settings writes, savepoints and rollback effects | `storage/sqlite.py` and its record/session modules | +| `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, browser/API/OAuth and token-recovery validation, transaction-bound resource authorization | `security/authentication.py`, `certificates.py`, `policy.py`, `oauth.py` | +| `msg-storage` | Transaction-bound authority reads; SQLite and PostgreSQL writers, identity/certificate/transfer/job records, atomic resource/revision/result/event/audit/settings writes, savepoints and rollback effects | `storage/sqlite.py`, `storage/postgres.py` and their record/session modules | Authentication validates service/digest/expiry, current credentials, SHA-256 bearer verifiers with constant-time comparison, actor/subject bindings, permitted @@ -173,13 +221,13 @@ cryptographic and database primitives are delegated to pinned libraries. - Dalek strict verification rejects weak-key forgeries. Historical key/signature compatibility still needs characterization before routing production traffic. - Storage row decoding uses `invalid_storage_record` rather than every Python - field-specific error. Core/domain validation (notably Unicode tag normalization), - hex-ID aliases and complete resource policy are pending. Native storage takes + field-specific error. Domain validation (notably Unicode tag normalization) remains pending. + Resource policy and PostgreSQL hexadecimal aliases are implemented above. Native storage takes canonical IDs/validated domain records; its DTOs are not public request schemas. The stricter poisoned-write and malformed-result checks are explicit safety differences, not assertions of identical behavior for every Python input. -- Full resource authorization, operation handlers, mutation-side identity/OAuth - issuance/refresh/recovery, encrypted-key wrapping, PostgreSQL and remaining stores, +- Operation handlers, mutation-side identity/OAuth + issuance/refresh/recovery, encrypted-key wrapping and remaining content stores, native `serve`/HTTP/MCP, `worker`, packaging and deployment remain pending. - No RSS, memory-reduction or throughput result is claimed. Benchmark equivalent functionality, data, concurrency and complete process trees before comparison. diff --git a/rust/crates/msg-identity/src/authorization.rs b/rust/crates/msg-identity/src/authorization.rs new file mode 100644 index 00000000..ac10bdb4 --- /dev/null +++ b/rust/crates/msg-identity/src/authorization.rs @@ -0,0 +1,942 @@ +//! Current-state resource authorization. Authentication is necessary, not sufficient. +//! +//! All facts are read through the same transaction as the operation. A request +//! cannot construct `Principal`, supply a registry, or bypass this policy with a +//! replayed result. The caller must check each replayed resource again. +use crate::{ + authentication::{Entry, Principal}, + certificates::CertificateValidator, + models::{ + decode, CertificateKind, CredentialKind, DelegationFact, Grant, ResourceAuthority, + ResourceRef, Timestamp, + }, + policy::{allows, grant_covers, parse_mode, scope_contains, CERTGATE}, + require, + store::{setting_truthy, truthy, AuthorityStore, Record, RuntimeGeneration}, +}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::{BTreeMap, BTreeSet}; + +const ROOT: &str = "u_root"; +const TOOLS: &str = "t_tools"; + +/// Policy metadata is read from stored resource rows, never a caller's arguments. +#[derive(Clone, Deserialize)] +pub struct PolicyResource { + #[serde(flatten)] + pub authority: ResourceAuthority, + pub name: String, + pub type_version: u32, +} +impl std::ops::Deref for PolicyResource { + type Target = ResourceAuthority; + fn deref(&self) -> &Self::Target { + &self.authority + } +} + +#[derive(Clone)] +pub struct DirectConversation { + pub resource_id: String, + pub participant_a: String, + pub participant_b: String, + pub state: String, +} +#[derive(Clone)] +pub struct LegacyShare { + pub grantor: String, + pub expires_at: String, + pub created_at: String, +} +#[derive(Clone)] +pub struct ShareSource { + pub resource_id: String, + pub grantor: String, + pub grantee: String, + pub grantee_kind: String, + pub parent_id: Option, + pub operations: String, + pub constraints: String, + pub allow_reshare: bool, + pub expires_at: String, + pub revoked_at: Option, + pub created_at: String, +} + +/// The complete read boundary used by resource policy. Implementations must not +/// silently turn storage errors into absent facts or ignore unreadable rows. +pub trait AuthorizationStore: AuthorityStore { + fn authority(&self) -> &dyn AuthorityStore; + fn memberships(&self, subject: &str) -> Result>; + fn organization_exists(&self, id: &str) -> Result; + fn topic_admin(&self, topic: &str, subject: &str) -> Result; + fn topic_ban(&self, topic: &str, subject: &str) -> Result>>; + fn direct_conversation(&self, resource: &str) -> Result>; + fn direct_blocked(&self, a: &str, b: &str) -> Result; + fn legacy_share(&self, resource: &str, grantee: &str) -> Result>; + fn share_candidates(&self, resource: &str, now: &str) -> Result>; + fn share_source(&self, id: &str) -> Result>; + fn csr_issuer(&self, id: &str) -> Result; + fn policy_resource(&self, id: &str) -> Result { + let resource: PolicyResource = decode(&self.record(Record::Resource, id)?)?; + require(resource.id == id, "storage_record_mismatch")?; + Ok(resource) + } +} + +/// Constructed by trusted handlers. This is not a deserializable network proof. +pub struct AccessRequirement { + pub resource_id: String, + pub operation: String, + pub check: Check, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Check { + Read, + List, + Traverse, + Write, + Create, + Remove, + Chmod, + Chgrp, + Chown, + Manage, + Certgate, + ToolUse, + Purge, +} +impl Check { + pub fn parse(value: &str) -> Result { + Ok(match value { + "read" => Self::Read, + "list" => Self::List, + "traverse" => Self::Traverse, + "write" => Self::Write, + "create" => Self::Create, + "remove" => Self::Remove, + "chmod" => Self::Chmod, + "chgrp" => Self::Chgrp, + "chown" => Self::Chown, + "manage" => Self::Manage, + "certgate" => Self::Certgate, + "tool_use" => Self::ToolUse, + "purge" => Self::Purge, + _ => return Err(Error("invalid_permission")), + }) + } + fn write(self) -> bool { + !matches!(self, Self::Read | Self::List | Self::Traverse) + } + fn override_capability(self) -> Option<&'static str> { + match self { + Self::Read | Self::List | Self::Traverse => Some("resource.read_override"), + Self::Write => Some("resource.write_override"), + Self::Chmod => Some("resource.chmod_override"), + Self::Chgrp => Some("resource.chgrp_override"), + Self::Chown => Some("resource.chown"), + Self::Purge => Some("resource.purge"), + Self::ToolUse => Some("tool.use"), + _ => None, + } + } +} + +/// Only the container flag is needed; type/version lookup is still mandatory. +#[derive(Clone, Deserialize)] +pub struct ResourceTypePolicy { + pub name: String, + pub version: u32, + pub container: bool, +} + +pub struct AuthorizationService { + certificates: CertificateValidator, + runtime: RuntimeGeneration, + resource_types: BTreeMap<(String, u32), bool>, + base_families: BTreeSet, +} +impl AuthorizationService { + pub fn new( + certificates: CertificateValidator, + store: &dyn AuthorityStore, + resource_types: Vec, + base_families: BTreeSet, + ) -> Result { + let mut types = BTreeMap::new(); + for spec in resource_types { + require( + types + .insert((spec.name, spec.version), spec.container) + .is_none(), + "duplicate_resource_type", + )?; + } + Ok(Self { + certificates, + runtime: RuntimeGeneration::capture(store)?, + resource_types: types, + base_families, + }) + } + fn live(&mut self, store: &dyn AuthorityStore) -> Result<()> { + self.runtime.require_current(store)?; + require( + !store.has_setting("recovery_quarantine")?, + "recovery_quarantined", + ) + } + pub fn grants( + &mut self, + principal: &Principal, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result> { + self.live(store)?; + let mut grants = Vec::new(); + for id in principal.certificates() { + let certificate = self.certificates.validate(id, store, now)?; + require( + Some(certificate.key_id.as_str()) == principal.credential_id() + && Some(certificate.subject_id.as_str()) == principal.actor(), + "certificate_subject_mismatch", + )?; + grants.extend(certificate.grants); + } + Ok(grants) + } + pub fn has( + &mut self, + principal: &Principal, + capability: &str, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + self.live(store)?; + if principal.method() == "local" && principal.subject() == Some(ROOT) { + return Ok(true); + } + let mut ceiling = false; + for grant in principal.ceiling() { + ceiling |= grant_covers(grant, capability, operation, resource, store)?; + } + if !ceiling { + return Ok(false); + } + for grant in self.grants(principal, store, now)? { + if grant_covers(&grant, capability, operation, resource, store)? { + return Ok(true); + } + } + Ok(false) + } + pub fn ceiling( + &mut self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result<()> { + self.live(store)?; + if principal.method() == "anonymous" { + return Ok(()); + } + let mut allowed = false; + for grant in principal.ceiling() { + allowed |= grant.operations.contains(operation) + && scope_contains(&grant.scope, resource, store)?; + } + if !allowed + && matches!( + operation, + "discovery.get@1" + | "discovery.raw@1" + | "job.get@1" + | "transfer.open@1" + | "transfer.part_get@1" + | "transfer.status@1" + | "transfer.seal@1" + | "transfer.cancel@1" + ) + { + allowed = self.tool_output(principal, operation, resource, store, now)?; + } + require(allowed, "credential_ceiling")?; + if principal.actor() != principal.subject() { + let mut delegated = false; + for id in principal.certificates() { + let certificate = self.certificates.validate(id, store, now)?; + if certificate.kind != CertificateKind::Delegation { + continue; + } + // validate() rechecks every authority source under this exact snapshot. + for source in &certificate.authority_sources { + let fact: DelegationFact = decode( + &store + .setting(&format!("delegation:{}", source.id))? + .ok_or(Error("authority_source_invalid"))?, + )?; + if Some(fact.grantor.as_str()) == principal.subject() { + for grant in &certificate.grants { + delegated |= grant.operations.contains(operation) + && scope_contains(&grant.scope, resource, store)?; + } + } + } + } + require(delegated, "delegation_scope")?; + } + Ok(()) + } + fn tool_output( + &mut self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result { + if let Some(output) = store.setting(&format!("tool_output:{resource}"))? { + if truthy(&output)? { + let object = output.as_object()?; + if object.get("subject").and_then(|v| v.as_str().ok()) == principal.subject() { + return self.has( + principal, + "tool.use", + operation, + object + .get("tool_id") + .ok_or(Error("invalid_storage_record"))? + .as_str()?, + store, + now, + ); + } + } + } + Ok(false) + } + pub fn ordinary( + &self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + ) -> Result { + if matches!(principal.method(), "anonymous" | "local") { + return Ok(true); + } + for grant in principal.ceiling() { + if self.base_families.contains(&grant.capability) + && grant.operations.contains(operation) + && scope_contains(&grant.scope, resource, store)? + { + return Ok(true); + } + } + Ok(false) + } + pub fn require_base( + &mut self, + principal: &Principal, + operation: &str, + resource: &str, + store: &dyn AuthorityStore, + now: Timestamp, + ) -> Result<()> { + self.ceiling(principal, operation, resource, store, now)?; + require( + self.ordinary(principal, operation, resource, store)?, + "credential_ceiling", + ) + } + fn chain( + &self, + resource: &PolicyResource, + store: &dyn AuthorizationStore, + ) -> Result> { + let mut chain = Vec::new(); + for item in store.ancestors(&resource.id)? { + chain.push(store.policy_resource(&item.id)?); + } + chain.push(resource.clone()); + Ok(chain) + } + pub fn share_target_error( + &self, + resource: &PolicyResource, + chain: &[PolicyResource], + store: &dyn AuthorizationStore, + ) -> Result> { + if resource.state != "active" || chain.iter().any(|r| r.state != "active") { + return Ok(Some("share_forbidden_resource")); + } + if *self + .resource_types + .get(&(resource.kind.clone(), resource.type_version)) + .ok_or(Error("unknown_resource_type"))? + { + return Ok(Some("share_container_forbidden")); + } + if chain.iter().any(|r| { + matches!(r.id.as_str(), "r_agents" | "r_rules" | "t_last_will") + || matches!( + r.kind.as_str(), + "tool" + | "csr" + | "certificate" + | "credential" + | "legacy_directive" + | "dm_conversation" + ) + }) || chain.windows(2).any(|p| { + p[0].kind == "user" && matches!(p[1].name.as_str(), "SOUL.md" | "AGENTS.md" | "todos") + }) { + return Ok(Some("share_forbidden_resource")); + } + for item in chain { + if setting_truthy(store.authority(), &format!("hosting_preview:{}", item.id))? + || setting_truthy( + store.authority(), + &format!("hosting_preview_file:{}", item.id), + )? + || store.direct_conversation(&item.id)?.is_some() + { + return Ok(Some("share_forbidden_resource")); + } + } + Ok(None) + } + pub fn share_source_active( + &self, + resource: &PolicyResource, + grant_id: &str, + subject: &str, + now: Timestamp, + store: &dyn AuthorizationStore, + mut reshare: bool, + ) -> Result { + let chain = self.chain(resource, store)?; + if self.share_target_error(resource, &chain, store)?.is_some() { + return Ok(false); + } + let (mut id, mut subject) = (grant_id.to_owned(), subject.to_owned()); + let mut seen = BTreeSet::new(); + let mut child_expiry = None; + loop { + if seen.len() >= 16 || !seen.insert(id.clone()) { + return Ok(false); + } + let Some(row) = store.share_source(&id)? else { + return Ok(false); + }; + if row.resource_id != resource.id || row.revoked_at.is_some() { + return Ok(false); + } + let (Ok(expires), Ok(created), Ok(operations), Ok(constraints)) = ( + Timestamp::parse(&row.expires_at), + Timestamp::parse(&row.created_at), + Json::parse(&row.operations), + Json::parse(&row.constraints), + ) else { + return Ok(false); + }; + let supported = operations + .as_array() + .is_ok_and(|a| a.len() == 1 && a[0].as_str().ok() == Some("read")) + && constraints.as_object().is_ok_and(|a| a.is_empty()); + if !supported + || created > now + || expires <= now + || expires <= created + || child_expiry.is_some_and(|child| child > expires) + || (reshare && !row.allow_reshare) + { + return Ok(false); + } + match row.grantee_kind.as_str() { + "user" if row.grantee == subject => (), + "group" => { + let group = match store.policy_resource(&row.grantee) { + Ok(group) => group, + Err(Error("not_found")) => return Ok(false), + Err(error) => return Err(error), + }; + if group.state != "active" + || !store.organization_exists(&row.grantee)? + || !store.memberships(&subject)?.contains(&row.grantee) + { + return Ok(false); + } + } + _ => return Ok(false), + } + let Some(parent) = row.parent_id else { + return Ok(row.grantor == resource.owner); + }; + subject = row.grantor; + id = parent; + child_expiry = Some(expires); + reshare = true; + } + } + pub fn shared_read( + &self, + principal: &Principal, + resource: &PolicyResource, + chain: &[PolicyResource], + now: Timestamp, + store: &dyn AuthorizationStore, + ) -> Result { + let Some(subject) = principal.subject() else { + return Ok(false); + }; + if principal.actor() != Some(subject) + || self.share_target_error(resource, chain, store)?.is_some() + { + return Ok(false); + } + if let Some(row) = store.legacy_share(&resource.id, subject)? { + if row.grantor == resource.owner && resource.state == "active" { + if let (Ok(expires), Ok(created)) = ( + Timestamp::parse(&row.expires_at), + Timestamp::parse(&row.created_at), + ) { + if created <= now && now < expires { + return Ok(true); + } + } + } + } + if resource.state != "active" { + return Ok(false); + } + let time = now.0.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string(); + for candidate in store.share_candidates(&resource.id, &time)? { + if self.share_source_active(resource, &candidate, subject, now, store, false)? { + return Ok(true); + } + } + Ok(false) + } + pub fn share_link_read_allowed( + &mut self, + grantor: &str, + credential_id: &str, + resource: &PolicyResource, + chain: &[PolicyResource], + now: Timestamp, + store: &dyn AuthorizationStore, + ) -> Result { + self.runtime.require_current(store.authority())?; + if store.has_setting("recovery_quarantine")? + || resource.owner != grantor + || self.share_target_error(resource, chain, store)?.is_some() + { + return Ok(false); + } + let memberships = store.memberships(grantor)?; + if !allows(resource, Some(grantor), &memberships, "read")? { + return Ok(false); + } + for ancestor in chain.iter().take(chain.len().saturating_sub(1)) { + if !allows(ancestor, Some(grantor), &memberships, "traverse")? { + return Ok(false); + } + } + let credential = match store.credential(credential_id) { + Ok(credential) => credential, + Err(Error("credential_not_found")) => return Ok(false), + Err(error) => return Err(error), + }; + if credential.subject_id != grantor + || credential.kind != CredentialKind::SigningKey + || !credential.current_at(now) + { + return Ok(false); + } + for grant in &credential.ceiling { + if grant.operations.contains("sharing.link_create@1") + && scope_contains(&grant.scope, &resource.id, store.authority())? + { + return Ok(true); + } + } + Ok(false) + } + /// Every check runs in order. `request_name` is only the literal `name` + /// argument needed for topic presentation policy; it grants no identity. + pub fn require( + &mut self, + principal: &Principal, + entry: Entry, + now: Timestamp, + checks: &[AccessRequirement], + request_name: Option<&str>, + store: &dyn AuthorizationStore, + ) -> Result> { + self.live(store.authority())?; + if principal.subject() == Some(ROOT) || principal.actor() == Some(ROOT) { + require( + matches!(entry, Entry::LocalAdmin) && principal.method() == "local", + "local_only", + )?; + } + let memberships = principal + .subject() + .map(|s| store.memberships(s)) + .transpose()? + .unwrap_or_default(); + for check in checks { + let resource = store.policy_resource(&check.resource_id)?; + let operation = check.operation.as_str(); + require(resource.kind != "watch", "permission_denied")?; + if resource.kind == "saved_query" { + require( + matches!(operation, "query.save@1" | "query.saved_archive@1") + && principal.subject() == principal.actor() + && principal.subject() == Some(&resource.owner), + "permission_denied", + )?; + } + self.ceiling(principal, operation, &resource.id, store.authority(), now)?; + let chain = self.chain(&resource, store)?; + let mut presentation = None; + if resource.kind == "file" + && matches!(resource.name.as_str(), "ABOUT.md" | "HEADER.svg") + { + if let Some(parent) = &resource.parent { + let parent = store.policy_resource(parent)?; + if parent.kind == "topic" { + presentation = Some(parent); + } + } + } + if resource.kind == "topic" + && check.check == Check::Create + && matches!(request_name, Some("ABOUT.md" | "HEADER.svg")) + { + presentation = Some(resource.clone()); + } + let presentation_edit = presentation.is_some() && check.check.write(); + if presentation_edit { + let topic = presentation + .as_ref() + .ok_or(Error("invalid_storage_record"))?; + let mut can_edit = false; + if let Some(subject) = principal.subject() { + can_edit = subject == topic.owner + || subject == ROOT + || store.topic_admin(&topic.id, subject)?; + } + require(can_edit, "topic_admin_required")?; + } + if check.check.write() && principal.subject() != Some(ROOT) { + protect_wiki(&chain, operation)?; + } + 'preview: for item in &chain { + for prefix in ["hosting_preview:", "hosting_preview_file:"] { + if let Some(marker) = store.setting(&format!("{prefix}{}", item.id))? { + if truthy(&marker)? { + let owner = marker + .as_object()? + .get("owner") + .ok_or(Error("invalid_storage_record"))? + .as_str()?; + require( + principal.subject() == Some(owner) + || (principal.subject() == Some(ROOT) + && matches!(entry, Entry::LocalAdmin) + && principal.method() == "local"), + "permission_denied", + )?; + break 'preview; + } + } + } + } + if check.check.write() { + require( + !chain + .iter() + .any(|r| matches!(r.id.as_str(), "r_agents" | "r_rules")), + "system_managed_resource", + )?; + if !presentation_edit && chain.iter().any(|r| r.id == "t_last_will") { + require( + matches!( + operation, + "identity.legacy_put@1" + | "identity.legacy_put@2" + | "identity.legacy_archive@1" + ) && principal.subject() == Some(&resource.owner), + "legacy_directive_only", + )?; + } + let managed: Vec<_> = chain + .windows(2) + .filter(|p| p[0].kind == "user") + .map(|p| p[1].name.as_str()) + .collect(); + if managed + .iter() + .any(|name| matches!(*name, "SOUL.md" | "AGENTS.md" | "notes" | "todos")) + { + let sharing_notes = matches!( + operation, + "sharing.grant@1" + | "sharing.grant@2" + | "sharing.revoke@1" + | "sharing.revoke@2" + | "sharing.link_create@1" + | "sharing.link_revoke@1" + ) && managed + .iter() + .all(|name| !matches!(*name, "SOUL.md" | "AGENTS.md" | "todos")); + require( + (sharing_notes + || matches!( + operation, + "identity.personal_put@1" + | "identity.note_put@1" + | "identity.personal_put@2" + | "identity.note_put@2" + | "identity.soul_visibility@1" + | "identity.note_archive@1" + | "identity.note_restore@1" + | "identity.todo_put@1" + | "identity.todo_put@2" + | "identity.todo_archive@1" + | "identity.todo_restore@1" + )) + && principal.subject() == Some(&resource.owner), + "personal_managed_resource", + )?; + } + if let Some(subject) = principal.subject() { + for ancestor in chain.iter().rev().filter(|r| r.kind == "topic") { + if let Some(expiry) = store.topic_ban(&ancestor.id, subject)? { + require( + expiry + .as_ref() + .map(|t| Timestamp::parse(t)) + .transpose()? + .is_some_and(|time| time <= now), + "topic_banned", + )?; + } + } + } + } + let mut direct = None; + for ancestor in &chain { + direct = store.direct_conversation(&ancestor.id)?; + if direct.is_some() { + break; + } + } + if let Some(direct) = direct { + require( + principal + .subject() + .is_some_and(|s| s == direct.participant_a || s == direct.participant_b), + "permission_denied", + )?; + if matches!(check.check, Check::Read | Check::List | Check::Traverse) { + continue; + } + if check.check == Check::Create + && resource.id == direct.resource_id + && matches!( + operation, + "communication.dm_send@1" | "communication.dm_send@2" + ) + && direct.state == "active" + { + require( + !store.direct_blocked(&direct.participant_a, &direct.participant_b)?, + "dm_blocked", + )?; + continue; + } + if check.check == Check::Write + && resource.kind == "post" + && principal.subject() == Some(&resource.owner) + && operation == "content.post_edit@1" + { + require( + direct.state == "active" + && !store + .direct_blocked(&direct.participant_a, &direct.participant_b)?, + "dm_blocked", + )?; + continue; + } + return Err(Error("dm_controlled_resource")); + } + let tool_access = resource.kind == "tool" + && self.has( + principal, + "tool.use", + operation, + &resource.id, + store.authority(), + now, + )?; + if resource.kind == "tool" { + require(tool_access, "tool_certificate_required")?; + } + let shared_read = check.check == Check::Read + && self.shared_read(principal, &resource, &chain, now, store)?; + for ancestor in chain.iter().take(chain.len().saturating_sub(1)) { + let mut readable = allows(ancestor, principal.subject(), &memberships, "traverse")?; + let minimal_tool = tool_access && ancestor.id == TOOLS; + if !(readable || minimal_tool || shared_read) { + readable = self.has( + principal, + "resource.read_override", + operation, + &ancestor.id, + store.authority(), + now, + )?; + } + require(readable || minimal_tool || shared_read, "permission_denied")?; + require(ancestor.state == "active", "ancestor_inactive")?; + } + if check.check.write() { + require(principal.subject().is_some(), "authentication_required")?; + for ancestor in &chain { + if parse_mode(&ancestor.mode)? & CERTGATE != 0 { + require( + self.has( + principal, + "resource.certified_write", + operation, + &resource.id, + store.authority(), + now, + )?, + "certificate_gate", + )?; + } + } + } + if resource.kind == "tool" { + require( + matches!(check.check, Check::Read | Check::ToolUse), + "tool_read_only", + )?; + continue; + } + if resource.kind == "csr" + && check.check == Check::Read + && principal.subject() != Some(&resource.owner) + { + require( + principal.subject() == Some(store.csr_issuer(&resource.id)?.as_str()) + && self.has( + principal, + "cert.issue", + operation, + &resource.id, + store.authority(), + now, + )?, + "permission_denied", + )?; + continue; + } + let mut allowed = match check.check { + Check::Certgate => self.has( + principal, + "resource.certified_write", + operation, + &resource.id, + store.authority(), + now, + )?, + Check::Chmod | Check::Chgrp | Check::Manage => { + principal.subject() == Some(&resource.owner) + } + Check::Create | Check::Remove => { + allows(&resource, principal.subject(), &memberships, "write")? + && allows(&resource, principal.subject(), &memberships, "traverse")? + } + Check::Chown | Check::Purge | Check::ToolUse => false, + Check::Read => { + allows(&resource, principal.subject(), &memberships, "read")? || shared_read + } + Check::List => allows(&resource, principal.subject(), &memberships, "list")?, + Check::Traverse => { + allows(&resource, principal.subject(), &memberships, "traverse")? + } + Check::Write => allows(&resource, principal.subject(), &memberships, "write")?, + }; + let mut ordinary = + self.ordinary(principal, operation, &resource.id, store.authority())?; + if !ordinary && check.check == Check::Read { + ordinary = + self.tool_output(principal, operation, &resource.id, store.authority(), now)?; + } + if presentation_edit && matches!(check.check, Check::Write | Check::Create) { + allowed = true; + } + allowed &= ordinary; + if !allowed { + if let Some(capability) = check.check.override_capability() { + allowed = self.has( + principal, + capability, + operation, + &resource.id, + store.authority(), + now, + )?; + } + } + require(allowed, "permission_denied")?; + } + Ok(principal + .certificates() + .iter() + .map(|id| ResourceRef { + id: id.clone(), + revision: None, + }) + .collect()) + } +} +fn protect_wiki(chain: &[PolicyResource], operation: &str) -> Result<()> { + if !chain.iter().any(|r| r.id == "t_wiki") { + return Ok(()); + } + let name = operation.split('@').next().unwrap_or(operation); + require( + !matches!( + name, + "content.move" + | "content.archive" + | "content.restore" + | "content.purge" + | "content.chmod" + | "content.chgrp" + | "content.chown" + | "content.topic_configure" + | "content.topic_policy_set" + | "file.delete" + | "content.topic_remove" + | "content.topic_promote" + | "content.topic_demote" + | "content.topic_ban" + | "content.topic_unban" + | "content.topic_invite" + | "content.topic_approve" + ) && !name.starts_with("content.topic_member_"), + "wiki_shared_resource", + ) +} diff --git a/rust/crates/msg-identity/src/lib.rs b/rust/crates/msg-identity/src/lib.rs index 51d782f0..2fcacaaf 100644 --- a/rust/crates/msg-identity/src/lib.rs +++ b/rust/crates/msg-identity/src/lib.rs @@ -3,6 +3,7 @@ //! An authenticated principal is NOT authorization to execute an operation. //! Full resource authorization and the daemon are separate cutover gates. pub mod authentication; +pub mod authorization; pub mod certificates; pub mod models; pub mod oauth; diff --git a/rust/crates/msg-storage/Cargo.toml b/rust/crates/msg-storage/Cargo.toml index ae6953fd..4dd93911 100644 --- a/rust/crates/msg-storage/Cargo.toml +++ b/rust/crates/msg-storage/Cargo.toml @@ -14,6 +14,11 @@ serde.workspace = true serde_json.workspace = true msg-crypto = { path = "../msg-crypto" } +postgres.workspace = true +postgres-native-tls.workspace = true +native-tls.workspace = true +bytes.workspace = true + [target.'cfg(unix)'.dependencies] fs2.workspace = true libc.workspace = true diff --git a/rust/crates/msg-storage/examples/authority_check.rs b/rust/crates/msg-storage/examples/authority_check.rs index ad3b20b5..49b7dad7 100644 --- a/rust/crates/msg-storage/examples/authority_check.rs +++ b/rust/crates/msg-storage/examples/authority_check.rs @@ -3,13 +3,16 @@ use msg_core::{digest, unb64, Error, Json, Result, MAX_BYTES}; use msg_identity::{ authentication::{Admission, AuthenticationService, Entry, OperationPolicy}, + authorization::{ + AccessRequirement, AuthorizationService, AuthorizationStore, Check, ResourceTypePolicy, + }, certificates::CertificateValidator, models::*, policy::*, - store::{AuthorityStore, Record}, + store::Record, }; use msg_protocol::Request; -use msg_storage::SqliteAuthority; +use msg_storage::{postgres::PgStore, SqliteAuthority}; use serde::Deserialize; use serde_json::{json, Value}; use std::{ @@ -27,6 +30,10 @@ struct Context { primary_ceiling: Vec, temporary_ceiling: Vec, oauth: msg_identity::oauth::OAuthPolicy, + #[serde(default)] + resource_types: Vec, + #[serde(default)] + base_families: BTreeSet, } impl Context { fn validator(&self) -> Result { @@ -59,8 +66,10 @@ fn run( input: Value, context: &Context, auth: &mut AuthenticationService, - store: &dyn AuthorityStore, + authorization: &mut AuthorizationService, + session: &dyn AuthorizationStore, ) -> Result { + let store = session.authority(); let now = || Timestamp::parse(text(&input, "now")?); match text(&input, "action")? { "authenticate" => { @@ -84,6 +93,78 @@ fn run( Admission::Replay(result) => Ok(json!({"kind":"replay", "result":result})), } } + "authorize" | "base" | "has" => { + let request = Request::parse(text(&input, "raw")?)?; + let policy: OperationPolicy = decode(&value(&input, "policy")?)?; + let entry = match text(&input, "entry")? { + "network" => Entry::Network, + "local_admin" => Entry::LocalAdmin, + "worker" => Entry::Worker, + _ => return Err(Error("invalid_test_input")), + }; + let Admission::Fresh(principal) = + auth.authenticate(&request, &policy, store, entry, now()?)? + else { + return Err(Error("test_fresh_principal_required")); + }; + match text(&input, "action")? { + "base" => { + authorization.require_base( + &principal, + text(&input, "operation")?, + text(&input, "id")?, + store, + now()?, + )?; + Ok(json!({"allowed":true})) + } + "has" => Ok( + json!({"allowed":authorization.has(&principal, text(&input,"capability")?, text(&input,"operation")?, text(&input,"id")?, store, now()?)?}), + ), + _ => { + let mut checks = Vec::new(); + for item in input + .get("checks") + .and_then(Value::as_array) + .ok_or(Error("invalid_test_input"))? + { + checks.push(AccessRequirement { + resource_id: text(item, "resource_id")?.to_owned(), + operation: text(item, "operation")?.to_owned(), + check: Check::parse(text(item, "check")?)?, + }); + } + let refs = authorization.require( + &principal, + entry, + now()?, + &checks, + input.get("request_name").and_then(Value::as_str), + session, + )?; + Ok(json!({"refs":refs})) + } + } + } + "memberships" => Ok(json!({"groups":session.memberships(text(&input,"subject")?)?})), + "share_source" => { + let resource = session.policy_resource(text(&input, "id")?)?; + Ok( + json!({"allowed":authorization.share_source_active(&resource,text(&input,"grant")?,text(&input,"subject")?,now()?,session,input.get("reshare").and_then(Value::as_bool).unwrap_or(false))?}), + ) + } + "share_link" => { + let resource = session.policy_resource(text(&input, "id")?)?; + let mut chain = store + .ancestors(&resource.id)? + .iter() + .map(|r| session.policy_resource(&r.id)) + .collect::>>()?; + chain.push(resource.clone()); + Ok( + json!({"allowed":authorization.share_link_read_allowed(text(&input,"grantor")?,text(&input,"credential")?,&resource,&chain,now()?,session)?}), + ) + } "certificate" => { let cert = context .validator()? @@ -191,53 +272,73 @@ fn main() -> io::Result<()> { let input = read_input(&mut reader)?.ok_or(io::Error::other("test_context_required"))?; let context: Context = serde_json::from_value(input).map_err(|_| io::Error::other("invalid_test_context"))?; - let mut store = match SqliteAuthority::open_existing(Path::new(&path)) { - Ok(store) => store, - Err(e) => { - output(&mut writer, Err(e))?; - return Ok(()); + enum Backend { + Sqlite(SqliteAuthority), + Postgres(Box), + } + impl Backend { + fn read( + &mut self, + body: impl FnOnce(&dyn AuthorizationStore) -> Result, + ) -> Result { + match self { + Self::Sqlite(store) => body(&store.snapshot()?), + Self::Postgres(store) => store + .transaction(false, |tx| body(tx)) + .map_err(|e| Error(e.code())), + } } + } + let mut store = match std::env::var("MSG_PARITY_POSTGRES_DSN") { + Ok(dsn) => PgStore::open_existing(&dsn).map(|store| Backend::Postgres(Box::new(store))), + Err(_) => SqliteAuthority::open_existing(Path::new(&path)).map(Backend::Sqlite), }; - let snapshot = match store.snapshot() { - Ok(snapshot) => snapshot, + let store = match &mut store { + Ok(store) => store, Err(e) => { - output(&mut writer, Err(e))?; + output(&mut writer, Err(*e))?; return Ok(()); } }; - let validator = context.validator().map_err(io::Error::other)?; - let mut auth = AuthenticationService::new( - validator, - &snapshot, - context.primary_ceiling.clone(), - context.temporary_ceiling.clone(), - ) - .map_err(io::Error::other)? - .with_oauth_policy(context.oauth.clone()); - drop(snapshot); + let (mut auth, mut authorization) = store + .read(|snapshot| { + let auth = AuthenticationService::new( + context.validator()?, + snapshot.authority(), + context.primary_ceiling.clone(), + context.temporary_ceiling.clone(), + )? + .with_oauth_policy(context.oauth.clone()); + let authorization = AuthorizationService::new( + context.validator()?, + snapshot.authority(), + context.resource_types.clone(), + context.base_families.clone(), + )?; + Ok((auth, authorization)) + }) + .map_err(io::Error::other)?; output(&mut writer, Ok(json!({"ready":true})))?; while let Some(input) = read_input(&mut reader)? { - let snapshot = match store.snapshot() { - Ok(snapshot) => snapshot, - Err(e) => { - output(&mut writer, Err(e))?; - continue; - } - }; if input.get("action").and_then(Value::as_str) == Some("hold_snapshot") { - let before = snapshot.setting("snapshot_probe"); - output(&mut writer, before.map(|v| json!({"before":v})))?; - if read_input(&mut reader)?.is_none() { - return Err(io::Error::other("test_resume_required")); - } - output( - &mut writer, - snapshot - .setting("snapshot_probe") - .map(|v| json!({"after":v})), - )?; + let result = store.read(|snapshot| { + output( + &mut writer, + snapshot + .setting("snapshot_probe") + .map(|v| json!({"before":v})), + ) + .map_err(|_| Error("test_io_error"))?; + read_input(&mut reader) + .map_err(|_| Error("test_io_error"))? + .ok_or(Error("test_resume_required"))?; + Ok(json!({"after":snapshot.setting("snapshot_probe")?})) + }); + output(&mut writer, result)?; } else { - output(&mut writer, run(input, &context, &mut auth, &snapshot))?; + let result = store + .read(|snapshot| run(input, &context, &mut auth, &mut authorization, snapshot)); + output(&mut writer, result)?; } } Ok(()) diff --git a/rust/crates/msg-storage/examples/pg_writer_check.rs b/rust/crates/msg-storage/examples/pg_writer_check.rs new file mode 100644 index 00000000..15784893 --- /dev/null +++ b/rust/crates/msg-storage/examples/pg_writer_check.rs @@ -0,0 +1,217 @@ +//! Disposable PostgreSQL failure-injection driver, NOT an authenticated executor. +//! No network listener, arbitrary SQL, production credentials or Python process. +use msg_core::{Error, Json, Result}; +use msg_identity::store::{AuthorityStore, Record}; +use msg_storage::{ + postgres::{PgFailure, PgSession, PgStore}, + records::{decode, encode, IdentityRecord}, +}; +use serde::de::DeserializeOwned; +use std::io::{self, Write}; +fn field<'a>(value: &'a Json, key: &str) -> Result<&'a Json> { + value.as_object()?.get(key).ok_or(Error("fixture_field")) +} +fn get(value: &Json, key: &str) -> Result { + decode(field(value, key)?) +} +fn flag(value: &Json, key: &str) -> Result { + value + .as_object()? + .get(key) + .map(Json::as_bool) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn emit(value: &Json) -> Result<()> { + println!("{}", value.canonical()?); + io::stdout().flush().map_err(|_| Error("fixture_output")) +} +fn hold(name: &str) -> Result<()> { + emit(&encode(&serde_json::json!({"checkpoint":name}))?)?; + let mut line = String::new(); + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + if line.trim() != "continue" { + return Err(Error("fixture_input")); + } + Ok(()) +} +fn steps(tx: &PgSession, values: &Json) -> Result> { + let mut output = Vec::new(); + for step in values.as_array()? { + match field(step, "action")?.as_str()? { + "set" => tx.set_setting(&get::(step, "key")?, field(step, "value")?)?, + "read" => output.push( + tx.setting(&get::(step, "key")?)? + .unwrap_or(Json::parse("null")?), + ), + "resource" => output.push(encode(&tx.resource_record(&get::(step, "id")?)?)?), + "identity" => { + let expected = get(step, "expected")?; + match get::(step, "record_kind")?.as_str() { + "subject" => tx.update_identity( + IdentityRecord::Subject(&get(step, "record")?), + expected, + )?, + "organization" => tx.update_identity( + IdentityRecord::Organization(&get(step, "record")?), + expected, + )?, + "membership" => tx.update_identity( + IdentityRecord::Membership(&get(step, "record")?), + expected, + )?, + "email" => { + tx.update_identity(IdentityRecord::Email(&get(step, "record")?), expected)? + } + _ => return Err(Error("fixture_action")), + } + } + "subject" => output.push(tx.record(Record::Subject, &get::(step, "id")?)?), + "credential" => tx.save_credential(&get(step, "record")?, get(step, "expected")?)?, + "get_credential" => { + output.push(tx.record(Record::Credential, &get::(step, "id")?)?) + } + "csr" => tx.save_csr(&get(step, "record")?)?, + "csr_state" => output.push(encode(&tx.csr_state(&get::(step, "id")?)?)?), + "transition_csr" => tx.transition_csr(&get(step, "record")?, get(step, "expected")?)?, + "certificate" => { + let id: Option = get(step, "csr")?; + tx.register_certificate( + &get(step, "record")?, + id.as_deref() + .map(|id| Ok((id, get(step, "expected")?))) + .transpose()?, + )?; + } + "revoke_certificate" => { + tx.revoke_certificate(&get::(step, "id")?, &get(step, "event")?)? + } + "get_certificate" => { + output.push(tx.record(Record::Certificate, &get::(step, "id")?)?) + } + "transfer" => tx.save_transfer(&get(step, "record")?, get(step, "expected")?)?, + "get_transfer" => output.push(encode(&tx.transfer(&get::(step, "id")?)?)?), + "chunk" => tx.put_chunk(&get(step, "record")?)?, + "missing" => output.push(encode(&tx.missing_ranges( + &get::(step, "id")?, + get::>(step, "cursor")?.as_deref(), + get(step, "limit")?, + )?)?), + "enqueue" => tx.enqueue(&get(step, "record")?)?, + "job" => output.push(encode(&tx.job(&get::(step, "id")?)?)?), + "save_job" => tx.save_job(&get(step, "record")?)?, + "path" => output.push(Json::string(tx.resource_path(&get::(step, "id")?)?)), + "resolve" => output.push(Json::string( + tx.resolve(&get::(step, "path")?, flag(step, "migrated")?)?, + )), + "children" => output.push(encode(&tx.children( + &get::(step, "id")?, + get::>(step, "cursor")?.as_deref(), + get(step, "limit")?, + )?)?), + "history" => output.push(encode(&tx.history( + &get::(step, "id")?, + get::>(step, "cursor")?.as_deref(), + get(step, "limit")?, + )?)?), + "get_revision" => output.push(encode(&tx.revision(&get(step, "reference")?)?)?), + "insert" => tx.insert_resource(&get(step, "record")?)?, + "replace" => tx.replace_resource(&get(step, "record")?, get(step, "expected")?)?, + "revision" => tx.append_revision(&get(step, "record")?)?, + "result" => tx.save_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "digest")?, + &get(step, "record")?, + )?, + "lookup" => output.push( + tx.request_result( + get::>(step, "subject")?.as_deref(), + &get::(step, "id")?, + &get::(step, "digest")?, + )? + .unwrap_or(Json::parse("null")?), + ), + "event" => tx.append_event(&get(step, "record")?)?, + "audit" => tx.append_audit(&get(step, "record")?)?, + "savepoint" => match tx.savepoint(|tx| steps(tx, field(step, "steps")?)) { + Ok(values) => output.push(encode(&values)?), + Err(error) if flag(step, "recover")? => { + output.push(encode(&serde_json::json!({"error":error.0}))?) + } + Err(error) => return Err(error), + }, + "ignore" => { + let _ = steps(tx, field(step, "steps")?); + } + "effect" => { + let name: String = get(step, "name")?; + let fail = flag(step, "fail")?; + let panic = flag(step, "panic")?; + let pause = flag(step, "hold")?; + tx.on_rollback(move || { + emit(&encode(&serde_json::json!({"effect":name}))?)?; + if pause { + hold("compensation")?; + } + assert!(!panic, "fixture cleanup panic"); + if fail { + return Err(Error("fixture_cleanup")); + } + Ok(()) + })?; + } + "hold" => hold("transaction")?, + "fail" => return Err(Error("fixture_failure")), + "panic" => panic!("fixture handler panic"), + // This is only a test of the storage building blocks. There is NO + // authentication or authorization here; never expose it as an API. + "once" => { + let subject: Option = get(step, "subject")?; + let digest: String = get(step, "digest")?; + let record: msg_storage::records::OperationResult = get(step, "record")?; + if let Some(result) = + tx.request_result(subject.as_deref(), &record.request_id, &digest)? + { + output.push(result); + } else { + steps(tx, field(step, "steps")?)?; + tx.save_result(subject.as_deref(), &digest, &record)?; + output.push(encode(&record)?); + } + } + _ => return Err(Error("fixture_action")), + } + } + Ok(output) +} +fn run(case: &Json) -> std::result::Result, PgFailure> { + let dsn = std::env::var("MSG_PARITY_POSTGRES_DSN").map_err(|_| Error("fixture_dsn"))?; + let store = PgStore::open_existing(&dsn)?; + let output = store.transaction(!flag(case, "read_only")?, |tx| { + steps(tx, field(case, "steps")?) + })?; + if flag(case, "hold_after_commit")? { + hold("committed")?; + } + Ok(output) +} +fn main() { + let mut line = String::new(); + let result = (|| { + io::stdin() + .read_line(&mut line) + .map_err(|_| Error("fixture_input"))?; + run(&Json::parse(&line)?) + })(); + let output = match result { + Ok(values) => serde_json::json!({"status":"ok","values":values}), + Err(error) => { + serde_json::json!({"status":"error","code":error.code(),"cleanup":if matches!(error,PgFailure::Rejected{cleanup_failed:true,..}) {vec!["storage_cleanup_failed"]} else {vec![]},"uncertain":matches!(error,PgFailure::Uncertain{..})}) + } + }; + if emit(&encode(&output).expect("fixture result is bounded")).is_err() { + std::process::exit(1); + } +} diff --git a/rust/crates/msg-storage/src/authorization.rs b/rust/crates/msg-storage/src/authorization.rs new file mode 100644 index 00000000..7d6bfdad --- /dev/null +++ b/rust/crates/msg-storage/src/authorization.rs @@ -0,0 +1,138 @@ +//! Transaction-bound authorization facts; no policy or implicit permission here. +use crate::records::{Membership, Organization}; +use crate::{database_error, Session}; +use msg_core::{Error, Json, Result, MAX_BYTES}; +use msg_identity::{ + authorization::{AuthorizationStore, DirectConversation, LegacyShare, ShareSource}, + models::{decode, SubjectKind}, + store::AuthorityStore, +}; +use rusqlite::{params, OptionalExtension, Params, Row}; +use std::collections::BTreeSet; + +const MAX_FACTS: usize = 4096; +fn bounded_json(raw: &str) -> Result { + if raw.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Json::parse(raw) +} +impl Session<'_, A> { + fn authorization_rows( + &self, + sql: &'static str, + params: impl Params, + decode_row: impl FnMut(&Row<'_>) -> rusqlite::Result, + ) -> Result> { + self.active()?; + let mut stmt = self + .transaction + .prepare_cached(sql) + .map_err(database_error)?; + let rows = stmt + .query_map(params, decode_row) + .map_err(database_error)? + .collect::, _>>() + .map_err(database_error)?; + if rows.len() > MAX_FACTS { + return Err(Error("authority_work_limit")); + } + Ok(rows) + } +} +impl AuthorizationStore for Session<'_, A> { + fn authority(&self) -> &dyn AuthorityStore { + self + } + fn memberships(&self, subject: &str) -> Result> { + let rows: Vec<(String, String)> = self.authorization_rows( + "SELECT m.org,CAST(substr(CAST(m.body AS BLOB),1,1048577) AS TEXT) FROM memberships m JOIN resources r ON r.id=m.org WHERE m.subject=?1 AND r.type='organization' AND r.state='active' ORDER BY m.org LIMIT 4097", + [subject], |r| Ok((r.get(0)?, r.get(1)?)))?; + let mut groups = BTreeSet::new(); + for (org, body) in rows { + let member: Membership = decode(&bounded_json(&body)?)?; + member.validate()?; + if member.status == "active" + && member.organization_id == org + && member.subject_id == subject + && org != "g_public" + { + groups.insert(org); + } + } + match self.subject(subject) { + Ok(identity) + if matches!( + identity.kind, + SubjectKind::Registered | SubjectKind::Custodial | SubjectKind::System + ) && !identity.local_only => + { + groups.insert("g_public".into()); + } + Ok(_) | Err(Error("subject_not_found")) => (), + Err(error) => return Err(error), + } + Ok(groups) + } + fn organization_exists(&self, id: &str) -> Result { + let value = self.json("SELECT CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM identities WHERE id=?1 AND kind='organization'", id)?; + match value { + None => Ok(false), + Some(value) => { + let row: Organization = decode(&value)?; + row.validate()?; + if row.resource_id != id { + return Err(Error("storage_record_mismatch")); + } + Ok(true) + } + } + } + fn topic_admin(&self, topic: &str, subject: &str) -> Result { + self.active()?; + self.transaction.query_row("SELECT EXISTS(SELECT 1 FROM topic_memberships WHERE topic=?1 AND subject=?2 AND role='admin' AND status='active')", + [topic, subject], |r| r.get(0)).map_err(database_error) + } + fn topic_ban(&self, topic: &str, subject: &str) -> Result>> { + self.active()?; + self.transaction.query_row("SELECT expires_at FROM topic_bans WHERE topic=?1 AND subject=?2 AND status='active'", [topic, subject], |r| r.get(0)) + .optional().map_err(database_error) + } + fn direct_conversation(&self, resource: &str) -> Result> { + self.active()?; + self.transaction.query_row("SELECT resource_id,participant_a,participant_b,state FROM dm_conversations WHERE resource_id=?1", [resource], + |r| Ok(DirectConversation { resource_id: r.get(0)?, participant_a: r.get(1)?, participant_b: r.get(2)?, state: r.get(3)? })) + .optional().map_err(database_error) + } + fn direct_blocked(&self, a: &str, b: &str) -> Result { + self.active()?; + self.transaction.query_row("SELECT EXISTS(SELECT 1 FROM dm_blocks WHERE (blocker=?1 AND blocked=?2) OR (blocker=?2 AND blocked=?1))", + [a, b], |r| r.get(0)).map_err(database_error) + } + fn legacy_share(&self, resource: &str, grantee: &str) -> Result> { + self.active()?; + self.transaction.query_row("SELECT grantor,expires_at,created_at FROM share_grants WHERE resource_id=?1 AND grantee=?2 AND revoked_at IS NULL", + [resource, grantee], |r| Ok(LegacyShare { grantor: r.get(0)?, expires_at: r.get(1)?, created_at: r.get(2)? })) + .optional().map_err(database_error) + } + fn share_candidates(&self, resource: &str, now: &str) -> Result> { + self.authorization_rows("SELECT id FROM share_grants_v2 WHERE resource_id=?1 AND revoked_at IS NULL AND expires_at>?2 ORDER BY id LIMIT 4097", + params![resource, now], |r| r.get(0)) + } + fn share_source(&self, id: &str) -> Result> { + self.active()?; + self.transaction.query_row("SELECT resource_id,grantor,grantee,grantee_kind,parent_id,substr(operations,1,1048577),substr(constraints,1,1048577),allow_reshare,expires_at,revoked_at,created_at FROM share_grants_v2 WHERE id=?1", + [id], |r| Ok(ShareSource { resource_id: r.get(0)?, grantor: r.get(1)?, grantee: r.get(2)?, grantee_kind: r.get(3)?, parent_id: r.get(4)?, + operations: r.get(5)?, constraints: r.get(6)?, allow_reshare: r.get::<_, i64>(7)? == 1, expires_at: r.get(8)?, revoked_at: r.get(9)?, created_at: r.get(10)? })) + .optional().map_err(database_error) + } + fn csr_issuer(&self, id: &str) -> Result { + let row = self + .json( + "SELECT CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM csrs WHERE id=?1", + id, + )? + .ok_or(Error("csr_not_found"))?; + Ok(decode::(&row)?.requested_issuer) + } +} diff --git a/rust/crates/msg-storage/src/lib.rs b/rust/crates/msg-storage/src/lib.rs index 352f9456..39cc7fa5 100644 --- a/rust/crates/msg-storage/src/lib.rs +++ b/rust/crates/msg-storage/src/lib.rs @@ -2,6 +2,8 @@ //! //! No schema creation/migration, network entry point or Python subprocess. //! Read snapshots and write sessions share the same transaction-bound authority reads. +mod authorization; +pub mod postgres; pub mod records; #[cfg(unix)] pub mod writer; diff --git a/rust/crates/msg-storage/src/postgres/authority.rs b/rust/crates/msg-storage/src/postgres/authority.rs new file mode 100644 index 00000000..b75af89b --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/authority.rs @@ -0,0 +1,200 @@ +//! Policy facts from the transaction's immutable PostgreSQL snapshot. +use super::PgSession; +use crate::records::{Membership, Organization}; +use msg_core::{Error, Json, Result}; +use msg_identity::{ + authorization::{AuthorizationStore, DirectConversation, LegacyShare, ShareSource}, + models::{decode, SubjectKind, Timestamp}, + oauth::OAuthState, + store::{AuthorityStore, Record, RecoveryDelivery}, +}; +use std::collections::BTreeSet; + +impl PgSession { + fn json_fact(&self, sql: &'static str, id: &str) -> Result> { + self.one(sql, &[id.into()])?.map(|r| r.json(0)).transpose() + } +} +impl AuthorityStore for PgSession { + fn record(&self, kind: Record, id: &str) -> Result { + let (sql, missing) = match kind { + Record::Subject => ( + "SELECT substring(body,1,1048577) FROM identities WHERE id=$1 AND kind='subject'", + "subject_not_found", + ), + Record::Credential => ( + "SELECT substring(body,1,1048577) FROM credentials WHERE id=$1", + "credential_not_found", + ), + Record::Certificate => ( + "SELECT substring(body,1,1048577) FROM certificates WHERE id=$1", + "certificate_not_found", + ), + Record::Resource => ( + "SELECT substring(body,1,1048577) FROM resources WHERE id=$1", + "not_found", + ), + }; + self.json_fact(sql, id)?.ok_or(Error(missing)) + } + fn setting(&self, key: &str) -> Result> { + self.json_fact( + "SELECT substring(value,1,1048577) FROM settings WHERE key=$1", + key, + ) + } + fn has_setting(&self, key: &str) -> Result { + self.one( + "SELECT EXISTS(SELECT 1 FROM settings WHERE key=$1)", + &[key.into()], + )? + .ok_or(Error("storage_error"))? + .boolean(0) + } + fn certificate_revoked(&self, id: &str) -> Result { + Ok(self + .one("SELECT revoked FROM certificates WHERE id=$1", &[id.into()])? + .ok_or(Error("certificate_not_found"))? + .integer(0)? + != 0) + } + fn request_result( + &self, + subject: Option<&str>, + id: &str, + digest: &str, + ) -> Result> { + let Some(row)=self.one("SELECT digest,substring(body,1,1048577) FROM results WHERE subject=$1 AND request_id=$2",&[subject.unwrap_or("").into(),id.into()])? else {return Ok(None)}; + if row.text(0)? != digest { + return Err(Error("idempotency_conflict")); + } + let result = row.json(1)?; + let fields = result.as_object()?; + let result_subject = fields.get("subject").and_then(|v| { + if v.is_null() { + Some(None) + } else { + v.as_str().ok().map(Some) + } + }); + if fields.get("request_id").and_then(|v| v.as_str().ok()) != Some(id) + || result_subject != Some(subject) + { + return Err(Error("storage_record_mismatch")); + } + Ok(Some(result)) + } + fn oauth_state(&self, id: &str) -> Result> { + self.one( + "SELECT expires,substring(body,1,1048577) FROM oauth_states WHERE id=$1", + &[id.into()], + )? + .map(|r| { + Ok(OAuthState { + expires_at: Timestamp::parse(r.text(0)?)?, + body: r.json(1)?, + }) + }) + .transpose() + } + fn custodial_binding(&self, subject: &str) -> Result> { + self.one( + "SELECT signing_key_id,status FROM custodial_vault WHERE subject=$1", + &[subject.into()], + )? + .map(|r| Ok((r.text(0)?.to_owned(), r.text(1)?.to_owned()))) + .transpose() + } + fn recovery_delivery( + &self, + credential: &str, + subject: &str, + ) -> Result> { + self.one("SELECT recovery_verifier,recovery_expires_at,consumed_at,request_id FROM token_deliveries WHERE credential_id=$1 AND subject=$2",&[credential.into(),subject.into()])? + .map(|r|Ok(RecoveryDelivery{verifier:r.text(0)?.to_owned(),expires_at:Timestamp::parse(r.text(1)?)?,consumed:r.optional_text(2)?.is_some(),request_id:r.text(3)?.to_owned()})).transpose() + } +} +impl AuthorizationStore for PgSession { + fn authority(&self) -> &dyn AuthorityStore { + self + } + fn memberships(&self, subject: &str) -> Result> { + let rows=self.rows("SELECT m.org,substring(m.body,1,1048577) FROM memberships m JOIN resources r ON r.id=m.org WHERE m.subject=$1 AND r.type='organization' AND r.state='active' ORDER BY m.org LIMIT 4097",&[subject.into()])?; + let mut result = BTreeSet::new(); + for r in rows { + let m: Membership = decode(&r.json(1)?)?; + m.validate()?; + if m.status == "active" + && m.organization_id == r.text(0)? + && m.subject_id == subject + && m.organization_id != "g_public" + { + result.insert(m.organization_id); + } + } + match self.subject(subject) { + Ok(s) + if matches!( + s.kind, + SubjectKind::Registered | SubjectKind::Custodial | SubjectKind::System + ) && !s.local_only => + { + result.insert("g_public".into()); + } + Ok(_) | Err(Error("subject_not_found")) => (), + Err(e) => return Err(e), + } + Ok(result) + } + fn organization_exists(&self, id: &str) -> Result { + let Some(value) = self.json_fact( + "SELECT substring(body,1,1048577) FROM identities WHERE id=$1 AND kind='organization'", + id, + )? + else { + return Ok(false); + }; + let row: Organization = decode(&value)?; + row.validate()?; + if row.resource_id != id { + return Err(Error("storage_record_mismatch")); + } + Ok(true) + } + fn topic_admin(&self, topic: &str, subject: &str) -> Result { + self.one("SELECT EXISTS(SELECT 1 FROM topic_memberships WHERE topic=$1 AND subject=$2 AND role='admin' AND status='active')",&[topic.into(),subject.into()])?.ok_or(Error("storage_error"))?.boolean(0) + } + fn topic_ban(&self, topic: &str, subject: &str) -> Result>> { + self.one( + "SELECT expires_at FROM topic_bans WHERE topic=$1 AND subject=$2 AND status='active'", + &[topic.into(), subject.into()], + )? + .map(|r| Ok(r.optional_text(0)?.map(str::to_owned))) + .transpose() + } + fn direct_conversation(&self, resource: &str) -> Result> { + self.one("SELECT resource_id,participant_a,participant_b,state FROM dm_conversations WHERE resource_id=$1",&[resource.into()])?.map(|r|Ok(DirectConversation{ + resource_id:r.text(0)?.to_owned(),participant_a:r.text(1)?.to_owned(),participant_b:r.text(2)?.to_owned(),state:r.text(3)?.to_owned()})).transpose() + } + fn direct_blocked(&self, a: &str, b: &str) -> Result { + self.one("SELECT EXISTS(SELECT 1 FROM dm_blocks WHERE (blocker=$1 AND blocked=$2) OR (blocker=$2 AND blocked=$1))",&[a.into(),b.into()])?.ok_or(Error("storage_error"))?.boolean(0) + } + fn legacy_share(&self, resource: &str, grantee: &str) -> Result> { + self.one("SELECT grantor,expires_at,created_at FROM share_grants WHERE resource_id=$1 AND grantee=$2 AND revoked_at IS NULL",&[resource.into(),grantee.into()])?.map(|r|Ok(LegacyShare{ + grantor:r.text(0)?.to_owned(),expires_at:r.text(1)?.to_owned(),created_at:r.text(2)?.to_owned()})).transpose() + } + fn share_candidates(&self, resource: &str, now: &str) -> Result> { + self.rows("SELECT id FROM share_grants_v2 WHERE resource_id=$1 AND revoked_at IS NULL AND expires_at>$2 ORDER BY id LIMIT 4097",&[resource.into(),now.into()])?.iter().map(|r|Ok(r.text(0)?.to_owned())).collect() + } + fn share_source(&self, id: &str) -> Result> { + self.one("SELECT resource_id,grantor,grantee,grantee_kind,parent_id,substring(operations,1,1048577),substring(constraints,1,1048577),allow_reshare,expires_at,revoked_at,created_at FROM share_grants_v2 WHERE id=$1",&[id.into()])?.map(|r|Ok(ShareSource{ + resource_id:r.text(0)?.to_owned(),grantor:r.text(1)?.to_owned(),grantee:r.text(2)?.to_owned(),grantee_kind:r.text(3)?.to_owned(),parent_id:r.optional_text(4)?.map(str::to_owned), + operations:r.text(5)?.to_owned(),constraints:r.text(6)?.to_owned(),allow_reshare:r.integer(7)?==1,expires_at:r.text(8)?.to_owned(),revoked_at:r.optional_text(9)?.map(str::to_owned),created_at:r.text(10)?.to_owned()})).transpose() + } + fn csr_issuer(&self, id: &str) -> Result { + let value = self + .json_fact("SELECT substring(body,1,1048577) FROM csrs WHERE id=$1", id)? + .ok_or(Error("csr_not_found"))?; + Ok(decode::(&value)?.requested_issuer) + } +} diff --git a/rust/crates/msg-storage/src/postgres/mod.rs b/rust/crates/msg-storage/src/postgres/mod.rs new file mode 100644 index 00000000..d0ddc3e6 --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/mod.rs @@ -0,0 +1,514 @@ +//! Native PostgreSQL transactions over the existing, Python-migrated schema. +//! +//! One dedicated connection owns each transaction. Writers use the same +//! session advisory lock as Python and retain it through all compensations. +//! Transaction pooling is unsupported. No schema is created or upgraded here. +mod authority; +mod query; +mod records; +use msg_core::{Error, Result}; +use postgres::{fallible_iterator::FallibleIterator, types::ToSql, Client, Config}; +use postgres_native_tls::MakeTlsConnector; +pub use query::{Row, Value}; +use std::{ + cell::{Cell, RefCell}, + fmt, + marker::PhantomData, + panic::{catch_unwind, resume_unwind, AssertUnwindSafe}, + rc::Rc, + str::FromStr, + time::Duration, +}; + +const MAX_ROWS: usize = 4096; +const MAX_QUERY_BYTES: usize = 8 * 1024 * 1024; +type Cleanup = Box Result<()>>; + +#[derive(Clone)] +pub struct PgStore { + config: Config, + tls: native_tls::TlsConnector, +} +impl fmt::Debug for PgStore { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("PgStore()") + } +} +#[derive(Debug)] +pub enum PgFailure { + Rejected { + error: Error, + cleanup_failed: bool, + }, + /// Do not retry under a new request ID, delete blobs, or issue secrets. + Uncertain { + error: Error, + }, +} +impl PgFailure { + pub fn code(&self) -> &'static str { + match self { + Self::Rejected { error, .. } | Self::Uncertain { error } => error.0, + } + } +} +impl From for PgFailure { + fn from(error: Error) -> Self { + Self::Rejected { + error, + cleanup_failed: false, + } + } +} +pub struct PgSession { + client: RefCell, + write: bool, + closed: Cell, + poisoned: Cell>, + rollback: RefCell>, + savepoints: Cell, + _owner: PhantomData>, +} +fn database_error(error: &postgres::Error) -> Error { + let code = error.code().map(|c| c.code()).unwrap_or(""); + if code.starts_with("23") { + Error("constraint_conflict") + } else if matches!( + code, + "40P01" | "40001" | "55P03" | "57014" | "53300" | "57P03" + ) { + Error("server_busy") + } else if code == "25006" { + Error("read_only_transaction") + } else { + Error("storage_error") + } +} +impl PgStore { + pub fn open_existing(dsn: &str) -> Result { + let mut config = Config::from_str(dsn).map_err(|_| Error("invalid_postgres_config"))?; + config.connect_timeout(Duration::from_secs(10)); + config.application_name("msgd-native"); + let local = config.get_hosts().iter().all(|host| match host { + postgres::config::Host::Tcp(host) => { + host == "localhost" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + } + #[cfg(unix)] + postgres::config::Host::Unix(_) => true, + }) && config + .get_hostaddrs() + .iter() + .all(std::net::IpAddr::is_loopback); + if !local { + if config.get_ssl_mode() == postgres::config::SslMode::Disable { + return Err(Error("postgres_tls_required")); + } + // A remote connection must not fall back to plaintext when the peer + // rejects TLS. Certificate/name validation is still enforced below. + config.ssl_mode(postgres::config::SslMode::Require); + } + // native-tls validates server certificates and names whenever TLS is used. + // There is no dangerous_accept_invalid_certs/hostnames setting. + let tls = native_tls::TlsConnector::builder() + .build() + .map_err(|_| Error("postgres_tls_error"))?; + let store = Self { config, tls }; + store + .transaction(false, |tx| tx.validate_schema()) + .map_err(|e| Error(e.code()))?; + Ok(store) + } + fn connect(&self) -> Result { + self.config + .connect(MakeTlsConnector::new(self.tls.clone())) + .map_err(|e| database_error(&e)) + } + pub fn transaction( + &self, + write: bool, + body: impl FnOnce(&PgSession) -> Result, + ) -> std::result::Result { + let mut client = self.connect()?; + client.batch_execute("SET search_path = pg_catalog, public; SET statement_timeout='30s'; SET lock_timeout='10s'; SET idle_in_transaction_session_timeout='60s';") + .map_err(|e|PgFailure::from(database_error(&e)))?; + if write { + client + .query_one("SELECT pg_advisory_lock(725274758,1886265951)", &[]) + .map_err(|e| PgFailure::from(database_error(&e)))?; + } + // Acquire the writer fence before taking the snapshot. Read-only requests + // also see one committed generation, not a mixture of policy revisions. + client + .batch_execute(if write { + "BEGIN ISOLATION LEVEL REPEATABLE READ" + } else { + "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY" + }) + .map_err(|e| PgFailure::from(database_error(&e)))?; + let session = PgSession { + client: RefCell::new(client), + write, + closed: Cell::new(false), + poisoned: Cell::new(None), + rollback: RefCell::new(Vec::new()), + savepoints: Cell::new(0), + _owner: PhantomData, + }; + let outcome = catch_unwind(AssertUnwindSafe(|| body(&session))); + match outcome { + Ok(Ok(value)) if session.poisoned.get().is_none() => { + let commit = session.client.borrow_mut().batch_execute("COMMIT"); + match commit { + Ok(()) => { + session.closed.set(true); + session.rollback.borrow_mut().clear(); + Ok(value) + } + Err(error) => { + let definitely_aborted = error.code().is_some_and(|c| { + c.code().starts_with("23") + || matches!(c.code(), "40001" | "40P01" | "25P02") + }); + if definitely_aborted + && session + .client + .borrow_mut() + .batch_execute("ROLLBACK") + .is_ok() + { + let failed = session.compensate(0); + session.closed.set(true); + Err(PgFailure::Rejected { + error: database_error(&error), + cleanup_failed: failed, + }) + } else { + session.closed.set(true); + session.rollback.borrow_mut().clear(); + Err(PgFailure::Uncertain { + error: Error("storage_commit_uncertain"), + }) + } + } + } + } + result => { + let confirmed = session + .client + .borrow_mut() + .batch_execute("ROLLBACK") + .is_ok(); + let cleanup_failed = if confirmed { + session.compensate(0) + } else { + session.rollback.borrow_mut().clear(); + false + }; + session.closed.set(true); + match result { + Err(panic) => resume_unwind(panic), + result => { + let error = match result { + Ok(Err(error)) => error, + _ => session + .poisoned + .get() + .unwrap_or(Error("transaction_aborted")), + }; + if confirmed { + Err(PgFailure::Rejected { + error, + cleanup_failed, + }) + } else { + Err(PgFailure::Uncertain { + error: Error("storage_rollback_uncertain"), + }) + } + } + } + } + } + // Closing this dedicated connection releases the advisory lock only after + // compensation completes. A transaction-pool proxy breaks that guarantee. + } +} +impl PgSession { + fn active(&self, write: bool) -> Result<()> { + if self.closed.get() { + return Err(Error("transaction_closed")); + } + if let Some(e) = self.poisoned.get() { + return Err(e); + } + if write && !self.write { + return Err(Error("read_only_transaction")); + } + Ok(()) + } + fn capture(&self, result: Result, write: bool) -> Result { + if let Err(e) = result { + if write { + self.poisoned.set(Some(e)); + } + return Err(e); + } + result + } + pub fn require_write(&self) -> Result<()> { + self.active(true) + } + /// SQL is a compiled, trusted string; only values may originate in a request. + pub fn rows(&self, sql: &'static str, params: &[Value]) -> Result> { + self.active(false)?; + let refs: Vec<&(dyn ToSql + Sync)> = + params.iter().map(|p| p as &(dyn ToSql + Sync)).collect(); + let result = (|| { + let mut client = self.client.borrow_mut(); + let mut iter = client + .query_raw(sql, refs) + .map_err(|e| database_error(&e))?; + let mut rows = Vec::new(); + let mut bytes = 0; + while let Some(row) = iter.next().map_err(|e| database_error(&e))? { + let row = Row::decode(row)?; + bytes += row.bytes(); + if rows.len() >= MAX_ROWS || bytes > MAX_QUERY_BYTES { + return Err(Error("storage_query_limit")); + } + rows.push(row); + } + Ok(rows) + })(); + self.capture(result, self.write) + } + pub fn one(&self, sql: &'static str, params: &[Value]) -> Result> { + let mut rows = self.rows(sql, params)?; + if rows.len() > 1 { + return self.capture(Err(Error("storage_record_ambiguous")), self.write); + } + Ok(rows.pop()) + } + pub fn execute(&self, sql: &'static str, params: &[Value]) -> Result { + self.active(true)?; + let refs: Vec<&(dyn ToSql + Sync)> = + params.iter().map(|p| p as &(dyn ToSql + Sync)).collect(); + let result = self + .client + .borrow_mut() + .execute(sql, &refs) + .map_err(|e| database_error(&e)); + self.capture(result, true) + } + pub fn on_rollback(&self, effect: impl FnOnce() -> Result<()> + 'static) -> Result<()> { + self.active(true)?; + self.rollback.borrow_mut().push(Box::new(effect)); + Ok(()) + } + fn compensate(&self, start: usize) -> bool { + let effects: Vec<_> = self.rollback.borrow_mut().drain(start..).collect(); + let mut failed = false; + for effect in effects.into_iter().rev() { + failed |= !matches!(catch_unwind(AssertUnwindSafe(effect)), Ok(Ok(()))); + } + failed + } + pub fn savepoint(&self, body: impl FnOnce(&Self) -> Result) -> Result { + self.active(false)?; + let id = self + .savepoints + .get() + .checked_add(1) + .ok_or(Error("storage_savepoint_limit"))?; + self.savepoints.set(id); + // Names contain only an internal integer. They never contain user strings. + self.client + .borrow_mut() + .batch_execute(&format!("SAVEPOINT msg_native_{id}")) + .map_err(|e| database_error(&e))?; + let start = self.rollback.borrow().len(); + let result = catch_unwind(AssertUnwindSafe(|| body(self))); + if matches!(&result, Ok(Ok(_))) && self.poisoned.get().is_none() { + let release = self + .client + .borrow_mut() + .batch_execute(&format!("RELEASE SAVEPOINT msg_native_{id}")) + .map_err(|e| database_error(&e)); + self.capture(release, self.write)?; + return match result { + Ok(value) => value, + Err(p) => resume_unwind(p), + }; + } + let rollback = self.client.borrow_mut().batch_execute(&format!( + "ROLLBACK TO SAVEPOINT msg_native_{id}; RELEASE SAVEPOINT msg_native_{id}" + )); + if let Err(error) = rollback { + let error = database_error(&error); + self.poisoned.set(Some(error)); + return Err(error); + } + let poison = self.poisoned.take(); + let cleanup_failed = self.compensate(start); + if cleanup_failed { + self.poisoned.set(Some(Error("storage_cleanup_failed"))); + } + match result { + Err(p) => resume_unwind(p), + Ok(Err(e)) => Err(e), + Ok(Ok(_)) => Err(poison.unwrap_or(Error("transaction_aborted"))), + } + } + fn validate_schema(&self) -> Result<()> { + let versions = self + .rows( + "SELECT version FROM public.schema_version ORDER BY version LIMIT 2", + &[], + ) + .map_err(|_| Error("unsupported_storage_schema"))?; + if versions.len() != 1 || versions[0].integer(0)? != 1 { + return Err(Error("unsupported_storage_schema")); + } + let required = [ + ( + "resources", + vec![ + "id", + "type", + "name", + "parent", + "owner", + "grp", + "mode", + "generation", + "revision", + "state", + "created_at", + "modified_at", + "body", + ], + ), + ("identities", vec!["id", "kind", "generation", "body"]), + ("credentials", vec!["id", "subject", "body"]), + ( + "certificates", + vec!["id", "subject", "parent", "revoked", "body"], + ), + ("settings", vec!["key", "value"]), + ("results", vec!["subject", "request_id", "digest", "body"]), + ("oauth_states", vec!["id", "kind", "expires", "body"]), + ("revisions", vec!["id", "resource_id", "created_at", "body"]), + ( + "relations", + vec![ + "revision_id", + "source_id", + "type", + "target_id", + "target_revision", + "body", + ], + ), + ( + "custodial_vault", + vec!["subject", "signing_key_id", "status"], + ), + ( + "token_deliveries", + vec![ + "credential_id", + "subject", + "request_id", + "recovery_verifier", + "recovery_expires_at", + "consumed_at", + ], + ), + ("memberships", vec!["org", "subject", "generation", "body"]), + ("emails", vec!["subject", "generation", "body"]), + ( + "csrs", + vec!["id", "generation", "state", "body", "state_body"], + ), + ( + "transfers", + vec!["id", "subject", "generation", "body", "limits"], + ), + ("chunks", vec!["transfer_id", "offset", "length", "body"]), + ( + "jobs", + vec!["id", "dedupe", "kind", "state", "next_at", "body"], + ), + ("resource_tags", vec!["resource_id", "tag"]), + ( + "resource_path_aliases", + vec!["parent_id", "name", "resource_id"], + ), + ("events", vec!["seq", "id", "body"]), + ("audit", vec!["seq", "digest", "previous", "body"]), + ]; + for (table, columns) in required { + let row=self.one("SELECT count(*) FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid=a.attrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname=$1 AND c.relkind='r' AND a.attnum>0 AND NOT a.attisdropped AND a.attname::text=ANY($2)",&[table.into(),Value::TextArray(columns.iter().map(|s|s.to_string()).collect())])?.ok_or(Error("unsupported_storage_schema"))?; + if row.integer(0)? != columns.len() as i64 { + return Err(Error("unsupported_storage_schema")); + } + } + for (table, columns) in [ + ("resources", vec!["id"]), + ("revisions", vec!["id"]), + ("identities", vec!["id"]), + ("credentials", vec!["id"]), + ("certificates", vec!["id"]), + ("csrs", vec!["id"]), + ("results", vec!["subject", "request_id"]), + ("settings", vec!["key"]), + ("memberships", vec!["org", "subject"]), + ("emails", vec!["subject"]), + ("transfers", vec!["id"]), + ("chunks", vec!["transfer_id", "offset"]), + ("jobs", vec!["id"]), + ("events", vec!["seq"]), + ("audit", vec!["seq"]), + ("resource_path_aliases", vec!["parent_id", "name"]), + ("resource_tags", vec!["tag", "resource_id"]), + ] { + self.require_constraint(table, "p", &columns)?; + } + for (table, columns) in [ + ("resources", vec!["parent", "name"]), + ("events", vec!["id"]), + ("audit", vec!["digest"]), + ("jobs", vec!["dedupe"]), + ] { + self.require_constraint(table, "u", &columns)?; + } + for (table, source, target, column) in [ + ("resources", "parent", "resources", "id"), + ("revisions", "resource_id", "resources", "id"), + ("relations", "revision_id", "revisions", "id"), + ("resource_tags", "resource_id", "resources", "id"), + ("resource_path_aliases", "resource_id", "resources", "id"), + ("resource_path_aliases", "parent_id", "resources", "id"), + ] { + let row=self.one("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_constraint c JOIN pg_catalog.pg_class t ON t.oid=c.conrelid JOIN pg_catalog.pg_namespace n ON n.oid=t.relnamespace JOIN pg_catalog.pg_class r ON r.oid=c.confrelid JOIN pg_catalog.pg_namespace rn ON rn.oid=r.relnamespace WHERE n.nspname='public' AND rn.nspname='public' AND t.relname=$1 AND r.relname=$3 AND c.contype='f' AND c.convalidated AND ARRAY(SELECT a.attname::text FROM unnest(c.conkey) WITH ORDINALITY k(attnum,ord) JOIN pg_catalog.pg_attribute a ON a.attrelid=t.oid AND a.attnum=k.attnum ORDER BY k.ord)=ARRAY[$2]::text[] AND ARRAY(SELECT a.attname::text FROM unnest(c.confkey) WITH ORDINALITY k(attnum,ord) JOIN pg_catalog.pg_attribute a ON a.attrelid=r.oid AND a.attnum=k.attnum ORDER BY k.ord)=ARRAY[$4]::text[])",&[table.into(),source.into(),target.into(),column.into()])?.ok_or(Error("unsupported_storage_schema"))?; + if !row.boolean(0)? { + return Err(Error("unsupported_storage_schema")); + } + } + let root=self.one("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class t ON t.oid=i.indrelid JOIN pg_catalog.pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname='public' AND t.relname='resources' AND i.indisunique AND i.indisvalid AND i.indisready AND i.indnkeyatts=1 AND pg_catalog.pg_get_expr(i.indexprs,i.indrelid)='1' AND pg_catalog.pg_get_expr(i.indpred,i.indrelid)='(parent IS NULL)')",&[])?.ok_or(Error("unsupported_storage_schema"))?; + if !root.boolean(0)? { + return Err(Error("unsupported_storage_schema")); + } + Ok(()) + } + fn require_constraint(&self, table: &str, kind: &str, columns: &[&str]) -> Result<()> { + let row=self.one("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_constraint c JOIN pg_catalog.pg_class t ON t.oid=c.conrelid JOIN pg_catalog.pg_namespace n ON n.oid=t.relnamespace WHERE n.nspname='public' AND t.relname=$1 AND c.contype::text=$2 AND c.convalidated AND ARRAY(SELECT a.attname::text FROM unnest(c.conkey) WITH ORDINALITY k(attnum,ord) JOIN pg_catalog.pg_attribute a ON a.attrelid=t.oid AND a.attnum=k.attnum ORDER BY k.ord)=$3)",&[table.into(),kind.into(),Value::TextArray(columns.iter().map(|s|s.to_string()).collect())])?.ok_or(Error("unsupported_storage_schema"))?; + if !row.boolean(0)? { + return Err(Error("unsupported_storage_schema")); + } + Ok(()) + } +} diff --git a/rust/crates/msg-storage/src/postgres/query.rs b/rust/crates/msg-storage/src/postgres/query.rs new file mode 100644 index 00000000..f4f6bdd8 --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/query.rs @@ -0,0 +1,188 @@ +//! Bound values for trusted, source-owned SQL. Neither rows nor parameters log secrets. +use bytes::BytesMut; +use msg_core::{Error, Json, Result, MAX_BYTES}; +use postgres::types::{to_sql_checked, IsNull, ToSql, Type}; +use std::{error::Error as StdError, fmt}; + +#[derive(Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Int(i64), + Text(String), + Bytes(Vec), + TextArray(Vec), +} +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Null => "NULL", + Self::Bool(_) => "", + Self::Int(_) => "", + Self::Text(_) => "", + Self::Bytes(_) => "", + Self::TextArray(_) => "", + }) + } +} +impl From<&str> for Value { + fn from(value: &str) -> Self { + Self::Text(value.into()) + } +} +impl From for Value { + fn from(value: String) -> Self { + Self::Text(value) + } +} +impl From for Value { + fn from(value: i64) -> Self { + Self::Int(value) + } +} +impl From for Value { + fn from(value: i32) -> Self { + Self::Int(value.into()) + } +} +impl From for Value { + fn from(value: u32) -> Self { + Self::Int(value.into()) + } +} +impl From for Value { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} +impl From> for Value { + fn from(value: Option<&str>) -> Self { + value.map(Self::from).unwrap_or(Self::Null) + } +} +impl From> for Value { + fn from(value: Option) -> Self { + value.map(Self::from).unwrap_or(Self::Null) + } +} +impl ToSql for Value { + fn to_sql( + &self, + ty: &Type, + out: &mut BytesMut, + ) -> std::result::Result> { + match self { + Self::Null => Ok(IsNull::Yes), + Self::Bool(value) if *ty == Type::BOOL => value.to_sql(ty, out), + Self::Int(value) => match *ty { + Type::INT2 => i16::try_from(*value) + .map_err(|_| "database_integer_range")? + .to_sql(ty, out), + Type::INT4 => i32::try_from(*value) + .map_err(|_| "database_integer_range")? + .to_sql(ty, out), + Type::INT8 => value.to_sql(ty, out), + _ => Err("database_parameter_type".into()), + }, + Self::Text(value) if ::accepts(ty) => value.to_sql(ty, out), + Self::Bytes(value) if *ty == Type::BYTEA => value.to_sql(ty, out), + Self::TextArray(value) if as ToSql>::accepts(ty) => value.to_sql(ty, out), + _ => Err("database_parameter_type".into()), + } + } + fn accepts(ty: &Type) -> bool { + matches!( + *ty, + Type::BOOL | Type::INT2 | Type::INT4 | Type::INT8 | Type::BYTEA + ) || ::accepts(ty) + || as ToSql>::accepts(ty) + } + to_sql_checked!(); +} +#[derive(Clone)] +pub struct Row(pub(crate) Vec); +impl Row { + pub fn len(&self) -> usize { + self.0.len() + } + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub fn value(&self, index: usize) -> Result<&Value> { + self.0.get(index).ok_or(Error("storage_column_missing")) + } + pub fn text(&self, index: usize) -> Result<&str> { + match self.value(index)? { + Value::Text(v) => Ok(v), + _ => Err(Error("storage_column_type")), + } + } + pub fn optional_text(&self, index: usize) -> Result> { + match self.value(index)? { + Value::Null => Ok(None), + Value::Text(v) => Ok(Some(v)), + _ => Err(Error("storage_column_type")), + } + } + pub fn integer(&self, index: usize) -> Result { + match self.value(index)? { + Value::Int(v) => Ok(*v), + _ => Err(Error("storage_column_type")), + } + } + pub fn boolean(&self, index: usize) -> Result { + match self.value(index)? { + Value::Bool(v) => Ok(*v), + _ => Err(Error("storage_column_type")), + } + } + pub fn json(&self, index: usize) -> Result { + let s = self.text(index)?; + if s.len() > MAX_BYTES { + return Err(Error("storage_record_too_large")); + } + Json::parse(s) + } + pub(crate) fn decode(raw: postgres::Row) -> Result { + let mut values = Vec::with_capacity(raw.len()); + for (i, column) in raw.columns().iter().enumerate() { + let value = match *column.type_() { + Type::BOOL => raw + .try_get::<_, Option>(i) + .map(|v| v.map(Value::Bool).unwrap_or(Value::Null)), + Type::INT2 => raw + .try_get::<_, Option>(i) + .map(|v| v.map(|v| Value::Int(v.into())).unwrap_or(Value::Null)), + Type::INT4 => raw + .try_get::<_, Option>(i) + .map(|v| v.map(|v| Value::Int(v.into())).unwrap_or(Value::Null)), + Type::INT8 => raw + .try_get::<_, Option>(i) + .map(|v| v.map(Value::Int).unwrap_or(Value::Null)), + Type::BYTEA => raw + .try_get::<_, Option>>(i) + .map(|v| v.map(Value::Bytes).unwrap_or(Value::Null)), + Type::TEXT_ARRAY | Type::VARCHAR_ARRAY => raw + .try_get::<_, Option>>(i) + .map(|v| v.map(Value::TextArray).unwrap_or(Value::Null)), + _ => raw + .try_get::<_, Option>(i) + .map(|v| v.map(Value::Text).unwrap_or(Value::Null)), + } + .map_err(|_| Error("storage_column_type"))?; + values.push(value); + } + Ok(Self(values)) + } + pub(crate) fn bytes(&self) -> usize { + self.0 + .iter() + .map(|v| match v { + Value::Text(v) => v.len(), + Value::Bytes(v) => v.len(), + Value::TextArray(v) => v.iter().map(String::len).sum(), + _ => 8, + }) + .sum() + } +} diff --git a/rust/crates/msg-storage/src/postgres/records.rs b/rust/crates/msg-storage/src/postgres/records.rs new file mode 100644 index 00000000..ac7a0f88 --- /dev/null +++ b/rust/crates/msg-storage/src/postgres/records.rs @@ -0,0 +1,645 @@ +//! Typed durable records. These methods do not grant business authority. +use super::{PgSession, Value}; +use crate::records::*; +use msg_core::{Error, Json, Result}; +use msg_identity::{ + models::{Certificate, CertificateRequest, Credential, ResourceRef, Timestamp}, + store::{AuthorityStore, Record}, +}; +use serde::Serialize; +use std::collections::BTreeSet; +fn time(value: Timestamp) -> String { + value.0.format("%Y-%m-%dT%H:%M:%S%.6fZ").to_string() +} +fn raw(value: &impl Serialize) -> Result { + Ok(encode(value)?.canonical()?.into()) +} +fn require(yes: bool, code: &'static str) -> Result<()> { + if yes { + Ok(()) + } else { + Err(Error(code)) + } +} +fn is_hex(id: &str) -> bool { + id.len() == 32 + && id + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +impl PgSession { + fn mutate(&self, body: impl FnOnce() -> Result) -> Result { + self.require_write()?; + self.capture(body(), true) + } + pub fn resource_record(&self, id: &str) -> Result { + if is_hex(id) { + let rows=self.rows("SELECT substring(body,1,1048577) FROM resources WHERE CASE WHEN id ~ '^[0-9a-f]{32}$' THEN id WHEN id ~ '^[A-Za-z][A-Za-z0-9_]*_([0-9a-f]{32})$' THEN right(id,32) ELSE left(encode(sha256(convert_to('msg.hex-reference/v1','UTF8')||decode('00','hex')||convert_to(id,'UTF8')),'hex'),32) END=$1 LIMIT 2",&[id.into()])?; + require(rows.len() <= 1, "ambiguous_resource_id")?; + return decode(&rows.first().ok_or(Error("not_found"))?.json(0)?); + } + let resource: Resource = decode(&self.record(Record::Resource, id)?)?; + require(resource.id == id, "storage_record_mismatch")?; + Ok(resource) + } + pub fn resolve(&self, path: &str, migrated: bool) -> Result { + require( + path.starts_with('/') && !path.contains(['\\', '\0']), + "invalid_path", + )?; + let parts: Vec<_> = path.trim_end_matches('/').split('/').skip(1).collect(); + require( + parts.len() <= 256 + && parts + .iter() + .all(|p| !p.is_empty() && !matches!(*p, "." | "..")), + "invalid_path", + )?; + if !migrated { + if parts.len() == 2 && parts[0] == "_id" { + return Ok(self.resource_record(parts[1])?.id); + } + if let Some(last) = parts.last().and_then(|p| p.strip_prefix('*')) { + let resource = self.resource_record(last)?; + if parts.len() > 1 { + let parent = + self.resolve(&format!("/{}", parts[..parts.len() - 1].join("/")), false)?; + require(resource.parent.as_deref() == Some(&parent), "not_found")?; + } + return Ok(resource.id); + } + } + let mut id = self + .one("SELECT id FROM resources WHERE parent IS NULL", &[])? + .ok_or(Error("not_initialized"))? + .text(0)? + .to_owned(); + if path == "/" { + return Ok(id); + } + require(!parts.is_empty(), "invalid_path")?; + for part in parts { + let mut found = self.one( + "SELECT id FROM resources WHERE parent=$1 AND name=$2", + &[id.clone().into(), part.into()], + )?; + if found.is_none() && migrated { + found = self.one( + "SELECT resource_id FROM resource_path_aliases WHERE parent_id=$1 AND name=$2", + &[id.clone().into(), part.into()], + )?; + } + id = found.ok_or(Error("not_found"))?.text(0)?.to_owned(); + } + Ok(self.resource_record(&id)?.id) + } + pub fn resource_path(&self, id: &str) -> Result { + let mut current = self.resource_record(id)?; + let mut names = Vec::new(); + let mut seen = BTreeSet::new(); + while let Some(parent) = ¤t.parent { + require(seen.insert(current.id.clone()), "parent_cycle")?; + require(seen.len() <= 1024, "authority_work_limit")?; + names.push(current.name.clone()); + current = self.resource_record(parent)?; + } + names.reverse(); + Ok(format!("/{}", names.join("/"))) + } + pub fn children( + &self, + parent: &str, + cursor: Option<&str>, + limit: u32, + ) -> Result> { + require((1..=500).contains(&limit), "invalid_limit")?; + let rows=self.rows("SELECT substring(body,1,1048577) FROM resources WHERE parent=$1 AND id>$2 ORDER BY id LIMIT $3",&[parent.into(),cursor.unwrap_or("").into(),(limit+1).into()])?; + let items: Vec = rows + .iter() + .take(limit as usize) + .map(|r| decode(&r.json(0)?)) + .collect::>()?; + let next_cursor = if rows.len() > limit as usize { + items.last().map(|v| v.id.clone()) + } else { + None + }; + Ok(Page { items, next_cursor }) + } + pub fn revision(&self, reference: &ResourceRef) -> Result { + let resource = self.resource_record(&reference.id)?; + let id = reference + .revision + .as_ref() + .or(resource.revision.as_ref()) + .ok_or(Error("revision_not_found"))?; + let rows = if is_hex(id) { + self.rows("SELECT substring(body,1,1048577) FROM revisions WHERE resource_id=$1 AND CASE WHEN id ~ '^[0-9a-f]{32}$' THEN id WHEN id ~ '^[A-Za-z][A-Za-z0-9_]*_([0-9a-f]{32})$' THEN right(id,32) ELSE left(encode(sha256(convert_to('msg.hex-reference/v1','UTF8')||decode('00','hex')||convert_to(id,'UTF8')),'hex'),32) END=$2 LIMIT 2",&[resource.id.clone().into(),id.as_str().into()])? + } else { + self.rows( + "SELECT substring(body,1,1048577) FROM revisions WHERE resource_id=$1 AND id=$2", + &[resource.id.clone().into(), id.as_str().into()], + )? + }; + require(rows.len() <= 1, "ambiguous_revision_id")?; + let revision: Revision = + decode(&rows.first().ok_or(Error("revision_not_found"))?.json(0)?)?; + require( + revision.resource_id == resource.id && (is_hex(id) || revision.id == *id), + "storage_record_mismatch", + )?; + Ok(revision) + } + pub fn history(&self, id: &str, cursor: Option<&str>, limit: u32) -> Result> { + require((1..=500).contains(&limit), "invalid_limit")?; + let rows=self.rows("SELECT substring(body,1,1048577) FROM revisions WHERE resource_id=$1 AND id>$2 ORDER BY id LIMIT $3",&[id.into(),cursor.unwrap_or("").into(),(limit+1).into()])?; + let items: Vec = rows + .iter() + .take(limit as usize) + .map(|r| decode(&r.json(0)?)) + .collect::>()?; + let next_cursor = if rows.len() > limit as usize { + items.last().map(|v| v.id.clone()) + } else { + None + }; + Ok(Page { items, next_cursor }) + } + pub fn set_setting(&self, key: &str, value: &Json) -> Result<()> { + self.mutate(||{ + self.execute("INSERT INTO settings(key,value) VALUES($1,$2) ON CONFLICT(key) DO UPDATE SET value=excluded.value",&[key.into(),value.canonical()?.into()])?;Ok(()) + }) + } + pub fn bump_authorization_epoch(&self) -> Result<()> { + self.mutate(|| { + let old = self + .setting("authorization_epoch")? + .map(|v| { + v.as_integer()? + .parse::() + .map_err(|_| Error("invalid_authorization_epoch")) + }) + .transpose()? + .unwrap_or(0); + let next = old + .checked_add(1) + .filter(|_| old >= 0) + .ok_or(Error("invalid_authorization_epoch"))?; + self.set_setting("authorization_epoch", &Json::parse(&next.to_string())?) + }) + } + pub fn insert_resource(&self, r: &Resource) -> Result<()> { + self.mutate(||{ + r.validate()?;if let Some(parent)=&r.parent{self.resource_record(parent)?;} + self.execute("INSERT INTO resources(id,type,name,parent,owner,grp,mode,generation,revision,state,created_at,modified_at,body) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)",&[ + r.id.as_str().into(),r.resource_type.as_str().into(),r.name.as_str().into(),r.parent.as_deref().into(),r.owner.as_str().into(),r.group.as_str().into(),(r.mode as i64).into(),r.generation.into(),r.revision.as_deref().into(),r.state.sql().into(),time(r.created_at).into(),time(r.modified_at).into(),raw(r)?])?; + for tag in &r.tags{self.execute("INSERT INTO resource_tags(resource_id,tag) VALUES($1,$2)",&[r.id.as_str().into(),tag.as_str().into()])?;}Ok(()) + }) + } + pub fn replace_resource(&self, r: &Resource, expected: i64) -> Result<()> { + self.mutate(||{ + r.validate()?;let old=self.resource_record(&r.id)?; + require(old.generation==expected,"generation_conflict")?;require(expected.checked_add(1)==Some(r.generation),"invalid_generation")?; + require(old.resource_type==r.resource_type&&old.created_at==r.created_at&&old.created_by==r.created_by,"immutable_creation_fact")?; + if let Some(parent)=&r.parent{require(parent!=&r.id&&!self.ancestors(parent)?.iter().any(|a|a.id==r.id),"parent_cycle")?;} + if old.parent!=r.parent||old.owner!=r.owner||old.group!=r.group||old.mode!=r.mode||old.state!=r.state{self.bump_authorization_epoch()?;} + if let Some(parent)=&old.parent{if old.parent!=r.parent||old.name!=r.name{self.execute("INSERT INTO resource_path_aliases(parent_id,name,resource_id) VALUES($1,$2,$3) ON CONFLICT(parent_id,name) DO UPDATE SET resource_id=excluded.resource_id",&[parent.as_str().into(),old.name.as_str().into(),old.id.as_str().into()])?;}} + let changed=self.execute("UPDATE resources SET name=$1,parent=$2,owner=$3,grp=$4,mode=$5,generation=$6,revision=$7,state=$8,modified_at=$9,body=$10 WHERE id=$11 AND generation=$12",&[ + r.name.as_str().into(),r.parent.as_deref().into(),r.owner.as_str().into(),r.group.as_str().into(),(r.mode as i64).into(),r.generation.into(),r.revision.as_deref().into(),r.state.sql().into(),time(r.modified_at).into(),raw(r)?,r.id.as_str().into(),expected.into()])?; + require(changed==1,"generation_conflict")?; + if old.tags!=r.tags{self.execute("DELETE FROM resource_tags WHERE resource_id=$1",&[r.id.as_str().into()])?;for tag in &r.tags{self.execute("INSERT INTO resource_tags(resource_id,tag) VALUES($1,$2)",&[r.id.as_str().into(),tag.as_str().into()])?;}}Ok(()) + }) + } + pub fn append_revision(&self, r: &Revision) -> Result<()> { + self.mutate(||{ + require(r.format_version>=1,"invalid_version")?;require(r.content.size>=0,"invalid_nonnegative_integer")?; + for parent in &r.parents{self.revision(&ResourceRef{id:r.resource_id.clone(),revision:Some(parent.clone())})?;} + self.execute("INSERT INTO revisions(id,resource_id,created_at,body) VALUES($1,$2,$3,$4)",&[r.id.as_str().into(),r.resource_id.as_str().into(),time(r.created_at).into(),raw(r)?])?; + for rel in &r.relations{ + require(!rel.excerpt.is_some_and(|[a,b]|a<0||a>b),"invalid_byte_range")?; + self.execute("INSERT INTO relations(revision_id,source_id,type,target_id,target_revision,body) VALUES($1,$2,$3,$4,$5,$6)",&[r.id.as_str().into(),r.resource_id.as_str().into(),rel.relation_type.as_str().into(),rel.target.id.as_str().into(),rel.target.revision.as_deref().into(),raw(rel)?])?; + }Ok(()) + }) + } + pub fn save_result( + &self, + subject: Option<&str>, + digest: &str, + result: &OperationResult, + ) -> Result<()> { + self.mutate(|| { + require( + subject == result.subject.as_deref() && subject != Some(""), + "storage_record_mismatch", + )?; + require( + digest.strip_prefix("sha256:").is_some_and(|s| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }), + "invalid_digest", + )?; + require( + self.one("SELECT count(*) FROM results", &[])? + .ok_or(Error("storage_error"))? + .integer(0)? + < 100_000, + "storage_capacity_exceeded", + )?; + self.execute( + "INSERT INTO results(subject,request_id,digest,body) VALUES($1,$2,$3,$4)", + &[ + subject.unwrap_or("").into(), + result.request_id.as_str().into(), + digest.into(), + raw(result)?, + ], + )?; + Ok(()) + }) + } + pub fn append_event(&self, event: &Event) -> Result<()> { + self.mutate(|| { + self.execute( + "INSERT INTO events(id,body) VALUES($1,$2)", + &[event.id.as_str().into(), raw(event)?], + )?; + Ok(()) + }) + } + pub fn append_audit(&self, event: &AuditEvent) -> Result<()> { + self.mutate(|| { + let previous = self + .one("SELECT digest FROM audit ORDER BY seq DESC LIMIT 1", &[])? + .map(|r| Ok(r.text(0)?.to_owned())) + .transpose()?; + let mut event = event.clone(); + event.previous_digest = previous; + event.entry_digest.clear(); + let mut fields = encode(&event)?.into_object()?; + fields.remove("entry_digest"); + event.entry_digest = + msg_core::digest(Json::from_object(fields).canonical()?.as_bytes()); + self.execute( + "INSERT INTO audit(digest,previous,body) VALUES($1,$2,$3)", + &[ + event.entry_digest.as_str().into(), + event.previous_digest.as_deref().into(), + raw(&event)?, + ], + )?; + Ok(()) + }) + } + pub fn update_identity(&self, record: IdentityRecord<'_>, expected: i64) -> Result<()> { + self.mutate(|| { + let (select, insert, update, mut keys, generation, body, kind) = match record { + IdentityRecord::Subject(s) => ( + "SELECT generation FROM identities WHERE id=$1", + "INSERT INTO identities(id,kind,generation,body) VALUES($1,$2,$3,$4)", + "UPDATE identities SET generation=$1,body=$2 WHERE id=$3", + vec![s.resource_id.as_str().into()], + i64::try_from(s.auth_version).map_err(|_| Error("invalid_generation"))?, + raw(s)?, + Some("subject"), + ), + IdentityRecord::Organization(g) => { + require( + g.membership_version >= 0 + && matches!( + g.membership_policy.as_str(), + "open" | "approval" | "invite" | "managed" + ) + && g.builtin + .as_ref() + .is_none_or(|s| matches!(s.as_str(), "public" | "admins")), + "invalid_identity_record", + )?; + ( + "SELECT generation FROM identities WHERE id=$1", + "INSERT INTO identities(id,kind,generation,body) VALUES($1,$2,$3,$4)", + "UPDATE identities SET generation=$1,body=$2 WHERE id=$3", + vec![g.resource_id.as_str().into()], + g.membership_version, + raw(g)?, + Some("organization"), + ) + } + IdentityRecord::Membership(m) => { + require( + m.version >= 0 + && matches!( + m.role.as_str(), + "owner" | "maintainer" | "member" | "admin" + ) + && matches!( + m.status.as_str(), + "active" | "pending" | "invited" | "rejected" + ), + "invalid_identity_record", + )?; + self.bump_authorization_epoch()?; + ( + "SELECT generation FROM memberships WHERE org=$1 AND subject=$2", + "INSERT INTO memberships(org,subject,generation,body) VALUES($1,$2,$3,$4)", + "UPDATE memberships SET generation=$1,body=$2 WHERE org=$3 AND subject=$4", + vec![ + m.organization_id.as_str().into(), + m.subject_id.as_str().into(), + ], + m.version, + raw(m)?, + None, + ) + } + IdentityRecord::Email(e) => ( + "SELECT generation FROM emails WHERE subject=$1", + "INSERT INTO emails(subject,generation,body) VALUES($1,$2,$3)", + "UPDATE emails SET generation=$1,body=$2 WHERE subject=$3", + vec![e.subject_id.as_str().into()], + expected.checked_add(1).ok_or(Error("invalid_generation"))?, + raw(e)?, + None, + ), + }; + let old = self.one(select, &keys)?; + require( + match &old { + None => expected == -1, + Some(row) => row.integer(0)? == expected, + }, + "generation_conflict", + )?; + if old.is_none() { + if let Some(kind) = kind { + keys.push(kind.into()); + } + keys.push(generation.into()); + keys.push(body); + self.execute(insert, &keys)?; + } else { + let mut params = vec![generation.into(), body]; + params.extend(keys); + self.execute(update, ¶ms)?; + } + Ok(()) + }) + } + pub fn save_credential(&self, c: &Credential, expected_auth_version: u64) -> Result<()> { + self.mutate(||{ + require(self.subject(&c.subject_id)?.auth_version==expected_auth_version,"auth_version_conflict")?; + if let Some(row)=self.one("SELECT substring(body,1,1048577) FROM credentials WHERE id=$1",&[c.id.as_str().into()])?{ + let old:Credential=msg_identity::models::decode(&row.json(0)?)?; + require(old.subject_id==c.subject_id&&old.kind==c.kind&&old.verifier.0==c.verifier.0&&old.source_credential_id==c.source_credential_id,"credential_identity_immutable")?; + } + self.execute("INSERT INTO credentials(id,subject,body) VALUES($1,$2,$3) ON CONFLICT(id) DO UPDATE SET body=excluded.body",&[c.id.as_str().into(),c.subject_id.as_str().into(),raw(c)?])?;Ok(()) + }) + } + pub fn csr(&self, id: &str) -> Result { + msg_identity::models::decode( + &self + .one( + "SELECT substring(body,1,1048577) FROM csrs WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("csr_not_found"))? + .json(0)?, + ) + } + pub fn csr_state(&self, id: &str) -> Result { + decode( + &self + .one( + "SELECT substring(state_body,1,1048577) FROM csrs WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("csr_not_found"))? + .json(0)?, + ) + } + pub fn save_csr(&self, request: &CertificateRequest) -> Result<()> { + self.mutate(||{ + let state=CertificateRequestState{request_id:request.resource_id.clone(),status:"pending".into(),generation:0,certificate_id:None}; + self.execute("INSERT INTO csrs(id,generation,state,body,state_body) VALUES($1,0,'pending',$2,$3)",&[request.resource_id.as_str().into(),raw(request)?,raw(&state)?])?;Ok(()) + }) + } + pub fn transition_csr(&self, state: &CertificateRequestState, expected: i64) -> Result<()> { + self.mutate(||{ + require(matches!(state.status.as_str(),"pending"|"issued"|"rejected"|"cancelled"|"expired"),"invalid_csr_state")?; + require(expected.checked_add(1)==Some(state.generation),"invalid_generation")?; + let old=self.csr_state(&state.request_id)?;require(old.status=="pending","csr_not_pending")?;require(old.generation==expected,"generation_conflict")?; + require(self.execute("UPDATE csrs SET generation=$1,state=$2,state_body=$3 WHERE id=$4 AND generation=$5",&[state.generation.into(),state.status.as_str().into(),raw(state)?,state.request_id.as_str().into(),expected.into()])?==1,"generation_conflict") + }) + } + pub fn register_certificate(&self, cert: &Certificate, csr: Option<(&str, i64)>) -> Result<()> { + self.mutate(|| { + self.execute( + "INSERT INTO certificates(id,subject,parent,revoked,body) VALUES($1,$2,$3,0,$4)", + &[ + cert.resource_id.as_str().into(), + cert.subject_id.as_str().into(), + cert.parent_certificate_id.as_deref().into(), + raw(cert)?, + ], + )?; + if let Some((id, generation)) = csr { + self.set_setting( + &format!("certificate_request:{}", cert.resource_id), + &Json::string(id), + )?; + self.transition_csr( + &CertificateRequestState { + request_id: id.into(), + status: "issued".into(), + generation: generation + .checked_add(1) + .ok_or(Error("invalid_generation"))?, + certificate_id: Some(cert.resource_id.clone()), + }, + generation, + )?; + } + Ok(()) + }) + } + pub fn revoke_certificate(&self, id: &str, event: &AuditEvent) -> Result<()> { + self.mutate(|| { + self.certificate(id)?; + self.execute( + "UPDATE certificates SET revoked=1 WHERE id=$1", + &[id.into()], + )?; + self.bump_authorization_epoch()?; + self.append_audit(event) + }) + } + pub fn transfer(&self, id: &str) -> Result { + decode( + &self + .one( + "SELECT substring(body,1,1048577) FROM transfers WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("transfer_not_found"))? + .json(0)?, + ) + } + pub fn save_transfer(&self, t: &TransferSession, expected: Option) -> Result<()> { + self.mutate(|| { + require( + matches!(t.direction.as_str(), "upload" | "download") + && matches!( + t.state.as_str(), + "open" | "sealed" | "cancelled" | "expired" + ) + && t.generation >= 0 + && t.expected_size.is_none_or(|n| n >= 0), + "invalid_transfer", + )?; + if let Some(expected) = expected { + require( + expected.checked_add(1) == Some(t.generation), + "invalid_generation", + )?; + require( + self.execute( + "UPDATE transfers SET generation=$1,body=$2 WHERE id=$3 AND generation=$4", + &[ + t.generation.into(), + raw(t)?, + t.id.as_str().into(), + expected.into(), + ], + )? == 1, + "generation_conflict", + )?; + } else { + self.execute( + "INSERT INTO transfers(id,subject,generation,body) VALUES($1,$2,$3,$4)", + &[ + t.id.as_str().into(), + t.subject_id.as_str().into(), + t.generation.into(), + raw(t)?, + ], + )?; + } + Ok(()) + }) + } + pub fn put_chunk(&self, c: &TransferChunk) -> Result<()> { + self.mutate(||{ + require(c.offset>=0&&c.content.size>=0,"invalid_nonnegative_integer")?; + let end=c.offset.checked_add(c.content.size).ok_or(Error("invalid_byte_range"))?; + let rows=self.rows("SELECT \"offset\",length,substring(body,1,1048577) FROM chunks WHERE transfer_id=$1 AND \"offset\"<$2 AND \"offset\"+length>$3",&[c.transfer_id.as_str().into(),end.into(),c.offset.into()])?; + if !rows.is_empty(){require(rows.len()==1&&rows[0].integer(0)?==c.offset&&rows[0].integer(1)?==c.content.size,"chunk_conflict")?; + let old:TransferChunk=decode(&rows[0].json(2)?)?;require(encode(&old.content)?.canonical()?==encode(&c.content)?.canonical()?,"chunk_conflict")?;return Ok(());} + self.execute("INSERT INTO chunks(transfer_id,\"offset\",length,body) VALUES($1,$2,$3,$4)",&[c.transfer_id.as_str().into(),c.offset.into(),c.content.size.into(),raw(c)?])?;Ok(()) + }) + } + pub fn missing_ranges( + &self, + id: &str, + cursor: Option<&str>, + limit: u32, + ) -> Result> { + let size = self + .transfer(id)? + .expected_size + .ok_or(Error("size_required"))?; + require((1..=500).contains(&limit), "invalid_limit")?; + let mut start = cursor + .unwrap_or("0") + .parse::() + .map_err(|_| Error("invalid_cursor"))?; + require(start >= 0 && start <= size, "invalid_cursor")?; + let rows=self.rows("SELECT \"offset\",length FROM chunks WHERE transfer_id=$1 AND \"offset\"+length>$2 ORDER BY \"offset\" LIMIT 4097",&[id.into(),start.into()])?; + let mut gaps = Vec::new(); + for row in rows { + let offset = row.integer(0)?; + let length = row.integer(1)?; + require(offset >= 0 && length >= 0, "invalid_byte_range")?; + if offset > start { + gaps.push([start, offset]); + if gaps.len() > limit as usize { + break; + } + } + start = start.max( + offset + .checked_add(length) + .ok_or(Error("invalid_byte_range"))?, + ); + } + if gaps.len() <= limit as usize && start < size { + gaps.push([start, size]); + } + let next_cursor = gaps.get(limit as usize).map(|v| v[0].to_string()); + gaps.truncate(limit as usize); + Ok(Page { + items: gaps, + next_cursor, + }) + } + pub fn job(&self, id: &str) -> Result { + let row = self + .one( + "SELECT state,next_at,substring(body,1,1048577) FROM jobs WHERE id=$1", + &[id.into()], + )? + .ok_or(Error("job_not_found"))?; + let job: EffectJob = decode(&row.json(2)?)?; + require( + job.id == id + && job.state == row.text(0)? + && time(job.next_attempt_at) == row.text(1)?, + "storage_record_mismatch", + )?; + Ok(job) + } + pub fn enqueue(&self, job: &EffectJob) -> Result<()> { + self.mutate(|| { + require( + job.attempts == 0 && job.state == "pending" && job.lease_until.is_none(), + "invalid_job_state", + )?; + self.execute( + "INSERT INTO jobs(id,dedupe,kind,state,next_at,body) VALUES($1,$2,$3,$4,$5,$6)", + &[ + job.id.as_str().into(), + job.dedupe_key.as_str().into(), + job.kind.as_str().into(), + job.state.as_str().into(), + time(job.next_attempt_at).into(), + raw(job)?, + ], + )?; + Ok(()) + }) + } + pub fn save_job(&self, job: &EffectJob) -> Result<()> { + self.mutate(|| { + require( + job.attempts >= 0 + && matches!( + job.state.as_str(), + "pending" | "running" | "done" | "failed" | "uncertain" + ), + "invalid_job_state", + )?; + require( + self.execute( + "UPDATE jobs SET state=$1,next_at=$2,body=$3 WHERE id=$4", + &[ + job.state.as_str().into(), + time(job.next_attempt_at).into(), + raw(job)?, + job.id.as_str().into(), + ], + )? == 1, + "job_not_found", + ) + }) + } +} diff --git a/rust/crates/msg-storage/src/records.rs b/rust/crates/msg-storage/src/records.rs index d33c9361..cb38d897 100644 --- a/rust/crates/msg-storage/src/records.rs +++ b/rust/crates/msg-storage/src/records.rs @@ -231,3 +231,191 @@ pub struct AuditEvent { pub entry_digest: String, pub result: String, } + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Organization { + pub resource_id: String, + pub membership_version: i64, + #[serde(default)] + pub builtin: Option, + #[serde(default = "invite")] + pub membership_policy: String, +} +fn invite() -> String { + "invite".into() +} +fn active() -> String { + "active".into() +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Membership { + pub organization_id: String, + pub subject_id: String, + pub role: String, + pub version: i64, + #[serde(default = "active")] + pub status: String, + #[serde(default)] + pub joined_at: Option, + #[serde(default)] + pub invited_by: Option, +} +impl Organization { + pub fn validate(&self) -> Result<()> { + if self.membership_version < 0 + || !matches!( + self.membership_policy.as_str(), + "open" | "approval" | "invite" | "managed" + ) + || self + .builtin + .as_ref() + .is_some_and(|s| !matches!(s.as_str(), "public" | "admins")) + { + return Err(Error("invalid_identity_record")); + } + Ok(()) + } +} +impl Membership { + pub fn validate(&self) -> Result<()> { + if self.version < 0 + || !matches!( + self.role.as_str(), + "owner" | "maintainer" | "member" | "admin" + ) + || !matches!( + self.status.as_str(), + "active" | "pending" | "invited" | "rejected" + ) + { + return Err(Error("invalid_identity_record")); + } + Ok(()) + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EmailSettings { + pub subject_id: String, + #[serde(deserialize_with = "required_option")] + pub address: Option, + #[serde(deserialize_with = "required_option")] + pub verified_at: Option, + #[serde(default)] + pub enabled_events: std::collections::BTreeSet, +} +pub enum IdentityRecord<'a> { + Subject(&'a msg_identity::models::Subject), + Organization(&'a Organization), + Membership(&'a Membership), + Email(&'a EmailSettings), +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CertificateRequestState { + pub request_id: String, + pub status: String, + pub generation: i64, + #[serde(default)] + pub certificate_id: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TransferSession { + pub id: String, + pub subject_id: String, + pub direction: String, + pub state: String, + #[serde(deserialize_with = "required_option")] + pub target: Option, + #[serde(deserialize_with = "required_option")] + pub expected_size: Option, + #[serde(deserialize_with = "required_option")] + pub expected_digest: Option, + pub expires_at: Timestamp, + pub generation: i64, + #[serde(default)] + pub output: Option, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TransferChunk { + pub transfer_id: String, + pub offset: i64, + pub content: BlobRef, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct EffectJob { + pub id: String, + pub event_id: String, + pub kind: String, + pub dedupe_key: String, + // A stored snapshot is not an authentication result. Workers must obtain a + // freshly validated Principal through the stored-job authentication port. + principal: Json, + pub operation: String, + pub arguments: BTreeMap, + pub state: String, + pub attempts: i64, + pub next_attempt_at: Timestamp, + #[serde(deserialize_with = "required_option")] + pub lease_until: Option, + #[serde(default)] + pub result: Option, +} +/// Trusted effect description. The identity is supplied separately after authentication. +pub struct JobDraft { + pub id: String, + pub event_id: String, + pub kind: String, + pub dedupe_key: String, + pub operation: String, + pub arguments: BTreeMap, +} +impl EffectJob { + pub fn new( + principal: &msg_identity::authentication::Principal, + draft: JobDraft, + now: Timestamp, + ) -> Result { + let JobDraft { + id, + event_id, + kind, + dedupe_key, + operation, + arguments, + } = draft; + Ok(Self { + id, + event_id, + kind, + dedupe_key, + principal: encode(principal)?, + operation, + arguments, + state: "pending".into(), + attempts: 0, + next_attempt_at: now, + lease_until: None, + result: None, + }) + } + pub fn subject(&self) -> Result<&str> { + self.principal + .as_object()? + .get("subject") + .ok_or(Error("worker_identity_required"))? + .as_str() + } +} +#[derive(Clone, Serialize)] +pub struct Page { + pub items: Vec, + pub next_cursor: Option, +} diff --git a/rust/tests/postgres_fixture.py b/rust/tests/postgres_fixture.py new file mode 100644 index 00000000..f60bd8f7 --- /dev/null +++ b/rust/tests/postgres_fixture.py @@ -0,0 +1,86 @@ +"""A disposable, Python-created PostgreSQL database for native conformance tests. + +Requires explicit MSG_PG_TEST_ADMIN_DSN. Only a freshly created, unpredictable +msg_rust_test_* database is populated and removed. Never truncates an existing DB. +""" + +from __future__ import annotations + +import os +import uuid +from contextlib import contextmanager + +import psycopg +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict, make_conninfo + +from msg.storage.postgres import PostgresMetadataStore, _postgres_sql + + +class Connection: + def __init__(self, dsn): + self.connection = psycopg.connect(dsn, autocommit=True) + + def execute(self, statement, parameters=()): + return self.connection.execute( + _postgres_sql(statement, has_parameters=bool(parameters)), parameters or None + ) + + def close(self): + self.connection.close() + + +@contextmanager +def database(): + admin = os.environ.get('MSG_PG_TEST_ADMIN_DSN') + if not admin: + raise RuntimeError('MSG_PG_TEST_ADMIN_DSN is required for isolated PostgreSQL tests') + config = conninfo_to_dict(admin) + host = config.get('host', '') + if host and not (host.startswith('/') or host in {'127.0.0.1', 'localhost', '::1'}): + raise RuntimeError('PostgreSQL conformance only connects to a local test instance') + name = 'msg_rust_test_' + uuid.uuid4().hex + with psycopg.connect(admin, autocommit=True) as connection: + connection.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name))) + dsn = make_conninfo(admin, dbname=name) + old = os.environ.get('MSG_PARITY_POSTGRES_DSN') + try: + os.environ['MSG_PARITY_POSTGRES_DSN'] = dsn + yield dsn + finally: + if old is None: + os.environ.pop('MSG_PARITY_POSTGRES_DSN', None) + else: + os.environ['MSG_PARITY_POSTGRES_DSN'] = old + with psycopg.connect(admin, autocommit=True) as connection: + connection.execute(sql.SQL('DROP DATABASE {}').format(sql.Identifier(name))) + + +async def replace_fixture_storage(fixture, dsn): + """Copy only deterministic public test records, then use real PG for both sides.""" + metadata = PostgresMetadataStore(dsn) + with psycopg.connect(dsn) as connection: + for table in ( + 'resources', + 'identities', + 'credentials', + 'certificates', + 'settings', + 'memberships', + 'dm_conversations', + ): + records = fixture.conn.execute('SELECT * FROM ' + table) + columns = [entry[0] for entry in records.description] + query = sql.SQL('INSERT INTO {} ({}) VALUES ({})').format( + sql.Identifier(table), + sql.SQL(',').join(map(sql.Identifier, columns)), + sql.SQL(',').join(sql.Placeholder() for _ in columns), + ) + if table == 'settings': + query += sql.SQL(' ON CONFLICT(key) DO UPDATE SET value=excluded.value') + for row in records: + connection.execute(query, row) + fixture.conn.close() + await fixture.metadata.close() + fixture.conn = Connection(dsn) + fixture.metadata = metadata diff --git a/rust/tests/postgres_transport_fixture.py b/rust/tests/postgres_transport_fixture.py new file mode 100644 index 00000000..d4620f84 --- /dev/null +++ b/rust/tests/postgres_transport_fixture.py @@ -0,0 +1,135 @@ +"""One local-test-only PostgreSQL proxy that loses a committed acknowledgement. + +The target must be an explicitly supplied loopback or Unix socket test instance. +It never sends authentication data anywhere other than that local test server. +""" + +from __future__ import annotations + +import socket +import threading +from contextlib import AbstractContextManager + +from psycopg.conninfo import conninfo_to_dict, make_conninfo + + +def exact(sock, n): + data = bytearray() + while len(data) < n: + block = sock.recv(n - len(data)) + if not block: + raise EOFError + data.extend(block) + return bytes(data) + + +class LoseCommitAck(AbstractContextManager): + def __init__(self, dsn): + self.config = conninfo_to_dict(dsn) + if not self.config.get('dbname', '').startswith('msg_rust_test_'): + raise ValueError('disposable database required') + host = self.config.get('host', '') + if not (host.startswith('/') or host in {'localhost', '127.0.0.1', '::1'}): + raise ValueError('local test host required') + self.listener = socket.socket() + self.listener.bind(('127.0.0.1', 0)) + self.listener.listen(8) + self.listener.settimeout(0.2) + self.dsn = make_conninfo( + dsn, host='127.0.0.1', port=self.listener.getsockname()[1], sslmode='disable' + ) + self.closed = threading.Event() + self.dropped = threading.Event() + self.sockets = [] + self.threads = [] + + def __enter__(self): + thread = threading.Thread(target=self.accept, daemon=True) + self.threads.append(thread) + thread.start() + return self + + def accept(self): + while not self.closed.is_set(): + try: + client, _ = self.listener.accept() + except TimeoutError: + continue + except OSError: + return + self.sockets.append(client) + thread = threading.Thread(target=self.connection, args=(client,), daemon=True) + self.threads.append(thread) + thread.start() + + def connection(self, client): + backend = None + try: + host = self.config['host'] + port = int(self.config.get('port', 5432)) + if host.startswith('/'): + backend = socket.socket(socket.AF_UNIX) + backend.connect(host + '/.s.PGSQL.' + str(port)) + else: + backend = socket.create_connection((host, port), timeout=5) + self.sockets.append(backend) + size = exact(client, 4) + n = int.from_bytes(size, 'big') + backend.sendall(size + exact(client, n - 4)) + wrote = threading.Event() + dropping = threading.Event() + + def responses(): + try: + while not self.closed.is_set(): + tag = exact(backend, 1) + size = exact(backend, 4) + n = int.from_bytes(size, 'big') + if not 4 <= n <= 2**24: + raise ValueError('test frame size') + body = exact(backend, n - 4) + if dropping.is_set(): + if tag == b'Z': + self.dropped.set() + client.shutdown(socket.SHUT_RDWR) + return + else: + client.sendall(tag + size + body) + except EOFError, OSError, ValueError: + pass + + reader = threading.Thread(target=responses, daemon=True) + self.threads.append(reader) + reader.start() + while not self.closed.is_set(): + tag = exact(client, 1) + size = exact(client, 4) + n = int.from_bytes(size, 'big') + if not 4 <= n <= 2**24: + raise ValueError('test frame size') + body = exact(client, n - 4) + if tag == b'P': + statement = body.split(b'\0', 2)[1].lstrip().upper() + if statement.startswith((b'INSERT ', b'UPDATE ', b'DELETE ')): + wrote.set() + if tag == b'Q' and body.rstrip(b'\0; \n').upper() == b'COMMIT' and wrote.is_set(): + dropping.set() + backend.sendall(tag + size + body) + except EOFError, OSError, ValueError: + pass + finally: + client.close() + if backend is not None: + backend.close() + + def __exit__(self, *_): + self.closed.set() + self.listener.close() + for sock in self.sockets: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + sock.close() + for thread in self.threads: + thread.join(timeout=1) diff --git a/rust/tests/test_authorization_parity.py b/rust/tests/test_authorization_parity.py new file mode 100644 index 00000000..6ca8df8d --- /dev/null +++ b/rust/tests/test_authorization_parity.py @@ -0,0 +1,640 @@ +"""Exercise complete resource authorization against Python, using public fixtures. + +The helper authenticates every request before authorization. It never accepts a +serialized Principal, and no test publishes data or reaches a production server. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import tempfile +from contextlib import contextmanager +from dataclasses import replace +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +from test_identity_parity import NOW, READ, WRITE, Check, Fixture + +from msg.core.codec import canonical, wire +from msg.core.errors import Failure +from msg.core.models import AccessRequirement, ResourceTypeSpec +from msg.security.authorization import AuthorizationService +from msg.security.capabilities import BASE_FAMILIES + +EXTRA = frozenset({ + 'communication.dm_send@1', + 'communication.dm_send@2', + 'query.save@1', + 'query.saved_archive@1', + 'sharing.link_create@1', + 'sharing.grant@2', + 'sharing.revoke@2', + 'identity.note_put@2', + 'identity.personal_put@2', + 'identity.todo_put@2', + 'identity.legacy_put@2', + 'content.archive@1', + 'content.move@1', + 'content.chmod@1', + 'content.topic_create@1', + 'content.file_put@1', + 'wiki.page_put@1', +}) + + +class PolicyFixture(Fixture): + def __init__(self, path): + super().__init__(path) + # Trusted test configuration, not part of a request. Use the real names + # of the built-in families; no synthetic bypass to ordinary checks. + self.primary = tuple(replace(g, operations=g.operations | EXTRA) for g in self.primary) + for key, spec in list(self.registry._capabilities.items()): + self.registry._capabilities[key] = replace(spec, operations=spec.operations | EXTRA) + for name, cred in self.credentials.items(): + cred = replace(cred, ceiling=self.primary) + self.credentials[name] = cred + self.conn.execute( + 'UPDATE credentials SET body=? WHERE id=?', (canonical(cred).decode(), cred.id) + ) + for kind in ( + 'space', + 'topic', + 'file', + 'post', + 'user', + 'organization', + 'tool', + 'csr', + 'certificate', + 'credential', + 'legacy_directive', + 'dm_conversation', + 'watch', + 'saved_query', + 'delegation', + 'todo', + ): + self.registry.add_resource_type( + ResourceTypeSpec( + name=kind, + version=1, + container=kind in {'space', 'topic', 'user', 'organization', 'dm_conversation'}, + content_schema=None, + operations=frozenset(), + relations=frozenset(), + ) + ) + self.policy = AuthorizationService(self.registry, self.validator) + self.put_resource('r_user', parent='r_root', kind='user', owner='alice') + self.put_resource('r_soul', parent='r_user', name='SOUL.md', owner='alice') + self.put_resource('r_notes', parent='r_user', name='notes', kind='topic', owner='alice') + self.put_resource('r_note', parent='r_notes', name='note.md', owner='alice') + self.put_resource('r_todos', parent='r_user', name='todos', kind='topic', owner='alice') + self.put_resource('r_todo', parent='r_todos', kind='todo', owner='alice') + self.put_resource('t_last_will', parent='r_root', kind='topic') + self.put_resource('r_legacy', parent='t_last_will', kind='legacy_directive', owner='alice') + self.put_resource('t_wiki', parent='r_root', kind='topic') + self.put_resource('r_wiki', parent='t_wiki', owner='alice') + self.put_resource('r_agents', parent='r_root', owner='alice') + self.put_resource('r_rules', parent='r_root', owner='alice') + self.put_resource('r_watch', parent='r_area', kind='watch', owner='alice') + self.put_resource('r_query', parent='r_area', kind='saved_query', owner='alice') + self.put_resource('r_about', parent='r_area', name='ABOUT.md', owner='alice') + self.put_resource('r_header', parent='r_area', name='HEADER.svg', owner='alice') + self.put_resource('t_tools', parent='r_root', kind='topic', mode=0o700) + self.put_resource('r_tool', parent='t_tools', kind='tool', mode=0o700) + self.put_resource('r_dm', parent='r_root', kind='topic', mode=0o777) + self.put_resource('r_message', parent='r_dm', kind='post', owner='alice', mode=0o777) + self.put_resource('g_test', parent='r_root', kind='organization') + self.conn.execute( + 'INSERT INTO identities VALUES (?,?,?,?)', + ( + 'g_test', + 'organization', + 0, + canonical({'resource_id': 'g_test', 'membership_version': 1}).decode(), + ), + ) + for member in ('alice', 'bob'): + self.conn.execute( + 'INSERT INTO memberships VALUES (?,?,?,?)', + ( + 'g_test', + self.users[member], + 1, + canonical({ + 'organization_id': 'g_test', + 'subject_id': self.users[member], + 'role': 'member', + 'version': 1, + }).decode(), + ), + ) + self.conn.execute( + """INSERT INTO dm_conversations + (pair,resource_id,participant_a,participant_b,initiator,conversation_kind,state,created_at,updated_at) + VALUES (?,?,?,?,?,'direct','active',?,?)""", + ( + 'pair_fixture', + 'r_dm', + *sorted((self.users['alice'], self.users['bob'])), + self.users['alice'], + wire(NOW), + wire(NOW), + ), + ) + + def put_resource(self, id, *, parent='r_area', kind='file', name=None, owner='bob', mode=0o755): + resource = replace( + self.resources['r_file'], + id=id, + parent=parent, + type=kind, + name=name or id, + owner=self.users.get(owner, owner), + mode=mode, + revision='v_' + id, + ) + self.resources[id] = resource + self.conn.execute( + 'INSERT INTO resources VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)', + ( + id, + kind, + resource.name, + parent, + resource.owner, + resource.group, + mode, + 0, + resource.revision, + 'active', + wire(NOW), + wire(NOW), + canonical(resource).decode(), + ), + ) + return resource + + def context(self): + return { + **super().context(), + 'resource_types': wire(self.registry.resource_types()), + 'base_families': list(BASE_FAMILIES), + } + + @contextmanager + def resource_change(self, id, **fields): + with self.body_change('resources', id, replace(self.resources[id], **fields)): + yield + + @contextmanager + def row_change(self, table, key, updates): + # Table, columns, and predicates are test-owned constants, not request data. + where = ' AND '.join(f'{k}=?' for k in key) + columns = ','.join(updates) + old = self.conn.execute( + f'SELECT {columns} FROM {table} WHERE {where}', tuple(key.values()) + ).fetchone() + assert old is not None + assign = ','.join(f'{k}=?' for k in updates) + self.conn.execute( + f'UPDATE {table} SET {assign} WHERE {where}', (*updates.values(), *key.values()) + ) + try: + yield + finally: + self.conn.execute(f'UPDATE {table} SET {assign} WHERE {where}', (*old, *key.values())) + + +class PolicyCheck(Check): + async def require_case( + self, + group, + *, + id='r_file', + check='read', + actor='alice', + operation=READ + '@1', + certificates=(), + name=None, + checks=None, + entry='network', + delegated=False, + ): + f = self.fixture + opname, version = operation.rsplit('@', 1) + policy = f.spec(opname, version=int(version), entries=('network', 'local_admin', 'worker')) + request = f.request( + name=opname, + version=int(version), + actor=actor, + anonymous=actor is None, + subject=None if actor is None else f.users['bob'] if delegated else 'default', + certificates=certificates, + arguments={} if name is None else {'name': name}, + ) + requirements = checks or [ + AccessRequirement(resource_id=id, operation=operation, check=check) + ] + async with f.metadata.transaction(write=False) as tx: + try: + principal = await f.auth.authenticate(request, tx, entry=entry) + context = SimpleNamespace(principal=principal, now=f.now, entry=entry) + refs = await f.policy.require(context, request, requirements, tx) + expected = {'ok': True, 'data': {'refs': wire(refs)}} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + actual = self.native({ + 'action': 'authorize', + 'raw': canonical(request).decode(), + 'policy': policy, + 'entry': entry, + 'now': wire(f.now), + 'checks': wire(requirements), + 'request_name': name, + }) + self.compare(group, actual, expected) + return actual + + async def source_case(self, group, id='r_file', grant='s_main', subject='alice', reshare=False): + f = self.fixture + async with f.metadata.transaction(write=False) as tx: + try: + resource = await tx.resource(id) + result = await f.policy.share_source_active( + resource, grant, f.users.get(subject, subject), f.now, tx, reshare=reshare + ) + expected = {'ok': True, 'data': {'allowed': result}} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare( + group, + self.native({ + 'action': 'share_source', + 'id': id, + 'grant': grant, + 'subject': f.users.get(subject, subject), + 'now': wire(f.now), + 'reshare': reshare, + }), + expected, + ) + + async def link_case(self, group, id='r_file', grantor='bob', credential=None): + f = self.fixture + credential = credential or f.keys[grantor] + async with f.metadata.transaction(write=False) as tx: + try: + r = await tx.resource(id) + result = await f.policy.share_link_read_allowed( + f.users[grantor], credential, r, (*await tx.ancestors(id), r), f.now, tx + ) + expected = {'ok': True, 'data': {'allowed': result}} + except Failure as exc: + expected = {'ok': False, 'code': exc.code} + self.compare( + group, + self.native({ + 'action': 'share_link', + 'id': id, + 'grantor': f.users[grantor], + 'credential': credential, + 'now': wire(f.now), + }), + expected, + ) + + +def insert_share( + f, + id='s_main', + *, + resource='r_file', + grantor='bob', + grantee='alice', + parent=None, + kind='user', + reshare=True, +): + f.conn.execute( + """INSERT INTO share_grants_v2 VALUES (?,?,?,?,?,?,?,?,?,?,?,?)""", + ( + id, + resource, + f.users.get(grantor, grantor), + f.users.get(grantee, grantee), + kind, + parent, + '["read"]', + '{}', + int(reshare), + wire(NOW - timedelta(minutes=1)), + wire(NOW + timedelta(hours=1)), + None, + ), + ) + + +async def exercise(f, c): + for actor in ('alice', 'bob', None): + for mode in ( + 0o000, + 0o100, + 0o400, + 0o600, + 0o640, + 0o700, + 0o750, + 0o755, + 0o777, + 0o1777, + 0o2777, + 0o4777, + ): + with f.resource_change('r_file', mode=mode): + for check in ( + 'read', + 'list', + 'traverse', + 'write', + 'create', + 'remove', + 'chmod', + 'chgrp', + 'chown', + 'manage', + 'certgate', + 'purge', + 'tool_use', + ): + await c.require_case('resource_policy', actor=actor, check=check) + for group_kind in ('active', 'pending', 'invited', 'rejected'): + body = { + 'organization_id': 'g_test', + 'subject_id': f.users['alice'], + 'role': 'member', + 'version': 1, + 'status': group_kind, + } + with f.row_change( + 'memberships', + {'org': 'g_test', 'subject': f.users['alice']}, + {'body': canonical(body).decode()}, + ): + async with f.metadata.transaction(write=False) as tx: + expected = { + 'ok': True, + 'data': { + 'groups': sorted( + m.organization_id for m in await tx.memberships(f.users['alice']) + ) + }, + } + c.compare( + 'memberships', + c.native({'action': 'memberships', 'subject': f.users['alice']}), + expected, + ) + with f.resource_change('r_file', mode=0o040): + await c.require_case('group_mode') + for parentmode in (0, 0o100, 0o400, 0o700, 0o710, 0o755): + with f.resource_change('r_area', mode=parentmode): + for actor in ('alice', 'bob', None): + await c.require_case('ancestor_traversal', actor=actor) + for state in ('archived', 'purged'): + with f.resource_change('r_area', state=state): + await c.require_case('inactive_ancestor') + for id in ( + 'r_watch', + 'r_query', + 'r_soul', + 'r_note', + 'r_todo', + 'r_legacy', + 'r_wiki', + 'r_agents', + 'r_rules', + ): + for actor in ('alice', 'bob', None): + for check in ('read', 'write', 'manage'): + await c.require_case('managed_resources', id=id, actor=actor, check=check) + for id, operations in { + 'r_query': ('query.save@1', 'query.saved_archive@1'), + 'r_soul': ('identity.personal_put@2', 'sharing.grant@2'), + 'r_note': ('identity.note_put@2', 'sharing.grant@2'), + 'r_todo': ('identity.todo_put@2', 'sharing.grant@2'), + 'r_legacy': ('identity.legacy_put@2', 'content.archive@1'), + 'r_wiki': ('content.archive@1', 'content.move@1', 'content.chmod@1', 'wiki.page_put@1'), + }.items(): + for op in operations: + for actor in ('alice', 'bob'): + await c.require_case( + 'dedicated_operations', id=id, operation=op, actor=actor, check='write' + ) + for resource in ('r_about', 'r_header', 'r_area'): + for actor in ('alice', 'bob', None): + for check in ('read', 'write', 'create', 'manage'): + await c.require_case( + 'topic_presentation', id=resource, actor=actor, check=check, name='ABOUT.md' + ) + f.conn.execute( + """INSERT INTO topic_memberships(topic,subject,role,status,joined_at,invited_by) + VALUES (?,?,'admin','active',?,?)""", + ('r_area', f.users['alice'], wire(NOW), None), + ) + for id in ('r_about', 'r_header', 'r_area'): + await c.require_case('topic_admin', id=id, check='write') + for expires in ( + None, + wire(NOW - timedelta(seconds=1)), + wire(NOW), + wire(NOW + timedelta(seconds=1)), + ): + f.conn.execute( + """INSERT INTO topic_bans(topic,subject,actor,created_at,expires_at,reason,status) + VALUES (?,?,?,?,?,'test','active') ON CONFLICT(topic,subject) DO UPDATE SET expires_at=excluded.expires_at""", + ('r_area', f.users['alice'], f.users['bob'], wire(NOW), expires), + ) + await c.require_case('topic_bans', id='r_about', check='write') + f.conn.execute('DELETE FROM topic_bans') + for prefix, id in (('hosting_preview:', 'r_area'), ('hosting_preview_file:', 'r_file')): + for marker in (False, {}, {'owner': f.users['alice']}, {'owner': f.users['bob']}): + with f.setting_change(prefix + id, marker): + for actor in ('alice', 'bob', None): + await c.require_case('hosting_preview', actor=actor) + for actor in ('alice', 'bob', 'ca', None): + for check in ('read', 'list', 'traverse', 'write', 'create', 'manage', 'chmod'): + await c.require_case('dm_participants', id='r_dm', actor=actor, check=check) + await c.require_case('dm_messages', id='r_message', actor=actor, check=check) + for state in ('active', 'pending', 'rejected'): + with f.row_change('dm_conversations', {'resource_id': 'r_dm'}, {'state': state}): + for actor in ('alice', 'bob'): + await c.require_case( + 'dm_send', + id='r_dm', + actor=actor, + check='create', + operation='communication.dm_send@2', + ) + await c.require_case( + 'dm_edit', id='r_message', actor=actor, check='write', operation=WRITE + '@1' + ) + f.conn.execute('INSERT INTO dm_blocks VALUES (?,?)', (f.users['alice'], f.users['bob'])) + for actor in ('alice', 'bob'): + await c.require_case( + 'dm_blocked', + id='r_dm', + actor=actor, + check='create', + operation='communication.dm_send@2', + ) + await c.require_case( + 'dm_blocked', id='r_message', actor=actor, check='write', operation=WRITE + '@1' + ) + f.conn.execute('DELETE FROM dm_blocks') + for mode in (0o2000, 0o2777, 0o6755): + with f.resource_change('r_area', mode=mode): + for check in ('read', 'write', 'create', 'manage'): + await c.require_case('certificate_gate', id='r_file', check=check, actor='bob') + for actor in ('alice', 'bob', None): + for check in ('read', 'tool_use', 'write'): + await c.require_case('tool_gate', id='r_tool', check=check, actor=actor) + for certificates in (('c_leaf',), ('c_online_leaf',), ('c_delegation',)): + await c.require_case('attached_certificates', certificates=certificates) + # A share authorizes one read, never traversal, writes, or descendant reads. + insert_share(f) + with f.resource_change('r_file', mode=0o600), f.resource_change('r_area', mode=0o700): + for actor in ('alice', 'bob', 'ca', None): + for check in ('read', 'list', 'traverse', 'write'): + await c.require_case('share_exact_read', actor=actor, check=check) + for field, values in { + 'resource_id': ('r_file', 'r_other'), + 'grantor': (f.users['bob'], f.users['alice']), + 'grantee': (f.users['alice'], f.users['bob']), + 'grantee_kind': ('user', 'group'), + 'operations': ('["read"]', '["read","write"]', '[]', 'null', '{bad', '["read","read"]'), + 'constraints': ('{}', '{"name":"secret"}', '[]', 'null', 'bad'), + 'expires_at': ( + wire(NOW), + wire(NOW - timedelta(seconds=1)), + wire(NOW + timedelta(seconds=1)), + 'invalid', + ), + 'created_at': ( + wire(NOW), + wire(NOW + timedelta(seconds=1)), + wire(NOW - timedelta(days=1)), + 'invalid', + ), + 'revoked_at': (None, wire(NOW)), + 'allow_reshare': (0, 1), + }.items(): + for value in values: + with f.row_change('share_grants_v2', {'id': 's_main'}, {field: value}): + await c.source_case('share_facts') + await c.source_case('share_reshare', reshare=True) + with f.row_change( + 'share_grants_v2', {'id': 's_main'}, {'grantee': 'g_test', 'grantee_kind': 'group'} + ): + for subject in ('alice', 'bob', 'ca'): + await c.source_case('group_share', subject=subject) + insert_share(f, 's_child', grantor='alice', grantee='ca', parent='s_main') + await c.source_case('share_chain', grant='s_child', subject='ca') + for change in ( + {'revoked_at': wire(NOW)}, + {'allow_reshare': 0}, + {'expires_at': wire(NOW + timedelta(minutes=2))}, + {'parent_id': 's_child'}, + ): + with f.row_change('share_grants_v2', {'id': 's_main'}, change): + await c.source_case('share_chain', grant='s_child', subject='ca') + for id in ( + 'r_dm', + 'r_message', + 'r_soul', + 'r_todo', + 'r_legacy', + 'r_agents', + 'r_rules', + 'r_area', + 'r_tool', + 'c_leaf', + ): + with f.row_change('share_grants_v2', {'id': 's_main'}, {'resource_id': id}): + await c.source_case('protected_share', id=id) + for grantor in ('alice', 'bob'): + await c.link_case('share_link', grantor=grantor) + for change in ( + {'revoked_at': NOW}, + {'not_before': NOW + timedelta(seconds=1)}, + {'expires_at': NOW}, + {'ceiling': ()}, + ): + with f.body_change('credentials', f.keys['bob'], replace(f.credentials['bob'], **change)): + await c.link_case('share_link_credential') + for key, value in ( + ('recovery_quarantine', False), + ('recovery_quarantine', None), + ('recovery_quarantine', {}), + ): + with f.setting_change(key, value): + await c.require_case('quarantine', actor=None) + await c.link_case('quarantined_share') + # Once either authority service observed a replaced generation it remains + # fenced even if a later restore reinstates the old database value. + with f.setting_change('recovery_runtime_generation', 'replacement'): + await c.require_case('runtime_generation') + await c.require_case('runtime_generation_latched') + + +async def run(binary, report, postgres=False): + from contextlib import nullcontext + + from postgres_fixture import database, replace_fixture_storage + + with tempfile.TemporaryDirectory(prefix='msg-rust-policy-') as temp: + f = PolicyFixture(Path(temp) / 'metadata.sqlite3') + with database() if postgres else nullcontext(None) as dsn: + c = None + try: + if dsn: + await replace_fixture_storage(f, dsn) + c = PolicyCheck(binary, f) + await exercise(f, c) + data = { + 'backend': 'postgresql' if postgres else 'sqlite', + 'cases': sum(c.groups.values()), + 'groups': dict(c.groups), + 'failures': c.failures, + } + report.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n') + print( + json.dumps({ + 'cases': data['cases'], + 'failures': len(c.failures), + 'report': str(report), + }) + ) + if c.failures: + print(json.dumps(c.failures[:8], indent=2)) + return bool(c.failures) + finally: + if c is not None: + c.process.stdin.close() + c.process.wait(timeout=10) + await f.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + parser.add_argument('--postgres', action='store_true') + options = parser.parse_args() + raise SystemExit( + asyncio.run(run(options.binary.resolve(), options.report.resolve(), options.postgres)) + ) diff --git a/rust/tests/test_postgres_parity.py b/rust/tests/test_postgres_parity.py new file mode 100644 index 00000000..dc1fc03c --- /dev/null +++ b/rust/tests/test_postgres_parity.py @@ -0,0 +1,645 @@ +"""Native PostgreSQL writes against the real Python PostgreSQL implementation. + +Only new, randomly named databases on an explicitly selected local test server +are used. SQL inputs in the native helper are compiled constants, not JSON data. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import subprocess +from collections import Counter +from dataclasses import replace +from pathlib import Path + +import psycopg +from postgres_fixture import database +from psycopg import sql +from psycopg.conninfo import conninfo_to_dict +from test_storage_parity import ( + NOW, + TABLES as METADATA_TABLES, + D, + action, + audit, + insert, + metadata_cases, + python_steps as base_python_steps, + resource, + revision, +) + +from msg.core.codec import canonical, decode, wire +from msg.core.errors import Failure +from msg.core.identifiers import hex_id +from msg.core.models import ( + BlobRef, + Certificate, + CertificateRequest, + CertificateRequestState, + Credential, + EffectJob, + EmailSettings, + Membership, + Organization, + Principal, + ResourceRef, + Signature, + Subject, + TransferChunk, + TransferSession, +) +from msg.storage.postgres import PostgresMetadataStore + +TABLES = ( + *METADATA_TABLES, + 'identities', + 'memberships', + 'emails', + 'credentials', + 'csrs', + 'certificates', + 'transfers', + 'chunks', + 'jobs', +) +REPORT = [] + + +def check(name, okay, detail=None, group='postgres_parity'): + REPORT.append({ + 'name': name, + 'group': group, + 'ok': bool(okay), + **({'detail': detail} if not okay else {}), + }) + + +def require_disposable(dsn): + if not conninfo_to_dict(dsn)['dbname'].startswith('msg_rust_test_'): + raise RuntimeError('Refusing to modify any existing application database') + + +def reset(dsn, baseline): + require_disposable(dsn) + with psycopg.connect(dsn) as c: + c.execute( + sql.SQL('TRUNCATE {} RESTART IDENTITY CASCADE').format( + sql.SQL(',').join(map(sql.Identifier, TABLES)) + ) + ) + for table, (columns, rows) in baseline.items(): + query = sql.SQL('INSERT INTO {} ({}) VALUES ({})').format( + sql.Identifier(table), + sql.SQL(',').join(map(sql.Identifier, columns)), + sql.SQL(',').join(sql.Placeholder() for _ in columns), + ) + for row in rows: + c.execute(query, row) + + +def snapshot(dsn): + with psycopg.connect(dsn) as c: + result = {} + for table in TABLES: + rows = c.execute(sql.SQL('SELECT * FROM {}').format(sql.Identifier(table))) + result[table] = ([x.name for x in rows.description], sorted(rows.fetchall(), key=repr)) + return result + + +async def python_steps(tx, steps, store, effects): + out = [] + classes = { + 'subject': Subject, + 'organization': Organization, + 'membership': Membership, + 'email': EmailSettings, + } + for step in steps: + kind = step['action'] + if kind == 'identity': + await tx.update_identity( + decode(classes[step['record_kind']], step['record']), step['expected'] + ) + elif kind == 'subject': + out.append(wire(await tx.subject(step['id']))) + elif kind == 'credential': + await tx.save_credential(decode(Credential, step['record']), step['expected']) + elif kind == 'get_credential': + out.append(wire(await tx.credential(step['id']))) + elif kind == 'csr': + await tx.save_csr(decode(CertificateRequest, step['record'])) + elif kind == 'csr_state': + out.append(wire(await tx.csr_state(step['id']))) + elif kind == 'transition_csr': + await tx.transition_csr( + decode(CertificateRequestState, step['record']), step['expected'] + ) + elif kind == 'certificate': + await tx.register_certificate( + decode(Certificate, step['record']), step['csr'], step.get('expected', 0) + ) + elif kind == 'revoke_certificate': + await tx.revoke_certificate(step['id'], decode(type(audit()), step['event'])) + elif kind == 'get_certificate': + out.append(wire(await tx.certificate(step['id']))) + elif kind == 'transfer': + await tx.save_transfer(decode(TransferSession, step['record']), step['expected']) + elif kind == 'get_transfer': + out.append(wire(await tx.transfer(step['id']))) + elif kind == 'chunk': + await tx.put_chunk(decode(TransferChunk, step['record'])) + elif kind == 'missing': + out.append(wire(await tx.missing_ranges(step['id'], step['cursor'], step['limit']))) + elif kind == 'enqueue': + await tx.enqueue(decode(EffectJob, step['record'])) + elif kind == 'save_job': + await tx.save_job(decode(EffectJob, step['record'])) + elif kind == 'job': + out.append(wire(await tx.job(step['id']))) + elif kind == 'path': + out.append(await tx.path(step['id'])) + elif kind == 'resolve': + out.append( + await ( + tx.resolve_migrated(step['path']) + if step.get('migrated') + else tx.resolve(step['path']) + ) + ) + elif kind == 'children': + out.append(wire(await tx.children(step['id'], step['cursor'], step['limit']))) + elif kind == 'history': + out.append(wire(await tx.history(step['id'], step['cursor'], step['limit']))) + elif kind == 'get_revision': + out.append(wire(await tx.revision(decode(ResourceRef, step['reference'])))) + elif kind == 'savepoint': + try: + async with store.transaction(write=True) as inner: + data = await python_steps(inner, step['steps'], store, effects) + out.append(data) + except Failure as exc: + if not step.get('recover'): + raise + out.append({'error': exc.code}) + else: + out.extend(await base_python_steps(tx, [step], store, effects)) + return out + + +async def run_python(store, steps): + effects = [] + try: + async with store.transaction(write=True) as tx: + out = await python_steps(tx, steps, store, effects) + return {'status': 'ok', 'values': out}, effects + except Failure as exc: + return {'status': 'error', 'code': exc.code, 'cleanup': []}, effects + + +class Native: + def __init__(self, binary): + self.binary = str(binary) + + def start(self, dsn, steps, **kwargs): + env = {**os.environ, 'MSG_PARITY_POSTGRES_DSN': dsn} + p = subprocess.Popen( + [self.binary], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + p.stdin.write(json.dumps({'steps': steps, **kwargs}, ensure_ascii=False) + '\n') + p.stdin.flush() + return p + + def run(self, dsn, steps, **kwargs): + p = self.start(dsn, steps, **kwargs) + stdout, stderr = p.communicate(timeout=30) + if p.returncode: + raise AssertionError('Native helper failed: ' + stderr) + lines = [json.loads(s) for s in stdout.splitlines()] + result = lines[-1] + if result.get('uncertain') is False: + result.pop('uncertain') + return result, [item['effect'] for item in lines if 'effect' in item] + + +def extended_cases(): + subject = Subject( + resource_id='u_alice', kind='registered', primary_group='g_public', auth_version=0 + ) + organization = Organization(resource_id='g_team', membership_version=0) + membership = Membership( + organization_id='g_team', subject_id='u_alice', role='member', version=0 + ) + email = EmailSettings( + subject_id='u_alice', address='public-test@example.invalid', verified_at=None + ) + credential = Credential( + id='key_a', + subject_id='u_alice', + kind='signing_key', + verifier=bytes([3]) * 32, + ceiling=(), + not_before=NOW, + expires_at=None, + revoked_at=None, + ) + sig = Signature(key_id='key_a', algorithm='Ed25519', value=bytes([1]) * 64) + csr = CertificateRequest( + resource_id='csr_a', + applicant='u_alice', + subject_id='u_alice', + requested_issuer='u_root', + public_key=credential.verifier, + kind='capability', + grants=(), + issuance=None, + requested_ttl_seconds=3600, + target_service='https://fixture.invalid', + delegation_depth=0, + authority_sources=(), + request_digest=D, + possession_proof=sig, + ) + cert = Certificate( + resource_id='cert_a', + serial='fixture', + subject_id='u_alice', + key_id='key_a', + issuer_id='u_root', + parent_certificate_id=None, + authority_sources=(), + kind='capability', + grants=(), + not_before=NOW, + expires_at=NOW.replace(year=2027), + target_service='https://fixture.invalid', + delegation_depth=0, + issuance=None, + signature=sig, + ) + transfer = TransferSession( + id='transfer', + subject_id='u_alice', + direction='upload', + state='open', + target=None, + expected_size=100, + expected_digest=D, + expires_at=NOW.replace(year=2027), + generation=0, + ) + chunk = TransferChunk( + transfer_id='transfer', + offset=10, + content=BlobRef(digest=D, size=20, media_type='text/plain'), + ) + principal = Principal( + actor='u_alice', + subject='u_alice', + credential_id='key_a', + method='signature', + ceiling=(), + certificates=(), + ) + job = EffectJob( + id='job', + event_id='event', + kind='tool', + dedupe_key='dedupe', + principal=principal, + operation='tool.run@1', + arguments={}, + state='pending', + attempts=0, + next_attempt_at=NOW, + lease_until=None, + ) + + def identity(record, kind='subject', expected=-1): + return action('identity', record_kind=kind, record=record, expected=expected) + + cases = [] + for kind, r, field in [ + ('subject', subject, 'auth_version'), + ('organization', organization, 'membership_version'), + ('membership', membership, 'version'), + ]: + cases.extend([ + (kind + '-insert', [identity(r, kind)]), + (kind + '-update', [identity(r, kind), identity(replace(r, **{field: 1}), kind, 0)]), + (kind + '-conflict', [identity(r, kind), identity(replace(r, **{field: 2}), kind, 1)]), + ]) + cases.extend([ + ('subject-read', [identity(subject), action('subject', id='u_alice')]), + ( + 'email-update', + [ + identity(email, 'email'), + identity( + replace(email, verified_at=NOW, enabled_events=frozenset({'reply', 'message'})), + 'email', + 0, + ), + ], + ), + ( + 'credential-read', + [ + identity(subject), + action('credential', record=credential, expected=0), + action('get_credential', id=credential.id), + ], + ), + ( + 'credential-revoke', + [ + identity(subject), + action('credential', record=credential, expected=0), + action('credential', record=replace(credential, revoked_at=NOW), expected=0), + ], + ), + ( + 'credential-version', + [identity(subject), action('credential', record=credential, expected=1)], + ), + ( + 'credential-immutable', + [ + identity(subject), + action('credential', record=credential, expected=0), + action( + 'credential', record=replace(credential, verifier=bytes([9]) * 32), expected=0 + ), + ], + ), + ('csr-pending', [action('csr', record=csr), action('csr_state', id=csr.resource_id)]), + ( + 'csr-rejected', + [ + action('csr', record=csr), + action( + 'transition_csr', + record=CertificateRequestState( + request_id=csr.resource_id, status='rejected', generation=1 + ), + expected=0, + ), + ], + ), + ( + 'certificate-with-csr', + [ + action('csr', record=csr), + action('certificate', record=cert, csr=csr.resource_id, expected=0), + action('csr_state', id=csr.resource_id), + action('get_certificate', id=cert.resource_id), + ], + ), + ( + 'certificate-without-csr', + [ + action('certificate', record=cert, csr=None), + action('get_certificate', id=cert.resource_id), + ], + ), + ( + 'certificate-csr-failed', + [ + action('csr', record=csr), + action('certificate', record=cert, csr=csr.resource_id, expected=1), + ], + ), + ( + 'certificate-revoked', + [ + action('certificate', record=cert, csr=None), + action('revoke_certificate', id=cert.resource_id, event=audit()), + ], + ), + ( + 'transfer-roundtrip', + [ + action('transfer', record=transfer, expected=None), + action('get_transfer', id=transfer.id), + ], + ), + ( + 'transfer-transition', + [ + action('transfer', record=transfer, expected=None), + action( + 'transfer', + record=replace( + transfer, + state='sealed', + generation=1, + output=ResourceRef(id='r_doc', revision='v1'), + ), + expected=0, + ), + ], + ), + ( + 'transfer-conflict', + [ + action('transfer', record=transfer, expected=None), + action('transfer', record=replace(transfer, generation=2), expected=1), + ], + ), + ( + 'chunk-repeat', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action('chunk', record=chunk), + ], + ), + ( + 'chunk-conflict', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action('chunk', record=replace(chunk, offset=20)), + ], + ), + ( + 'chunk-content-conflict', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action( + 'chunk', + record=replace( + chunk, content=replace(chunk.content, digest='sha256:' + 'a' * 64) + ), + ), + ], + ), + ('job-roundtrip', [action('enqueue', record=job), action('job', id=job.id)]), + ( + 'job-update', + [ + action('enqueue', record=job), + action( + 'save_job', + record=replace( + job, state='running', attempts=1, lease_until=NOW.replace(hour=1) + ), + ), + action('job', id=job.id), + ], + ), + ( + 'job-dedupe', + [action('enqueue', record=job), action('enqueue', record=replace(job, id='other'))], + ), + ]) + for cursor in (None, '0', '30', '100'): + for limit in (1, 2, 50): + cases.append(( + f'missing-{cursor}-{limit}', + [ + action('transfer', record=transfer, expected=None), + action('chunk', record=chunk), + action('chunk', record=replace(chunk, offset=50)), + action('missing', id=transfer.id, cursor=cursor, limit=limit), + ], + )) + for path, migrated in [ + ('/', False), + ('/document', False), + ('/_id/r_doc', False), + ('/_id/' + hex_id('r_doc'), False), + ('/*' + hex_id('r_doc'), False), + ('/document', True), + ]: + cases.append(( + 'resolve-' + path, + [insert(), action('resolve', path=path, migrated=migrated)], + )) + cases.extend([ + ('path', [insert(), action('path', id='r_doc')]), + ('children', [insert(), action('children', id='r_root', cursor=None, limit=1)]), + ( + 'history', + [ + insert(), + action('revision', record=revision()), + action('history', id='r_doc', cursor=None, limit=1), + ], + ), + ( + 'revision-read', + [ + insert(), + action('revision', record=revision()), + action('get_revision', reference=ResourceRef(id='r_doc', revision='v1')), + ], + ), + ( + 'revision-hex', + [ + insert(), + action('revision', record=revision()), + action( + 'get_revision', reference=ResourceRef(id=hex_id('r_doc'), revision=hex_id('v1')) + ), + ], + ), + ( + 'read-after-rename', + [ + insert(), + action('replace', record=replace(resource(), name='new', generation=1), expected=0), + action('resolve', path='/document', migrated=True), + action('resolve', path='/new', migrated=False), + ], + ), + ]) + return cases + + +async def run(args): + native = Native(args.binary.resolve()) + with database() as py_dsn, database() as rs_dsn: + py_store = PostgresMetadataStore(py_dsn) + rs_store = PostgresMetadataStore(rs_dsn) + for store in [py_store, rs_store]: + async with store.transaction(write=True) as tx: + await tx.insert(resource('r_root', parent=None, name='')) + baseline = snapshot(py_dsn) + try: + for name, steps in metadata_cases() + extended_cases(): + reset(py_dsn, baseline) + reset(rs_dsn, baseline) + expected, py_effects = await run_python(py_store, steps) + actual, rs_effects = native.run(rs_dsn, steps) + py_db, rs_db = snapshot(py_dsn), snapshot(rs_dsn) + different = [t for t in TABLES if py_db[t] != rs_db[t]] + check( + name, + canonical(actual) == canonical(expected) + and not different + and py_effects == rs_effects, + { + 'python': expected, + 'rust': actual, + 'tables': different, + 'effects': [py_effects, rs_effects], + }, + ) + reset(rs_dsn, baseline) + actual, _ = native.run( + rs_dsn, [action('set', key='read-only', value=True)], read_only=True + ) + check( + 'read-only-write-rejected', + actual.get('code') == 'read_only_transaction' and snapshot(rs_dsn) == baseline, + actual, + group='postgres_safety', + ) + actual, _ = native.run( + rs_dsn, + [ + action('set', key='before', value=True), + action('ignore', steps=[insert(resource(parent='absent'))]), + ], + ) + check( + 'ignored-write-error-cannot-commit', + actual.get('status') == 'error' and snapshot(rs_dsn) == baseline, + actual, + group='postgres_safety', + ) + finally: + await py_store.close() + await rs_store.close() + report = { + 'backend': 'postgresql', + 'checks': len(REPORT), + 'failures': sum(not x['ok'] for x in REPORT), + 'groups': dict(Counter(x['group'] for x in REPORT)), + 'cases': REPORT, + } + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({k: v for k, v in report.items() if k != 'cases'})) + return report['failures'] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--binary', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + raise SystemExit(bool(asyncio.run(run(parser.parse_args())))) + + +if __name__ == '__main__': + main() diff --git a/rust/tests/test_postgres_process.py b/rust/tests/test_postgres_process.py new file mode 100644 index 00000000..9311145e --- /dev/null +++ b/rust/tests/test_postgres_process.py @@ -0,0 +1,268 @@ +"""Real process/transaction failure checks against a disposable PostgreSQL DB.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import selectors +import time +from collections import Counter +from pathlib import Path + +import psycopg +from postgres_fixture import database +from postgres_transport_fixture import LoseCommitAck +from test_postgres_parity import REPORT, Native, check, reset, snapshot +from test_storage_parity import action, audit, event, insert, ledger, once, resource + +from msg.storage.postgres import PostgresMetadataStore + + +def line(p): + with selectors.DefaultSelector() as selected: + selected.register(p.stdout, selectors.EVENT_READ) + if not selected.select(timeout=15): + raise AssertionError('missing process checkpoint') + return json.loads(p.stdout.readline()) + + +def finish(p): + p.stdin.write('continue\n') + p.stdin.flush() + out, err = p.communicate(timeout=15) + if p.returncode: + raise AssertionError('native process failed: ' + err) + return json.loads(out.splitlines()[-1]) + + +def kill(p): + if p.poll() is None: + p.kill() + p.communicate(timeout=10) + + +def waiting_writer(dsn): + deadline = time.monotonic() + 5 + with psycopg.connect(dsn, autocommit=True) as c: + while time.monotonic() < deadline: + if c.execute( + "SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND application_name='msgd-native' AND wait_event_type='Lock' AND wait_event='advisory')" + ).fetchone()[0]: + return True + time.sleep(0.01) + return False + + +async def run(args): + native = Native(args.binary.resolve()) + with database() as dsn: + store = PostgresMetadataStore(dsn) + async with store.transaction(write=True) as tx: + await tx.insert(resource('r_root', parent=None, name='')) + baseline = snapshot(dsn) + body = [insert(), action('event', record=event()), action('audit', record=audit())] + try: + for stage in ['before', 'after']: + reset(dsn, baseline) + p = native.start( + dsn, + [once(body + ([action('hold')] if stage == 'before' else []))], + hold_after_commit=stage == 'after', + ) + try: + got = line(p) + visible = snapshot(dsn) + check( + 'kill-' + stage + '-visibility', + got == {'checkpoint': 'transaction' if stage == 'before' else 'committed'} + and bool(visible['results'][1]) == (stage == 'after'), + got, + group='postgres_process', + ) + finally: + kill(p) + actual, _ = native.run(dsn, [once(body)]) + after = snapshot(dsn) + check( + 'kill-' + stage + '-retry-once', + actual['status'] == 'ok' + and len(after['resources'][1]) == 2 + and all(len(after[t][1]) == 1 for t in ['results', 'events', 'audit']), + actual, + group='postgres_process', + ) + reset(dsn, baseline) + first = native.start(dsn, [once(body + [action('hold')])]) + second = None + try: + check( + 'first-writer-acquired', + line(first) == {'checkpoint': 'transaction'}, + group='postgres_process', + ) + second = native.start(dsn, [once(body)]) + blocked = waiting_writer(dsn) + first_result = finish(first) + out, err = second.communicate(timeout=15) + check( + 'concurrent-writers-one-commit', + blocked + and second.returncode == 0 + and first_result == json.loads(out.splitlines()[-1]) + and len(snapshot(dsn)['audit'][1]) == 1, + err, + group='postgres_process', + ) + finally: + kill(first) + if second is not None: + kill(second) + reset(dsn, baseline) + second = None + async with store.transaction(write=True) as tx: + tx.set_setting('python-held', True) + second = native.start(dsn, [action('set', key='native-held', value=True)]) + check('python-writer-blocks-native', waiting_writer(dsn), group='postgres_process') + try: + out, err = second.communicate(timeout=15) + check( + 'native-continues-after-python-commit', + json.loads(out.splitlines()[-1])['status'] == 'ok', + err, + group='postgres_process', + ) + finally: + kill(second) + p = native.start(dsn, [action('set', key='native-held', value=True), action('hold')]) + try: + line(p) + + async def python_contender(): + async with store.transaction(write=True) as tx: + tx.set_setting('python-after', True) + + task = asyncio.create_task(python_contender()) + await asyncio.sleep(0.1) + check('native-writer-blocks-python', not task.done(), group='postgres_process') + finish(p) + await asyncio.wait_for(task, 15) + finally: + kill(p) + reset(dsn, baseline) + p = native.start( + dsn, + [ + action('effect', name='cleanup', hold=True), + action('set', key='abort', value=True), + action('fail'), + ], + ) + try: + got = line(p) + checkpoint = json.loads(p.stdout.readline()) + with psycopg.connect(dsn, autocommit=True) as c: + lock = c.execute( + 'SELECT pg_try_advisory_lock(725274758,1886265951)' + ).fetchone()[0] + if lock: + c.execute('SELECT pg_advisory_unlock(725274758,1886265951)') + check( + 'fence-survives-sql-rollback', + got == {'effect': 'cleanup'} + and checkpoint == {'checkpoint': 'compensation'} + and not lock + and snapshot(dsn) == baseline, + group='postgres_process', + ) + check( + 'compensation-completes', + finish(p).get('code') == 'fixture_failure', + group='postgres_process', + ) + finally: + kill(p) + reset(dsn, baseline) + with LoseCommitAck(dsn) as proxy: + actual, effects = native.run( + proxy.dsn, [action('effect', name='must-not-delete'), *body, ledger()] + ) + check( + 'lost-commit-response-is-uncertain', + proxy.dropped.is_set() + and actual.get('code') == 'storage_commit_uncertain' + and actual.get('uncertain') is True + and not effects, + actual, + group='postgres_process', + ) + after = snapshot(dsn) + actual, _ = native.run(dsn, [once(body)]) + check( + 'lost-commit-response-retry-no-duplicate', + actual['status'] == 'ok' and after == snapshot(dsn) and len(after['audit'][1]) == 1, + actual, + group='postgres_process', + ) + # Every schema mutation is isolated to this new test database and + # reverted, so later tests never inherit a deliberately broken DB. + reset(dsn, baseline) + schema_changes = [ + ( + 'missing-dedupe', + 'ALTER TABLE jobs DROP CONSTRAINT jobs_dedupe_key', + 'ALTER TABLE jobs ADD CONSTRAINT jobs_dedupe_key UNIQUE(dedupe)', + ), + ( + 'missing-root-index', + 'DROP INDEX one_root', + 'CREATE UNIQUE INDEX one_root ON resources((1)) WHERE parent IS NULL', + ), + ( + 'wrong-primary-key', + 'ALTER TABLE results DROP CONSTRAINT results_pkey; ALTER TABLE results ADD PRIMARY KEY(subject)', + 'ALTER TABLE results DROP CONSTRAINT results_pkey; ALTER TABLE results ADD PRIMARY KEY(subject,request_id)', + ), + ( + 'missing-foreign-key', + 'ALTER TABLE revisions DROP CONSTRAINT revisions_resource_id_fkey', + 'ALTER TABLE revisions ADD CONSTRAINT revisions_resource_id_fkey FOREIGN KEY(resource_id) REFERENCES resources(id)', + ), + ] + for name, break_sql, repair_sql in schema_changes: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(break_sql) + try: + actual, _ = native.run(dsn, [action('set', key='must-not-write', value=True)]) + check( + name + '-rejected', + actual.get('code') == 'unsupported_storage_schema' + and snapshot(dsn) == baseline, + actual, + group='postgres_schema', + ) + finally: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(repair_sql) + finally: + await store.close() + report = { + 'checks': len(REPORT), + 'failures': sum(not r['ok'] for r in REPORT), + 'groups': dict(Counter(r['group'] for r in REPORT)), + 'cases': REPORT, + } + args.report.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') + print(json.dumps({k: v for k, v in report.items() if k != 'cases'})) + return report['failures'] + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--binary', type=Path, required=True) + p.add_argument('--report', type=Path, required=True) + raise SystemExit(bool(asyncio.run(run(p.parse_args())))) + + +if __name__ == '__main__': + main() diff --git a/rust/tests/test_storage_parity.py b/rust/tests/test_storage_parity.py index b70ba4a3..669f98de 100644 --- a/rust/tests/test_storage_parity.py +++ b/rust/tests/test_storage_parity.py @@ -306,7 +306,7 @@ def kill(process): process.communicate(timeout=10) -async def parity_cases(native, directory): +def metadata_cases(): doc = resource() rev = revision() cases = [ @@ -597,6 +597,11 @@ async def parity_cases(native, directory): ) ], )) + return cases + + +async def parity_cases(native, directory): + cases = metadata_cases() for index, (name, steps) in enumerate(cases): py = directory / f'py-{index}.db' rs = directory / f'rs-{index}.db' From ac08e7f503574719876079fda54842dab5aff990 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 18:37:51 +0800 Subject: [PATCH 15/22] ci(rust): verify native policy and PostgreSQL on exact sources; remove editor exports --- .github/workflows/rust-editor-tools.yml | 34 ------ .github/workflows/rust-native-domain-deps.yml | 44 ------- .github/workflows/rust-phase4-import.yml | 110 ------------------ .github/workflows/rust-postgres-editor.yml | 65 ----------- .github/workflows/rust-runtime-deps.yml | 89 -------------- .github/workflows/rust-wire.yml | 27 ++++- 6 files changed, 24 insertions(+), 345 deletions(-) delete mode 100644 .github/workflows/rust-editor-tools.yml delete mode 100644 .github/workflows/rust-native-domain-deps.yml delete mode 100644 .github/workflows/rust-phase4-import.yml delete mode 100644 .github/workflows/rust-postgres-editor.yml delete mode 100644 .github/workflows/rust-runtime-deps.yml diff --git a/.github/workflows/rust-editor-tools.yml b/.github/workflows/rust-editor-tools.yml deleted file mode 100644 index 372f2bbc..00000000 --- a/.github/workflows/rust-editor-tools.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Rust offline editor tools -on: - push: - branches: ['migration/rust-*'] - paths: ['.github/workflows/rust-editor-tools.yml'] -permissions: - contents: read -jobs: - export: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - fetch-depth: 1 - - name: Export only public compiler and repository source - shell: bash - run: | - set -euo pipefail - rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy - mkdir -p /tmp/msg-editor-tools - git rev-parse HEAD > /tmp/msg-editor-tools/source-commit.txt - git rev-parse HEAD^{tree} > /tmp/msg-editor-tools/source-tree.txt - git archive --format=tar.gz HEAD > /tmp/msg-editor-tools/source.tar.gz - tar -cJf /tmp/msg-editor-tools/rust-toolchain.tar.xz -C "$HOME/.rustup/toolchains/1.85.0-x86_64-unknown-linux-gnu" . - (cd /tmp/msg-editor-tools && sha256sum *.tar.* > SHA256SUMS) - - uses: actions/upload-artifact@v7 - with: - name: msg-editor-tools-${{ github.sha }} - path: /tmp/msg-editor-tools/ - retention-days: 1 - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/rust-native-domain-deps.yml b/.github/workflows/rust-native-domain-deps.yml deleted file mode 100644 index 25c01cb1..00000000 --- a/.github/workflows/rust-native-domain-deps.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Rust native domain dependencies -on: - push: - branches: ['migration/rust-*'] - paths: ['.github/workflows/rust-native-domain-deps.yml'] -permissions: - contents: read -jobs: - snapshot: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Resolve pinned public dependencies for the offline editor - shell: bash - run: | - set -euo pipefail - rustup toolchain install 1.85.0 --profile minimal - mkdir -p /tmp/domain-deps/src /tmp/domain-deps/.cargo /tmp/domain-export - cat > /tmp/domain-deps/Cargo.toml <<'TOML' - [package] - name="msg-domain-editor-dependencies" - version="0.0.0" - edition="2021" - rust-version="1.85" - [dependencies] - roxmltree="=0.20.0" - chrono-tz="=0.10.4" - postgres-openssl="=0.5.2" - ssh-key={version="=0.6.7",features=["crypto"]} - openssl="=0.10.75" - TOML - printf '[resolver]\nincompatible-rust-versions="fallback"\n' > /tmp/domain-deps/.cargo/config.toml - echo 'fn main() {}' > /tmp/domain-deps/src/main.rs - (cd /tmp/domain-deps && cargo +1.85.0 vendor --versioned-dirs /tmp/domain-vendor > /tmp/domain-vendor-config.toml) - cp /tmp/domain-deps/Cargo.toml /tmp/domain-deps/Cargo.lock /tmp/domain-export/ - tar -czf /tmp/domain-export/vendor.tar.gz -C /tmp domain-vendor domain-vendor-config.toml - (cd /tmp/domain-export && sha256sum vendor.tar.gz Cargo.lock > SHA256SUMS) - - uses: actions/upload-artifact@v7 - with: - name: msg-domain-editor-${{ github.sha }} - path: /tmp/domain-export/ - retention-days: 1 - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/rust-phase4-import.yml b/.github/workflows/rust-phase4-import.yml deleted file mode 100644 index 2689bfd7..00000000 --- a/.github/workflows/rust-phase4-import.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: Import verified Rust checkpoint -on: - push: - branches: ['migration/rust-wire-phase1'] - paths: ['.github/workflows/rust-phase4-import.yml'] -permissions: - contents: write -concurrency: - group: msg-rust-checkpoint-import - cancel-in-progress: false -jobs: - import: - if: github.repository == 'TokenNotIncluded/msg' && github.ref == 'refs/heads/migration/rust-wire-phase1' - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 1 - persist-credentials: false - - name: Verify exact source, apply Rust-only patch, and publish without force - env: - GH_TOKEN: ${{ github.token }} - EXPECTED_HEAD: ${{ github.sha }} - shell: bash - run: | - set -euo pipefail - python3 - <<'PY' - import base64 - import hashlib - import json - import lzma - import os - import pathlib - import subprocess - import urllib.request - - repo = 'TokenNotIncluded/msg' - branch = 'migration/rust-wire-phase1' - blobs = [ - ('f7075d03c96731f0555c2b341ddf89a13801311c', 16000), - ('85578e5f30d48f7623abfa0bc107d2989ac7965e', 16000), - ('2e9c61ba133df50aaff5faac80efbe0f9c4705e7', 16000), - ('4c33f5f5f3446167ca2dde2d10a82c4cd54a5eb9', 16000), - ('1dc1a240972e2db80898b2c6fe3519858ff6e5d7', 264), - ] - def git(*args): - return subprocess.check_output(['git', *args], text=True).strip() - def require(condition, message): - if not condition: - raise RuntimeError(message) - require(git('rev-parse', 'HEAD') == os.environ['EXPECTED_HEAD'], 'head mismatch') - require(git('rev-parse', 'HEAD:rust') == 'd83f9d6c9e837ba8ccb78c226122874b23bce520', 'Rust baseline changed') - require(not git('status', '--porcelain'), 'worktree is not clean') - chunks = [] - for sha, size in blobs: - request = urllib.request.Request( - f'https://api.github.com/repos/{repo}/git/blobs/{sha}', - headers={'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28'}, - ) - with urllib.request.urlopen(request, timeout=30) as response: - record = json.loads(response.read(100000)) - require(record['sha'] == sha and record['size'] == size, 'blob metadata mismatch') - content = base64.b64decode(record['content'].replace('\n', ''), validate=True) - require(len(content) == size, 'chunk size mismatch') - require(hashlib.sha1(b'blob ' + str(size).encode() + b'\0' + content).hexdigest() == sha, 'blob digest mismatch') - chunks.append(content) - packed = base64.b64decode(b''.join(chunks), validate=True) - require(hashlib.sha256(packed).hexdigest() == '2e57559abcdbbb2fe56791880f9db034a853aec4306acd7f9b0ad1ac455b9117', 'archive digest mismatch') - decoder = lzma.LZMADecompressor(memlimit=134217728) - patch = decoder.decompress(packed, max_length=250448) - require(decoder.eof and not decoder.unused_data and len(patch) == 250447, 'patch size or format mismatch') - require(hashlib.sha256(patch).hexdigest() == '84a79dc86f1fb725c316f4d2139c93e5eaa1d70e81b8b543812c10d385ca9011', 'patch digest mismatch') - path = pathlib.Path('/tmp/msg-verified-rust.patch') - path.write_bytes(patch) - subprocess.run(['git', 'apply', '--check', '--index', str(path)], check=True) - subprocess.run(['git', 'apply', '--index', str(path)], check=True) - changed = git('diff', '--cached', '--name-only').splitlines() - require(changed and all(p.startswith('rust/') and '..' not in p.split('/') for p in changed), 'non-Rust path rejected') - tree = git('write-tree') - require(git('rev-parse', tree + ':rust') == 'c0901a33e6cb0ba22126540be4b2a2fb667ac0a8', 'resulting Rust tree mismatch') - git('config', 'user.name', 'github-actions[bot]') - git('config', 'user.email', '41898282+github-actions[bot]@users.noreply.github.com') - git('commit', '-m', 'feat(rust): enforce resource policy and native PostgreSQL transactions') - header = 'AUTHORIZATION: basic ' + base64.b64encode(('x-access-token:' + os.environ['GH_TOKEN']).encode()).decode() - env = dict(os.environ) - env.update(GIT_CONFIG_COUNT='1', GIT_CONFIG_KEY_0='http.https://github.com/.extraheader', GIT_CONFIG_VALUE_0=header) - remote = subprocess.check_output(['git', 'ls-remote', 'origin', 'refs/heads/' + branch], env=env, text=True).split()[0] - require(remote == os.environ['EXPECTED_HEAD'], 'branch advanced; refusing to overwrite') - subprocess.run(['git', 'push', 'origin', 'HEAD:refs/heads/' + branch], env=env, check=True) - out = pathlib.Path('/tmp/msg-rust-checkpoint') - out.mkdir() - metadata = {'source_commit': git('rev-parse', 'HEAD'), 'source_tree': git('rev-parse', 'HEAD^{tree}'), - 'rust_tree': git('rev-parse', 'HEAD:rust'), 'patch_sha256': hashlib.sha256(patch).hexdigest(), - 'files': changed, 'note': 'Import evidence only; independent CI has not run on this commit.'} - (out / 'metadata.json').write_text(json.dumps(metadata, indent=2) + '\n') - (out / 'Cargo.lock').write_bytes(pathlib.Path('rust/Cargo.lock').read_bytes()) - with (out / 'source.tar.gz').open('wb') as archive: - subprocess.run(['git', 'archive', '--format=tar.gz', 'HEAD'], stdout=archive, check=True) - print(json.dumps(metadata, indent=2)) - PY - - uses: actions/upload-artifact@v7 - with: - name: msg-rust-phase4-import - path: /tmp/msg-rust-checkpoint/ - retention-days: 7 - if-no-files-found: error diff --git a/.github/workflows/rust-postgres-editor.yml b/.github/workflows/rust-postgres-editor.yml deleted file mode 100644 index d7201f8b..00000000 --- a/.github/workflows/rust-postgres-editor.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Rust PostgreSQL editor snapshot -on: - push: - branches: ['migration/rust-*'] - paths: ['.github/workflows/rust-postgres-editor.yml'] -permissions: - contents: read -jobs: - snapshot: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - name: Export public TLS crates and disposable PostgreSQL binaries - shell: bash - run: | - set -euo pipefail - mkdir -p /tmp/pg-editor /tmp/pg-deps/src /tmp/pg-deps/.cargo - rustup toolchain install 1.85.0 --profile minimal - cat > /tmp/pg-deps/Cargo.toml <<'TOML' - [package] - name="msg-pg-editor-dependencies" - version="0.0.0" - edition="2021" - rust-version="1.85" - [dependencies] - postgres="=0.19.12" - postgres-native-tls="=0.5.2" - native-tls="=0.2.14" - openssl="=0.10.75" - serde_json="=1.0.145" - TOML - printf '[resolver]\nincompatible-rust-versions="fallback"\n' > /tmp/pg-deps/.cargo/config.toml - echo 'fn main() {}' > /tmp/pg-deps/src/main.rs - (cd /tmp/pg-deps && cargo +1.85.0 vendor --versioned-dirs /tmp/pg-vendor > /tmp/pg-vendor-config.toml) - tar -czf /tmp/pg-editor/vendor.tar.gz -C /tmp pg-vendor pg-vendor-config.toml - cp /tmp/pg-deps/Cargo.lock /tmp/pg-editor/ - PG=$(find /usr/lib/postgresql -type f -path '*/bin/postgres' | sort -V | tail -n1) - test -n "$PG" - BINDIR=$(dirname "$PG") - SHARED=$($BINDIR/pg_config --sharedir) - PKGLIB=$($BINDIR/pg_config --pkglibdir) - mkdir -p /tmp/pg-export/bin /tmp/pg-export/share /tmp/pg-export/lib /tmp/pg-export/deps - cp -a "$BINDIR"/. /tmp/pg-export/bin/ - cp -a "$SHARED"/. /tmp/pg-export/share/ - cp -a "$PKGLIB"/. /tmp/pg-export/lib/ - printf '%s\n%s\n%s\n' "$BINDIR" "$SHARED" "$PKGLIB" > /tmp/pg-export/original-paths.txt - "$PG" --version > /tmp/pg-export/version.txt - find /tmp/pg-export/bin /tmp/pg-export/lib -type f -print0 | xargs -0 -r -n1 sh -c 'ldd "$0" 2>/dev/null || true' | awk '/=> \/|^\s*\// {for(i=1;i<=NF;i++)if($i ~ /^\//)print $i}' | sort -u > /tmp/pg-export/dependencies.txt - while IFS= read -r lib; do - case "$(basename "$lib")" in libc.so.*|libm.so.*|libpthread.so.*|libdl.so.*|librt.so.*|libresolv.so.*|ld-linux*) continue ;; esac - cp -L "$lib" /tmp/pg-export/deps/ - done < /tmp/pg-export/dependencies.txt - tar -cJf /tmp/pg-editor/postgres.tar.xz -C /tmp pg-export - git rev-parse HEAD > /tmp/pg-editor/source-commit.txt - (cd /tmp/pg-editor && sha256sum *.tar.* Cargo.lock > SHA256SUMS) - - uses: actions/upload-artifact@v7 - with: - name: msg-pg-editor-${{ github.sha }} - path: /tmp/pg-editor/ - retention-days: 1 - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/rust-runtime-deps.yml b/.github/workflows/rust-runtime-deps.yml deleted file mode 100644 index 8ce00f2f..00000000 --- a/.github/workflows/rust-runtime-deps.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: Rust runtime dependency snapshot -on: - push: - branches: ['migration/rust-*'] - paths: ['.github/workflows/rust-runtime-deps.yml'] -permissions: - contents: read -jobs: - snapshot: - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v7 - with: - persist-credentials: false - - uses: actions/setup-python@v7 - with: - python-version: '3.15' - allow-prereleases: true - - name: Export pinned public dependencies and disposable test tools - shell: bash - run: | - set -euo pipefail - mkdir -p /tmp/msg-runtime-editor /tmp/runtime-deps/src /tmp/runtime-deps/.cargo - git rev-parse HEAD > /tmp/msg-runtime-editor/source-commit.txt - git archive --format=tar.gz HEAD > /tmp/msg-runtime-editor/source.tar.gz - rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy - printf '[resolver]\nincompatible-rust-versions="fallback"\n' > /tmp/runtime-deps/.cargo/config.toml - cat > /tmp/runtime-deps/Cargo.toml <<'TOML' - [package] - name="msg-runtime-offline-dependencies" - version="0.0.0" - edition="2021" - rust-version="1.85" - [dependencies] - axum={version="=0.8.6",features=["ws","http2"]} - tokio={version="=1.48.0",features=["full"]} - tower={version="=0.5.2",features=["limit","timeout","util"]} - tower-http={version="=0.6.6",features=["limit","timeout","trace","catch-panic","compression-gzip"]} - reqwest={version="=0.12.24",default-features=false,features=["rustls-tls","json","stream"]} - clap={version="=4.5.51",features=["derive","env"]} - toml="=0.8.23" - hmac="=0.12.1" - rand="=0.8.5" - aes-gcm="=0.10.3" - hkdf="=0.12.4" - x25519-dalek={version="=2.0.1",features=["static_secrets"]} - age={version="=0.11.1",default-features=false} - zeroize="=1.8.2" - hex="=0.4.3" - url="=2.5.7" - percent-encoding="=2.3.2" - mime_guess="=2.0.5" - uuid={version="=1.18.1",features=["v4"]} - regex="=1.12.2" - unicode-normalization="=0.1.24" - pulldown-cmark="=0.13.0" - ammonia="=4.1.2" - minijinja={version="=2.12.0",features=["loader","json"]} - jsonschema={version="=0.33.0",default-features=false} - postgres={version="=0.19.12",features=["with-serde_json-1","with-chrono-0_4"]} - redis={version="=0.32.7",default-features=false,features=["tokio-comp"]} - flate2="=1.1.5" - rustix={version="=1.1.2",features=["fs","process"]} - tracing="=0.1.41" - tracing-subscriber={version="=0.3.20",features=["env-filter","json"]} - futures-util="=0.3.31" - tokio-stream={version="=0.1.17",features=["sync"]} - graphql-parser="=0.4.1" - lettre={version="=0.11.19",default-features=false,features=["builder","smtp-transport","tokio1-rustls-tls"]} - similar="=2.7.0" - hickory-proto="=0.24.4" - TOML - printf 'msg-storage={path="%s/rust/crates/msg-storage"}\nmsg-protocol={path="%s/rust/crates/msg-protocol"}\n' "$PWD" "$PWD" >> /tmp/runtime-deps/Cargo.toml - echo 'fn main() {}' > /tmp/runtime-deps/src/main.rs - (cd /tmp/runtime-deps && cargo +1.85.0 vendor --sync "$GITHUB_WORKSPACE/rust/Cargo.toml" --versioned-dirs /tmp/runtime-vendor > /tmp/runtime-vendor-config.toml) - cp /tmp/runtime-deps/Cargo.toml /tmp/runtime-deps/Cargo.lock /tmp/msg-runtime-editor/ - tar -czf /tmp/msg-runtime-editor/vendor.tar.gz -C /tmp runtime-vendor runtime-vendor-config.toml - python -m pip install '.[server]' pytest pytest-asyncio pytest-cov 'ruff==0.16.9' - python -m pip freeze > /tmp/msg-runtime-editor/python-requirements.txt - tar -cJf /tmp/msg-runtime-editor/python-toolchain.tar.xz -C "$(python -c 'import sys;print(sys.prefix)')" . - (cd /tmp/msg-runtime-editor && sha256sum *.tar.* Cargo.lock > SHA256SUMS) - - uses: actions/upload-artifact@v7 - with: - name: msg-runtime-editor-${{ github.sha }} - path: /tmp/msg-runtime-editor/ - retention-days: 1 - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index 775a4f92..9b7552a9 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -1,4 +1,4 @@ -name: Rust wire compatibility +name: Rust native conformance on: pull_request: paths: @@ -30,7 +30,20 @@ concurrency: jobs: conformance: runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: msg_test + POSTGRES_PASSWORD: msg_test_local_only + POSTGRES_DB: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U msg_test -d postgres" + --health-interval 5s --health-timeout 5s --health-retries 12 + env: + MSG_PG_TEST_ADMIN_DSN: 'host=127.0.0.1 port=5432 user=msg_test password=msg_test_local_only dbname=postgres sslmode=disable' steps: - uses: actions/checkout@v7 with: @@ -42,7 +55,7 @@ jobs: - name: Install Rust toolchain and Python reference dependencies run: | rustup toolchain install 1.85.0 --profile minimal --component rustfmt,clippy - python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 + python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 "psycopg[binary]==3.3.6" - name: Record exact tested revision and environment run: | mkdir -p rust/artifacts @@ -78,6 +91,10 @@ jobs: python rust/tests/test_python_parity.py --binary rust/target/debug/examples/wire_check --report rust/artifacts/parity.json python rust/tests/test_identity_parity.py --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json python rust/tests/test_storage_parity.py --binary rust/target/debug/examples/writer_check --report rust/artifacts/storage-parity.json + python rust/tests/test_authorization_parity.py --binary rust/target/debug/examples/authority_check --report rust/artifacts/authorization.json + python rust/tests/test_authorization_parity.py --postgres --binary rust/target/debug/examples/authority_check --report rust/artifacts/authorization-pg.json + python rust/tests/test_postgres_parity.py --binary rust/target/debug/examples/pg_writer_check --report rust/artifacts/postgres.json + python rust/tests/test_postgres_process.py --binary rust/target/debug/examples/pg_writer_check --report rust/artifacts/postgres-process.json - name: Optimized native differential checks if: ${{ !cancelled() }} env: @@ -87,6 +104,10 @@ jobs: python rust/tests/test_python_parity.py --binary rust/target/release/examples/wire_check --report rust/artifacts/parity-release.json python rust/tests/test_identity_parity.py --binary rust/target/release/examples/authority_check --report rust/artifacts/identity-parity-release.json python rust/tests/test_storage_parity.py --binary rust/target/release/examples/writer_check --report rust/artifacts/storage-parity-release.json + python rust/tests/test_authorization_parity.py --binary rust/target/release/examples/authority_check --report rust/artifacts/authorization-release.json + python rust/tests/test_authorization_parity.py --postgres --binary rust/target/release/examples/authority_check --report rust/artifacts/authorization-pg-release.json + python rust/tests/test_postgres_parity.py --binary rust/target/release/examples/pg_writer_check --report rust/artifacts/postgres-release.json + python rust/tests/test_postgres_process.py --binary rust/target/release/examples/pg_writer_check --report rust/artifacts/postgres-process-release.json - name: Preserve evidence and dependency lock if: ${{ always() }} uses: actions/upload-artifact@v7 From de810d15da747b7be34b64f4e2324e5f5e7da6f7 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Mon, 5 Oct 2026 18:51:17 +0800 Subject: [PATCH 16/22] ci(rust): export exact merged source for native migration tests --- .github/workflows/rust-source-checkpoint.yml | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/rust-source-checkpoint.yml diff --git a/.github/workflows/rust-source-checkpoint.yml b/.github/workflows/rust-source-checkpoint.yml new file mode 100644 index 00000000..079b500f --- /dev/null +++ b/.github/workflows/rust-source-checkpoint.yml @@ -0,0 +1,32 @@ +name: Rust source checkpoint +on: + push: + branches: ['migration/rust-wire-phase1'] + paths: ['.github/workflows/rust-source-checkpoint.yml'] +permissions: + contents: read +jobs: + source: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + with: + ref: bfe46024ab7d396f979688c57077d2c2dd3c0d7e + fetch-depth: 1 + persist-credentials: false + - name: Archive tracked public source only + run: | + set -euo pipefail + test "$(git rev-parse HEAD^{tree})" = 56511c906e4efbc7097aa1262c944f3f6824471d + mkdir -p /tmp/msg-source + git rev-parse HEAD > /tmp/msg-source/source-commit.txt + git rev-parse HEAD^{tree} > /tmp/msg-source/source-tree.txt + git archive --format=tar.gz HEAD > /tmp/msg-source/source.tar.gz + (cd /tmp/msg-source && sha256sum source.tar.gz > SHA256SUMS) + - uses: actions/upload-artifact@v7 + with: + name: msg-rust-merged-source + path: /tmp/msg-source/ + retention-days: 2 + if-no-files-found: error From 0ae08f64993355adfb5b65f1fedbd68f81fe3ec3 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Wed, 7 Oct 2026 04:41:03 +0800 Subject: [PATCH 17/22] ci(rust): prepare pinned offline toolchain for content migration --- .github/workflows/rust-source-checkpoint.yml | 29 ++++++++++++-------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/.github/workflows/rust-source-checkpoint.yml b/.github/workflows/rust-source-checkpoint.yml index 079b500f..c3d63205 100644 --- a/.github/workflows/rust-source-checkpoint.yml +++ b/.github/workflows/rust-source-checkpoint.yml @@ -8,25 +8,32 @@ permissions: jobs: source: runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 15 steps: - uses: actions/checkout@v7 with: - ref: bfe46024ab7d396f979688c57077d2c2dd3c0d7e + ref: de810d15da747b7be34b64f4e2324e5f5e7da6f7 fetch-depth: 1 persist-credentials: false - - name: Archive tracked public source only + - name: Prepare pinned public build inputs run: | set -euo pipefail - test "$(git rev-parse HEAD^{tree})" = 56511c906e4efbc7097aa1262c944f3f6824471d - mkdir -p /tmp/msg-source - git rev-parse HEAD > /tmp/msg-source/source-commit.txt - git rev-parse HEAD^{tree} > /tmp/msg-source/source-tree.txt - git archive --format=tar.gz HEAD > /tmp/msg-source/source.tar.gz - (cd /tmp/msg-source && sha256sum source.tar.gz > SHA256SUMS) + rustup toolchain install 1.85.0 --profile minimal --component rustfmt --component clippy + mkdir -p /tmp/msg-offline + cargo +1.85.0 vendor --locked --manifest-path rust/Cargo.toml /tmp/msg-offline/vendor > /tmp/msg-offline/vendor-config.toml + toolchain="$(rustup run 1.85.0 rustc --print sysroot)" + tar -C "$toolchain" -czf /tmp/msg-offline/rust-toolchain.tar.gz . + tar -C /tmp/msg-offline -czf /tmp/msg-offline/vendor.tar.gz vendor vendor-config.toml + rm -rf /tmp/msg-offline/vendor + git rev-parse HEAD > /tmp/msg-offline/source-commit.txt + git rev-parse HEAD^{tree} > /tmp/msg-offline/source-tree.txt + git archive --format=tar.gz HEAD > /tmp/msg-offline/source.tar.gz + rustup run 1.85.0 rustc -Vv > /tmp/msg-offline/rust-version.txt + (cd /tmp/msg-offline && sha256sum *.tar.gz > SHA256SUMS) - uses: actions/upload-artifact@v7 with: - name: msg-rust-merged-source - path: /tmp/msg-source/ + name: msg-rust-offline-content-build + path: /tmp/msg-offline/ retention-days: 2 + compression-level: 0 if-no-files-found: error From 71ec74e484bca856da386174429f108d721b8f89 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Wed, 7 Oct 2026 04:42:06 +0800 Subject: [PATCH 18/22] ci(rust): include matching Python oracle in offline test inputs --- .github/workflows/rust-source-checkpoint.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/rust-source-checkpoint.yml b/.github/workflows/rust-source-checkpoint.yml index c3d63205..ce0b98c4 100644 --- a/.github/workflows/rust-source-checkpoint.yml +++ b/.github/workflows/rust-source-checkpoint.yml @@ -15,14 +15,20 @@ jobs: ref: de810d15da747b7be34b64f4e2324e5f5e7da6f7 fetch-depth: 1 persist-credentials: false + - uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true - name: Prepare pinned public build inputs run: | set -euo pipefail rustup toolchain install 1.85.0 --profile minimal --component rustfmt --component clippy + python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 'psycopg[binary]==3.3.6' mkdir -p /tmp/msg-offline cargo +1.85.0 vendor --locked --manifest-path rust/Cargo.toml /tmp/msg-offline/vendor > /tmp/msg-offline/vendor-config.toml toolchain="$(rustup run 1.85.0 rustc --print sysroot)" tar -C "$toolchain" -czf /tmp/msg-offline/rust-toolchain.tar.gz . + tar -C "$(python -c 'import sys; print(sys.prefix)')" -czf /tmp/msg-offline/python-toolchain.tar.gz . tar -C /tmp/msg-offline -czf /tmp/msg-offline/vendor.tar.gz vendor vendor-config.toml rm -rf /tmp/msg-offline/vendor git rev-parse HEAD > /tmp/msg-offline/source-commit.txt From 17f378cee9693f660a4a9c96e05e4ff832aa923b Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Wed, 7 Oct 2026 05:13:29 +0800 Subject: [PATCH 19/22] ci(rust): prepare exact tested content tree without moving refs --- .github/workflows/rust-source-checkpoint.yml | 101 ++++++++++++++----- 1 file changed, 73 insertions(+), 28 deletions(-) diff --git a/.github/workflows/rust-source-checkpoint.yml b/.github/workflows/rust-source-checkpoint.yml index ce0b98c4..3f987cde 100644 --- a/.github/workflows/rust-source-checkpoint.yml +++ b/.github/workflows/rust-source-checkpoint.yml @@ -1,45 +1,90 @@ -name: Rust source checkpoint +name: Rust content commit preparation on: push: branches: ['migration/rust-wire-phase1'] paths: ['.github/workflows/rust-source-checkpoint.yml'] permissions: - contents: read + contents: write jobs: - source: + prepare: + if: github.repository == 'TokenNotIncluded/msg' && github.ref == 'refs/heads/migration/rust-wire-phase1' runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 5 steps: - uses: actions/checkout@v7 with: - ref: de810d15da747b7be34b64f4e2324e5f5e7da6f7 + ref: ${{ github.sha }} fetch-depth: 1 persist-credentials: false - - uses: actions/setup-python@v7 - with: - python-version: '3.15' - allow-prereleases: true - - name: Prepare pinned public build inputs + - name: Prepare content-addressed objects only, never update refs + env: + REPOSITORY: ${{ github.repository }} + EVENT_SHA: ${{ github.sha }} + ACTUAL_BEFORE: ${{ github.event.before }} + API_TOKEN: ${{ github.token }} run: | - set -euo pipefail - rustup toolchain install 1.85.0 --profile minimal --component rustfmt --component clippy - python -m pip install cryptography==46.0.3 jsonschema==4.25.1 ruff==0.16.9 'psycopg[binary]==3.3.6' - mkdir -p /tmp/msg-offline - cargo +1.85.0 vendor --locked --manifest-path rust/Cargo.toml /tmp/msg-offline/vendor > /tmp/msg-offline/vendor-config.toml - toolchain="$(rustup run 1.85.0 rustc --print sysroot)" - tar -C "$toolchain" -czf /tmp/msg-offline/rust-toolchain.tar.gz . - tar -C "$(python -c 'import sys; print(sys.prefix)')" -czf /tmp/msg-offline/python-toolchain.tar.gz . - tar -C /tmp/msg-offline -czf /tmp/msg-offline/vendor.tar.gz vendor vendor-config.toml - rm -rf /tmp/msg-offline/vendor - git rev-parse HEAD > /tmp/msg-offline/source-commit.txt - git rev-parse HEAD^{tree} > /tmp/msg-offline/source-tree.txt - git archive --format=tar.gz HEAD > /tmp/msg-offline/source.tar.gz - rustup run 1.85.0 rustc -Vv > /tmp/msg-offline/rust-version.txt - (cd /tmp/msg-offline && sha256sum *.tar.gz > SHA256SUMS) + python - <<'PY' + import base64, hashlib, json, lzma, os, pathlib, subprocess, urllib.request + repo = os.environ['REPOSITORY'] + parent = os.environ['EVENT_SHA'] + assert repo == 'TokenNotIncluded/msg' + assert os.environ['ACTUAL_BEFORE'] == '71ec74e484bca856da386174429f108d721b8f89' + expected_tree = '94c6c50eddeaa0c37bcd416417323c1eea8877f3' + patch_digest = '2d82567ddbad9e8ebf7ca424928a3815e167eda6319fc687eabfaf56fbbac7bf' + parts = [ + '{Wp48S^xk9=GL@E0stWa8~^|S5YJf5;Z|T%bzJ~58c7Ml{1JUn-U?4g%<)t4wb~W0)DdL_LWmAQ>GJ0ba5)+?(zF^xc^T7Qho0P!z95uHS9@E2K*z$j7w($GC6F7tdcMf3=nZoVx*h~o$T*e%00a;PL1@_YaH>Oy<(h$x-(lm(fir41Q#MxS53}9roE?f?Qrl`SF(KV&sd=aayj&>3uyW}tB5=+c=}xNPBnYvaF>+|kvJ+}ZPzD6Z^6Vb?jKp2aYxYzDiN-pqX7_zqDt`L0(Vb4j@}@5JQX@|gy3Ga|-lt)j&)MQ6M9FD&(Wyj(P=%(1-Pi%vXc_skl4ryy&xfgcDJb{5e}f&_bHGb~-`FsaF6Jh>^Xv5jKQd&+$ted#Fx`r-~AV%noKV~NYPPC5HdUz#;*wd-_}3(o1e?g%&_DakaWGDBR!9!4EW#?EOFo;OIaFW79ndi5}}uuw|LpeBn2r`ZqKAdrcz9rXfYDYKX~Q-jQJ7E9S8l)=rkY`JfvI6G54+{0>OLNaj>7rD!jo36z8WTE?lp}{EUz-%0v{LZO$EhoR(%$ouhj}gzKf|^6d+NL`QwDbZQXN2}IW)0c%PH}~2bGa@-*9ghx(~=`IOR^7+tsZM@J;@RIwQ=0Q;pSe(`=>~x{1#r5{B4$leLyDn!Ebd5Z&wLWlrruP#`a+q$tn@KWSwe&i`f}Q8MH|{jGtJ^ifFs`f3$KlALRe(~RM;-~|k)v)T$Qni{a(aDrd_iF%z<+a2vtt0#Vdp`?h)I(c3m&0%Py`FSV4R0{Te$`KAL(1~2z^y-Z=TpDXZvj8*03wE<4SE4c6=gc!v=}>$mZR*?%^W=KDnh?_KSWm7H=Lq`$M0XZ0h>t&eXS;mfsUPV4RT33t0kMcpKJNlcxCfaflp7lWNlIxr+h=dvlo0u4T(Ir96Q(TE%$dhMB+kt{IrYAIj)w<@&qk7Cn`&+>qQr>Yg@}XEr0eKnu~6eY2;t@B&}g8KmKf6bq{ZDCF1^z6l5^*HZKi+U*ek=9lQ2eaO(&aZJ+3e(-4X9h9yO*7o3$P7+amGEcsoSo4!0J#IKr#~|j@9&_yS!*{2oOuhRSNVO`Xm8$(R;Ih^2smad=$-1GvsMj*V{13^SBTezarr5}qVwFgLyit}kY0XLZavno-nXzAKu;;HuWD2vulGG&XSv;J}+2#-1K)JQD%(7&4`k>_7{~>@poehd{q&V?ssS#UB}DM~hNxot6I6)@v-mBtbZvJogr{?x<>#S&EkdMnRK_+t2t)_db9DORG9$(?HuWc8kY+7PIN71U?H`J}r0&vh+g7^EY*pWmqHS4;$^1G5@s^EMASd=@RDFA3QSYfG9iRy1PN1l325|vH|tdRA`s-w(+W$^l*_CbWBfuh636b|1rxR%&p2jHrSLt?Myqt?3K=M@{rY@lkU^5(7b=412DCe!4F$2xzt0>3aM!G{~{0R10TD|v=w|)_8$FLc{-T}tgn^_Fm;vWD_jkNDbK9#jM1mG{J#Gbu{)478rG&+hsH-cBh(Ed_rF_C#vRlrA-h+YZZW@XR$U{I-5Vz~z+73aV3s}<1H5OV?bKd6ruaqQ%C9pO&$UkHl4FjBR3wjot$2D<~x(?%PN81@0D*+Q&{UO^L`|gBAS58HSHPeE`0Ba5u>BPMKNOw8Z!-vp&GKs<~Nc{2&Q+WU$@n0gCT6Oh<_KUV-W;o5efAWRzwzGH{xHgnR>0tnjjsB$?*k5_1wu}BG7E-_MSCB1ktpHsY=5iyEzefb-xoBf;%u3I7Ln{9_{L#}_&?6@+Ns@x^VsETH{~ap^8S=8wV5s}jSIEsGZJIM+`BTj7Q)65iB4T({e3gYNEUY}0_X09gauZ~Q+f|H38HK-to6av=D7jvN2b|yApD(Wac+P_d}Li8eY+T|nG#57-W?I^S9QY9pt7Gz<(iD|ulquQ1G^wUDSsy(ZBmiX5cz}RCP)rt*)P3KZe^SxFJ4Zevs+X%%h((pX1_7i=99P!+%?#vCPPIw-!c;30DMPUqlR$xGPNd(hq+5Frz=}UG$w~Qg+=4LMLN#DPQrC0t+b-`-iSlXbCaBeg@EcA@YUh9r74nNMZ{5j5wtH}y;}Tm%;sd2dT=+A*Hpq_oXYaB-NYC=TWzPgok5Pasy23lpvQMusJT@kJOCids2xDi?X$?LD_aj7-wU(+_?Hsx{Bsj6vb%%V3-7VnGU3&V>XmuX3M&~Moh;S#^I=qWhR^xZ<~YkV5Jw$5k=*5J-{)-L1&qQV?xl)5Y2631M65J9qrQhf*F7qZVW34+^6iPm8>7vB`yBv|h9z*T%t-LrYf5grR0_M7)BZP1BSvz0W)mG8yxjy8M;%Uo8wTB}K&<}3qlc-q$Ya8GB|nmCRMf=ERFvK({LaC5AZ7X5rIf9aDD=7O91App-=f1l5@s2%FK*7_7b1)KCXCwt4)S{qfpC}7hW%#{!M<*>O(4QWK!MwFmewXz#s$IW2#B({OzebO!1dvID?qI+y{mxsc?<&oS)QdNg`H=DlIV{{!Y@M!0qeubb+yh#q#U{Nbv_@?MUEQRh*QecNr9M^^DYpX}B;(u^EzoOVwJfB6UNbxQ&~KvAmczRk^2)USSfq3`eU6m6>o<-WxP&r6UAUNhE$E?5dG4zo&ummU{nlR%=BBDE_ngjoPfkt#pme6{FF=J`>paw`{sq{W@Nd!-Pc1!hw}{@HWb9X>+m3t@mo6~dn}LWR(gM^$AZnMuiMIX^qR_82I8a{vUu7te3!#y%FNLx@Jtg7|Fb&gA*-yq<_ynZJ_FBl+e9wVIx{MWW9RL3{wqvZ6E_{T>ars~(kEm@{QK6h8?-Oz*5sh&?7;6TdqT2dPP-|lYnm3@J=*pg&QD`#gYe|TaC=fP|DEz1f^ovXS9kqmw1=wJID_`>XjeX@uX9?1lv6^qvXnO-FU&I6g$`7QxaU+Z&K?T<|re=f6)Ir#7C909mrB(X0i1`uonDbl}ESmWS{>GYBUv)iUV7*u+D$FS4agl(uRVb#k1$;_wCF5;V^MFK51#CwCyHfgGx;a|mKd#JKO`51}^GqDcn`jvgd;~kX&>e)S4R>s%{EN*U~oaNX*Gbym7jvw~i4d%qb6>UlPa-{psEa;Pde@?OY`+uA;MtO${;s(a%9^owxv?YoZ2_NK9GS#Nw)opIyt@}mXrTsIW`vfk_v&s|8CBm_ViggF}_8H7WB*UvcBz`nUEGRP+i(~k{@Vd|%unqkAC#`xMH~eYwoa0bwRkmv$(i#|ZhcQ-lwtETY9U(U!2ra4XQj!KT*$j4(?)I(48-dQ-q}}4^1l-)b<(Tb{#{IE?4TTDV{#yWde`x}9bIU0ae4MUy=u)`U2Oj;`*A-5p@8)eEL5Qd3AdVJJOg_w5Bp`ZS^r;sF@fWPsESCEkP0f`1btJ$qxBn9zW}SbynC6b4w}BWV;mai3&7POfd|q?1SlcBYggP(0CcISaL+C;DoVazvfhc@eWaO}En&5i2I?#?PygZp1xRW(I7Eop2X?RVAnfLfe;GtTQJM3T*SYznI{)JBvOcGY)eRfR4j*OS0lekTF)2r>@?QHsvwRYa$c-aF=Q>nA~N`0DDW!+Y4)+x1JucDE^E2)}69c$imNgUZDrR?rMPaBt)q<6%iWXLkr%n?6Em6CG=UpJMrJDRJk)mLyCIG#&pPW0py844)Lf998ZD9@^K0`>!g;SkibC5A*O3I&7e@#%ptBIT}>l3koYFv-C4f*YNZ)UzjJ2Y8e4aC!WYixo)pNAKYd9IRXGm6NvDzZx+{Wcjg|5}sC^)5ZgQkm(4_xzl-+HoaHCx{J_R42qeS|`_Espb{h-H1-BIu}F^wIGMi=j{!%1>hzZm+pFn$*V|@YPq~JZYvsE~PC9S55J&k@XI%eW`^^*uo3A`={a=O+pO1Ab8^)W^L9y>a*TV>zNH0aT2L|Lp+RCCg6(mqM689Q$2?OQS@gc^^6J-kECoyCHCzTC))0#%s()+5=eZG~mP!&0*z|U)YDJ+$#F`P=Zo)ik(ByT)a+_`;+R<2i$#o`D4$e;kS~usQUpnd-l|?taCNb$%^v4iRQNVf~xbnab%&CmU;iw@_qbTfs64LOIrUExL2!$f9|smY~_w9+dd*&%UM7#!>Ipv@>=@Cc-$VJ2{{I8j0kwiMAh(27Xh=ni^zc3@>8ZpH)HCu-tIP2{Cj`g6LY!UEjEE%3OQe{_fTkD7h}wE!JLM&kzx=!Pcoy%u*`V>{fFsMEc1j@NgAc`JXx-g9U|V&q-Fa8sWK~PI5we8j`v~#LF{eRb&aZC%&=>vSXpCVrWy(|DNl^rwO8ZqV1h7nMu2)cW{n{Z%XRjXm;N}tHT#$5J6a8@exx0$6=dwq1$~-CqRagh=C2jB5k9U9~M@_)dH2>_|%GbL)g4Y;o3PMZMN5?I7n|T=DUKwN|}WzU=E=?Y%!j^-7H4l4Fh!a8{QGSV&p+Xv^vp}5IcZo2X?Z@Shq$i%?)1R73FCL50z^3BHt)Nln#)naPdkdxQ<#HxwfN*D-`oX-Cw9b*CA;U`x2WtyvRXq#Z3Yip$w^@y**wBs7H205h#^%((Iev$uq%6$X8PL=L9WaPWb+m1nPbl#iRJ4xq7)(JA3dj6TqnF^4fE|-c>GeoJ`?nvJSsY;WX<7nDZ@Z#<(N+)$y!pa&A<^;_;qzhTCphsM!Bo&jA$S(@sALs`{1Kd!r%p}lmdI;JYp@MAKl`&=>6WSI)_F8?sE@IJ{PCHOKEBg2S(U4_VKUS+S-R?fix(sA2z^7&V=&SRKp2bzsnh!h6(USw(?vRY&{@QsDNiEr1^Uj7cIa`?%E=|-Taujs@%AeAIE);ojfzHM#+Z1V)pGrRfrr>qme%|$~r?RCgf^Em|{k>yP_F~l9`)(0q|)_?(1{Y7!=wY^b`*GP*J&8>vZTL;`3LQc{yT6>{^a>5Rx3qO7>u%U^o#{HOQBtpm6Rj|jaN7x96JI~;Ki*-3N1%=2U9b%8S<5gW{6cl`BA>{1OC%eJDTM4s)d$UZ$m=7^rVD?#?PWsm)Q3hlhi9*Z=h+n%}-?cl*G&qDbRG0gvS=iM8*D$WuNRd!sl=itdLveFeR=LXC=@yt}mh>mbkNGO6*>k0|+ZlI;Qm51MXQP7kb5@6k(-I@+{~z#2Y;cw%I)hUdSlM_?Kqx2oThYv(?t3E<7VdIG?7+K>b!Jz|eYOv^${@Nj(A*t+c+IuG4%UQe7!okS)(uTA>5R0CGDeX;l0r^y#Ey^|?RY{(-h', + 'MQneOek{u$U=nIua;_%MT#ctY4izd1}21p1R{yNs0b$Hq*BO2Ha%+5H+6LjV~HkwT44{r1t_BbMWz?2(52o(B48;QOyqawYj+mM{0FXcru?!0K9ZE86xWFdPX3ALdt$4V+x9L>g#?iwz1UoNZaM3$9WhhCBt-Yh)F*V_LVhItCB}e;|Yp$+l@>r4O;Repybr=c4RHfTVA&z3RB7E8_pL0EbEzf>!ROgIh+7+s-g?lh%4heK%`pDYXG`+mv)I{1%hDmunHmmf(lBE9^o1(J{z5OPuiH`v(e%YQKiPtiD8S_@HI5L!fI^hfSKJ4QA8O8h1ogFsBIQzQDCy9f~1{dz=cMTwzaLM~+7cL2p^Ij)cWrY5Ib0FO^#hTJpjxhzuP3okF<7tUhEzmomr`czSq}|V@;>@myOeWE9q|2`53F-U)t6{vk%b5Kio6?Y5e!XWglv{GVcPAd9PMV!c)<&iHY#A|`^=&KW*--!jgdhwYE<%Kvyv@3)&K1`nabV$NV8L2`S6;3Ybu5lsK;*N`qWF}`x;;ERS7+_YW(B-PGjCnf4C@`6%zpb@}06Mc-9z(lfk1_uj)d$rq+{KuxzHX#|xbl!n6aGMCkibWKt9tWsr=<@_uy>3P!$B>0ry`SckXT)dzR`B*^RSO%WbCy7*i@VL49txDaw6I*KEvT`F^22cev$wS9SY<6ja*=ZnTrn_cL0_FmvyIw{yO;=SXW(NlMw^Qb(vkGJfOhR>CvUu3?o+(g?mTDw{A?C-JH621V|Q(vSU$QBM9*(?~Fuom{BLV2F;(U!x7vuq1A-p?sVtE_`Q({zrQu$;g9*s@TXtUmNLpdSUGkx!}Ys$K$Y3i~i&CbUHA^>$!0oe0$2fcUYkHcBKHJ!VaP)S;<_N7=HQlnUt5J8$H|rK6xGPk<3JMIO{ZHj_NTmAlNV90-W-jAr9;t(2}MkZv<)x@8#t`%L!S>+nYZUOBC@%2`0r<*E0xRY=AhEy9U{%NqC%{CldDyJ1tEm!4?VL8E5xMetXzGiz&c-Ox1zF?ub-EBR55%{lD6ztnWJ(FT`T;D_i6+oLe^HFBXp_P%<;{_{R-BV0vdiV`>mPFdW(6BT=_~v69~d+GCerL^4@&zgv~;peiIK%AS}XqJv0X@=N^T%3@Qq%G2vlB=L1g~!8&|3hdON5yA#tuVl8nfdpE)Pa%iYTthiJ#&8gNL%P7HVxm(v$Q!K;i!<0@Rze@G@gYLRL9#fxX;Fq-yV>yW7D(s0?G`jRBC&r?BEtp3>*k0mAa@|Y6J$ttbwTnUBTlbX+H;7}Vn+iqzeMEe4_IN2&6cWh5HV70q=dgLjdZ>kq$tvgU?c;8Qpn*w1^yJ)@t@ldfVCJ1KkTl+=>mjC(nl--2a9BOvxmN!bzig}*Dug-T=*7>l~{=2;0$e$;iQ{1n~Vdyor+(T$twnKt~j32RzJ0&dJnnC^PQf@04ab$=|HDE)o;lFvx$Oa<~N5?~^&D^_Opz$3pT+T{HdKNX}aw#W+Mww_jonjQkJuWvd3!VP=#WUORqhl>52yKSlpI@bSDJvfzYXu~87)0qEcb^)w0$-3Ah!~S5L%KXSn>Yy`gA=B6Jwr%!QI<5;d+NDRC3h`|CSguhVhCB9}tsEPDESGsWps59izWme)&kI)An@Z>jeT*_KAX>g7#u!!ZXW5ao6w-c;oYX=F4Ue>8U$p+@H%7BtdnA#FqA0_!2#|%L;go0JIN@-YNzH$b1Q6^4$r*7!DhFCx%#fRpt@Su)8V|V5$IjY-GB36N^z*-hHi|#9yy7z#oBnb6l*7f#yb+9nwK=N$4u6?B|v4SSG#hXoTGhAa$hnV4B)dB(A2V^B)^RdQvdLXRb@aMNAjd)NbPi(M(2)P1IPy>hGI*q>Rf+>`{Uj9$^ty3C=ff_XLh4(q!GXZ~UtBY5=vz_?s8j|AfEe;;usEEN{^AkY5pO94FDa?t!kWgQ-^O7R!k7ai_3_jBfJl3D;43#EJd>?iaQn#KK)I)t>z)6eji{x|mPY>a9TtcwPi!j6)rS`w~T+Z6ppuOJLNO=x@-eTegfpYI4$#*g7J*ve!`zS>2^%!icc$L^@Ijgg?r_;U-kr3*=U&t&yhIt{ITkthm1%bF`#i32_(%H#3UwJRymzZXq=Mj<`xSLVi)p)KqOFVTRB9y^eOO%ubSWJ$w~FwHOl#1v|T8)2KmzMl`^w+EiieZ!_fKVfdqu{WfuCmp#k{M)0vUI~Le^(*%b!T+v%MuEE^bUY>Wf@;+o=GlqD4fMe%#@hU|22ON(EExMc%*zTp^=p_8v(=Rw68hbO{%t}Kn*L}QNWurLEi+1RJ$?>cEL?3PWl75^=9{a@+l~SQukfX_eUlj9g!=KR9(@a=i4p|L^|>64UShFnh~W_VI5pN@HV*R5eP?iO-E;E8Q*pU9;ByvpLX;K;2ZjTM(&`}!Gc@>T7?WQ@j0sKYX15~F^<^bc*R;I2@?fQx0EhJxgZ8@~Z=5>HcK8}BYut7&PO2`U<74ZrB$WvwhEnYJ&>SoS*-Y;(h^OR>DdPtj%UX-F$J|hVgDE2gOQ9~aZ|H8@v)SFVC#jw^0IQH_sb}mT@#b;U~ywLV$9BmZ3Pi2Q|rpfTPPNSxr-;K{{#hJwY87|9!V4mIq|?!FEaIBuov03%DRezC0j3bx;z%Wf#@ttJgFY${pZT3OLpU6QVNvcn{rd{jkY69;2&x5skSsx&S{{sFr+L|?*=o#pZiyy~w#IC73z%T-`PEt}Fy3}Wc(coyzi1XSFzpF*&QWLPQHA2r6=D48k~_YCcON2Ys|3)Gtpec9_!EE_+AI1GuvX-yw-`#|7%@jl2mX&Q_ZBXmo4x%jos0Lb;*jbgwz8J!6SdsFHod7U6K6X0MEcR1s>qfCB>_#d`VwUrIp)d{f#sXKGHNB&?U5oD&u3`Tg7!SyGFBBI?^Lrwa}T*Pgw75C_%KaQ*#gYJ&FiLe4QDV17YX8Jjjoz!YNhwnw!r~Z}iN0N92?2~mkj0FSW1fjYZy>n%zD#cPJZ}%0bR8BSv{Wb*=QqN!+1b=Ab-?Os(o<9{J=sm1A>-@{yWJaH`ve3n+aXbo>@79ZEgas0(;(RPBLM=9WK@L7fC^zIpWP!8W{6z=T>=-Q{3;?osJfJiRXI9=qG~W37|}RQN`$!O-F<8ce)0YnRZq9htyM@_lVqR{+p|h{0iBqqKRVWC;YjRSE{?*gBiT)6McCl>{3IPTPyBnKFzQnuNs8$8fLIbLyndcqO8Lz3J;Fr}}bV^XK|gqmBBn1udywsY?H&_IOROE#GmjWH!*xqa%UdBA%Z`u|_f~Bmi7;j2hpgP0bD~swo_YW18Zin3Z;to*uoLhOQ>>%5%+nP;Z*peh7+8HaCL~!cLL<``vN}+rz#~{dJ&QbJv4nE!N-sDY}^QH2mjbu+FfW94snz&&J)TbBlZA$`Uhw&O(onWv`0wwoPBRiG&`!z~C|!pu&>-Pl!$96Tf>z;YtX_=mrf}F+I9>9l+6b@ZfZxs%9#oFvkbKW3I#_gO9I|Gto2g4KZp>X}I2OfX$-TnvP7SdKT$NK`c<|BW%>vn_uET;Ko{)Z^#AArKnPmo>n5XmXr}>=$GoJDG;a<2ywT0YlWZgCDCLpWue4K#g>!r^FWDrI_Mos#+T{5&9G2%!@QFkBYcqwDnom(0jNyd?Fm5%sngo7=LTunkx7&>kGL!JH|739;ulg?v6n*237XRcy}5BJk{oSu~a5&`QFt#A9L=#r3!`u>)pgYxeh#xoQMcko#{bvk{MzRH%h3HhIMTVd(;g1m$EyMsb|<;0t0tnmjSm{)~!imfmJr&ls@2-+~w4ivr#kL@3hb>THo;c5BU-^>*N*GXKaisPNqi_g=fzkLeYdNnTt|N$LY>VAzvHckFqW=x1WJT^D-}tY$OLPYxGm2A_w_*4h*mqnZpn-X%X9K+n2nk`8*O05AQ)0T$yQr}Vgp;DUG)jBNuAM2RX^_!);A-G|=xM!E!rr!KP~~YRZ3{tIAf@#NcGG}U2BJKxY0zGT+xw6kW4IPYO~`3k-z@7&pv9~%`)p8>`s!!$xky}UE}gJ{Z_LTKw`;96c^A~AYrQou4$oEeiX3XWyqMXYbeM|d2iHvt3%^erMuEAyUbQZWc)s=dE+K~>@n$~^ih6K5%Ux`6wDCl+MJ2x^w(YzxH;CNuOJ^H_&iRF|f1|W&y`08H09jiHOIHKI!m$*ZtnGe6&Sw3pfRE{K2KRtNv~VicfZCUww!}pcjaUD0wrA7R;pUI|F;~M9;?O*lGAL=$0~LB_1fc?iza{_eL%@no&h!mp_s^VZ#_PDL?@-SQ9yUzuxT?%rR@Qgb!cnJdFWLd|AEh=@jBZYB;dbAe`!XK%jFuyoQ-LjCbFdF_^-qflgCy&o>kU{`mJA2%65Y%APjg(GR~9TRK;izP!FxGB#hm-Bo$3KJ$cE*DDW~Pjc0o!(nKhu38IFrTT%B6!@1pmW3b3UY%!uL^Hux%`&7N3=I-P3EsHeaHiH18C~bK>8!6lqV^pfPim}s5K|m0>$rLXcxU_|H)fF%D8g>dm!24JP^jvx=rA~39B&3PhaT~&VDDHz!#bY)aA#pfEDC&R9gA^jW1cKtTaGikT6$>vA-1a@7xdD=9OPymfT1Q#-tyS5CA-6}Ua?A%b)xT{e?{&=YJFMaH~Z0NYk=Sp**gPN%%0NbSua}8UO+-w4tpps~>NLpWI}J+3?@sdQ@K>O}0~zy1Shz{nHc1yjkuRz}>%6n!&zpp0`^6zD_&DuGzs#B98neIQfrwrV2_wVvQOYqZ?SPWWhlyY~3Cih~|_At9Opd*bRki#b3nx@LAHMRxCau1RV%vC%>SVf9CF9Yh($I<<#7%w~*(KBE(a8SPkiQy&z1f#^Fb%y_L)pV1Z(HC|}*n}6iY_yH?)(FwDDW(sVm?937O{lA|w81^NYn!TOf33c@=2!8_IFMimsPz(GQP@LK=WP=+AF0S3tFx5ol87u`Y6E0u|9)hgO=G*cHr(TufMoBJN0YlKnZix7wh|#IZ!b>hBUNpB?q)@p)lk)Ywe4~HxhruuB6L+D-|HB1wV>5Krl&6oKs5MLZI5pC)RC?^RTOmcR(QF-c8*`n85W$v~23qgG0nD>ft2WqQU8CIk=OBp?gp18Tf_5q!Yfc@Jvp5bB1&|X`#lvpol)esAI!hqxEuw9c)$i#DR;ED2qTsapO6XtzNJDkMT?_fq90N*c4H9PAgc}=OE@?TWv5mE>lPg^zc2dqk>vkMsFvS32+cGu8=(yby|&UT_+r2`b;UK_i^^@|n8-W7a_No4q%I@rl=BUyo}LlnC8P#EfZYImlMJh%}X|Dhor(*a2Z5!xQ~(XIrw-2?fJtZLAHHqMg1lwWm)#akAsg}YGQ=#Uoj3`6>bPSH}=v4`vpO$R-_rE5H&l6UGmGDYKo8)>gHgep9*@^amh27|H8Ud1Xqdcfdl2>Ph=>N(RJ*KHk>?GIcb*8s|s21Co?;JEGdD<}Rjfc<>&f00C+HQAwG`h?)40Kia}e1eUwc}qr-{ExWFS3ucdR%T;a{AvzzSt=YXl|Diofmhoee270PhhB2Pc-4&GZzk1e30=aSaBVD{;VA?8;9XEOi@mUn++T(WC|$>j}$u8M+V^1kUZ#zuw{{iPw(CTUI6Dm^lqRUh6ViZfl4zi!r1@Yo@Y8Z`!#g36LHRrXH)F3mJz2t(+X7}ei~&7ytPSURYaNRMRH~BSgr+^PV+gcMF4)Js{mIuTQ;%q`@3!$A-wnG!EHfuPew6&DjAoL!)s60T|#Hqi*?Moh}F4O2ydgKu%EP3Jflb0LIN9GZ>4WTZCtWAp{QF7ZAg~Uc4z#m<|>@u)KpB)_ibWC>vp=xe$Y~E2m}#q;wPy&Pv>XjMETt-ZT05)-mfzDwJ)*2{IW{h}d+O#B?3EDSlRAleU!PMaT>;AE$9}sZc-@DCgsYvBrK-%;hZCLj)OgtNx$-qV{_(g6{OLi87W)D?uwnV#ixc>1wCL|CNBP_%O!KX*t`EAr7|RM*}s7(vt;U3DIhs=qD=#Oqj0P9EWU90r#xTvk=+GrGheLPwq)BZM8JQgEz_X8$FnrBj@lLg6Q$h11O04ulxYLN}>xLi@L^W+c)f+7UVK2XaoCfOGZ*D>R2a{}qZ>f#h4z{Z=iN@}o^3{&ORh>)RupR&c1gX0SS0k#Vg-jyj}#VbCk$s#D#z$#@-dJutc*eHFt&u%I!&?W=J80Nah>2m1~!f4`%>@$X}I0{WdnwqjGQ$oCsW+Pa%X-OE>{WB}H@K}ex>NC4VjDe-AN6p6YUU(|U2Psur&T}>`2?5zKOsFwM&equ)u;CYxZsK*2zJD&8kR4n57*6dQHGlK5!X@B&&O*0*f%5KB|+sKF(P}<|poracY(P9M3DX(lMsybd=nO(jj$nnP?ch@3t~+z@{I>QHv2)3fRth2EFujQ@*-kS{N8yXbdKmff`r?WU_$`eW{j8F^-vvM~O+s5f$CI3?PY?O6l>aDpM*z5w?jt_>kiqVTJ}WJ>)iL$?_KlM+FxuoOD8G_4tX%lHFmui$QfD3;r=(=FRzXU$*xex9cFllw)a~8_9pzw#i)TAat^NT*96F4@6RT=wCHOLnC%LZ|cw^i?A}UaFj-oG{f@@;KPq{!j8Yu!zvI3Fa|u9IJ<2fYNt}t38@j=C(Utu8y%tpqiek%$?r%Gu5HQOg{LS#+Vfd-^1lI2|JyKatYXPL8d7{JI4@at9*3WqB?X$85FreVYOSQz#}Q2Hc;9^iCwKGEjywW1&ZaFf-%PNsQ#h@{Gpx}yxj492&W-S*G2dEtyuB>lLZ|;Z#ig-JpNh#RFQb{o~x!9!DDSCm33NC(tBnQAs^RWDmj6(Exq83%5S9oWs;~0E^G5R5x4l$!c)d!cS7iv(iyIQs`n&yVzUaopc?<5k!Obr32&H7{u$&ug|W#l=OB>Q0MHMu7#}%CGj{Gxv^j=x7~u4LtyB%8a(3ta)f3WNs|TeSIV`0y|m7LIkZtgd(8-tAe-smlOKya%_b3I-{qJ1mY%_FX@*$}orH4EUE{Vh3*ndp79q|${0@-BGEtb%-A&Ra0Pi+~!QV0&by!CBRV9iTKHTw~o=|KN7ig^P>&PzGMu^1S}Y(ZKy|{0yyZ9#4TI@`(yqm6}IyJvMC%C?CqEN_WX>V#+Yr(u1)=i+fbK=eeqAc_MT*z=E#1RJausj;A5bvLwr9vYL*V8xlpZ@A}3=oHm0oa^UaWqw66s8m5-<`3@U1Dszw=27?%V{}Lt`4Ml%^o4yvmZE;2MP=V3QNKq|Hy(b0{23CtY%POc!Nodh8p(#T0x*%j}n7yB9V#-9YlFmP;ToTdZR`%a4U@6vjt|e?hpObmHw&kXAAKsV9|h&sHd-TJ6M^ln@wFuhNDCYk>>klUenCjE!@4of(S3)8?Z3+R`SA#8_qvq55*}dB=kLwv_nwyD+lN*x#5vj#>r#D+a@KCvWMAaBbXvChwIR_Qyg>cWv+#UVF>}MQTGG(H*NFmoWyDEBjL2X3$I>lP%1s{ZUJ2%4FVax_ht5JW|`r!eUZS1PCeU#zJH_2-HjT{vqN`;+&OM!buM&njaLyKH+w!f5Lt%WV613{d1F`r%zM9jLuPL1&nZkJB2TTUfH#Nw!rgAo*x3A0F>{cF{dO+p%CGUJlt?p1{Vb8)G`lk?}oLtZy1U4#a_qCCEG?d))*02yH;9g2Bq{U$6-{EnB;9_euzdH+2t5fV@AE-tZXu^y1yW0gy3{!Dl*%zn4dTXLLgrt7j!Gs(jSk%3ebJsK1na2ek+M7j7+=J;^GZMJNDTI6T5(ByoOVR^Sgb$C=w_a~nCY&UD$o5|n1E<<#O>XoXsPeU10wYHpe3>0L!^CCy-6BpX5h|iw+iUkA9lg0^K-{5VvIwoV6uB{8@hcID%A?DzF)bzVe6%IzBZBHvx~qG>^}GkHncQnoPG$2I?VNTQa|L0Al^^EtOg^sC1F-Ju+lSdyCVeA@+(~3%Lt&Z_S5DX_yE9W_{VN(Bt3F`l_}#KQQW%CPxWa`brunVC-Ywd|3`Eo$8;e~#d-zZcpyJ80kA5wFw^r{H-x3))GHF{o_Mn4_nfU@95Q6Ipup#{aau4|<;}`=?Acghy>OOPFL@*Bh*l*%Y0(euP2x352g)Uz3Ag#F}pmxVSRANIwLx|pd*uPszH==wSGOPDHCpYB3LNY-|l3BZ8I`|<3V(UW@3ifSeV$Gr+-VoLY(v3@vQi>x8GO)?miAHPZPySUwt%OusBuhWhbJFBbp@kG>uQ4>pY$zq?3}%DIk-_V*IToByEh{Vq)JUCe;#IHIrp)^)2)DF4JkW5w2CN^o_QXjHtiZ`fj3?YgwzxjazYAq2us0Cs9b=%@xPhCLqE`oCB|9KtEO1AZVCMU`LntQ6&Y2W|&6W;ZErW3{m7L@D_xqc~px+u5nqiGkRfPTJOysbI(E440SYVG*H(qc1=M{sgy4!MTqmF?+|DEALDZzUQi~qjuu1uCB{03wyU5j^m}7uOBuiwi3@fNVxfbI{_mfn;eu5R_?Ko_q@K)%&NC6Wn?)KeuShJy$|qR=5n|U0FYmU43QUJ>em%)*>*2m8?yklJV+9XnUsK$BBe5rD$m6Pwiww_3o>u2KH15--##2hGRt-m@gX8CV<-020oN@1hpG$g8k7kW>>Vonw6QvZ8eQxaJ6?LgXas<&-i-@90C~Irwr5_;o_lg84pAalD{?CTF+U)gOafaqUmzbRu+pCe$c_km9K*v+wb|obj*B!$F)3FqMFrjF}rk!S+v+f4Ca?B}>J|5`hV?UFbFF*wy5H({34bkmLHNE+2_vyecI`1BK>^{9a`FixKG{>_gyWqxEYVxuY0j++yD}B|Kf}@mT|mOC;w(5-j;K3gJm|aZuX1Z4jWP$l~FOY&Oc6-EX`L1CK)zX?(sWJf@}Nik+kA3qc_#tGxR(sfY^ca5?HP0iNj;K^>NaT37b&<(A;YE9YAJ+Y|^S4N%t&8k5R_RfSVPU%Bk)(K1`H+_*As8N=?5EqG_#R*EWQsUfi~&7GU&jlw0`I7TK3!p?7tlWS2OWdgNZ4mOcC2t}_fk_fMdoZj5dFUj*2?E?Z87Ci1yS>z8DsqjBvuoI-EfXnVq6Ec9aCfbX4))hOZpnV8K<`_LR`r1oWy`$>U_IY*NXegg8V1mY1s_2Hy`hP27Ieg~48!U16c1sy`wg%TbmZ!ya9LwsL*6W;VRg4XXuQVGTWbeAeJM`USqq5nE*8whE+#Na*P&6hYSZe#FSLj}}d)0EHSi+;41^i?Q#c}?1)+>CRq>3>79ts~Oskw!6=zB{WpDJcYj@w7k|(%RycM7Tc%t7p`Slv$%wC9_D&x}I`!m#O!n<@8FCpF^7kA4NX2-_noGHLJR@RP|RK{2u%K}>Q4>U%hfud;vrE{^WAOfxvl!?2^2uNYp)MDczixLG%FUNjh|w{+^H1MM0LFBvciR*_es#70NW1W*-Nt6J`f9Z*3Ru4kT@5A_HE#mQDV4acmT}kfIIrQQ2<8ei8LHJYrZp61g$w$RNyqFd?caPKc`$a6x874#+(&e0HnGC35u4#P1p%#oF3+YqdP^c+k1xRhz}|X7%7iVx;l{HXTUsquqJgMrb@WbR^5*z${a3~c%TKa6|I~~^8_F%m7L;DziB{vsPR0g3l|Y%%G$3(eVeREL`iERN9tRZZhhB^D4H~Bkxe8GNDod4VN2FJd~)OZ6K-J)Km;>7)R;D8krq~wC+n|2uAs9QfsNG1jWH}uq0NF~elg>1Bnp{H$A1$?9aafZ0N(3KwZAw1N$Mg19E7DWTnmbM-r;GGk?r^SMw9IOxEwvR>ooH4ba90m@f#c$P|2Y0;5bi6WL(Ik6Sfu8_|7h?#ozkDV3@VDg>27Thb#*KOeQ=zwBIAKh$?$>A?bnL5t;Q^Rr{LfYO4<5j@L$kYo|r0&#bC5aO2)gXXJJe5@$^BwAW^l%0SHeiu82R$^I(7D`OPAh77&f$hfRmp2Xov1mo&~d)CUPP3q9iNbEDbsLTsjmu-Bb0+qCII~DR(n7hA45i2dZP@z?d5O&eYC;PZ&%CVdqJ}prEJb3ODIU7TYm{@O)B?e-0^2e-{>pVV?9wK6VvCrP1|Tc(=q`v%_tM+xmorcKU-W5I7!wI-2-P^VARn@ZO=(iMl~bWntBPMlY+7aGLLYs5?W-Gxt~K-HyhR4w6(y+!Op{b8Yl7tlTOFRmUyid)61x0g6Z2^eH?)cFnb#;$gBD}r7*{Kw5RUbWi5N&t_Lz?L5?Z)>2#RD*Xbi3Z6seEOG!4yQircqHtxA;@g*)&C=3nP9fhGsM6dlx%q}|jk0C|DU8$JV;i|*0Tsxw>WyUqI#(j<2WjE)sKMcJex4b9r3LqS)r2QCGo{g@)dvqP;LLebsRjd4gN0MbwQ|j^{hWOufr&VZUJ>_>D;a=*+x$oUVmOt3Zy{-mjpjoMw5eeDJVj%k%9p{|PsNs8lvZe=EkUF#C^-f;}uM!UI!~N3#O68ysOzLfTQ=7Ab-O$fEyicIdtLy2qC&MSOb?+%vbNYvZ##u6QEAp{XAI`9pR5XmPUmH7^Bl-#l`)5UxSnnhWRUW{HFi-jJA-LqJlzsNV6Ph?=?Kgj_!y3TjN7t=w?WGzFSMzSCk;G-V2I+i58ORz9l{*fSm_g)*6dF$yYQAR6h71-9J`6S%r=nD|aJ_KP5)ExhD)UE8!v^)$kx3KTZ<<P$52^TCT$6!b~tH(0I~~)>DIlngMP@By4f6!>5B7~iJ^T@Qz~>7gK@SKSWD3#2!-95Nn5?9q>@Xq9{u0GS{V(seZwMXPEXe&r5Fm@AX~6jlm98FtFxZ0-_3SX4Izbn*pxUgdy@i_SnB|7qD&IcdHC!n*!kWVx+OR?n0)6sP$3#I_3uF!g`C?{>dR|8M-m*6d7&mEUSVJXD6R@T57buBxkWhb_8sc@Pc3I+(Ft`xq$P21B82cTarQ@g15w*S?LmH6GpLLaEK_=GsUDfVpjb+rgQU7aSG^4C=ebB+&NLnK&gbeIw)kte&cx(6HEgTNb#8LRE;5+XBBYA6x(9xsc(7Iay8;LwIn@k&CRwH2ZeH`>d%kp@e1y)@BGNd$u2abxIy!000007;uq8t8eT=00EJw0pY9#C5SAwvBYQl0ssI200dcD' + ] + hashes = ['e655621583054d8df50323e12ab1a5026745f70e35dfb63c3450a248a6a3dc4c', '68e080ebd4659f50c106996edd16929ee48f8631b09741071d766d201827e2ae', 'e61c927771aa4bbd6c0eddafc3728c72c3d6c0f145637b1ec39fc670cd06d9a4', '6c3477b298fa3bd7de947e938717ca801abc95701c7d5cf042249e0d88f71951'] + for i, (part, digest) in enumerate(zip(parts, hashes, strict=True)): + assert hashlib.sha256(part.encode()).hexdigest() == digest, f'payload chunk {i}' + patch = lzma.decompress(base64.b85decode(''.join(parts))) + assert len(patch) == 87649 and hashlib.sha256(patch).hexdigest() == patch_digest + def git(*args): + return subprocess.check_output(['git', *args], text=True).strip() + assert git('rev-parse', 'HEAD') == parent + subprocess.run(['git', 'apply', '--index', '--binary', '-'], input=patch, check=True) + checkpoint = '.github/workflows/rust-source-checkpoint.yml' + subprocess.run(['git', 'rm', '--', checkpoint], check=True) + allowed = set('''.github/workflows/rust-wire.yml + rust/Cargo.lock + rust/MIGRATION.md + rust/README.md + rust/crates/msg-storage/Cargo.toml + rust/crates/msg-storage/examples/content_check.rs + rust/crates/msg-storage/src/content/files.rs + rust/crates/msg-storage/src/content/git.rs + rust/crates/msg-storage/src/content/mod.rs + rust/crates/msg-storage/src/content/tests.rs + rust/crates/msg-storage/src/lib.rs + rust/tests/test_content_parity.py'''.splitlines()) + assert set(git('diff', '--cached', '--name-only').splitlines()) == allowed | {checkpoint} + assert git('write-tree') == expected_tree, 'tested source tree mismatch' + def api(path, data=None): + request = urllib.request.Request( + 'https://api.github.com/repos/' + repo + path, + data=None if data is None else json.dumps(data).encode(), + headers={'Authorization': 'Bearer ' + os.environ['API_TOKEN'], 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', 'User-Agent': 'msg-content-preparation'}, + method='GET' if data is None else 'POST') + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + assert api('/git/ref/heads/migration/rust-wire-phase1')['object']['sha'] == parent + entries = [{'path': path, 'mode': '100644', 'type': 'blob', 'content': pathlib.Path(path).read_text()} for path in sorted(allowed)] + entries.append({'path': checkpoint, 'mode': '100644', 'type': 'blob', 'sha': None}) + tree = api('/git/trees', {'base_tree': git('rev-parse', 'HEAD^{tree}'), 'tree': entries}) + assert tree['sha'] == expected_tree + commit = api('/git/commits', {'message': 'feat(rust): add verified native Git/blob content storage', 'tree': expected_tree, 'parents': [parent]}) + receipt = {'parent_sha': parent, 'tree_sha': expected_tree, 'commit_sha': commit['sha'], 'patch_sha256': patch_digest, 'refs_updated': False} + destination = pathlib.Path('/tmp/msg-prepared') + destination.mkdir() + (destination / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(json.dumps(receipt)) + PY - uses: actions/upload-artifact@v7 with: - name: msg-rust-offline-content-build - path: /tmp/msg-offline/ + name: msg-rust-content-prepared + path: /tmp/msg-prepared/receipt.json retention-days: 2 - compression-level: 0 if-no-files-found: error From 17fb4e03b89089e7a46eea72ec12dbabb2008bf6 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Wed, 7 Oct 2026 05:14:49 +0800 Subject: [PATCH 20/22] ci(rust): restrict source preparation to non-workflow objects --- .github/workflows/rust-source-checkpoint.yml | 58 ++++++++++---------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/rust-source-checkpoint.yml b/.github/workflows/rust-source-checkpoint.yml index 3f987cde..6c5430f2 100644 --- a/.github/workflows/rust-source-checkpoint.yml +++ b/.github/workflows/rust-source-checkpoint.yml @@ -1,4 +1,4 @@ -name: Rust content commit preparation +name: Rust content source preparation on: push: branches: ['migration/rust-wire-phase1'] @@ -14,9 +14,9 @@ jobs: - uses: actions/checkout@v7 with: ref: ${{ github.sha }} - fetch-depth: 1 + fetch-depth: 2 persist-credentials: false - - name: Prepare content-addressed objects only, never update refs + - name: Prepare Rust subtree only, without workflows or ref updates env: REPOSITORY: ${{ github.repository }} EVENT_SHA: ${{ github.sha }} @@ -24,32 +24,29 @@ jobs: API_TOKEN: ${{ github.token }} run: | python - <<'PY' - import base64, hashlib, json, lzma, os, pathlib, subprocess, urllib.request + import ast, base64, hashlib, json, lzma, os, pathlib, subprocess, urllib.error, urllib.request repo = os.environ['REPOSITORY'] parent = os.environ['EVENT_SHA'] + payload_commit = '17f378cee9693f660a4a9c96e05e4ff832aa923b' assert repo == 'TokenNotIncluded/msg' - assert os.environ['ACTUAL_BEFORE'] == '71ec74e484bca856da386174429f108d721b8f89' + assert os.environ['ACTUAL_BEFORE'] == payload_commit expected_tree = '94c6c50eddeaa0c37bcd416417323c1eea8877f3' + expected_rust = '13f795c809c1b328775ca010e5f34cd78348cd13' patch_digest = '2d82567ddbad9e8ebf7ca424928a3815e167eda6319fc687eabfaf56fbbac7bf' - parts = [ - '{Wp48S^xk9=GL@E0stWa8~^|S5YJf5;Z|T%bzJ~58c7Ml{1JUn-U?4g%<)t4wb~W0)DdL_LWmAQ>GJ0ba5)+?(zF^xc^T7Qho0P!z95uHS9@E2K*z$j7w($GC6F7tdcMf3=nZoVx*h~o$T*e%00a;PL1@_YaH>Oy<(h$x-(lm(fir41Q#MxS53}9roE?f?Qrl`SF(KV&sd=aayj&>3uyW}tB5=+c=}xNPBnYvaF>+|kvJ+}ZPzD6Z^6Vb?jKp2aYxYzDiN-pqX7_zqDt`L0(Vb4j@}@5JQX@|gy3Ga|-lt)j&)MQ6M9FD&(Wyj(P=%(1-Pi%vXc_skl4ryy&xfgcDJb{5e}f&_bHGb~-`FsaF6Jh>^Xv5jKQd&+$ted#Fx`r-~AV%noKV~NYPPC5HdUz#;*wd-_}3(o1e?g%&_DakaWGDBR!9!4EW#?EOFo;OIaFW79ndi5}}uuw|LpeBn2r`ZqKAdrcz9rXfYDYKX~Q-jQJ7E9S8l)=rkY`JfvI6G54+{0>OLNaj>7rD!jo36z8WTE?lp}{EUz-%0v{LZO$EhoR(%$ouhj}gzKf|^6d+NL`QwDbZQXN2}IW)0c%PH}~2bGa@-*9ghx(~=`IOR^7+tsZM@J;@RIwQ=0Q;pSe(`=>~x{1#r5{B4$leLyDn!Ebd5Z&wLWlrruP#`a+q$tn@KWSwe&i`f}Q8MH|{jGtJ^ifFs`f3$KlALRe(~RM;-~|k)v)T$Qni{a(aDrd_iF%z<+a2vtt0#Vdp`?h)I(c3m&0%Py`FSV4R0{Te$`KAL(1~2z^y-Z=TpDXZvj8*03wE<4SE4c6=gc!v=}>$mZR*?%^W=KDnh?_KSWm7H=Lq`$M0XZ0h>t&eXS;mfsUPV4RT33t0kMcpKJNlcxCfaflp7lWNlIxr+h=dvlo0u4T(Ir96Q(TE%$dhMB+kt{IrYAIj)w<@&qk7Cn`&+>qQr>Yg@}XEr0eKnu~6eY2;t@B&}g8KmKf6bq{ZDCF1^z6l5^*HZKi+U*ek=9lQ2eaO(&aZJ+3e(-4X9h9yO*7o3$P7+amGEcsoSo4!0J#IKr#~|j@9&_yS!*{2oOuhRSNVO`Xm8$(R;Ih^2smad=$-1GvsMj*V{13^SBTezarr5}qVwFgLyit}kY0XLZavno-nXzAKu;;HuWD2vulGG&XSv;J}+2#-1K)JQD%(7&4`k>_7{~>@poehd{q&V?ssS#UB}DM~hNxot6I6)@v-mBtbZvJogr{?x<>#S&EkdMnRK_+t2t)_db9DORG9$(?HuWc8kY+7PIN71U?H`J}r0&vh+g7^EY*pWmqHS4;$^1G5@s^EMASd=@RDFA3QSYfG9iRy1PN1l325|vH|tdRA`s-w(+W$^l*_CbWBfuh636b|1rxR%&p2jHrSLt?Myqt?3K=M@{rY@lkU^5(7b=412DCe!4F$2xzt0>3aM!G{~{0R10TD|v=w|)_8$FLc{-T}tgn^_Fm;vWD_jkNDbK9#jM1mG{J#Gbu{)478rG&+hsH-cBh(Ed_rF_C#vRlrA-h+YZZW@XR$U{I-5Vz~z+73aV3s}<1H5OV?bKd6ruaqQ%C9pO&$UkHl4FjBR3wjot$2D<~x(?%PN81@0D*+Q&{UO^L`|gBAS58HSHPeE`0Ba5u>BPMKNOw8Z!-vp&GKs<~Nc{2&Q+WU$@n0gCT6Oh<_KUV-W;o5efAWRzwzGH{xHgnR>0tnjjsB$?*k5_1wu}BG7E-_MSCB1ktpHsY=5iyEzefb-xoBf;%u3I7Ln{9_{L#}_&?6@+Ns@x^VsETH{~ap^8S=8wV5s}jSIEsGZJIM+`BTj7Q)65iB4T({e3gYNEUY}0_X09gauZ~Q+f|H38HK-to6av=D7jvN2b|yApD(Wac+P_d}Li8eY+T|nG#57-W?I^S9QY9pt7Gz<(iD|ulquQ1G^wUDSsy(ZBmiX5cz}RCP)rt*)P3KZe^SxFJ4Zevs+X%%h((pX1_7i=99P!+%?#vCPPIw-!c;30DMPUqlR$xGPNd(hq+5Frz=}UG$w~Qg+=4LMLN#DPQrC0t+b-`-iSlXbCaBeg@EcA@YUh9r74nNMZ{5j5wtH}y;}Tm%;sd2dT=+A*Hpq_oXYaB-NYC=TWzPgok5Pasy23lpvQMusJT@kJOCids2xDi?X$?LD_aj7-wU(+_?Hsx{Bsj6vb%%V3-7VnGU3&V>XmuX3M&~Moh;S#^I=qWhR^xZ<~YkV5Jw$5k=*5J-{)-L1&qQV?xl)5Y2631M65J9qrQhf*F7qZVW34+^6iPm8>7vB`yBv|h9z*T%t-LrYf5grR0_M7)BZP1BSvz0W)mG8yxjy8M;%Uo8wTB}K&<}3qlc-q$Ya8GB|nmCRMf=ERFvK({LaC5AZ7X5rIf9aDD=7O91App-=f1l5@s2%FK*7_7b1)KCXCwt4)S{qfpC}7hW%#{!M<*>O(4QWK!MwFmewXz#s$IW2#B({OzebO!1dvID?qI+y{mxsc?<&oS)QdNg`H=DlIV{{!Y@M!0qeubb+yh#q#U{Nbv_@?MUEQRh*QecNr9M^^DYpX}B;(u^EzoOVwJfB6UNbxQ&~KvAmczRk^2)USSfq3`eU6m6>o<-WxP&r6UAUNhE$E?5dG4zo&ummU{nlR%=BBDE_ngjoPfkt#pme6{FF=J`>paw`{sq{W@Nd!-Pc1!hw}{@HWb9X>+m3t@mo6~dn}LWR(gM^$AZnMuiMIX^qR_82I8a{vUu7te3!#y%FNLx@Jtg7|Fb&gA*-yq<_ynZJ_FBl+e9wVIx{MWW9RL3{wqvZ6E_{T>ars~(kEm@{QK6h8?-Oz*5sh&?7;6TdqT2dPP-|lYnm3@J=*pg&QD`#gYe|TaC=fP|DEz1f^ovXS9kqmw1=wJID_`>XjeX@uX9?1lv6^qvXnO-FU&I6g$`7QxaU+Z&K?T<|re=f6)Ir#7C909mrB(X0i1`uonDbl}ESmWS{>GYBUv)iUV7*u+D$FS4agl(uRVb#k1$;_wCF5;V^MFK51#CwCyHfgGx;a|mKd#JKO`51}^GqDcn`jvgd;~kX&>e)S4R>s%{EN*U~oaNX*Gbym7jvw~i4d%qb6>UlPa-{psEa;Pde@?OY`+uA;MtO${;s(a%9^owxv?YoZ2_NK9GS#Nw)opIyt@}mXrTsIW`vfk_v&s|8CBm_ViggF}_8H7WB*UvcBz`nUEGRP+i(~k{@Vd|%unqkAC#`xMH~eYwoa0bwRkmv$(i#|ZhcQ-lwtETY9U(U!2ra4XQj!KT*$j4(?)I(48-dQ-q}}4^1l-)b<(Tb{#{IE?4TTDV{#yWde`x}9bIU0ae4MUy=u)`U2Oj;`*A-5p@8)eEL5Qd3AdVJJOg_w5Bp`ZS^r;sF@fWPsESCEkP0f`1btJ$qxBn9zW}SbynC6b4w}BWV;mai3&7POfd|q?1SlcBYggP(0CcISaL+C;DoVazvfhc@eWaO}En&5i2I?#?PygZp1xRW(I7Eop2X?RVAnfLfe;GtTQJM3T*SYznI{)JBvOcGY)eRfR4j*OS0lekTF)2r>@?QHsvwRYa$c-aF=Q>nA~N`0DDW!+Y4)+x1JucDE^E2)}69c$imNgUZDrR?rMPaBt)q<6%iWXLkr%n?6Em6CG=UpJMrJDRJk)mLyCIG#&pPW0py844)Lf998ZD9@^K0`>!g;SkibC5A*O3I&7e@#%ptBIT}>l3koYFv-C4f*YNZ)UzjJ2Y8e4aC!WYixo)pNAKYd9IRXGm6NvDzZx+{Wcjg|5}sC^)5ZgQkm(4_xzl-+HoaHCx{J_R42qeS|`_Espb{h-H1-BIu}F^wIGMi=j{!%1>hzZm+pFn$*V|@YPq~JZYvsE~PC9S55J&k@XI%eW`^^*uo3A`={a=O+pO1Ab8^)W^L9y>a*TV>zNH0aT2L|Lp+RCCg6(mqM689Q$2?OQS@gc^^6J-kECoyCHCzTC))0#%s()+5=eZG~mP!&0*z|U)YDJ+$#F`P=Zo)ik(ByT)a+_`;+R<2i$#o`D4$e;kS~usQUpnd-l|?taCNb$%^v4iRQNVf~xbnab%&CmU;iw@_qbTfs64LOIrUExL2!$f9|smY~_w9+dd*&%UM7#!>Ipv@>=@Cc-$VJ2{{I8j0kwiMAh(27Xh=ni^zc3@>8ZpH)HCu-tIP2{Cj`g6LY!UEjEE%3OQe{_fTkD7h}wE!JLM&kzx=!Pcoy%u*`V>{fFsMEc1j@NgAc`JXx-g9U|V&q-Fa8sWK~PI5we8j`v~#LF{eRb&aZC%&=>vSXpCVrWy(|DNl^rwO8ZqV1h7nMu2)cW{n{Z%XRjXm;N}tHT#$5J6a8@exx0$6=dwq1$~-CqRagh=C2jB5k9U9~M@_)dH2>_|%GbL)g4Y;o3PMZMN5?I7n|T=DUKwN|}WzU=E=?Y%!j^-7H4l4Fh!a8{QGSV&p+Xv^vp}5IcZo2X?Z@Shq$i%?)1R73FCL50z^3BHt)Nln#)naPdkdxQ<#HxwfN*D-`oX-Cw9b*CA;U`x2WtyvRXq#Z3Yip$w^@y**wBs7H205h#^%((Iev$uq%6$X8PL=L9WaPWb+m1nPbl#iRJ4xq7)(JA3dj6TqnF^4fE|-c>GeoJ`?nvJSsY;WX<7nDZ@Z#<(N+)$y!pa&A<^;_;qzhTCphsM!Bo&jA$S(@sALs`{1Kd!r%p}lmdI;JYp@MAKl`&=>6WSI)_F8?sE@IJ{PCHOKEBg2S(U4_VKUS+S-R?fix(sA2z^7&V=&SRKp2bzsnh!h6(USw(?vRY&{@QsDNiEr1^Uj7cIa`?%E=|-Taujs@%AeAIE);ojfzHM#+Z1V)pGrRfrr>qme%|$~r?RCgf^Em|{k>yP_F~l9`)(0q|)_?(1{Y7!=wY^b`*GP*J&8>vZTL;`3LQc{yT6>{^a>5Rx3qO7>u%U^o#{HOQBtpm6Rj|jaN7x96JI~;Ki*-3N1%=2U9b%8S<5gW{6cl`BA>{1OC%eJDTM4s)d$UZ$m=7^rVD?#?PWsm)Q3hlhi9*Z=h+n%}-?cl*G&qDbRG0gvS=iM8*D$WuNRd!sl=itdLveFeR=LXC=@yt}mh>mbkNGO6*>k0|+ZlI;Qm51MXQP7kb5@6k(-I@+{~z#2Y;cw%I)hUdSlM_?Kqx2oThYv(?t3E<7VdIG?7+K>b!Jz|eYOv^${@Nj(A*t+c+IuG4%UQe7!okS)(uTA>5R0CGDeX;l0r^y#Ey^|?RY{(-h', - 'MQneOek{u$U=nIua;_%MT#ctY4izd1}21p1R{yNs0b$Hq*BO2Ha%+5H+6LjV~HkwT44{r1t_BbMWz?2(52o(B48;QOyqawYj+mM{0FXcru?!0K9ZE86xWFdPX3ALdt$4V+x9L>g#?iwz1UoNZaM3$9WhhCBt-Yh)F*V_LVhItCB}e;|Yp$+l@>r4O;Repybr=c4RHfTVA&z3RB7E8_pL0EbEzf>!ROgIh+7+s-g?lh%4heK%`pDYXG`+mv)I{1%hDmunHmmf(lBE9^o1(J{z5OPuiH`v(e%YQKiPtiD8S_@HI5L!fI^hfSKJ4QA8O8h1ogFsBIQzQDCy9f~1{dz=cMTwzaLM~+7cL2p^Ij)cWrY5Ib0FO^#hTJpjxhzuP3okF<7tUhEzmomr`czSq}|V@;>@myOeWE9q|2`53F-U)t6{vk%b5Kio6?Y5e!XWglv{GVcPAd9PMV!c)<&iHY#A|`^=&KW*--!jgdhwYE<%Kvyv@3)&K1`nabV$NV8L2`S6;3Ybu5lsK;*N`qWF}`x;;ERS7+_YW(B-PGjCnf4C@`6%zpb@}06Mc-9z(lfk1_uj)d$rq+{KuxzHX#|xbl!n6aGMCkibWKt9tWsr=<@_uy>3P!$B>0ry`SckXT)dzR`B*^RSO%WbCy7*i@VL49txDaw6I*KEvT`F^22cev$wS9SY<6ja*=ZnTrn_cL0_FmvyIw{yO;=SXW(NlMw^Qb(vkGJfOhR>CvUu3?o+(g?mTDw{A?C-JH621V|Q(vSU$QBM9*(?~Fuom{BLV2F;(U!x7vuq1A-p?sVtE_`Q({zrQu$;g9*s@TXtUmNLpdSUGkx!}Ys$K$Y3i~i&CbUHA^>$!0oe0$2fcUYkHcBKHJ!VaP)S;<_N7=HQlnUt5J8$H|rK6xGPk<3JMIO{ZHj_NTmAlNV90-W-jAr9;t(2}MkZv<)x@8#t`%L!S>+nYZUOBC@%2`0r<*E0xRY=AhEy9U{%NqC%{CldDyJ1tEm!4?VL8E5xMetXzGiz&c-Ox1zF?ub-EBR55%{lD6ztnWJ(FT`T;D_i6+oLe^HFBXp_P%<;{_{R-BV0vdiV`>mPFdW(6BT=_~v69~d+GCerL^4@&zgv~;peiIK%AS}XqJv0X@=N^T%3@Qq%G2vlB=L1g~!8&|3hdON5yA#tuVl8nfdpE)Pa%iYTthiJ#&8gNL%P7HVxm(v$Q!K;i!<0@Rze@G@gYLRL9#fxX;Fq-yV>yW7D(s0?G`jRBC&r?BEtp3>*k0mAa@|Y6J$ttbwTnUBTlbX+H;7}Vn+iqzeMEe4_IN2&6cWh5HV70q=dgLjdZ>kq$tvgU?c;8Qpn*w1^yJ)@t@ldfVCJ1KkTl+=>mjC(nl--2a9BOvxmN!bzig}*Dug-T=*7>l~{=2;0$e$;iQ{1n~Vdyor+(T$twnKt~j32RzJ0&dJnnC^PQf@04ab$=|HDE)o;lFvx$Oa<~N5?~^&D^_Opz$3pT+T{HdKNX}aw#W+Mww_jonjQkJuWvd3!VP=#WUORqhl>52yKSlpI@bSDJvfzYXu~87)0qEcb^)w0$-3Ah!~S5L%KXSn>Yy`gA=B6Jwr%!QI<5;d+NDRC3h`|CSguhVhCB9}tsEPDESGsWps59izWme)&kI)An@Z>jeT*_KAX>g7#u!!ZXW5ao6w-c;oYX=F4Ue>8U$p+@H%7BtdnA#FqA0_!2#|%L;go0JIN@-YNzH$b1Q6^4$r*7!DhFCx%#fRpt@Su)8V|V5$IjY-GB36N^z*-hHi|#9yy7z#oBnb6l*7f#yb+9nwK=N$4u6?B|v4SSG#hXoTGhAa$hnV4B)dB(A2V^B)^RdQvdLXRb@aMNAjd)NbPi(M(2)P1IPy>hGI*q>Rf+>`{Uj9$^ty3C=ff_XLh4(q!GXZ~UtBY5=vz_?s8j|AfEe;;usEEN{^AkY5pO94FDa?t!kWgQ-^O7R!k7ai_3_jBfJl3D;43#EJd>?iaQn#KK)I)t>z)6eji{x|mPY>a9TtcwPi!j6)rS`w~T+Z6ppuOJLNO=x@-eTegfpYI4$#*g7J*ve!`zS>2^%!icc$L^@Ijgg?r_;U-kr3*=U&t&yhIt{ITkthm1%bF`#i32_(%H#3UwJRymzZXq=Mj<`xSLVi)p)KqOFVTRB9y^eOO%ubSWJ$w~FwHOl#1v|T8)2KmzMl`^w+EiieZ!_fKVfdqu{WfuCmp#k{M)0vUI~Le^(*%b!T+v%MuEE^bUY>Wf@;+o=GlqD4fMe%#@hU|22ON(EExMc%*zTp^=p_8v(=Rw68hbO{%t}Kn*L}QNWurLEi+1RJ$?>cEL?3PWl75^=9{a@+l~SQukfX_eUlj9g!=KR9(@a=i4p|L^|>64UShFnh~W_VI5pN@HV*R5eP?iO-E;E8Q*pU9;ByvpLX;K;2ZjTM(&`}!Gc@>T7?WQ@j0sKYX15~F^<^bc*R;I2@?fQx0EhJxgZ8@~Z=5>HcK8}BYut7&PO2`U<74ZrB$WvwhEnYJ&>SoS*-Y;(h^OR>DdPtj%UX-F$J|hVgDE2gOQ9~aZ|H8@v)SFVC#jw^0IQH_sb}mT@#b;U~ywLV$9BmZ3Pi2Q|rpfTPPNSxr-;K{{#hJwY87|9!V4mIq|?!FEaIBuov03%DRezC0j3bx;z%Wf#@ttJgFY${pZT3OLpU6QVNvcn{rd{jkY69;2&x5skSsx&S{{sFr+L|?*=o#pZiyy~w#IC73z%T-`PEt}Fy3}Wc(coyzi1XSFzpF*&QWLPQHA2r6=D48k~_YCcON2Ys|3)Gtpec9_!EE_+AI1GuvX-yw-`#|7%@jl2mX&Q_ZBXmo4x%jos0Lb;*jbgwz8J!6SdsFHod7U6K6X0MEcR1s>qfCB>_#d`VwUrIp)d{f#sXKGHNB&?U5oD&u3`Tg7!SyGFBBI?^Lrwa}T*Pgw75C_%KaQ*#gYJ&FiLe4QDV17YX8Jjjoz!YNhwnw!r~Z}iN0N92?2~mkj0FSW1fjYZy>n%zD#cPJZ}%0bR8BSv{Wb*=QqN!+1b=Ab-?Os(o<9{J=sm1A>-@{yWJaH`ve3n+aXbo>@79ZEgas0(;(RPBLM=9WK@L7fC^zIpWP!8W{6z=T>=-Q{3;?osJfJiRXI9=qG~W37|}RQN`$!O-F<8ce)0YnRZq9htyM@_lVqR{+p|h{0iBqqKRVWC;YjRSE{?*gBiT)6McCl>{3IPTPyBnKFzQnuNs8$8fLIbLyndcqO8Lz3J;Fr}}bV^XK|gqmBBn1udywsY?H&_IOROE#GmjWH!*xqa%UdBA%Z`u|_f~Bmi7;j2hpgP0bD~swo_YW18Zin3Z;to*uoLhOQ>>%5%+nP;Z*peh7+8HaCL~!cLL<``vN}+rz#~{dJ&QbJv4nE!N-sDY}^QH2mjbu+FfW94snz&&J)TbBlZA$`Uhw&O(onWv`0wwoPBRiG&`!z~C|!pu&>-Pl!$96Tf>z;YtX_=mrf}F+I9>9l+6b@ZfZxs%9#oFvkbKW3I#_gO9I|Gto2g4KZp>X}I2OfX$-TnvP7SdKT$NK`c<|BW%>vn_uET;Ko{)Z^#AArKnPmo>n5XmXr}>=$GoJDG;a<2ywT0YlWZgCDCLpWue4K#g>!r^FWDrI_Mos#+T{5&9G2%!@QFkBYcqwDnom(0jNyd?Fm5%sngo7=LTunkx7&>kGL!JH|739;ulg?v6n*237XRcy}5BJk{oSu~a5&`QFt#A9L=#r3!`u>)pgYxeh#xoQMcko#{bvk{MzRH%h3HhIMTVd(;g1m$EyMsb|<;0t0tnmjSm{)~!imfmJr&ls@2-+~w4ivr#kL@3hb>THo;c5BU-^>*N*GXKaisPNqi_g=fzkLeYdNnTt|N$LY>VAzvHckFqW=x1WJT^D-}tY$OLPYxGm2A_w_*4h*mqnZpn-X%X9K+n2nk`8*O05AQ)0T$yQr}Vgp;DUG)jBNuAM2RX^_!);A-G|=xM!E!rr!KP~~YRZ3{tIAf@#NcGG}U2BJKxY0zGT+xw6kW4IPYO~`3k-z@7&pv9~%`)p8>`s!!$xky}UE}gJ{Z_LTKw`;96c^A~AYrQou4$oEeiX3XWyqMXYbeM|d2iHvt3%^erMuEAyUbQZWc)s=dE+K~>@n$~^ih6K5%Ux`6wDCl+MJ2x^w(YzxH;CNuOJ^H_&iRF|f1|W&y`08H09jiHOIHKI!m$*ZtnGe6&Sw3pfRE{K2KRtNv~VicfZCUww!}pcjaUD0wrA7R;pUI|F;~M9;?O*lGAL=$0~LB_1fc?iza{_eL%@no&h!mp_s^VZ#_PDL?@-SQ9yUzuxT?%rR@Qgb!cnJdFWLd|AEh=@jBZYB;dbAe`!XK%jFuyoQ-LjCbFdF_^-qflgCy&o>kU{`mJA2%65Y%APjg(GR~9TRK;izP!FxGB#hm-Bo$3KJ$cE*DDW~Pjc0o!(nKhu38IFrTT%B6!@1pmW3b3UY%!uL^Hux%`&7N3=I-P3EsHeaHiH18C~bK>8!6lqV^pfPim}s5K|m0>$rLXcxU_|H)fF%D8g>dm!24JP^jvx=rA~39B&3PhaT~&VDDHz!#bY)aA#pfEDC&R9gA^jW1cKtTaGikT6$>vA-1a@7xdD=9OPymfT1Q#-tyS5CA-6}Ua?A%b)xT{e?{&=YJFMaH~Z0NYk=Sp**gPN%%0NbSua}8UO+-w4tpps~>NLpWI}J+3?@sdQ@K>O}0~zy1Shz{nHc1yjkuRz}>%6n!&zpp0`^6zD_&DuGzs#B98neIQfrwrV2_wVvQOYqZ?SPWWhlyY~3Cih~|_At9Opd*bRki#b3nx@LAHMRxCau1RV%vC%>SVf9CF9Yh($I<<#7%w~*(KBE(a8SPkiQy&z1f#^Fb%y_L)pV1Z(HC|}*n}6iY_yH?)(FwDDW(sVm?937O{lA|w81^NYn!TOf33c@=2!8_IFMimsPz(GQP@LK=WP=+AF0S3tFx5ol87u`Y6E0u|9)hgO=G*cHr(TufMoBJN0YlKnZix7wh|#IZ!b>hBUNpB?q)@p)lk)Ywe4~HxhruuB6L+D-|HB1wV>5Krl&6oKs5MLZI5pC)RC?^RTOmcR(QF-c8*`n85W$v~23qgG0nD>ft2WqQU8CIk=OBp?gp18Tf_5q!Yfc@Jvp5bB1&|X`#lvpol)esAI!hqxEuw9c)$i#DR;ED2qTsapO6XtzNJDkMT?_fq90N*c4H9PAgc}=OE@?TWv5mE>lPg^zc2dqk>vkMsFvS32+cGu8=(yby|&UT_+r2`b;UK_i^^@|n8-W7a_No4q%I@rl=BUyo}LlnC8P#EfZYImlMJh%}X|Dhor(*a2Z5!xQ~(XIrw-2?fJtZLAHHqMg1lwWm)#akAsg}YGQ=#Uoj3`6>bPSH}=v4`vpO$R-_rE5H&l6UGmGDYKo8)>gHgep9*@^amh27|H8Ud1Xqdcfdl2>Ph=>N(RJ*KHk>?GIcb*8s|s21Co?;JEGdD<}Rjfc<>&f00C+HQAwG`h?)40Kia}e1eUwc}qr-{ExWFS3ucdR%T;a{AvzzSt=YXl|Diofmhoee270PhhB2Pc-4&GZzk1e30=aSaBVD{;VA?8;9XEOi@mUn++T(WC|$>j}$u8M+V^1kUZ#zuw{{iPw(CTUI6Dm^lqRUh6ViZfl4zi!r1@Yo@Y8Z`!#g36LHRrXH)F3mJz2t(+X7}ei~&7ytPSURYaNRMRH~BSgr+^PV+gcMF4)Js{mIuTQ;%q`@3!$A-wnG!EHfuPew6&DjAoL!)s60T|#Hqi*?Moh}F4O2ydgKu%EP3Jflb0LIN9GZ>4WTZCtWAp{QF7ZAg~Uc4z#m<|>@u)KpB)_ibWC>vp=xe$Y~E2m}#q;wPy&Pv>XjMETt-ZT05)-mfzDwJ)*2{IW{h}d+O#B?3EDSlRAleU!PMaT>;AE$9}sZc-@DCgsYvBrK-%;hZCLj)OgtNx$-qV{_(g6{OLi87W)D?uwnV#ixc>1wCL|CNBP_%O!KX*t`EAr7|RM*}s7(vt;U3DIhs=qD=#Oqj0P9EWU90r#xTvk=+GrGheLPwq)BZM8JQgEz_X8$FnrBj@lLg6Q$h11O04ulxYLN}>xLi@L^W+c)f+7UVK2XaoCfOGZ*D>R2a{}qZ>f#h4z{Z=iN@}o^3{&ORh>)RupR&c1gX0SS0k#Vg-jyj}#VbCk$s#D#z$#@-dJutc*eHFt&u%I!&?W=J80Nah>2m1~!f4`%>@$X}I0{WdnwqjGQ$oCsW+Pa%X-OE>{WB}H@K}ex>NC4VjDe-AN6p6YUU(|U2Psur&T}>`2?5zKOsFwM&equ)u;CYxZsK*2zJD&8kR4n57*6dQHGlK5!X@B&&O*0*f%5KB|+sKF(P}<|poracY(P9M3DX(lMsybd=nO(jj$nnP?ch@3t~+z@{I>QHv2)3fRth2EFujQ@*-kS{N8yXbdKmff`r?WU_$`eW{j8F^-vvM~O+s5f$CI3?PY?O6l>aDpM*z5w?jt_>kiqVTJ}WJ>)iL$?_KlM+FxuoOD8G_4tX%lHFmui$QfD3;r=(=FRzXU$*xex9cFllw)a~8_9pzw#i)TAat^NT*96F4@6RT=wCHOLnC%LZ|cw^i?A}UaFj-oG{f@@;KPq{!j8Yu!zvI3Fa|u9IJ<2fYNt}t38@j=C(Utu8y%tpqiek%$?r%Gu5HQOg{LS#+Vfd-^1lI2|JyKatYXPL8d7{JI4@at9*3WqB?X$85FreVYOSQz#}Q2Hc;9^iCwKGEjywW1&ZaFf-%PNsQ#h@{Gpx}yxj492&W-S*G2dEtyuB>lLZ|;Z#ig-JpNh#RFQb{o~x!9!DDSCm33NC(tBnQAs^RWDmj6(Exq83%5S9oWs;~0E^G5R5x4l$!c)d!cS7iv(iyIQs`n&yVzUaopc?<5k!Obr32&H7{u$&ug|W#l=OB>Q0MHMu7#}%CGj{Gxv^j=x7~u4LtyB%8a(3ta)f3WNs|TeSIV`0y|m7LIkZtgd(8-tAe-smlOKya%_b3I-{qJ1mY%_FX@*$}orH4EUE{Vh3*ndp79q|${0@-BGEtb%-A&Ra0Pi+~!QV0&by!CBRV9iTKHTw~o=|KN7ig^P>&PzGMu^1S}Y(ZKy|{0yyZ9#4TI@`(yqm6}IyJvMC%C?CqEN_WX>V#+Yr(u1)=i+fbK=eeqAc_MT*z=E#1RJausj;A5bvLwr9vYL*V8xlpZ@A}3=oHm0oa^UaWqw66s8m5-<`3@U1Dszw=27?%V{}Lt`4Ml%^o4yvmZE;2MP=V3QNKq|Hy(b0{23CtY%POc!Nodh8p(#T0x*%j}n7yB9V#-9YlFmP;ToTdZR`%a4U@6vjt|e?hpObmHw&kXAAKsV9|h&sHd-TJ6M^ln@wFuhNDCYk>>klUenCjE!@4of(S3)8?Z3+R`SA#8_qvq55*}dB=kLwv_nwyD+lN*x#5vj#>r#D+a@KCvWMAaBbXvChwIR_Qyg>cWv+#UVF>}MQTGG(H*NFmoWyDEBjL2X3$I>lP%1s{ZUJ2%4FVax_ht5JW|`r!eUZS1PCeU#zJH_2-HjT{vqN`;+&OM!buM&njaLyKH+w!f5Lt%WV613{d1F`r%zM9jLuPL1&nZkJB2TTUfH#Nw!rgAo*x3A0F>{cF{dO+p%CGUJlt?p1{Vb8)G`lk?}oLtZy1U4#a_qCCEG?d))*02yH;9g2Bq{U$6-{EnB;9_euzdH+2t5fV@AE-tZXu^y1yW0gy3{!Dl*%zn4dTXLLgrt7j!Gs(jSk%3ebJsK1na2ek+M7j7+=J;^GZMJNDTI6T5(ByoOVR^Sgb$C=w_a~nCY&UD$o5|n1E<<#O>XoXsPeU10wYHpe3>0L!^CCy-6BpX5h|iw+iUkA9lg0^K-{5VvIwoV6uB{8@hcID%A?DzF)bzVe6%IzBZBHvx~qG>^}GkHncQnoPG$2I?VNTQa|L0Al^^EtOg^sC1F-Ju+lSdyCVeA@+(~3%Lt&Z_S5DX_yE9W_{VN(Bt3F`l_}#KQQW%CPxWa`brunVC-Ywd|3`Eo$8;e~#d-zZcpyJ80kA5wFw^r{H-x3))GHF{o_Mn4_nfU@95Q6Ipup#{aau4|<;}`=?Acghy>OOPFL@*Bh*l*%Y0(euP2x352g)Uz3Ag#F}pmxVSRANIwLx|pd*uPszH==wSGOPDHCpYB3LNY-|l3BZ8I`|<3V(UW@3ifSeV$Gr+-VoLY(v3@vQi>x8GO)?miAHPZPySUwt%OusBuhWhbJFBbp@kG>uQ4>pY$zq?3}%DIk-_V*IToByEh{Vq)JUCe;#IHIrp)^)2)DF4JkW5w2CN^o_QXjHtiZ`fj3?YgwzxjazYAq2us0Cs9b=%@xPhCLqE`oCB|9KtEO1AZVCMU`LntQ6&Y2W|&6W;ZErW3{m7L@D_xqc~px+u5nqiGkRfPTJOysbI(E440SYVG*H(qc1=M{sgy4!MTqmF?+|DEALDZzUQi~qjuu1uCB{03wyU5j^m}7uOBuiwi3@fNVxfbI{_mfn;eu5R_?Ko_q@K)%&NC6Wn?)KeuShJy$|qR=5n|U0FYmU43QUJ>em%)*>*2m8?yklJV+9XnUsK$BBe5rD$m6Pwiww_3o>u2KH15--##2hGRt-m@gX8CV<-020oN@1hpG$g8k7kW>>Vonw6QvZ8eQxaJ6?LgXas<&-i-@90C~Irwr5_;o_lg84pAalD{?CTF+U)gOafaqUmzbRu+pCe$c_km9K*v+wb|obj*B!$F)3FqMFrjF}rk!S+v+f4Ca?B}>J|5`hV?UFbFF*wy5H({34bkmLHNE+2_vyecI`1BK>^{9a`FixKG{>_gyWqxEYVxuY0j++yD}B|Kf}@mT|mOC;w(5-j;K3gJm|aZuX1Z4jWP$l~FOY&Oc6-EX`L1CK)zX?(sWJf@}Nik+kA3qc_#tGxR(sfY^ca5?HP0iNj;K^>NaT37b&<(A;YE9YAJ+Y|^S4N%t&8k5R_RfSVPU%Bk)(K1`H+_*As8N=?5EqG_#R*EWQsUfi~&7GU&jlw0`I7TK3!p?7tlWS2OWdgNZ4mOcC2t}_fk_fMdoZj5dFUj*2?E?Z87Ci1yS>z8DsqjBvuoI-EfXnVq6Ec9aCfbX4))hOZpnV8K<`_LR`r1oWy`$>U_IY*NXegg8V1mY1s_2Hy`hP27Ieg~48!U16c1sy`wg%TbmZ!ya9LwsL*6W;VRg4XXuQVGTWbeAeJM`USqq5nE*8whE+#Na*P&6hYSZe#FSLj}}d)0EHSi+;41^i?Q#c}?1)+>CRq>3>79ts~Oskw!6=zB{WpDJcYj@w7k|(%RycM7Tc%t7p`Slv$%wC9_D&x}I`!m#O!n<@8FCpF^7kA4NX2-_noGHLJR@RP|RK{2u%K}>Q4>U%hfud;vrE{^WAOfxvl!?2^2uNYp)MDczixLG%FUNjh|w{+^H1MM0LFBvciR*_es#70NW1W*-Nt6J`f9Z*3Ru4kT@5A_HE#mQDV4acmT}kfIIrQQ2<8ei8LHJYrZp61g$w$RNyqFd?caPKc`$a6x874#+(&e0HnGC35u4#P1p%#oF3+YqdP^c+k1xRhz}|X7%7iVx;l{HXTUsquqJgMrb@WbR^5*z${a3~c%TKa6|I~~^8_F%m7L;DziB{vsPR0g3l|Y%%G$3(eVeREL`iERN9tRZZhhB^D4H~Bkxe8GNDod4VN2FJd~)OZ6K-J)Km;>7)R;D8krq~wC+n|2uAs9QfsNG1jWH}uq0NF~elg>1Bnp{H$A1$?9aafZ0N(3KwZAw1N$Mg19E7DWTnmbM-r;GGk?r^SMw9IOxEwvR>ooH4ba90m@f#c$P|2Y0;5bi6WL(Ik6Sfu8_|7h?#ozkDV3@VDg>27Thb#*KOeQ=zwBIAKh$?$>A?bnL5t;Q^Rr{LfYO4<5j@L$kYo|r0&#bC5aO2)gXXJJe5@$^BwAW^l%0SHeiu82R$^I(7D`OPAh77&f$hfRmp2Xov1mo&~d)CUPP3q9iNbEDbsLTsjmu-Bb0+qCII~DR(n7hA45i2dZP@z?d5O&eYC;PZ&%CVdqJ}prEJb3ODIU7TYm{@O)B?e-0^2e-{>pVV?9wK6VvCrP1|Tc(=q`v%_tM+xmorcKU-W5I7!wI-2-P^VARn@ZO=(iMl~bWntBPMlY+7aGLLYs5?W-Gxt~K-HyhR4w6(y+!Op{b8Yl7tlTOFRmUyid)61x0g6Z2^eH?)cFnb#;$gBD}r7*{Kw5RUbWi5N&t_Lz?L5?Z)>2#RD*Xbi3Z6seEOG!4yQircqHtxA;@g*)&C=3nP9fhGsM6dlx%q}|jk0C|DU8$JV;i|*0Tsxw>WyUqI#(j<2WjE)sKMcJex4b9r3LqS)r2QCGo{g@)dvqP;LLebsRjd4gN0MbwQ|j^{hWOufr&VZUJ>_>D;a=*+x$oUVmOt3Zy{-mjpjoMw5eeDJVj%k%9p{|PsNs8lvZe=EkUF#C^-f;}uM!UI!~N3#O68ysOzLfTQ=7Ab-O$fEyicIdtLy2qC&MSOb?+%vbNYvZ##u6QEAp{XAI`9pR5XmPUmH7^Bl-#l`)5UxSnnhWRUW{HFi-jJA-LqJlzsNV6Ph?=?Kgj_!y3TjN7t=w?WGzFSMzSCk;G-V2I+i58ORz9l{*fSm_g)*6dF$yYQAR6h71-9J`6S%r=nD|aJ_KP5)ExhD)UE8!v^)$kx3KTZ<<P$52^TCT$6!b~tH(0I~~)>DIlngMP@By4f6!>5B7~iJ^T@Qz~>7gK@SKSWD3#2!-95Nn5?9q>@Xq9{u0GS{V(seZwMXPEXe&r5Fm@AX~6jlm98FtFxZ0-_3SX4Izbn*pxUgdy@i_SnB|7qD&IcdHC!n*!kWVx+OR?n0)6sP$3#I_3uF!g`C?{>dR|8M-m*6d7&mEUSVJXD6R@T57buBxkWhb_8sc@Pc3I+(Ft`xq$P21B82cTarQ@g15w*S?LmH6GpLLaEK_=GsUDfVpjb+rgQU7aSG^4C=ebB+&NLnK&gbeIw)kte&cx(6HEgTNb#8LRE;5+XBBYA6x(9xsc(7Iay8;LwIn@k&CRwH2ZeH`>d%kp@e1y)@BGNd$u2abxIy!000007;uq8t8eT=00EJw0pY9#C5SAwvBYQl0ssI200dcD' - ] - hashes = ['e655621583054d8df50323e12ab1a5026745f70e35dfb63c3450a248a6a3dc4c', '68e080ebd4659f50c106996edd16929ee48f8631b09741071d766d201827e2ae', 'e61c927771aa4bbd6c0eddafc3728c72c3d6c0f145637b1ec39fc670cd06d9a4', '6c3477b298fa3bd7de947e938717ca801abc95701c7d5cf042249e0d88f71951'] - for i, (part, digest) in enumerate(zip(parts, hashes, strict=True)): - assert hashlib.sha256(part.encode()).hexdigest() == digest, f'payload chunk {i}' - patch = lzma.decompress(base64.b85decode(''.join(parts))) - assert len(patch) == 87649 and hashlib.sha256(patch).hexdigest() == patch_digest + checkpoint = '.github/workflows/rust-source-checkpoint.yml' def git(*args): return subprocess.check_output(['git', *args], text=True).strip() assert git('rev-parse', 'HEAD') == parent + source = git('show', payload_commit + ':' + checkpoint) + # Parse only a literal data list; never execute the earlier script. + encoded = source.split(' parts = [\n', 1)[1].split('\n ]', 1)[0] + parts = ast.literal_eval('[\n' + encoded + '\n]') + assert len(parts) == 4 and all(isinstance(part, str) for part in parts) + patch = lzma.decompress(base64.b85decode(''.join(parts))) + assert len(patch) == 87649 and hashlib.sha256(patch).hexdigest() == patch_digest subprocess.run(['git', 'apply', '--index', '--binary', '-'], input=patch, check=True) - checkpoint = '.github/workflows/rust-source-checkpoint.yml' subprocess.run(['git', 'rm', '--', checkpoint], check=True) - allowed = set('''.github/workflows/rust-wire.yml - rust/Cargo.lock + allowed = set('''rust/Cargo.lock rust/MIGRATION.md rust/README.md rust/crates/msg-storage/Cargo.toml @@ -60,23 +57,28 @@ jobs: rust/crates/msg-storage/src/content/tests.rs rust/crates/msg-storage/src/lib.rs rust/tests/test_content_parity.py'''.splitlines()) - assert set(git('diff', '--cached', '--name-only').splitlines()) == allowed | {checkpoint} + assert set(git('diff', '--cached', '--name-only').splitlines()) == allowed | {checkpoint, '.github/workflows/rust-wire.yml'} assert git('write-tree') == expected_tree, 'tested source tree mismatch' + assert git('rev-parse', expected_tree + ':rust') == expected_rust def api(path, data=None): request = urllib.request.Request( 'https://api.github.com/repos/' + repo + path, data=None if data is None else json.dumps(data).encode(), headers={'Authorization': 'Bearer ' + os.environ['API_TOKEN'], 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', 'User-Agent': 'msg-content-preparation'}, method='GET' if data is None else 'POST') - with urllib.request.urlopen(request, timeout=30) as response: - return json.load(response) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + print(error.code, error.read(4096).decode(errors='replace')) + raise assert api('/git/ref/heads/migration/rust-wire-phase1')['object']['sha'] == parent - entries = [{'path': path, 'mode': '100644', 'type': 'blob', 'content': pathlib.Path(path).read_text()} for path in sorted(allowed)] - entries.append({'path': checkpoint, 'mode': '100644', 'type': 'blob', 'sha': None}) - tree = api('/git/trees', {'base_tree': git('rev-parse', 'HEAD^{tree}'), 'tree': entries}) - assert tree['sha'] == expected_tree - commit = api('/git/commits', {'message': 'feat(rust): add verified native Git/blob content storage', 'tree': expected_tree, 'parents': [parent]}) - receipt = {'parent_sha': parent, 'tree_sha': expected_tree, 'commit_sha': commit['sha'], 'patch_sha256': patch_digest, 'refs_updated': False} + entries = [{'path': path.removeprefix('rust/'), 'mode': '100644', 'type': 'blob', 'content': pathlib.Path(path).read_text()} for path in sorted(allowed)] + # Actions may write source objects, not workflow files. The authorized + # repository connector will separately compose the final root tree. + tree = api('/git/trees', {'base_tree': git('rev-parse', 'HEAD:rust'), 'tree': entries}) + assert tree['sha'] == expected_rust + receipt = {'parent_sha': parent, 'base_tree_sha': git('rev-parse', 'HEAD^{tree}'), 'rust_tree_sha': expected_rust, 'final_tree_sha': expected_tree, 'patch_sha256': patch_digest, 'refs_updated': False} destination = pathlib.Path('/tmp/msg-prepared') destination.mkdir() (destination / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') From fead32ffa7e0ca1bd85930097a376b939024e9df Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Wed, 7 Oct 2026 05:15:59 +0800 Subject: [PATCH 21/22] feat(rust): add verified native Git/blob content storage Implement streamed native content storage, verified range reads, compatible pins and immutable revision graphs. Preserve binary hardlinks and fence publication behind digest checks, fsync and no-clobber installation. Add 14 native tests and 170 bidirectional Python/Rust content checks per profile, including real process interruption cases. Keep production Python unchanged, remove temporary preparation workflows, and record the fixed migration acceptance checklist (completed/remaining: 5/6). Validated source tree: 94c6c50eddeaa0c37bcd416417323c1eea8877f3 Local Rust 1.85.0 / Python 3.15.0rc3 checks passed; fresh cloud CI required. This is a storage primitive, not a complete daemon or deployment. --- .github/workflows/rust-source-checkpoint.yml | 92 ---- .github/workflows/rust-wire.yml | 2 + rust/Cargo.lock | 1 + rust/MIGRATION.md | 81 +++ rust/README.md | 72 ++- rust/crates/msg-storage/Cargo.toml | 1 + .../msg-storage/examples/content_check.rs | 156 ++++++ rust/crates/msg-storage/src/content/files.rs | 282 ++++++++++ rust/crates/msg-storage/src/content/git.rs | 108 ++++ rust/crates/msg-storage/src/content/mod.rs | 521 ++++++++++++++++++ rust/crates/msg-storage/src/content/tests.rs | 397 +++++++++++++ rust/crates/msg-storage/src/lib.rs | 2 + rust/tests/test_content_parity.py | 392 +++++++++++++ 13 files changed, 2005 insertions(+), 102 deletions(-) delete mode 100644 .github/workflows/rust-source-checkpoint.yml create mode 100644 rust/MIGRATION.md create mode 100644 rust/crates/msg-storage/examples/content_check.rs create mode 100644 rust/crates/msg-storage/src/content/files.rs create mode 100644 rust/crates/msg-storage/src/content/git.rs create mode 100644 rust/crates/msg-storage/src/content/mod.rs create mode 100644 rust/crates/msg-storage/src/content/tests.rs create mode 100644 rust/tests/test_content_parity.py diff --git a/.github/workflows/rust-source-checkpoint.yml b/.github/workflows/rust-source-checkpoint.yml deleted file mode 100644 index 6c5430f2..00000000 --- a/.github/workflows/rust-source-checkpoint.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Rust content source preparation -on: - push: - branches: ['migration/rust-wire-phase1'] - paths: ['.github/workflows/rust-source-checkpoint.yml'] -permissions: - contents: write -jobs: - prepare: - if: github.repository == 'TokenNotIncluded/msg' && github.ref == 'refs/heads/migration/rust-wire-phase1' - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v7 - with: - ref: ${{ github.sha }} - fetch-depth: 2 - persist-credentials: false - - name: Prepare Rust subtree only, without workflows or ref updates - env: - REPOSITORY: ${{ github.repository }} - EVENT_SHA: ${{ github.sha }} - ACTUAL_BEFORE: ${{ github.event.before }} - API_TOKEN: ${{ github.token }} - run: | - python - <<'PY' - import ast, base64, hashlib, json, lzma, os, pathlib, subprocess, urllib.error, urllib.request - repo = os.environ['REPOSITORY'] - parent = os.environ['EVENT_SHA'] - payload_commit = '17f378cee9693f660a4a9c96e05e4ff832aa923b' - assert repo == 'TokenNotIncluded/msg' - assert os.environ['ACTUAL_BEFORE'] == payload_commit - expected_tree = '94c6c50eddeaa0c37bcd416417323c1eea8877f3' - expected_rust = '13f795c809c1b328775ca010e5f34cd78348cd13' - patch_digest = '2d82567ddbad9e8ebf7ca424928a3815e167eda6319fc687eabfaf56fbbac7bf' - checkpoint = '.github/workflows/rust-source-checkpoint.yml' - def git(*args): - return subprocess.check_output(['git', *args], text=True).strip() - assert git('rev-parse', 'HEAD') == parent - source = git('show', payload_commit + ':' + checkpoint) - # Parse only a literal data list; never execute the earlier script. - encoded = source.split(' parts = [\n', 1)[1].split('\n ]', 1)[0] - parts = ast.literal_eval('[\n' + encoded + '\n]') - assert len(parts) == 4 and all(isinstance(part, str) for part in parts) - patch = lzma.decompress(base64.b85decode(''.join(parts))) - assert len(patch) == 87649 and hashlib.sha256(patch).hexdigest() == patch_digest - subprocess.run(['git', 'apply', '--index', '--binary', '-'], input=patch, check=True) - subprocess.run(['git', 'rm', '--', checkpoint], check=True) - allowed = set('''rust/Cargo.lock - rust/MIGRATION.md - rust/README.md - rust/crates/msg-storage/Cargo.toml - rust/crates/msg-storage/examples/content_check.rs - rust/crates/msg-storage/src/content/files.rs - rust/crates/msg-storage/src/content/git.rs - rust/crates/msg-storage/src/content/mod.rs - rust/crates/msg-storage/src/content/tests.rs - rust/crates/msg-storage/src/lib.rs - rust/tests/test_content_parity.py'''.splitlines()) - assert set(git('diff', '--cached', '--name-only').splitlines()) == allowed | {checkpoint, '.github/workflows/rust-wire.yml'} - assert git('write-tree') == expected_tree, 'tested source tree mismatch' - assert git('rev-parse', expected_tree + ':rust') == expected_rust - def api(path, data=None): - request = urllib.request.Request( - 'https://api.github.com/repos/' + repo + path, - data=None if data is None else json.dumps(data).encode(), - headers={'Authorization': 'Bearer ' + os.environ['API_TOKEN'], 'Accept': 'application/vnd.github+json', 'Content-Type': 'application/json', 'User-Agent': 'msg-content-preparation'}, - method='GET' if data is None else 'POST') - try: - with urllib.request.urlopen(request, timeout=30) as response: - return json.load(response) - except urllib.error.HTTPError as error: - print(error.code, error.read(4096).decode(errors='replace')) - raise - assert api('/git/ref/heads/migration/rust-wire-phase1')['object']['sha'] == parent - entries = [{'path': path.removeprefix('rust/'), 'mode': '100644', 'type': 'blob', 'content': pathlib.Path(path).read_text()} for path in sorted(allowed)] - # Actions may write source objects, not workflow files. The authorized - # repository connector will separately compose the final root tree. - tree = api('/git/trees', {'base_tree': git('rev-parse', 'HEAD:rust'), 'tree': entries}) - assert tree['sha'] == expected_rust - receipt = {'parent_sha': parent, 'base_tree_sha': git('rev-parse', 'HEAD^{tree}'), 'rust_tree_sha': expected_rust, 'final_tree_sha': expected_tree, 'patch_sha256': patch_digest, 'refs_updated': False} - destination = pathlib.Path('/tmp/msg-prepared') - destination.mkdir() - (destination / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') - print(json.dumps(receipt)) - PY - - uses: actions/upload-artifact@v7 - with: - name: msg-rust-content-prepared - path: /tmp/msg-prepared/receipt.json - retention-days: 2 - if-no-files-found: error diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index 9b7552a9..b913bf10 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -95,6 +95,7 @@ jobs: python rust/tests/test_authorization_parity.py --postgres --binary rust/target/debug/examples/authority_check --report rust/artifacts/authorization-pg.json python rust/tests/test_postgres_parity.py --binary rust/target/debug/examples/pg_writer_check --report rust/artifacts/postgres.json python rust/tests/test_postgres_process.py --binary rust/target/debug/examples/pg_writer_check --report rust/artifacts/postgres-process.json + python rust/tests/test_content_parity.py rust/target/debug/examples/content_check --report rust/artifacts/content-debug.json - name: Optimized native differential checks if: ${{ !cancelled() }} env: @@ -108,6 +109,7 @@ jobs: python rust/tests/test_authorization_parity.py --postgres --binary rust/target/release/examples/authority_check --report rust/artifacts/authorization-pg-release.json python rust/tests/test_postgres_parity.py --binary rust/target/release/examples/pg_writer_check --report rust/artifacts/postgres-release.json python rust/tests/test_postgres_process.py --binary rust/target/release/examples/pg_writer_check --report rust/artifacts/postgres-process-release.json + python rust/tests/test_content_parity.py rust/target/release/examples/content_check --report rust/artifacts/content-release.json - name: Preserve evidence and dependency lock if: ${{ always() }} uses: actions/upload-artifact@v7 diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 08d6a441..afc9bed7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -653,6 +653,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha2 0.10.9", ] [[package]] diff --git a/rust/MIGRATION.md b/rust/MIGRATION.md new file mode 100644 index 00000000..64f557f6 --- /dev/null +++ b/rust/MIGRATION.md @@ -0,0 +1,81 @@ +# Native migration acceptance checklist + +**Completed / remaining: 5 / 6.** This counts the fixed eleven packages below, +not a percentage of effort, code lines, operations or production readiness. +A foundation package is not the corresponding end-to-end server feature. +Keep the IDs stable in later progress reports; split/redefine one only explicitly. + +| ID | Package and completion boundary | Status | +|---|---|---| +| M01 | Strict canonical wire/packet/crypto primitives and Python parity | Implemented | +| M02 | Transaction-bound identity admission, certificate and resource-policy primitives; not issuance handlers | Implemented | +| M03 | Native existing-schema SQLite metadata sessions, writers and process fencing | Implemented | +| M04 | Native existing-schema PostgreSQL metadata sessions, writers and process fencing | Implemented | +| M05 | Native Git/blob content, ranges, pins and revision graphs; valid-data interoperability | Implemented in slice 5 | +| M06 | LFS shared-object lifecycle and durable metadata/content recovery; no ambiguous-commit data loss | Remaining | +| M07 | Real operation/capability registry, authorized executor, business handlers, receipts and secret delivery | Remaining | +| M08 | Native identity/OAuth/browser-session issuance, rotation, refresh, revocation and encrypted-key wrapping | Remaining | +| M09 | Native `msgd serve`, HTTP/MCP/GET and other supported transports with current contracts | Remaining | +| M10 | Native `msgd worker`, leases/outbox, retries, recovery fences and actual side effects | Remaining | +| M11 | Installable native daemon, isolated cutover/rollback rehearsal, full end-to-end acceptance and equivalent benchmarks | Remaining | + +Python retains `msg` CLI, administrator/migration scripts and developer tooling. +There is still no native production daemon in this PR. Root signing remains local +only. Do not merge/deploy solely because these primitive suites pass. + +## Slice 5 acceptance and boundaries + +- 14 new native content tests; current workspace: 46 unit tests and 5 doc tests. +- 170 Python/Rust content checks per debug/release profile. These include actual + shared storage, 64 KiB boundary ranges, hard-link retention, canonical revision + manifests and ancestry, pin interoperability, corrupted objects, ignored Git + environment/hooks, and process kills before/after publication. +- Existing wire, identity, SQLite writer and SQLite authorization differential + suites remain enabled: respectively 18,447 / 61,734 / 71 / 878 checks. The large + identity count includes the documented simple mode matrix, not independent + end-to-end identity scenarios. PostgreSQL suites remain mandatory in cloud CI. +- `cargo fmt --check`, Ruff check/format, Clippy `-D warnings` and locked builds + remain mandatory. No warning suppression or production Python changes were + used to obtain a native pass. +- CI writes source commit/tree, compiler version, lockfile and per-profile JSON + reports to its artifact. Inspect the actual run for acceptance; a workflow + trigger or a previous commit's green status is not evidence for a new tree. + +Stricter invalid-data behavior is intentional: corrupt content is not delivered, +paths/indices are bounded and checked, reused revision IDs are immutable, and +native commit timestamps are deterministic. This is valid-data compatibility, +not a claim to reproduce every Python error or accept every malformed old record. +Directories are trusted service-owned configuration, not caller-selected paths +or an openat capability sandbox. Full reads use verified disk snapshots; request +admission must account for their disk/I/O/concurrency costs. + +No partial operation may destroy externally published data merely because an SQL +COMMIT acknowledgement was lost. M06/M07 must integrate explicit reconciliation +with the existing writer fence and ledger before serving real requests. Staging +left by SIGKILL is retained intentionally; this slice does not add unsafe GC. + +## Pre-existing full-regression blocker + +Before this slice, Rewrite contracts run **37299218496**, source head +`de810d15da747b7be34b64f4e2324e5f5e7da6f7`, failed all eight started test shards. +Their JUnit artifacts were inspected (4,759 test entries): **32 failures/errors**, +of which **29** end in `recovery_schema_unsupported`. The other three are: + +| Test | Existing failure location | +|---|---| +| `test_default_build_checks_packaged_publication_snapshot` | `tests/test_dictionary.py:178` | +| `test_universe_account_hint_is_minimal_current_identity_without_private_reads` | `tests/test_universe.py:51` | +| `test_accepts_postgres_and_optional_valkey_urls_without_revealing_secrets` | `tests/test_pg_config.py:26` | + +These are observed baseline failures, not fixed by changing native content code. +Do not label a new failure as pre-existing without matching its actual evidence. +The full regression/cutover gate remains open independently of native conformance. + +## Coordination and evidence + +Read repository `AGENTS.md`, the entry skill and deployed `/AGENTS.md` before this +slice. The editor has no selected local MSG signing identity; no authorized +private coordination thread could be created. Use the existing draft PR #235 and +this checklist as the fallback. No production account, public coordination post, +production data write or deployment was created. The temporary public-toolchain +export workflow is removed from the final source tree. diff --git a/rust/README.md b/rust/README.md index 42e98bab..02d48ae4 100644 --- a/rust/README.md +++ b/rust/README.md @@ -2,9 +2,57 @@ Python keeps the CLI, administrator/migration scripts, and development helpers. Native Rust code is not yet a replacement for the running Python service. Do not -change production entry points until the remaining executor, transport, content, +change production entry points until the remaining executor, transport, content-recovery, worker, and deployment gates below pass. +## Fifth implementation slice: native Git/blob content storage + +`msg-storage::content::GitContentStore` ports the existing `private.git`, `index`, +`binary`, `pins` and `revisions` layout without invoking Python. It streams +admitted input through 64 KiB buffers, checks optional SHA-256 expectations before +publication, uses direct Git plumbing for text/JSON/templates, and preserves +existing binary inodes (including LFS hard links). Index replacement and published +objects/ref updates are durable before success; failed input cannot publish an +index. Binary publication is on the destination filesystem, including when +staging is configured elsewhere. + +Reads verify a private snapshot before delivering any requested range. This uses +constant buffered memory but O(blob size) temporary disk and full-object I/O, even +for a small range. The future adapter must enforce admission, disk and concurrency +quotas; this is not a free random-access optimization. Pins use the existing +hashed lease/digest names. Revision trees contain the same canonical manifests, +content and parent commits. Existing Python commits are reused unchanged; newly +created native commits use the revision time for deterministic retries. A reused +revision ID with different content/parents is rejected rather than overwritten. + +Git runs with a cleared environment, no inherited/global Git configuration or +hooks, bounded captured output and a kill-and-wait timeout. Git stdin/stdout use +private files rather than deadlock-prone pipes. Paths/indices reject traversal, +symlinks, non-regular files and malformed object IDs. Configured directories must +be service-owned and protected from hostile writers: this is **not an openat +sandbox against same-UID races**. Existing shared-reader layouts must already +have the required modes; no implicit permission migration is performed. + +The API is synchronous, Unix-only and a **trusted storage primitive**. Callers +must supply byte quotas and retain the metadata writer fence until the call has +actually finished; abandoning a blocking task does not cancel Git. Input-reader +cancellation and request scheduling belong to the still-pending adapters. +No handler, listener, root administration or production entry point is changed. + +Fourteen native tests exercise success, corrupt objects/indices, quotas, failed +input, ranges, hard links, path confinement checks, shared modes, pins, immutable +revision retries and actual Git timeout/reaping. The Python differential suite +has **170 checks per profile**, importing the actual Python store and reading and +writing the same disposable directories. It covers both directions, Git objects, +canonical manifests/parent graphs, cross-language pins, a 4 MiB streamed blob, +hermetic Git and real process kills before publication and after a lost response. +`content_check` is a marker-gated local fixture, never an operation endpoint. + +These tests do **not** implement LFS shared-object publication/collection, a +cross-store metadata/content recovery journal, SIGKILL cleanup or an authorized +executor. Interrupted staging is deliberately retained, not blindly collected. +See [the fixed migration checklist](MIGRATION.md) for completed/remaining scope. + ## Fourth implementation slice: resource authority and PostgreSQL `msg-identity::authorization` implements transaction-bound resource policy: @@ -72,7 +120,7 @@ The original inspected main baseline is | `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing/verification | `security/crypto.py` | | `msg-protocol` | Request shape/defaults, UTC time, business digest/signature bytes, immutable field access | `core/packet.py`, `core/requests.py`, `core/models.py` | | `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, browser/API/OAuth and token-recovery validation, transaction-bound resource authorization | `security/authentication.py`, `certificates.py`, `policy.py`, `oauth.py` | -| `msg-storage` | Transaction-bound authority reads; SQLite and PostgreSQL writers, identity/certificate/transfer/job records, atomic resource/revision/result/event/audit/settings writes, savepoints and rollback effects | `storage/sqlite.py`, `storage/postgres.py` and their record/session modules | +| `msg-storage` | Transaction-bound authority reads; SQLite/PostgreSQL writers and typed records, savepoints/rollback effects; native Git/blob content, ranges, pins and revision trees | `storage/sqlite.py`, `storage/postgres.py`, `storage/git.py` and their record/session modules | Authentication validates service/digest/expiry, current credentials, SHA-256 bearer verifiers with constant-time comparison, actor/subject bindings, permitted @@ -147,10 +195,12 @@ result ledger and reconcile external work before reopening. Reopening alone is not recovery. Durable service-wide quarantine and external-work reconciliation are not implemented by this slice. -The process-kill tests cover **SQLite metadata only**. Rollback callbacks are not -run after SIGKILL and are not a durable blob/Git recovery journal. Content/Git -adapters, identity/OAuth mutations, outbox/lease handling, PostgreSQL and complete -request execution recovery still need to be ported. Do not introduce dual writes. +The earlier process-kill tests cover **SQL metadata**, not a complete operation. +The content tests above additionally cover standalone blob publication. Neither +rollback callbacks nor private staging implement a durable cross-store recovery +journal: callbacks do not run after SIGKILL. Identity/OAuth workflows, outbox/lease +execution and complete operation recovery still need to be ported. PostgreSQL +metadata primitives are already implemented above. Do not introduce dual writes. ## Reproducible validation @@ -171,6 +221,8 @@ PYTHONPATH=src python rust/tests/test_identity_parity.py \ --binary rust/target/debug/examples/authority_check --report rust/artifacts/identity-parity.json PYTHONPATH=src python rust/tests/test_storage_parity.py \ --binary rust/target/debug/examples/writer_check --report rust/artifacts/storage-parity.json +PYTHONPATH=src python rust/tests/test_content_parity.py \ + rust/target/debug/examples/content_check --report rust/artifacts/content-debug.json ``` Repeat the build with `--release` and the same suites against @@ -196,8 +248,8 @@ COMMIT, result caps, real process kills and Python/Rust fence interoperability. Native tests additionally inject panics, cleanup failures and an uncertain commit result. The last case is deterministic error injection, not a physical disk test. -`wire_check`, `authority_check` and `writer_check` are local JSONL **test examples**, never HTTP or -MCP endpoints. They use only deterministic public test keys and disposable data. +`wire_check`, `authority_check`, `writer_check`, `pg_writer_check` and +`content_check` are local JSONL **test examples**, never HTTP or MCP endpoints. They use only deterministic public test keys and disposable data. The writer helper deliberately includes unauthenticated storage orchestration (`once`) and failure/checkpoint controls only for disposable tests. It is not a production executor and must never be exposed as an endpoint. These examples @@ -236,7 +288,7 @@ cryptographic and database primitives are delegated to pinned libraries. 1. Integrate the real operation/capability registry and complete resource authorization, including private resources, sharing, memberships and tools. -2. Complete mutation stores, blob/Git staging and durable external recovery; +2. Complete remaining workflow mutations, LFS and durable cross-store recovery; connect the writer to the real authorized executor, receipt signing and secret delivery. Retain Python migration/administration scripts. Extend the current metadata crash/retry checks to complete operation effects. @@ -253,5 +305,5 @@ selected MSG signing identity was available in the editor, so no authorized private coordination thread could be created. No new production identity or public coordination post was created. Work is recorded in the migration branch/PR; use an existing authorized identity when private coordination resumes. -The temporary writer-dependency export workflow is removed by this slice. +The temporary offline-input export workflow is removed by the fifth slice. No selected production MSG identity was invented or registered for testing. diff --git a/rust/crates/msg-storage/Cargo.toml b/rust/crates/msg-storage/Cargo.toml index 4dd93911..b0722f12 100644 --- a/rust/crates/msg-storage/Cargo.toml +++ b/rust/crates/msg-storage/Cargo.toml @@ -18,6 +18,7 @@ postgres.workspace = true postgres-native-tls.workspace = true native-tls.workspace = true bytes.workspace = true +sha2.workspace = true [target.'cfg(unix)'.dependencies] fs2.workspace = true diff --git a/rust/crates/msg-storage/examples/content_check.rs b/rust/crates/msg-storage/examples/content_check.rs new file mode 100644 index 00000000..8ca58491 --- /dev/null +++ b/rust/crates/msg-storage/examples/content_check.rs @@ -0,0 +1,156 @@ +//! Disposable JSONL parity/crash driver, NOT an authenticated operation API. +//! The root must contain an explicit test marker. No listener or Python adapter. +use msg_core::{b64, unb64, Error, Json, Result}; +use msg_storage::{ + content::{ContentOptions, GitContentStore}, + records::{decode, encode, BlobRef}, +}; +use serde::de::DeserializeOwned; +use sha2::{Digest, Sha256}; +use std::{ + io::{self, Read, Write}, + path::Path, +}; +fn get(v: &Json, key: &str) -> Result { + decode(v.as_object()?.get(key).ok_or(Error("fixture_field"))?) +} +fn emit(value: &Json) -> Result<()> { + println!("{}", value.canonical()?); + io::stdout().flush().map_err(|_| Error("fixture_output")) +} +fn hold() -> Result<()> { + emit(&Json::parse(r#"{"checkpoint":"held"}"#)?)?; + let mut input = String::new(); + io::stdin() + .read_line(&mut input) + .map_err(|_| Error("fixture_input"))?; + if input.trim() != "continue" { + return Err(Error("fixture_input")); + } + Ok(()) +} +struct Generated { + remaining: u64, + pause: bool, + started: bool, +} +impl Read for Generated { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if self.pause && self.started { + self.pause = false; + hold().map_err(io::Error::other)?; + } + self.started = true; + let length = bytes.len().min(self.remaining as usize); + bytes[..length].fill(b'x'); + self.remaining -= length as u64; + Ok(length) + } +} +struct HashSink { + hash: Sha256, + count: u64, +} +impl Write for HashSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.hash.update(bytes); + self.count += bytes.len() as u64; + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +fn dispatch(store: &GitContentStore, value: &Json) -> Result { + match get::(value, "action")?.as_str() { + "put" => { + let bytes = unb64(&get::(value, "bytes")?, 524_288)?; + let expected: Option = get(value, "expected")?; + encode(&store.put_bytes( + &bytes, + &get::(value, "media_type")?, + expected.as_deref(), + )?) + } + "generated" => { + let length: u64 = get(value, "length")?; + if length > 16_777_216 { + return Err(Error("fixture_limit")); + } + let mut input = Generated { + remaining: length, + pause: get(value, "hold_during")?, + started: false, + }; + let blob = store.put( + &mut input, + &get::(value, "media_type")?, + None, + length, + )?; + if get::(value, "hold_after")? { + hold()?; + } + encode(&blob) + } + "read" => { + let blob: BlobRef = get(value, "blob")?; + let range: Option<(u64, u64)> = get(value, "range")?; + if range + .map(|(a, b)| b.saturating_sub(a)) + .unwrap_or(blob.size.max(0) as u64) + > 524_288 + { + return Err(Error("fixture_limit")); + } + let mut bytes = Vec::new(); + store.read_to(&blob, range, &mut bytes)?; + Ok(Json::string(b64(&bytes))) + } + "hash" => { + let mut sink = HashSink { + hash: Sha256::new(), + count: 0, + }; + store.read_to(&get(value, "blob")?, None, &mut sink)?; + encode( + &serde_json::json!({"digest":format!("sha256:{:x}",sink.hash.finalize()),"size":sink.count}), + ) + } + "pin" | "unpin" | "pinned" => { + let blob = get(value, "blob")?; + let lease: String = get(value, "lease")?; + match get::(value, "action")?.as_str() { + "pin" => store.pin(&blob, &lease)?, + "unpin" => store.unpin(&blob, &lease)?, + _ => (), + } + encode(&store.pinned(&blob, &lease)?) + } + "revision" => encode( + &store.commit_revision(&get::(value, "topic")?, &get(value, "revision")?)?, + ), + _ => Err(Error("fixture_action")), + } +} +fn main() -> std::result::Result<(), Box> { + let root = std::env::args().nth(1).ok_or("fixture_root")?; + let root = Path::new(&root); + if std::fs::read(root.join(".msg-native-content-test"))? != b"disposable test content\n" { + return Err("fixture_marker".into()); + } + let store = GitContentStore::open(root, ContentOptions::default())?; + loop { + let mut line = String::new(); + if io::stdin().read_line(&mut line)? == 0 { + break; + } + let result = Json::parse(line.trim_end()).and_then(|value| dispatch(&store, &value)); + let output = match result { + Ok(data) => encode(&serde_json::json!({"data":data}))?, + Err(e) => encode(&serde_json::json!({"error":e.0}))?, + }; + emit(&output)?; + } + Ok(()) +} diff --git a/rust/crates/msg-storage/src/content/files.rs b/rust/crates/msg-storage/src/content/files.rs new file mode 100644 index 00000000..fad2d072 --- /dev/null +++ b/rust/crates/msg-storage/src/content/files.rs @@ -0,0 +1,282 @@ +//! Filesystem primitives for service-owned content directories, never user paths. +use msg_core::{Error, Result}; +use sha2::{Digest, Sha256}; +use std::{ + fs::{self, DirBuilder, File, Metadata, OpenOptions, Permissions}, + io::{Read, Seek, SeekFrom, Write}, + os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}, + path::{Component, Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, +}; + +pub(super) fn io(_: std::io::Error) -> Error { + Error("content_store_error") +} +pub(super) fn hex(data: &[u8]) -> String { + format!("{:x}", Sha256::digest(data)) +} +pub(super) fn key(digest: &str) -> Result<&str> { + let key = digest + .strip_prefix("sha256:") + .ok_or(Error("invalid_digest"))?; + if key.len() != 64 + || !key + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(Error("invalid_digest")); + } + Ok(key) +} +pub(super) fn oid(value: &str) -> Result<&str> { + if ![40, 64].contains(&value.len()) + || !value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return Err(Error("invalid_content_index")); + } + Ok(value) +} +pub(super) fn segment(value: &str) -> Result<&str> { + if value.is_empty() + || value.len() > 160 + || [".", ".."].contains(&value) + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._:-".contains(&b)) + { + return Err(Error("invalid_revision_id")); + } + Ok(value) +} + +/// Reject symlink path components. Directory ownership and protection against +/// hostile same-UID processes remain an administrator responsibility; this is +/// not an openat sandbox. No request may choose these configured directories. +pub(super) fn absolute(path: &Path) -> Result { + let path = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir().map_err(io)?.join(path) + }; + let mut checked = PathBuf::new(); + for component in path.components() { + match component { + Component::ParentDir => return Err(Error("unsafe_content_path")), + Component::CurDir => continue, + _ => checked.push(component), + } + match fs::symlink_metadata(&checked) { + Ok(meta) if meta.file_type().is_symlink() => return Err(Error("unsafe_content_path")), + Ok(_) => (), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => (), + Err(e) => return Err(io(e)), + } + } + Ok(checked) +} +pub(super) fn directory(path: &Path, group: bool) -> Result<()> { + absolute(path)?; + let created = match fs::symlink_metadata(path) { + Ok(_) => false, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + DirBuilder::new().mode(0o700).create(path).map_err(io)?; + true + } + Err(e) => return Err(io(e)), + }; + if created && group { + fs::set_permissions(path, Permissions::from_mode(0o2750)).map_err(io)?; + } + check_directory(path, group)?; + if created { + sync_dir(path)?; + if let Some(parent) = path.parent() { + sync_dir(parent)?; + } + } + Ok(()) +} +pub(super) fn check_directory(path: &Path, group: bool) -> Result<()> { + let meta = fs::symlink_metadata(path).map_err(io)?; + if !meta.is_dir() || meta.file_type().is_symlink() || meta.mode() & 0o022 != 0 { + return Err(Error("unsafe_content_path")); + } + if group && meta.mode() & 0o2077 != 0o2050 { + return Err(Error("content_permission_migration_required")); + } + Ok(()) +} +pub(super) fn sync_dir(path: &Path) -> Result<()> { + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_DIRECTORY | libc::O_CLOEXEC) + .open(path) + .and_then(|f| f.sync_all()) + .map_err(io) +} +pub(super) fn open(path: &Path) -> Result { + let file = OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK | libc::O_CLOEXEC) + .open(path) + .map_err(|e| { + if e.kind() == std::io::ErrorKind::NotFound { + Error("content_missing") + } else { + Error("unsafe_content_path") + } + })?; + if !file.metadata().map_err(io)?.is_file() { + return Err(Error("unsafe_content_path")); + } + Ok(file) +} +pub(super) fn bounded(path: &Path, limit: usize) -> Result> { + let file = open(path)?; + if file.metadata().map_err(io)?.len() > limit as u64 { + return Err(Error("invalid_content_index")); + } + let mut bytes = Vec::new(); + file.take(limit as u64 + 1) + .read_to_end(&mut bytes) + .map_err(io)?; + if bytes.len() > limit { + return Err(Error("invalid_content_index")); + } + Ok(bytes) +} +fn unchanged(a: &Metadata, b: &Metadata) -> bool { + ( + a.dev(), + a.ino(), + a.len(), + a.mtime(), + a.mtime_nsec(), + a.ctime(), + a.ctime_nsec(), + ) == ( + b.dev(), + b.ino(), + b.len(), + b.mtime(), + b.mtime_nsec(), + b.ctime(), + b.ctime_nsec(), + ) +} +/// Read and hash with constant memory, stopping at a caller-supplied quota. +pub(super) fn stream( + source: &mut impl Read, + target: &mut impl Write, + limit: u64, +) -> Result<(String, u64)> { + let mut hash = Sha256::new(); + let mut total = 0_u64; + let mut buffer = [0; 65_536]; + loop { + let count = match source.read(&mut buffer) { + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + other => other.map_err(io)?, + }; + if count == 0 { + break; + } + total = total + .checked_add(count as u64) + .ok_or(Error("content_too_large"))?; + if total > limit.min(i64::MAX as u64) { + return Err(Error("content_too_large")); + } + hash.update(&buffer[..count]); + target.write_all(&buffer[..count]).map_err(io)?; + } + Ok((format!("sha256:{:x}", hash.finalize()), total)) +} +pub(super) fn verified_copy( + source: &mut File, + target: &mut impl Write, + digest: &str, + size: u64, +) -> Result<()> { + let before = source.metadata().map_err(io)?; + if before.len() != size { + return Err(Error("content_size_mismatch")); + } + source.seek(SeekFrom::Start(0)).map_err(io)?; + let (actual, length) = stream(source, target, size)?; + if actual != digest || length != size || !unchanged(&before, &source.metadata().map_err(io)?) { + return Err(Error("content_digest_mismatch")); + } + Ok(()) +} +static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); +pub(super) struct Temporary { + pub path: PathBuf, + pub file: File, +} +impl Temporary { + pub fn new(parent: &Path) -> Result { + check_directory(parent, false)?; + for _ in 0..128 { + let path = parent.join(format!( + ".pending-rust-{}-{}", + std::process::id(), + NEXT_TEMP.fetch_add(1, Ordering::Relaxed) + )); + let result = OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .mode(0o600) + .custom_flags(libc::O_NOFOLLOW | libc::O_CLOEXEC) + .open(&path); + match result { + Ok(file) => return Ok(Self { path, file }), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(io(e)), + } + } + Err(Error("content_store_error")) + } + pub fn rewind(&mut self) -> Result<()> { + self.file.seek(SeekFrom::Start(0)).map(|_| ()).map_err(io) + } + pub fn permissions(&self, group: bool) -> Result<()> { + self.file + .set_permissions(Permissions::from_mode(if group { 0o640 } else { 0o600 })) + .map_err(io)?; + self.file.sync_all().map_err(io) + } + /// Publish without overwriting an existing inode (also preserves LFS links). + pub fn install(&self, path: &Path) -> Result { + match fs::hard_link(&self.path, path) { + Ok(()) => { + sync_dir(path.parent().ok_or(Error("unsafe_content_path"))?)?; + Ok(true) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(false), + Err(e) => Err(io(e)), + } + } +} +impl Drop for Temporary { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +pub(super) fn replace(path: &Path, bytes: &[u8], group: bool) -> Result<()> { + if let Ok(meta) = fs::symlink_metadata(path) { + if !meta.is_file() || meta.file_type().is_symlink() { + return Err(Error("unsafe_content_path")); + } + } + let parent = path.parent().ok_or(Error("unsafe_content_path"))?; + let mut temp = Temporary::new(parent)?; + temp.file.write_all(bytes).map_err(io)?; + temp.permissions(group)?; + fs::rename(&temp.path, path).map_err(io)?; + sync_dir(parent) +} diff --git a/rust/crates/msg-storage/src/content/git.rs b/rust/crates/msg-storage/src/content/git.rs new file mode 100644 index 00000000..ccdf2ec6 --- /dev/null +++ b/rust/crates/msg-storage/src/content/git.rs @@ -0,0 +1,108 @@ +//! Direct Git plumbing only: no shell, Python, hooks, inherited Git variables, +//! global configuration, stdin pipe deadlock, or unbounded in-memory output. +use super::{ + files::{self, Temporary}, + GitContentStore, +}; +use msg_core::{Error, Result}; +use std::{ + fs::File, + io::{Read, Write}, + process::{Child, Command, Stdio}, + thread, + time::Instant, +}; + +struct Running(Child); +impl Drop for Running { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} +impl GitContentStore { + pub(super) fn command(&self) -> Command { + let mut command = Command::new(&self.options.git_binary); + command + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("LANG", "C.UTF-8") + .env("HOME", &self.root) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_NO_REPLACE_OBJECTS", "1") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_AUTHOR_NAME", "msg") + .env("GIT_COMMITTER_NAME", "msg") + .env("GIT_AUTHOR_EMAIL", "msg@localhost") + .env("GIT_COMMITTER_EMAIL", "msg@localhost") + .arg("--git-dir") + .arg(&self.repo) + .args([ + "-c", + "core.fsync=all", + "-c", + "core.logAllRefUpdates=false", + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + ]) + .current_dir(&self.root); + command + } + pub(super) fn run( + &self, + command: &mut Command, + input: Option, + max_output: u64, + ) -> Result { + let mut output = Temporary::new(&self.staging)?; + command + .stdin(input.map(Stdio::from).unwrap_or_else(Stdio::null)) + .stdout(output.file.try_clone().map_err(files::io)?) + .stderr(Stdio::null()); + let mut child = Running(command.spawn().map_err(files::io)?); + let started = Instant::now(); + loop { + if output.file.metadata().map_err(files::io)?.len() > max_output { + return Err(Error("content_output_too_large")); + } + if let Some(status) = child.0.try_wait().map_err(files::io)? { + if !status.success() { + return Err(Error("content_store_error")); + } + // The child can finish between the first size check and try_wait. + if output.file.metadata().map_err(files::io)?.len() > max_output { + return Err(Error("content_output_too_large")); + } + output.rewind()?; + return Ok(output); + } + if started.elapsed() >= self.options.git_timeout { + return Err(Error("content_store_timeout")); + } + thread::sleep(std::time::Duration::from_millis(5)); + } + } + pub(super) fn git(&self, args: &[&str], bytes: Option<&[u8]>) -> Result { + let mut input = Temporary::new(&self.staging)?; + if let Some(bytes) = bytes { + input.file.write_all(bytes).map_err(files::io)?; + } + input.rewind()?; + let mut result = self.run( + self.command().args(args), + Some(input.file.try_clone().map_err(files::io)?), + 1_048_576, + )?; + let mut text = String::new(); + result.file.read_to_string(&mut text).map_err(files::io)?; + Ok(text.trim_end_matches('\n').to_owned()) + } + pub(super) fn ref_set(&self, reference: &str, oid: &str) -> Result<()> { + files::oid(oid)?; + self.git(&["update-ref", "--no-deref", reference, oid], None) + .map(|_| ()) + } +} diff --git a/rust/crates/msg-storage/src/content/mod.rs b/rust/crates/msg-storage/src/content/mod.rs new file mode 100644 index 00000000..44ae385a --- /dev/null +++ b/rust/crates/msg-storage/src/content/mod.rs @@ -0,0 +1,521 @@ +//! Native Git/blob content storage, compatible with Python's private.git, +//! index, binary, pins and revisions layout. Synchronous and Unix-only. +//! +//! This is a trusted storage port, not an authorization boundary or an executor. +//! The caller supplies quotas and holds the metadata writer fence while coupling +//! mutations to metadata. Wait for an in-flight call to finish before releasing +//! that fence; abandoning an async blocking task is not cancellation. +//! +//! Files and Git refs are durable before successful return. This is NOT an +//! atomic metadata/content commit or a durable cross-store recovery journal. +//! Errors may leave unreferenced content, which must not be blindly deleted. +mod files; +mod git; +#[cfg(test)] +mod tests; +use crate::records::{encode, BlobRef, Revision}; +use files::{io, Temporary}; +use msg_core::{Error, Json, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + time::Duration, +}; + +/// Administrator-supplied configuration, never deserialized from a request. +#[derive(Clone)] +pub struct ContentOptions { + pub binary_dir: Option, + pub staging_dir: Option, + pub group_read: bool, + pub git_binary: PathBuf, + pub git_timeout: Duration, +} +impl Default for ContentOptions { + fn default() -> Self { + Self { + binary_dir: None, + staging_dir: None, + group_read: false, + git_binary: "/usr/bin/git".into(), + git_timeout: Duration::from_secs(120), + } + } +} +pub struct GitContentStore { + root: PathBuf, + repo: PathBuf, + index: PathBuf, + binary: PathBuf, + staging: PathBuf, + options: ContentOptions, +} +#[derive(Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "lowercase", deny_unknown_fields)] +enum Entry { + Git { oid: String, size: u64 }, + Binary { size: u64 }, +} +impl Entry { + fn size(&self) -> u64 { + match self { + Self::Git { size, .. } | Self::Binary { size } => *size, + } + } +} +impl GitContentStore { + /// Initialize or open a content store in protected, service-owned paths. + /// Existing directory permissions are never repaired implicitly. Parent + /// directories must already exist; shared layouts require explicit migration. + pub fn open(root: &Path, options: ContentOptions) -> Result { + if !options.git_binary.is_absolute() || options.git_timeout.is_zero() { + return Err(Error("invalid_content_configuration")); + } + let root = files::absolute(root)?; + let store = Self { + repo: root.join("private.git"), + index: root.join("index"), + binary: files::absolute( + options + .binary_dir + .as_deref() + .unwrap_or(&root.join("binary")), + )?, + staging: files::absolute( + options + .staging_dir + .as_deref() + .unwrap_or(&root.join("staging")), + )?, + root, + options, + }; + // Do not chmod/create anything before rejecting an existing bad layout. + for path in [&store.root, &store.index, &store.binary] { + if fs::symlink_metadata(path).is_ok() { + files::check_directory(path, store.options.group_read)?; + } + } + for path in [&store.repo, &store.staging] { + if fs::symlink_metadata(path).is_ok() { + files::check_directory(path, false)?; + } + } + files::directory(&store.root, store.options.group_read)?; + files::directory(&store.staging, false)?; + if store.repo.exists() + && store.options.group_read + && store.git(&["config", "--local", "core.sharedRepository"], None)? != "0640" + { + return Err(Error("content_permission_migration_required")); + } + files::directory(&store.index, store.options.group_read)?; + files::directory(&store.binary, store.options.group_read)?; + if !store.repo.exists() { + let mut command = store.command(); + command.args([ + "-c", + "init.templateDir=", + "init", + "--bare", + "--object-format=sha1", + ]); + if store.options.group_read { + command.arg("--shared=0640"); + } + command.arg(&store.repo); + store.run(&mut command, None, 65_536)?; + files::sync_dir(&store.root)?; + } + store.layout()?; + if store.git(&["rev-parse", "--is-bare-repository"], None)? != "true" { + return Err(Error("invalid_content_repository")); + } + Ok(store) + } + fn layout(&self) -> Result<()> { + for path in [ + &self.root, + &self.index, + &self.binary, + &self.staging, + &self.repo, + ] { + files::absolute(path)?; + files::check_directory(path, false)?; + } + Ok(()) + } + fn entry(&self, blob: &BlobRef) -> Result { + self.layout()?; + let key = files::key(&blob.digest)?; + if blob.size < 0 { + return Err(Error("invalid_blob_size")); + } + let bytes = files::bounded(&self.index.join(key), 4096)?; + let json = + Json::parse(std::str::from_utf8(&bytes).map_err(|_| Error("invalid_content_index"))?)?; + let entry: Entry = + serde_json::from_str(&json.canonical()?).map_err(|_| Error("invalid_content_index"))?; + if entry.size() != blob.size as u64 { + return Err(Error("content_size_mismatch")); + } + if let Entry::Git { oid, .. } = &entry { + files::oid(oid)?; + } + Ok(entry) + } + /// Stream into private staging. A wrong expected digest or source error + /// cannot publish an index/ref. `limit` is the caller's admitted byte quota. + pub fn put( + &self, + source: &mut impl Read, + media_type: &str, + expected: Option<&str>, + limit: u64, + ) -> Result { + self.layout()?; + if let Some(expected) = expected { + files::key(expected)?; + } + let mut staged = Temporary::new(&self.staging)?; + let (digest, size) = files::stream(source, &mut staged.file, limit)?; + if expected.is_some_and(|expected| expected != digest) { + return Err(Error("digest_mismatch")); + } + staged.file.sync_all().map_err(io)?; + let key = files::key(&digest)?; + let entry = if media_type.starts_with("text/") + || ["application/json", "application/msg-template"].contains(&media_type) + { + staged.rewind()?; + let mut output = self.run( + self.command().args(["hash-object", "-w", "--stdin"]), + Some(staged.file.try_clone().map_err(io)?), + 128, + )?; + let mut oid = String::new(); + output.file.read_to_string(&mut oid).map_err(io)?; + let oid = files::oid(oid.trim())?.to_owned(); + self.ref_set(&format!("refs/staging/{key}"), &oid)?; + Entry::Git { oid, size } + } else { + let destination = self.binary.join(key); + if fs::symlink_metadata(&destination).is_ok() { + files::verified_copy( + &mut files::open(&destination)?, + &mut std::io::sink(), + &digest, + size, + )?; + } else { + // Create on the destination filesystem; this also inherits the + // intended setgid directory rather than the staging directory's gid. + let mut published = Temporary::new(&self.binary)?; + staged.rewind()?; + files::verified_copy(&mut staged.file, &mut published.file, &digest, size)?; + published.permissions(self.options.group_read)?; + if !published.install(&destination)? { + files::verified_copy( + &mut files::open(&destination)?, + &mut std::io::sink(), + &digest, + size, + )?; + } + } + Entry::Binary { size } + }; + files::replace( + &self.index.join(key), + encode(&entry)?.canonical()?.as_bytes(), + self.options.group_read, + )?; + Ok(BlobRef { + digest, + size: size as i64, + media_type: media_type.into(), + }) + } + pub fn put_bytes( + &self, + bytes: &[u8], + media_type: &str, + expected: Option<&str>, + ) -> Result { + let mut reader = bytes; + self.put(&mut reader, media_type, expected, bytes.len() as u64) + } + fn snapshot(&self, blob: &BlobRef) -> Result { + let entry = self.entry(blob)?; + let mut snapshot = Temporary::new(&self.staging)?; + match entry { + Entry::Binary { .. } => { + files::verified_copy( + &mut files::open(&self.binary.join(files::key(&blob.digest)?))?, + &mut snapshot.file, + &blob.digest, + blob.size as u64, + )?; + } + Entry::Git { oid, .. } => { + snapshot = self.run( + self.command().args(["cat-file", "blob", &oid]), + None, + blob.size as u64, + )?; + files::verified_copy( + &mut snapshot.file, + &mut std::io::sink(), + &blob.digest, + blob.size as u64, + )?; + } + } + snapshot.rewind()?; + Ok(snapshot) + } + /// Deliver from a verified private snapshot using 64 KiB buffers. Integrity + /// is checked before any bytes reach the sink, even for an empty/small range. + /// Snapshot disk usage is O(blob size), memory O(1); the caller enforces + /// concurrency/disk quotas. A failed sink may have accepted a partial range. + pub fn read_to( + &self, + blob: &BlobRef, + range: Option<(u64, u64)>, + sink: &mut impl Write, + ) -> Result<()> { + if blob.size < 0 { + return Err(Error("invalid_blob_size")); + } + let (start, end) = range.unwrap_or((0, blob.size as u64)); + if start > end || end > blob.size as u64 { + return Err(Error("invalid_byte_range")); + } + let mut snapshot = self.snapshot(blob)?; + snapshot.file.seek(SeekFrom::Start(start)).map_err(io)?; + let copied = + std::io::copy(&mut (&mut snapshot.file).take(end - start), sink).map_err(io)?; + if copied != end - start { + return Err(Error("content_truncated")); + } + Ok(()) + } + pub fn read_bytes(&self, blob: &BlobRef, limit: u64) -> Result> { + if blob.size < 0 { + return Err(Error("invalid_blob_size")); + } + if blob.size as u64 > limit { + return Err(Error("use_transfer")); + } + let mut bytes = Vec::new(); + self.read_to(blob, None, &mut bytes)?; + Ok(bytes) + } + fn pin_path(&self, blob: &BlobRef, lease: &str, create: bool) -> Result { + let root = self.root.join("pins"); + let parent = root.join(files::hex(lease.as_bytes())); + if create { + files::directory(&root, false)?; + files::directory(&parent, false)?; + } + files::absolute(&parent)?; + Ok(parent.join(files::key(&blob.digest)?)) + } + pub fn pin(&self, blob: &BlobRef, lease: &str) -> Result<()> { + // Verify that a pin retains precisely the promised bytes, not just a + // well-formed index pointing to a corrupt or different object. + self.snapshot(blob)?; + match self.entry(blob)? { + Entry::Git { oid, .. } => self.ref_set( + &format!( + "refs/pins/{}/{}", + files::hex(lease.as_bytes()), + files::key(&blob.digest)? + ), + &oid, + ), + Entry::Binary { .. } => { + files::replace(&self.pin_path(blob, lease, true)?, b"1\n", false) + } + } + } + pub fn unpin(&self, blob: &BlobRef, lease: &str) -> Result<()> { + match self.entry(blob)? { + Entry::Git { .. } => self + .git( + &[ + "update-ref", + "--no-deref", + "-d", + &format!( + "refs/pins/{}/{}", + files::hex(lease.as_bytes()), + files::key(&blob.digest)? + ), + ], + None, + ) + .map(|_| ()), + Entry::Binary { .. } => { + let path = self.pin_path(blob, lease, false)?; + match fs::symlink_metadata(&path) { + Ok(meta) if meta.file_type().is_symlink() || !meta.is_file() => { + Err(Error("unsafe_content_path")) + } + Ok(_) => { + fs::remove_file(&path).map_err(io)?; + files::sync_dir(path.parent().ok_or(Error("unsafe_content_path"))?) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(io(e)), + } + } + } + } + pub fn pinned(&self, blob: &BlobRef, lease: &str) -> Result { + match self.entry(blob)? { + Entry::Git { oid, .. } => { + // show-ref's missing-ref status is distinct from a corrupt index; + // check existence via for-each-ref, which succeeds for no matches. + let reference = format!( + "refs/pins/{}/{}", + files::hex(lease.as_bytes()), + files::key(&blob.digest)? + ); + let text = self.git( + &[ + "for-each-ref", + "--format=%(refname) %(objectname)", + &reference, + ], + None, + )?; + Ok(text + .lines() + .any(|line| line == format!("{reference} {oid}"))) + } + Entry::Binary { .. } => match files::bounded(&self.pin_path(blob, lease, false)?, 2) { + Ok(bytes) if bytes == b"1\n" => Ok(true), + Ok(_) => Err(Error("invalid_content_pin")), + Err(Error("content_missing")) => Ok(false), + Err(e) => Err(e), + }, + } + } + /// Commit canonical revision bytes and content to the existing topic refs. + /// The domain layer still validates signatures, parent authority and quotas. + /// Existing IDs are immutable: a same-manifest retry reuses the stored commit; + /// conflicting reuse fails instead of overwriting history. + pub fn commit_revision(&self, topic: &str, revision: &Revision) -> Result { + files::segment(&revision.id)?; + if revision.parents.len() > 64 { + return Err(Error("too_many_revision_parents")); + } + for parent in &revision.parents { + files::segment(parent)?; + } + self.snapshot(&revision.content)?; + let manifest = encode(revision)?.canonical()?; + let manifest_oid = + self.git(&["hash-object", "-w", "--stdin"], Some(manifest.as_bytes()))?; + files::oid(&manifest_oid)?; + let mut tree_input = format!("100644 blob {manifest_oid}\tmanifest.json\n"); + if let Entry::Git { oid, .. } = self.entry(&revision.content)? { + tree_input.push_str(&format!("100644 blob {oid}\tcontent\n")); + } + let tree = self.git(&["mktree"], Some(tree_input.as_bytes()))?; + files::oid(&tree)?; + let directory = self.root.join("revisions"); + files::directory(&directory, false)?; + let parents: Vec = revision + .parents + .iter() + .map(|parent| { + let bytes = files::bounded(&directory.join(parent), 128).map_err(|e| { + if e == Error("content_missing") { + Error("revision_content_missing") + } else { + e + } + })?; + Ok(files::oid( + std::str::from_utf8(&bytes) + .map_err(|_| Error("invalid_content_index"))? + .trim(), + )? + .to_owned()) + }) + .collect::>()?; + let path = directory.join(&revision.id); + let mut commit = match files::bounded(&path, 128) { + Ok(bytes) => files::oid( + std::str::from_utf8(&bytes) + .map_err(|_| Error("invalid_content_index"))? + .trim(), + )? + .to_owned(), + Err(Error("content_missing")) => { + let mut command = self.command(); + command.args(["commit-tree", &tree]); + for parent in &parents { + command.args(["-p", parent]); + } + // Deterministic native commits; Python-created existing commits + // retain their original wall-clock timestamps and object IDs. + let date = revision.created_at.0.to_rfc3339(); + command + .env("GIT_AUTHOR_DATE", &date) + .env("GIT_COMMITTER_DATE", &date); + let mut input = Temporary::new(&self.staging)?; + writeln!(input.file, "{}", revision.id).map_err(io)?; + input.rewind()?; + let mut output = + self.run(&mut command, Some(input.file.try_clone().map_err(io)?), 128)?; + let mut text = String::new(); + output.file.read_to_string(&mut text).map_err(io)?; + files::oid(text.trim())?.to_owned() + } + Err(e) => return Err(e), + }; + self.check_commit(&commit, &tree, &parents)?; + let reference = format!( + "refs/topics/{}/{}/{}", + files::hex(topic.as_bytes()), + files::hex(revision.resource_id.as_bytes()), + files::hex(revision.id.as_bytes()) + ); + self.ref_set(&reference, &commit)?; + let mut mapping = Temporary::new(&directory)?; + writeln!(mapping.file, "{commit}").map_err(io)?; + mapping.permissions(false)?; + if !mapping.install(&path)? { + let existing = files::bounded(&path, 128)?; + commit = files::oid( + std::str::from_utf8(&existing) + .map_err(|_| Error("invalid_content_index"))? + .trim(), + )? + .to_owned(); + self.check_commit(&commit, &tree, &parents)?; + self.ref_set(&reference, &commit)?; + } + Ok(commit) + } + fn check_commit(&self, commit: &str, tree: &str, parents: &[String]) -> Result<()> { + let body = self.git(&["cat-file", "commit", files::oid(commit)?], None)?; + let headers: Vec<&str> = body.split("\n\n").next().unwrap_or("").lines().collect(); + let actual: Vec<&str> = headers + .iter() + .filter_map(|line| line.strip_prefix("parent ")) + .collect(); + if headers.first() != Some(&format!("tree {tree}").as_str()) + || actual != parents.iter().map(String::as_str).collect::>() + { + return Err(Error("revision_content_conflict")); + } + Ok(()) + } +} diff --git a/rust/crates/msg-storage/src/content/tests.rs b/rust/crates/msg-storage/src/content/tests.rs new file mode 100644 index 00000000..ebf0abb0 --- /dev/null +++ b/rust/crates/msg-storage/src/content/tests.rs @@ -0,0 +1,397 @@ +use super::*; +use crate::records::decode; +use std::{ + fs::{DirBuilder, Permissions}, + io, + os::unix::fs::{symlink, DirBuilderExt, MetadataExt, PermissionsExt}, + sync::atomic::{AtomicU64, Ordering}, +}; +static NEXT: AtomicU64 = AtomicU64::new(0); +struct Fixture(PathBuf); +impl Fixture { + fn new() -> Self { + let path = std::env::temp_dir().join(format!( + "msg-native-content-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + DirBuilder::new().mode(0o700).create(&path).unwrap(); + Self(path) + } + fn store(&self) -> GitContentStore { + GitContentStore::open(&self.0.join("store"), ContentOptions::default()).unwrap() + } +} +impl Drop for Fixture { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} +fn revision(blob: &BlobRef, id: &str, parents: &[&str]) -> Revision { + decode( + &encode( + &serde_json::json!({"format_version":1,"id":id,"resource_id":"r_test", + "parents":parents,"content":blob,"relations":[],"actor":"u_alice","subject":"u_alice", + "author":"u_alice","created_at":"2026-10-06T00:00:00.000000Z", + "manifest_digest":format!("sha256:{}", "a".repeat(64)),"signature":null}), + ) + .unwrap(), + ) + .unwrap() +} +#[test] +fn storage_kinds_and_empty_content_round_trip() { + let f = Fixture::new(); + let store = f.store(); + for media in [ + "text/markdown", + "text/plain", + "application/json", + "application/msg-template", + "application/octet-stream", + ] { + for data in [b"".as_slice(), "Hello\0中文\n".as_bytes()] { + let blob = store.put_bytes(data, media, None).unwrap(); + assert_eq!(store.read_bytes(&blob, 1024).unwrap(), data); + assert_eq!(blob.digest, msg_core::digest(data)); + } + } + assert_eq!(fs::read_dir(&store.staging).unwrap().count(), 0); +} +#[test] +fn ranges_cross_buffers_and_reject_invalid_bounds() { + let f = Fixture::new(); + let store = f.store(); + let data: Vec = (0..200_001).map(|n| (n % 251) as u8).collect(); + for media in ["text/plain", "application/octet-stream"] { + let blob = store.put_bytes(&data, media, None).unwrap(); + for (start, end) in [ + (0, 0), + (0, 1), + (65_532, 65_540), + (5, 180_000), + (200_001, 200_001), + ] { + let mut output = Vec::new(); + store + .read_to(&blob, Some((start, end)), &mut output) + .unwrap(); + assert_eq!(output, data[start as usize..end as usize]); + } + assert_eq!( + store.read_to(&blob, Some((2, 1)), &mut Vec::new()).err(), + Some(Error("invalid_byte_range")) + ); + assert_eq!( + store + .read_to(&blob, Some((0, 200_002)), &mut Vec::new()) + .err(), + Some(Error("invalid_byte_range")) + ); + assert_eq!( + store.read_bytes(&blob, 200_000).err(), + Some(Error("use_transfer")) + ); + } +} +#[test] +fn failed_source_quota_and_digest_do_not_publish() { + struct Broken; + impl Read for Broken { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("fixture")) + } + } + let f = Fixture::new(); + let store = f.store(); + assert_eq!( + store.put(&mut Broken, "text/plain", None, 1024).err(), + Some(Error("content_store_error")) + ); + assert_eq!( + store.put(&mut &b"abc"[..], "text/plain", None, 2).err(), + Some(Error("content_too_large")) + ); + assert_eq!( + store + .put_bytes(b"abc", "text/plain", Some(&msg_core::digest(b"different"))) + .err(), + Some(Error("digest_mismatch")) + ); + assert_eq!( + store + .put_bytes(b"abc", "text/plain", Some("sha256:../wrong")) + .err(), + Some(Error("invalid_digest")) + ); + for path in [&store.staging, &store.index, &store.binary] { + assert_eq!(fs::read_dir(path).unwrap().count(), 0); + } + assert_eq!(store.git(&["for-each-ref"], None).unwrap(), ""); +} +#[test] +fn binary_duplicate_keeps_existing_hard_link_inode() { + let f = Fixture::new(); + let store = f.store(); + let blob = store + .put_bytes(b"same binary", "application/octet-stream", None) + .unwrap(); + let path = store.binary.join(files::key(&blob.digest).unwrap()); + let link = f.0.join("lfs-link"); + fs::hard_link(&path, &link).unwrap(); + let before = fs::metadata(&path).unwrap(); + store + .put_bytes(b"same binary", "application/octet-stream", None) + .unwrap(); + assert_eq!(fs::metadata(&path).unwrap().ino(), before.ino()); + assert_eq!(fs::metadata(link).unwrap().nlink(), 2); +} +#[test] +fn corrupt_binary_never_reaches_sink_or_gets_replaced() { + let f = Fixture::new(); + let store = f.store(); + let blob = store + .put_bytes(b"correct", "application/octet-stream", None) + .unwrap(); + let path = store.binary.join(files::key(&blob.digest).unwrap()); + fs::write(&path, b"corrupt").unwrap(); + let mut sink = Vec::new(); + assert_eq!( + store.read_to(&blob, None, &mut sink).err(), + Some(Error("content_digest_mismatch")) + ); + assert!(sink.is_empty()); + assert_eq!( + store + .put_bytes(b"correct", "application/octet-stream", None) + .err(), + Some(Error("content_digest_mismatch")) + ); + assert_eq!(fs::read(path).unwrap(), b"corrupt"); +} +#[test] +fn corrupt_index_and_truncated_binary_fail_closed() { + let f = Fixture::new(); + let store = f.store(); + let blob = store + .put_bytes(b"correct", "application/octet-stream", None) + .unwrap(); + fs::write(store.binary.join(files::key(&blob.digest).unwrap()), b"x").unwrap(); + assert_eq!( + store.read_bytes(&blob, 1024).err(), + Some(Error("content_size_mismatch")) + ); + let index = store.index.join(files::key(&blob.digest).unwrap()); + for value in [ + r#"{"kind":"git","oid":"--help","size":7}"#, + r#"{"kind":"binary","size":7,"size":7}"#, + r#"{"kind":"binary","size":-1}"#, + ] { + fs::write(&index, value).unwrap(); + assert!(store.read_bytes(&blob, 1024).is_err()); + } +} +#[test] +fn bad_digest_and_negative_size_cannot_escape_root() { + let f = Fixture::new(); + let store = f.store(); + for digest in [ + "../outside", + "sha256:../../etc/passwd", + "sha256:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ] { + let blob = BlobRef { + digest: digest.into(), + size: 1, + media_type: "text/plain".into(), + }; + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("invalid_digest")) + ); + } + let blob = BlobRef { + digest: msg_core::digest(b""), + size: -1, + media_type: "text/plain".into(), + }; + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("invalid_blob_size")) + ); +} +#[test] +fn symlink_roots_indices_and_binary_objects_are_rejected() { + let f = Fixture::new(); + let store = f.store(); + symlink(&store.root, f.0.join("alias")).unwrap(); + assert!(GitContentStore::open(&f.0.join("alias"), ContentOptions::default()).is_err()); + let blob = store + .put_bytes(b"abc", "application/octet-stream", None) + .unwrap(); + let path = store.binary.join(files::key(&blob.digest).unwrap()); + let external = f.0.join("external"); + fs::write(&external, b"abc").unwrap(); + fs::remove_file(&path).unwrap(); + symlink(&external, &path).unwrap(); + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("unsafe_content_path")) + ); + assert_eq!( + store + .put_bytes(b"abc", "application/octet-stream", None) + .err(), + Some(Error("unsafe_content_path")) + ); + let index = store.index.join(files::key(&blob.digest).unwrap()); + fs::remove_file(&index).unwrap(); + symlink(&external, index).unwrap(); + assert_eq!( + store.read_bytes(&blob, 10).err(), + Some(Error("unsafe_content_path")) + ); + assert_eq!(fs::read(external).unwrap(), b"abc"); +} +#[test] +fn pins_are_hashed_and_idempotent() { + let f = Fixture::new(); + let store = f.store(); + for media in ["text/plain", "application/octet-stream"] { + let blob = store.put_bytes(b"pin me", media, None).unwrap(); + let lease = "../../untrusted/lease\0中文"; + assert!(!store.pinned(&blob, lease).unwrap()); + store.pin(&blob, lease).unwrap(); + store.pin(&blob, lease).unwrap(); + assert!(store.pinned(&blob, lease).unwrap()); + store.unpin(&blob, lease).unwrap(); + store.unpin(&blob, lease).unwrap(); + assert!(!store.pinned(&blob, lease).unwrap()); + } +} +#[test] +fn revisions_are_canonical_parented_and_retryable() { + let f = Fixture::new(); + let store = f.store(); + let blob = store.put_bytes(b"revision", "text/plain", None).unwrap(); + let first = revision(&blob, "v_first", &[]); + let oid = store.commit_revision("t_中文", &first).unwrap(); + assert_eq!(store.commit_revision("t_中文", &first).unwrap(), oid); + assert_eq!( + store + .git(&["show", &format!("{oid}:manifest.json")], None) + .unwrap(), + encode(&first).unwrap().canonical().unwrap() + ); + let child = store + .commit_revision("t_中文", &revision(&blob, "v_second", &["v_first"])) + .unwrap(); + assert!(store + .git(&["cat-file", "commit", &child], None) + .unwrap() + .contains(&format!("parent {oid}\n"))); + let mut conflicting = first.clone(); + conflicting.author = "u_other".into(); + assert_eq!( + store.commit_revision("t_中文", &conflicting).err(), + Some(Error("revision_content_conflict")) + ); + assert_eq!( + fs::read_to_string(store.root.join("revisions/v_first")).unwrap(), + format!("{oid}\n") + ); +} +#[test] +fn revision_paths_and_missing_parents_are_checked() { + let f = Fixture::new(); + let store = f.store(); + let blob = store.put_bytes(b"x", "text/plain", None).unwrap(); + assert_eq!( + store + .commit_revision("t", &revision(&blob, "../../escape", &[])) + .err(), + Some(Error("invalid_revision_id")) + ); + assert_eq!( + store + .commit_revision("t", &revision(&blob, "v_second", &["missing"])) + .err(), + Some(Error("revision_content_missing")) + ); + assert!(!store.root.join("revisions/v_second").exists()); +} +#[test] +fn shared_layout_requires_explicit_permission_migration() { + let f = Fixture::new(); + let store = f.store(); + let old = fs::metadata(&store.root).unwrap().mode(); + assert!(GitContentStore::open( + &store.root, + ContentOptions { + group_read: true, + ..ContentOptions::default() + } + ) + .is_err()); + assert_eq!(fs::metadata(&store.root).unwrap().mode(), old); +} +#[test] +fn newly_created_shared_files_have_the_intended_mode() { + let f = Fixture::new(); + let store = GitContentStore::open( + &f.0.join("shared"), + ContentOptions { + group_read: true, + ..ContentOptions::default() + }, + ) + .unwrap(); + let blob = store + .put_bytes(b"group", "application/octet-stream", None) + .unwrap(); + for dir in [&store.root, &store.binary, &store.index] { + assert_eq!(fs::metadata(dir).unwrap().mode() & 0o2777, 0o2750); + } + for dir in [&store.binary, &store.index] { + assert_eq!( + fs::metadata(dir.join(files::key(&blob.digest).unwrap())) + .unwrap() + .mode() + & 0o777, + 0o640 + ); + } + GitContentStore::open( + &store.root, + ContentOptions { + group_read: true, + ..ContentOptions::default() + }, + ) + .unwrap(); +} +#[test] +fn git_timeout_kills_and_reaps_the_child() { + let f = Fixture::new(); + let store = f.store(); + let script = f.0.join("slow-git"); + // Disposable test executable, not a runtime shell adapter. + fs::write(&script, "#!/bin/sh\necho $$ > child.pid\nexec sleep 30\n").unwrap(); + fs::set_permissions(&script, Permissions::from_mode(0o700)).unwrap(); + let store = GitContentStore { + options: ContentOptions { + git_binary: script, + git_timeout: Duration::from_millis(100), + ..ContentOptions::default() + }, + ..store + }; + assert_eq!( + store.put_bytes(b"timeout", "text/plain", None).err(), + Some(Error("content_store_timeout")) + ); + let pid = fs::read_to_string(store.root.join("child.pid")).unwrap(); + assert!(!Path::new(&format!("/proc/{}", pid.trim())).exists()); + assert_eq!(fs::read_dir(&store.staging).unwrap().count(), 0); + assert_eq!(fs::read_dir(&store.index).unwrap().count(), 0); +} diff --git a/rust/crates/msg-storage/src/lib.rs b/rust/crates/msg-storage/src/lib.rs index 39cc7fa5..07230a37 100644 --- a/rust/crates/msg-storage/src/lib.rs +++ b/rust/crates/msg-storage/src/lib.rs @@ -3,6 +3,8 @@ //! No schema creation/migration, network entry point or Python subprocess. //! Read snapshots and write sessions share the same transaction-bound authority reads. mod authorization; +#[cfg(unix)] +pub mod content; pub mod postgres; pub mod records; #[cfg(unix)] diff --git a/rust/tests/test_content_parity.py b/rust/tests/test_content_parity.py new file mode 100644 index 00000000..7990699e --- /dev/null +++ b/rust/tests/test_content_parity.py @@ -0,0 +1,392 @@ +"""Real Python/Rust content interoperability on disposable, shared directories. + +Not an authenticated executor, full server acceptance or a recovery journal test. +All test identities, paths and hooks are synthetic. No production service access. +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import hashlib +import json +import os +import selectors +import subprocess +import tempfile +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +from msg.core.codec import canonical, decode, wire +from msg.core.models import BlobRef, Revision +from msg.storage.git import GitContentStore + +REPORT = [] +NOW = datetime(2026, 10, 6, tzinfo=UTC) + + +def check(condition, name): + if not condition: + raise AssertionError(name) + REPORT.append(name) + + +def b64(value): + return base64.urlsafe_b64encode(value).rstrip(b'=').decode() + + +class Native: + def __init__(self, binary, root, *, env=None): + root.mkdir(exist_ok=True) + (root / '.msg-native-content-test').write_bytes(b'disposable test content\n') + self.process = subprocess.Popen( + [str(binary), str(root)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + def send(self, **value): + self.process.stdin.write(json.dumps(value) + '\n') + self.process.stdin.flush() + with selectors.DefaultSelector() as selector: + selector.register(self.process.stdout, selectors.EVENT_READ) + if not selector.select(timeout=20): + self.stop(kill=True) + raise AssertionError('native content fixture timed out') + line = self.process.stdout.readline() + if not line: + self.process.wait(timeout=5) + raise AssertionError(self.process.stderr.read()) + return json.loads(line) + + def call(self, **value): + result = self.send(**value) + if 'error' in result: + raise AssertionError(result) + return result['data'] + + def stop(self, *, kill=False): + if self.process.poll() is None: + if kill: + self.process.kill() + else: + self.process.stdin.close() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + raise + for stream in [self.process.stdin, self.process.stdout, self.process.stderr]: + stream.close() + + +def revision(blob, name, parents=()): + return Revision( + format_version=1, + id=name, + resource_id='r_content', + parents=list(parents), + content=blob, + relations=[], + actor='u_alice', + subject='u_alice', + author='u_alice', + created_at=NOW, + manifest_digest='sha256:' + 'a' * 64, + signature=None, + ) + + +def git(store, *args): + return subprocess.run( + ['git', '--git-dir', str(store.repo), *args], + check=True, + capture_output=True, + env=store.env, + timeout=20, + ).stdout + + +async def interoperate(binary, root): + store = GitContentStore(root) + native = Native(binary, root) + try: + kinds = [ + 'text/markdown', + 'application/json', + 'application/msg-template', + 'application/octet-stream', + ] + payloads = [b'', 'Hello\0中文\n'.encode(), bytes(range(251)) * 600] + for index, media in enumerate(kinds): + for number, data in enumerate(payloads): + label = f'{index}-{number}' + py_blob = await store.put_bytes(data, media) + check( + native.call(action='read', blob=wire(py_blob), range=None) == b64(data), + 'python-to-rust-' + label, + ) + native_blob = native.call( + action='put', bytes=b64(data), media_type=media, expected=None + ) + check(native_blob == wire(py_blob), 'blob-ref-' + label) + check( + await store.read_bytes(decode(BlobRef, native_blob)) == data, + 'rust-to-python-' + label, + ) + index_path = store.index / py_blob.digest.removeprefix('sha256:') + entry = json.loads(index_path.read_bytes()) + if entry['kind'] == 'git': + check( + git(store, 'cat-file', 'blob', entry['oid']) == data, 'git-object-' + label + ) + else: + check( + (store.binary / py_blob.digest.removeprefix('sha256:')).read_bytes() + == data, + 'binary-object-' + label, + ) + lease = '../../opaque/lease中文' + check( + native.call(action='pin', blob=native_blob, lease=lease), 'native-pin-' + label + ) + check(await store.pinned(py_blob, lease), 'python-sees-pin-' + label) + await store.unpin(py_blob, lease) + check( + not native.call(action='pinned', blob=native_blob, lease=lease), + 'native-sees-unpin-' + label, + ) + await store.pin(py_blob, lease) + check( + not native.call(action='unpin', blob=native_blob, lease=lease), + 'native-unpin-' + label, + ) + check(not await store.pinned(py_blob, lease), 'python-sees-unpin-' + label) + ranges = [(0, len(data)), (len(data), len(data))] + if len(data) > 65536: + ranges += [(65532, 65540), (5, 70000)] + for begin, end in ranges: + py_bytes = b''.join([ + chunk async for chunk in store.read(py_blob, (begin, end)) + ]) + check( + native.call(action='read', blob=native_blob, range=[begin, end]) + == b64(py_bytes), + f'range-{label}-{begin}-{end}', + ) + for media in ['text/plain', 'application/octet-stream']: + blob = await store.put_bytes(b'parent', media) + suffix = 'text' if media.startswith('text') else 'binary' + parent = revision(blob, 'v_python_' + suffix) + parent_oid = await store.commit_revision('t_fixture', parent) + check( + native.call(action='revision', topic='t_fixture', revision=wire(parent)) + == parent_oid, + 'reuse-python-commit-' + suffix, + ) + child = revision(blob, 'v_rust_' + suffix, [parent.id]) + child_oid = native.call(action='revision', topic='t_fixture', revision=wire(child)) + check( + git(store, 'show', f'{child_oid}:manifest.json') == canonical(child), + 'canonical-manifest-' + suffix, + ) + check( + git(store, 'rev-parse', f'{child_oid}^').strip().decode() == parent_oid, + 'native-parent-' + suffix, + ) + descendant = revision(blob, 'v_python_after_' + suffix, [child.id]) + descendant_oid = await store.commit_revision('t_fixture', descendant) + check( + git(store, 'rev-parse', f'{descendant_oid}^').strip().decode() == child_oid, + 'python-parent-' + suffix, + ) + conflict = replace(child, author='u_different') + check( + native.send(action='revision', topic='t_fixture', revision=wire(conflict)) + == {'error': 'revision_content_conflict'}, + 'immutable-revision-' + suffix, + ) + check( + native.call(action='revision', topic='t_fixture', revision=wire(child)) + == child_oid, + 'retry-native-commit-' + suffix, + ) + large = native.call( + action='generated', + length=4_194_305, + media_type='application/octet-stream', + hold_during=False, + hold_after=False, + ) + expected = 'sha256:' + hashlib.sha256(b'x' * large['size']).hexdigest() + check( + native.call(action='hash', blob=large) == {'digest': expected, 'size': large['size']}, + 'streaming-native-large-blob', + ) + py_hash = hashlib.sha256() + async for chunk in store.read(decode(BlobRef, large)): + py_hash.update(chunk) + check('sha256:' + py_hash.hexdigest() == expected, 'streaming-python-large-blob') + bad = native.send( + action='put', + bytes=b64(b'wrong'), + media_type='text/plain', + expected='sha256:' + '0' * 64, + ) + check(bad == {'error': 'digest_mismatch'}, 'digest-mismatch') + check(not (store.index / ('0' * 64)).exists(), 'digest-mismatch-no-index') + check(not list(store.staging.iterdir()), 'normal-paths-clean-staging') + finally: + native.stop() + + +async def hardlinks_and_corruption(binary, root): + store = GitContentStore(root) + blob = await store.put_bytes(b'correct', 'application/octet-stream') + path = store.binary / blob.digest.removeprefix('sha256:') + link = root / 'lfs-shared-link' + os.link(path, link) + before = path.stat().st_ino + native = Native(binary, root) + try: + native.call(action='put', bytes=b64(b'correct'), media_type=blob.media_type, expected=None) + check( + path.stat().st_ino == before and link.stat().st_nlink == 2, + 'binary-inode-and-lfs-hardlink-preserved', + ) + path.write_bytes(b'corrupt') + check( + native.send(action='read', blob=wire(blob), range=[0, 1]) + == {'error': 'content_digest_mismatch'}, + 'corrupt-range-rejected', + ) + check( + native.send( + action='put', bytes=b64(b'correct'), media_type=blob.media_type, expected=None + ) + == {'error': 'content_digest_mismatch'}, + 'corrupt-duplicate-not-replaced', + ) + check(link.read_bytes() == b'corrupt', 'existing-inode-never-rewritten') + path.unlink() + path.symlink_to(link) + check( + native.send(action='read', blob=wire(blob), range=None) + == {'error': 'unsafe_content_path'}, + 'binary-symlink-rejected', + ) + finally: + native.stop() + + +def hermetic_git(binary, root): + root.mkdir() + env = os.environ | { + 'GIT_DIR': str(root / 'wrong'), + 'GIT_OBJECT_DIRECTORY': str(root / 'wrong-objects'), + 'GIT_CONFIG_COUNT': '1', + 'GIT_CONFIG_KEY_0': 'core.bare', + 'GIT_CONFIG_VALUE_0': 'false', + } + native = Native(binary, root, env=env) + try: + native.call(action='put', bytes=b64(b'ready'), media_type='text/plain', expected=None) + hooks = root / 'private.git' / 'hooks' + hooks.mkdir(exist_ok=True) + marker = root / 'hook-executed' + hook = hooks / 'reference-transaction' + hook.write_text('#!/bin/sh\nprintf bad > "' + str(marker) + '"\n') + hook.chmod(0o700) + native.call( + action='put', bytes=b64(b'no inherited config'), media_type='text/plain', expected=None + ) + check(not marker.exists(), 'git-hooks-disabled') + check(not (root / 'wrong-objects').exists(), 'inherited-git-environment-ignored') + finally: + native.stop() + + +async def crashes(binary, root): + root.mkdir() + native = Native(binary, root) + try: + checkpoint = native.send( + action='generated', + length=131_072, + media_type='text/plain', + hold_during=True, + hold_after=False, + ) + check(checkpoint == {'checkpoint': 'held'}, 'crash-before-publication-checkpoint') + finally: + native.stop(kill=True) + check(not list((root / 'index').iterdir()), 'crash-before-publication-no-index') + # SIGKILL leaves staging bytes intentionally: no unsafe GC/recovery claim. + check(bool(list((root / 'staging').iterdir())), 'crash-staging-retained-for-recovery') + native = Native(binary, root) + length = 131_073 + blob = BlobRef( + digest='sha256:' + hashlib.sha256(b'x' * length).hexdigest(), + size=length, + media_type='text/plain', + ) + try: + checkpoint = native.send( + action='generated', + length=length, + media_type=blob.media_type, + hold_during=False, + hold_after=True, + ) + check(checkpoint == {'checkpoint': 'held'}, 'crash-after-publication-checkpoint') + finally: + native.stop(kill=True) + store = GitContentStore(root) + check(await store.read_bytes(blob) == b'x' * length, 'python-reads-after-lost-native-response') + native = Native(binary, root) + try: + check( + native.call( + action='generated', + length=length, + media_type=blob.media_type, + hold_during=False, + hold_after=False, + ) + == wire(blob), + 'retry-after-lost-response', + ) + finally: + native.stop() + + +async def main(binary): + with tempfile.TemporaryDirectory(prefix='msg-content-parity-') as name: + root = Path(name) + await interoperate(binary, root / 'interop') + await hardlinks_and_corruption(binary, root / 'hardlinks') + hermetic_git(binary, root / 'hermetic') + await crashes(binary, root / 'crashes') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('binary', type=Path) + parser.add_argument('--report', type=Path) + args = parser.parse_args() + asyncio.run(main(args.binary.resolve())) + result = { + 'checks': len(REPORT), + 'passed': REPORT, + 'scope': 'content storage primitives; not a server or recovery journal', + } + if args.report: + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') + print(json.dumps(result, ensure_ascii=False)) From 08c87c14e0186ed511e2371bf0699ef4e1127425 Mon Sep 17 00:00:00 2001 From: LIghtJUNction Date: Tue, 13 Oct 2026 08:53:45 +0800 Subject: [PATCH 22/22] fix: align native admission and stores with current key-only identity --- .github/workflows/rust-wire.yml | 22 +- rust/README.md | 31 ++- .../crates/msg-identity/src/authentication.rs | 32 +-- rust/crates/msg-identity/src/lib.rs | 2 +- rust/crates/msg-identity/src/oauth.rs | 258 ------------------ rust/crates/msg-identity/src/sessions.rs | 170 ++++++++++++ rust/crates/msg-identity/src/store.rs | 2 +- rust/crates/msg-identity/tests/sessions.rs | 141 ++++++++++ .../msg-storage/examples/authority_check.rs | 13 +- rust/crates/msg-storage/src/lib.rs | 10 +- .../msg-storage/src/postgres/authority.rs | 8 +- rust/crates/msg-storage/src/postgres/mod.rs | 2 +- rust/crates/msg-storage/src/writer/tests.rs | 2 +- rust/tests/test_identity_parity.py | 223 ++++++++------- 14 files changed, 472 insertions(+), 444 deletions(-) delete mode 100644 rust/crates/msg-identity/src/oauth.rs create mode 100644 rust/crates/msg-identity/src/sessions.rs create mode 100644 rust/crates/msg-identity/tests/sessions.rs diff --git a/.github/workflows/rust-wire.yml b/.github/workflows/rust-wire.yml index b913bf10..d282c278 100644 --- a/.github/workflows/rust-wire.yml +++ b/.github/workflows/rust-wire.yml @@ -1,26 +1,6 @@ name: Rust native conformance on: - pull_request: - paths: - - 'rust/**' - - 'src/msg/core/**' - - 'src/msg/security/**' - - 'src/msg/storage/**' - - 'src/msg/oauth_config.py' - - 'src/msg/constants.py' - - 'pyproject.toml' - - '.github/workflows/rust-wire.yml' - push: - branches: [main, 'migration/rust-*'] - paths: - - 'rust/**' - - 'src/msg/core/**' - - 'src/msg/security/**' - - 'src/msg/storage/**' - - 'src/msg/oauth_config.py' - - 'src/msg/constants.py' - - 'pyproject.toml' - - '.github/workflows/rust-wire.yml' + # Daily changes are accepted locally; retain explicit conformance reproduction. workflow_dispatch: permissions: contents: read diff --git a/rust/README.md b/rust/README.md index 02d48ae4..444645d8 100644 --- a/rust/README.md +++ b/rust/README.md @@ -119,7 +119,7 @@ The original inspected main baseline is | `msg-core` | Bounded strict JSON, duplicate rejection, canonical numbers/Unicode, lossless typed JSON, SHA-256/base64url | `core/codec.py` | | `msg-crypto` | Existing key/subject IDs, purpose-framed Ed25519 signing/verification | `security/crypto.py` | | `msg-protocol` | Request shape/defaults, UTC time, business digest/signature bytes, immutable field access | `core/packet.py`, `core/requests.py`, `core/models.py` | -| `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, browser/API/OAuth and token-recovery validation, transaction-bound resource authorization | `security/authentication.py`, `certificates.py`, `policy.py`, `oauth.py` | +| `msg-identity` | Typed identity/credential/certificate/CSR records, credential and current-state admission, pinned CA chains, live delegation, scope/ceiling/mode primitives, key-approved browser/API binding and token-recovery validation, transaction-bound resource authorization | `security/authentication.py`, `certificates.py`, `policy.py`, `sessions.py` | | `msg-storage` | Transaction-bound authority reads; SQLite/PostgreSQL writers and typed records, savepoints/rollback effects; native Git/blob content, ranges, pins and revision trees | `storage/sqlite.py`, `storage/postgres.py`, `storage/git.py` and their record/session modules | Authentication validates service/digest/expiry, current credentials, SHA-256 @@ -137,10 +137,12 @@ registry projection is injected locally; requests cannot supply authority rules. Only the existing network-constraint schema is implemented, not a generic schema engine or plugin loader. -OAuth validation follows current signing parents, captured/current ceilings, -auth versions, browser sessions, refresh-family authority and custodial vault -status. MCP audience is passed from a trusted adapter, not from an envelope -field. This is **binding validation**, not an OAuth or MCP server. The helper's +Key-approved session validation follows current signing parents, captured/current +ceilings, auth versions, nested session revocation and custodial vault status. +The current `credential_states` schema is used by both native stores. Historical +`t_oauth_`/`t_browser_` credentials and login bindings remain rejected even if +old rows are restored. This is **binding validation**, not credential issuance +or an OAuth/MCP server; no removed module, OAuth scope or Bearer authority is revived. The helper's operation/capability declarations are synthetic test configuration, not a production registry adapter. @@ -198,7 +200,7 @@ are not implemented by this slice. The earlier process-kill tests cover **SQL metadata**, not a complete operation. The content tests above additionally cover standalone blob publication. Neither rollback callbacks nor private staging implement a durable cross-store recovery -journal: callbacks do not run after SIGKILL. Identity/OAuth workflows, outbox/lease +journal: callbacks do not run after SIGKILL. Identity/session issuance workflows, outbox/lease execution and complete operation recovery still need to be ported. PostgreSQL metadata primitives are already implemented above. Do not introduce dual writes. @@ -226,17 +228,16 @@ PYTHONPATH=src python rust/tests/test_content_parity.py \ ``` Repeat the build with `--release` and the same suites against -`rust/target/release/examples/`. CI performs both profiles, records exact source -SHA/toolchains, and preserves reports plus `Cargo.lock`. Source changes to Python -identity/storage/OAuth also trigger differential checks. The local offline editor -now has the same pinned compiler and Python reference dependencies; independent -cloud results must still be inspected before claiming CI acceptance. +`rust/target/release/examples/`. The opt-in conformance workflow performs both profiles and records exact source +SHA/toolchains plus `Cargo.lock`. Daily identity/storage/session changes are +validated locally; no automatic CI acceptance is claimed. Use the pinned compiler +and current Python reference, never a compatibility shim for a removed auth API. The wire suite covers canonical floating-point boundaries, Unicode, large integers, malformed packets and bidirectional signatures. The authority suite creates disposable databases using the actual Python `SqliteMetadataStore`, then compares native and Python decisions over the same records. It covers live -revocation, CA constraints, OAuth source changes, recovery, exact retries and a +revocation, CA constraints, key-approved session/source changes, rejected legacy authority, recovery, exact retries and a real concurrent WAL snapshot test. It also checks all 4096 modes across three classes and five permissions: **61,440 simple mode decisions**, not 61,440 independent identity scenarios. Report groups separate these from other cases. @@ -267,7 +268,7 @@ cryptographic and database primitives are delegated to pinned libraries. use a unified `invalid_identity_record` code instead of Python's field-specific decoder errors. Corrupt result body/namespace mismatches are explicitly rejected. - Ancestor traversal is capped at 1024, certificate traversal at 32 with a shared - 4096-work budget, and OAuth source chains at 32. Cycles/oversized restored + 4096-work budget, and session source chains at 32. Cycles/oversized restored authority fail closed. These defensive limits are not silently described as accepting every Python input. - Dalek strict verification rejects weak-key forgeries. Historical key/signature @@ -278,8 +279,8 @@ cryptographic and database primitives are delegated to pinned libraries. canonical IDs/validated domain records; its DTOs are not public request schemas. The stricter poisoned-write and malformed-result checks are explicit safety differences, not assertions of identical behavior for every Python input. -- Operation handlers, mutation-side identity/OAuth - issuance/refresh/recovery, encrypted-key wrapping and remaining content stores, +- Operation handlers, mutation-side key-approved identity/session + issuance/recovery, encrypted-key wrapping and remaining content stores, native `serve`/HTTP/MCP, `worker`, packaging and deployment remain pending. - No RSS, memory-reduction or throughput result is claimed. Benchmark equivalent functionality, data, concurrency and complete process trees before comparison. diff --git a/rust/crates/msg-identity/src/authentication.rs b/rust/crates/msg-identity/src/authentication.rs index b4b8c0e9..6c01e7ea 100644 --- a/rust/crates/msg-identity/src/authentication.rs +++ b/rust/crates/msg-identity/src/authentication.rs @@ -84,7 +84,7 @@ pub struct AuthenticationService { runtime: RuntimeGeneration, primary_ceiling: Vec, temporary_ceiling: Vec, - oauth_policy: Option, + browser_policy: Option, } impl AuthenticationService { pub fn new( @@ -98,11 +98,11 @@ impl AuthenticationService { runtime: RuntimeGeneration::capture(store)?, primary_ceiling, temporary_ceiling, - oauth_policy: None, + browser_policy: None, }) } - pub fn with_oauth_policy(mut self, policy: crate::oauth::OAuthPolicy) -> Self { - self.oauth_policy = Some(policy); + pub fn with_browser_policy(mut self, policy: crate::sessions::BrowserPolicy) -> Self { + self.browser_policy = Some(policy); self } pub fn authenticate( @@ -112,19 +112,6 @@ impl AuthenticationService { store: &dyn AuthorityStore, entry: Entry, now: Timestamp, - ) -> Result { - self.authenticate_for_resource(request, spec, store, entry, now, None) - } - /// `resource` must be supplied by the adapter from its validated request URL, - /// never copied from a field in the operation envelope. - pub fn authenticate_for_resource( - &mut self, - request: &Request, - spec: &OperationPolicy, - store: &dyn AuthorityStore, - entry: Entry, - now: Timestamp, - resource: Option<&str>, ) -> Result { let requested_subject = optional_string(request.field("subject")?)?; check_archived(store, requested_subject)?; @@ -233,13 +220,12 @@ impl AuthenticationService { "invalid_recovery_proof", )?; let old = store.credential(argument(request, "credential_id")?.as_str()?)?; - crate::oauth::require_binding( + crate::sessions::require_binding( store, &old, now, - self.oauth_policy.as_ref(), + self.browser_policy.as_ref(), &self.temporary_ceiling, - resource, )?; require( old.kind == CredentialKind::Token && requested_subject == Some(&old.subject_id), @@ -378,13 +364,12 @@ impl AuthenticationService { bool::from(credential.verifier.0.as_slice().ct_eq(hash.as_slice())), "invalid_token", )?; - crate::oauth::require_binding( + crate::sessions::require_binding( store, &credential, now, - self.oauth_policy.as_ref(), + self.browser_policy.as_ref(), &self.temporary_ceiling, - resource, )?; ("token", vec![]) } @@ -573,7 +558,6 @@ fn custodial_write(name: &str) -> bool { matches!( name, "identity.rename" - | "identity.oauth_approve" | "content.post_create" | "content.post_edit" | "content.file_put" diff --git a/rust/crates/msg-identity/src/lib.rs b/rust/crates/msg-identity/src/lib.rs index 2fcacaaf..481a3907 100644 --- a/rust/crates/msg-identity/src/lib.rs +++ b/rust/crates/msg-identity/src/lib.rs @@ -6,8 +6,8 @@ pub mod authentication; pub mod authorization; pub mod certificates; pub mod models; -pub mod oauth; pub mod policy; +pub mod sessions; pub mod store; use msg_core::{Error, Result}; diff --git a/rust/crates/msg-identity/src/oauth.rs b/rust/crates/msg-identity/src/oauth.rs deleted file mode 100644 index 63cbcc03..00000000 --- a/rust/crates/msg-identity/src/oauth.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! Current-state browser/API/OAuth bindings; no issuance or refresh writes. -//! The resource audience is a trusted adapter value, never a packet field. -use crate::{ - models::*, - policy::grant_subset, - require, - store::{truthy, AuthorityStore}, -}; -use msg_core::{Error, Json, Result}; -use serde::Deserialize; -use std::collections::BTreeSet; - -#[derive(Clone, Deserialize)] -pub struct OAuthClientPolicy { - pub client_id: String, - pub scopes: BTreeSet, -} -#[derive(Clone, Deserialize)] -pub struct OAuthPolicy { - pub enabled: bool, - pub clients: Vec, -} -#[derive(Clone)] -pub struct OAuthState { - pub expires_at: Timestamp, - pub body: Json, -} -#[derive(Deserialize)] -struct Source { - parent: String, - subject: String, - auth_version: u64, - #[serde(default)] - custodial: bool, - #[serde(default)] - ceiling: Vec, - #[serde(default)] - session: Option, -} -#[derive(Deserialize)] -struct Family { - parent: String, - subject: String, - client_id: String, - scopes: BTreeSet, - #[serde(default)] - resource: Option, - ceiling: Vec, -} - -fn field<'a>(body: &'a Json, name: &str) -> Result<&'a Json> { - body.as_object()?.get(name).ok_or(Error("invalid_grant")) -} -fn revoked(body: &Json) -> Result { - body.as_object()? - .get("revoked") - .map(truthy) - .transpose() - .map(|v| v.unwrap_or(false)) -} -fn get(store: &dyn AuthorityStore, id: &str, now: Timestamp) -> Result { - let state = store.oauth_state(id)?.ok_or(Error("invalid_grant"))?; - require(state.expires_at > now, "invalid_grant")?; - Ok(state.body) -} -fn require_ceiling(store: &dyn AuthorityStore, ceiling: &[Grant], parent: &[Grant]) -> Result<()> { - for grant in ceiling { - let mut allowed = false; - for source in parent { - allowed |= grant_subset(grant, source, store)?; - } - require(allowed, "invalid_grant")?; - } - Ok(()) -} -fn require_source( - store: &dyn AuthorityStore, - source: &Json, - now: Timestamp, - custodial_ceiling: &[Grant], - seen: &mut BTreeSet, -) -> Result<()> { - let source: Source = decode(source).map_err(|_| Error("invalid_grant"))?; - let parent = store.credential(&source.parent)?; - let subject = store.subject(&source.subject)?; - require( - parent.subject_id == subject.resource_id - && !subject.local_only - && subject.auth_version == source.auth_version - && parent.current_at(now), - "invalid_grant", - )?; - if source.custodial { - let vault = store.custodial_binding(&subject.resource_id)?; - require( - subject.kind == SubjectKind::Custodial - && vault - .as_ref() - .is_some_and(|(key, status)| key == &parent.id && status == "active"), - "invalid_grant", - )?; - } - require_ceiling( - store, - &source.ceiling, - if source.custodial { - custodial_ceiling - } else { - &parent.ceiling - }, - )?; - if let Some(session) = source.session.filter(|s| !s.is_empty()) { - // The Python path currently recurses without a bound. Restored cyclic - // or oversized chains fail closed rather than exhausting the stack. - require( - seen.len() < 32 && seen.insert(session.clone()), - "oauth_source_cycle", - )?; - let body = get(store, &session, now)?; - require(!revoked(&body)?, "invalid_grant")?; - require_source(store, &body, now, custodial_ceiling, seen)?; - } - Ok(()) -} -fn enabled(policy: Option<&OAuthPolicy>) -> Result<&OAuthPolicy> { - policy.filter(|p| p.enabled).ok_or(Error("oauth_disabled")) -} - -pub fn require_binding( - store: &dyn AuthorityStore, - credential: &Credential, - now: Timestamp, - policy: Option<&OAuthPolicy>, - custodial_ceiling: &[Grant], - resource: Option<&str>, -) -> Result<()> { - if let Some(browser) = store.oauth_state(&format!("browser:{}", credential.id))? { - enabled(policy)?; - let source: Source = decode(&browser.body).map_err(|_| Error("invalid_grant"))?; - require( - credential.source_credential_id.as_deref() == Some(&source.parent) - && credential.subject_id == source.subject, - "invalid_grant", - )?; - require_source( - store, - &browser.body, - now, - custodial_ceiling, - &mut BTreeSet::new(), - )?; - return require_ceiling( - store, - &credential.ceiling, - &decode::>(field(&browser.body, "ceiling")?)?, - ); - } - let api = store.oauth_state(&format!("api:{}", credential.id))?; - if let Some(api) = &api { - let source: Source = decode(&api.body).map_err(|_| Error("invalid_grant"))?; - require( - credential.source_credential_id.as_deref() == Some(&source.parent), - "invalid_grant", - )?; - require_source( - store, - &api.body, - now, - custodial_ceiling, - &mut BTreeSet::new(), - )?; - require_ceiling( - store, - &credential.ceiling, - &store.credential(&source.parent)?.ceiling, - )?; - } - let access = store.oauth_state(&format!("access:{}", credential.id))?; - let Some(access) = access else { - return require( - !credential.id.starts_with("t_oauth_") - && !credential.id.starts_with("t_browser_") - && (credential.source_credential_id.is_none() || api.is_some()), - "invalid_grant", - ); - }; - let body = get(store, field(&access.body, "family")?.as_str()?, now)?; - let family: Family = decode(&body).map_err(|_| Error("invalid_grant"))?; - require( - credential.source_credential_id.as_deref() == Some(&family.parent), - "invalid_grant", - )?; - let client = enabled(policy)? - .clients - .iter() - .find(|c| c.client_id == family.client_id) - .ok_or(Error("invalid_client"))?; - require( - family.scopes.is_subset(&client.scopes) && !revoked(&body)?, - "invalid_grant", - )?; - let has_mcp = family.scopes.iter().any(|s| { - matches!( - s.as_str(), - "msg.mcp.read" | "msg.mcp.message" | "msg.mcp.post" - ) - }); - if let Some(audience) = &family.resource { - require( - credential.subject_id == family.subject && resource == Some(audience), - "invalid_grant", - )?; - require( - has_mcp && !family.scopes.contains("msg.read") && !family.scopes.contains("msg.write"), - "invalid_grant", - )?; - for grant in &credential.ceiling { - require( - grant - .operations - .iter() - .all(|op| family.scopes.iter().any(|scope| mcp_operation(scope, op))), - "invalid_grant", - )?; - } - } else { - require(!has_mcp, "invalid_grant")?; - } - require_source(store, &body, now, custodial_ceiling, &mut BTreeSet::new())?; - require_ceiling(store, &credential.ceiling, &family.ceiling) -} - -fn mcp_operation(scope: &str, operation: &str) -> bool { - match scope { - "msg.mcp.read" => matches!( - operation, - "discovery.get@1" - | "discovery.resolve@1" - | "discovery.read_segment@1" - | "discovery.read_query@3" - | "discovery.lexical_search@5" - | "communication.inbox@1" - | "communication.dm_list@1" - | "communication.conversation_get@1" - | "communication.changes@1" - ), - "msg.mcp.message" => matches!( - operation, - "communication.dm_request@3" - | "communication.dm_send@2" - | "communication.dm_accept@2" - | "communication.dm_reject@2" - | "discussion.ack@1" - ), - "msg.mcp.post" => matches!(operation, "content.post_create@2" | "discussion.reply@2"), - _ => false, - } -} diff --git a/rust/crates/msg-identity/src/sessions.rs b/rust/crates/msg-identity/src/sessions.rs new file mode 100644 index 00000000..945ce9a1 --- /dev/null +++ b/rust/crates/msg-identity/src/sessions.rs @@ -0,0 +1,170 @@ +//! Current key-approved browser/API bindings. No OAuth/Bearer revival or issuance. +use crate::{ + models::*, + policy::grant_subset, + require, + store::{truthy, AuthorityStore}, +}; +use msg_core::{Error, Json, Result}; +use serde::Deserialize; +use std::collections::BTreeSet; + +#[derive(Clone, Deserialize)] +pub struct BrowserPolicy { + pub enabled: bool, +} +#[derive(Clone)] +pub struct CredentialState { + pub expires_at: Timestamp, + pub body: Json, +} +#[derive(Deserialize)] +struct Source { + parent: String, + subject: String, + auth_version: u64, + #[serde(default)] + custodial: bool, + #[serde(default)] + ceiling: Vec, + #[serde(default)] + session: Option, +} +fn rejected_family(id: &str) -> bool { + id.starts_with("t_oauth_") || id.starts_with("t_browser_") +} +fn revoked(body: &Json) -> Result { + body.as_object()? + .get("revoked") + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn login_binding(body: &Json) -> Result { + body.as_object()? + .get("login_binding") + .map(truthy) + .transpose() + .map(|v| v.unwrap_or(false)) +} +fn get(store: &dyn AuthorityStore, id: &str, now: Timestamp) -> Result { + let state = store.credential_state(id)?.ok_or(Error("invalid_grant"))?; + require(state.expires_at > now, "invalid_grant")?; + Ok(state.body) +} +fn require_ceiling(store: &dyn AuthorityStore, ceiling: &[Grant], parent: &[Grant]) -> Result<()> { + for grant in ceiling { + let mut allowed = false; + for source in parent { + allowed |= grant_subset(grant, source, store)?; + } + require(allowed, "invalid_grant")?; + } + Ok(()) +} +fn require_source( + store: &dyn AuthorityStore, + body: &Json, + now: Timestamp, + custodial_ceiling: &[Grant], + seen: &mut BTreeSet, +) -> Result<()> { + let source: Source = decode(body).map_err(|_| Error("invalid_grant"))?; + let parent = store.credential(&source.parent)?; + let subject = store.subject(&source.subject)?; + require( + parent.subject_id == subject.resource_id + && !rejected_family(&parent.id) + && !subject.local_only + && subject.auth_version == source.auth_version + && parent.current_at(now) + && !login_binding(body)?, + "invalid_grant", + )?; + if source.custodial { + let vault = store.custodial_binding(&subject.resource_id)?; + require( + subject.kind == SubjectKind::Custodial + && vault + .as_ref() + .is_some_and(|(key, status)| key == &parent.id && status == "active"), + "invalid_grant", + )?; + } + require_ceiling( + store, + &source.ceiling, + if source.custodial { + custodial_ceiling + } else { + &parent.ceiling + }, + )?; + if let Some(session) = source.session.filter(|s| !s.is_empty()) { + // Corrupt restored cycles fail closed; they never exhaust a native stack. + require( + seen.len() < 32 && seen.insert(session.clone()), + "invalid_grant", + )?; + let body = get(store, &session, now)?; + require(!revoked(&body)?, "invalid_grant")?; + require_source(store, &body, now, custodial_ceiling, seen)?; + } + Ok(()) +} +pub fn require_binding( + store: &dyn AuthorityStore, + credential: &Credential, + now: Timestamp, + policy: Option<&BrowserPolicy>, + custodial_ceiling: &[Grant], +) -> Result<()> { + // Restoring historical access/family rows must not reactivate those protocols. + require(!rejected_family(&credential.id), "invalid_grant")?; + if let Some(browser) = store.credential_state(&format!("browser:{}", credential.id))? { + require( + policy.is_some_and(|p| p.enabled), + "browser_sessions_disabled", + )?; + let source: Source = decode(&browser.body).map_err(|_| Error("invalid_grant"))?; + require( + credential.id.starts_with("t_session_") + && credential.source_credential_id.as_deref() == Some(&source.parent) + && credential.subject_id == source.subject + && !source.custodial + && !login_binding(&browser.body)?, + "invalid_grant", + )?; + require_source( + store, + &browser.body, + now, + custodial_ceiling, + &mut BTreeSet::new(), + )?; + return require_ceiling(store, &credential.ceiling, &source.ceiling); + } + if let Some(api) = store.credential_state(&format!("api:{}", credential.id))? { + let source: Source = decode(&api.body).map_err(|_| Error("invalid_grant"))?; + require( + credential.source_credential_id.as_deref() == Some(&source.parent), + "invalid_grant", + )?; + require_source( + store, + &api.body, + now, + custodial_ceiling, + &mut BTreeSet::new(), + )?; + return require_ceiling( + store, + &credential.ceiling, + &store.credential(&source.parent)?.ceiling, + ); + } + require( + !credential.id.starts_with("t_session_") && credential.source_credential_id.is_none(), + "invalid_grant", + ) +} diff --git a/rust/crates/msg-identity/src/store.rs b/rust/crates/msg-identity/src/store.rs index 078abce6..39fe9571 100644 --- a/rust/crates/msg-identity/src/store.rs +++ b/rust/crates/msg-identity/src/store.rs @@ -26,7 +26,7 @@ pub trait AuthorityStore { fn certificate_revoked(&self, id: &str) -> Result; fn request_result(&self, subject: Option<&str>, id: &str, digest: &str) -> Result>; - fn oauth_state(&self, id: &str) -> Result>; + fn credential_state(&self, id: &str) -> Result>; fn custodial_binding(&self, subject: &str) -> Result>; fn recovery_delivery( &self, diff --git a/rust/crates/msg-identity/tests/sessions.rs b/rust/crates/msg-identity/tests/sessions.rs new file mode 100644 index 00000000..1e1510ff --- /dev/null +++ b/rust/crates/msg-identity/tests/sessions.rs @@ -0,0 +1,141 @@ +use msg_core::{Error, Json, Result}; +use msg_identity::{ + models::*, + sessions::{require_binding, BrowserPolicy, CredentialState}, + store::{AuthorityStore, Record, RecoveryDelivery}, +}; +use std::collections::BTreeMap; + +struct Memory { + states: BTreeMap, +} +fn now() -> Timestamp { + Timestamp::parse("2026-10-05T00:00:00Z").unwrap() +} +fn credential(id: &str, parent: Option<&str>) -> Credential { + Credential { + id: id.into(), + subject_id: "u_fixture".into(), + kind: CredentialKind::Token, + verifier: Verifier(vec![0; 32]), + ceiling: vec![], + not_before: now(), + expires_at: Some(Timestamp::parse("2030-01-01T00:00:00Z").unwrap()), + revoked_at: None, + source_credential_id: parent.map(String::from), + } +} +fn source(session: Option<&str>) -> CredentialState { + let body = serde_json::json!({"parent":"t_parent","subject":"u_fixture","auth_version":1,"ceiling":[],"session":session}); + CredentialState { + expires_at: Timestamp::parse("2030-01-01T00:00:00Z").unwrap(), + body: Json::parse(&body.to_string()).unwrap(), + } +} +impl AuthorityStore for Memory { + fn record(&self, kind: Record, _id: &str) -> Result { + match kind { + Record::Subject => wire(&Subject { + resource_id: "u_fixture".into(), + kind: SubjectKind::Registered, + primary_group: "g_fixture".into(), + auth_version: 1, + local_only: false, + }), + Record::Credential => wire(&credential("t_parent", None)), + _ => Err(Error("not_found")), + } + } + fn setting(&self, _key: &str) -> Result> { + Ok(None) + } + fn has_setting(&self, _key: &str) -> Result { + Ok(false) + } + fn certificate_revoked(&self, _id: &str) -> Result { + Ok(false) + } + fn request_result( + &self, + _subject: Option<&str>, + _id: &str, + _digest: &str, + ) -> Result> { + Ok(None) + } + fn credential_state(&self, id: &str) -> Result> { + Ok(self.states.get(id).cloned()) + } + fn custodial_binding(&self, _subject: &str) -> Result> { + Ok(None) + } + fn recovery_delivery( + &self, + _credential: &str, + _subject: &str, + ) -> Result> { + Ok(None) + } +} +#[test] +fn browser_disabled_does_not_accept_a_valid_stored_binding() { + let memory = Memory { + states: BTreeMap::from([("browser:t_session_fixture".into(), source(None))]), + }; + let c = credential("t_session_fixture", Some("t_parent")); + for policy in [None, Some(BrowserPolicy { enabled: false })] { + assert!(matches!( + require_binding(&memory, &c, now(), policy.as_ref(), &[]), + Err(Error("browser_sessions_disabled")) + )); + } + require_binding( + &memory, + &c, + now(), + Some(&BrowserPolicy { enabled: true }), + &[], + ) + .unwrap(); +} +#[test] +fn restored_session_cycle_is_bounded_and_denied() { + let memory = Memory { + states: BTreeMap::from([ + ("api:t_api_fixture".into(), source(Some("session:cycle"))), + ("session:cycle".into(), source(Some("session:cycle"))), + ]), + }; + assert!(matches!( + require_binding( + &memory, + &credential("t_api_fixture", Some("t_parent")), + now(), + None, + &[] + ), + Err(Error("invalid_grant")) + )); +} +#[test] +fn restored_excessive_source_depth_is_bounded_and_denied() { + let mut memory = Memory { + states: BTreeMap::from([("api:t_api_fixture".into(), source(Some("session:0")))]), + }; + for i in 0..33 { + let next = (i < 32).then(|| format!("session:{}", i + 1)); + memory + .states + .insert(format!("session:{i}"), source(next.as_deref())); + } + assert!(matches!( + require_binding( + &memory, + &credential("t_api_fixture", Some("t_parent")), + now(), + None, + &[] + ), + Err(Error("invalid_grant")) + )); +} diff --git a/rust/crates/msg-storage/examples/authority_check.rs b/rust/crates/msg-storage/examples/authority_check.rs index 49b7dad7..af6480c7 100644 --- a/rust/crates/msg-storage/examples/authority_check.rs +++ b/rust/crates/msg-storage/examples/authority_check.rs @@ -29,7 +29,7 @@ struct Context { capabilities: Vec, primary_ceiling: Vec, temporary_ceiling: Vec, - oauth: msg_identity::oauth::OAuthPolicy, + browser: msg_identity::sessions::BrowserPolicy, #[serde(default)] resource_types: Vec, #[serde(default)] @@ -81,14 +81,7 @@ fn run( "worker" => Entry::Worker, _ => return Err(Error("invalid_test_input")), }; - match auth.authenticate_for_resource( - &request, - &policy, - store, - entry, - now()?, - input.get("resource").and_then(Value::as_str), - )? { + match auth.authenticate(&request, &policy, store, entry, now()?)? { Admission::Fresh(p) => Ok(json!({"kind":"fresh", "principal":p})), Admission::Replay(result) => Ok(json!({"kind":"replay", "result":result})), } @@ -308,7 +301,7 @@ fn main() -> io::Result<()> { context.primary_ceiling.clone(), context.temporary_ceiling.clone(), )? - .with_oauth_policy(context.oauth.clone()); + .with_browser_policy(context.browser.clone()); let authorization = AuthorizationService::new( context.validator()?, snapshot.authority(), diff --git a/rust/crates/msg-storage/src/lib.rs b/rust/crates/msg-storage/src/lib.rs index 07230a37..97732060 100644 --- a/rust/crates/msg-storage/src/lib.rs +++ b/rust/crates/msg-storage/src/lib.rs @@ -12,7 +12,7 @@ pub mod writer; use msg_core::{Error, Json, Result, MAX_BYTES}; use msg_identity::{ models::Timestamp, - oauth::OAuthState, + sessions::CredentialState, store::{AuthorityStore, Record, RecoveryDelivery}, }; use rusqlite::{Connection, ErrorCode, OpenFlags, OptionalExtension, Transaction}; @@ -89,7 +89,7 @@ impl Session<'_, Access> { ("identities", "id,kind,body"), ("credentials", "id,subject,body"), ("certificates", "id,subject,parent,revoked,body"), ("resources", "id,parent,body"), ("results", "subject,request_id,digest,body"), - ("oauth_states", "id,expires,body"), + ("credential_states", "id,expires,body"), ("custodial_vault", "subject,signing_key_id,status"), ("token_deliveries", "credential_id,subject,recovery_verifier,recovery_expires_at,consumed_at,request_id"), ]; @@ -217,16 +217,16 @@ impl AuthorityStore for Session<'_, Access> { } Ok(Some(result)) } - fn oauth_state(&self, id: &str) -> Result> { + fn credential_state(&self, id: &str) -> Result> { self.active()?; let row: Option<(String, String)> = self.transaction.query_row( - "SELECT expires,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM oauth_states WHERE id=?1", + "SELECT expires,CAST(substr(CAST(body AS BLOB),1,1048577) AS TEXT) FROM credential_states WHERE id=?1", [id], |r| Ok((r.get(0)?, r.get(1)?))).optional().map_err(database_error)?; row.map(|(expires, body)| { if body.len() > MAX_BYTES { return Err(Error("storage_record_too_large")); } - Ok(OAuthState { + Ok(CredentialState { expires_at: Timestamp::parse(&expires)?, body: Json::parse(&body)?, }) diff --git a/rust/crates/msg-storage/src/postgres/authority.rs b/rust/crates/msg-storage/src/postgres/authority.rs index b75af89b..abe2a57b 100644 --- a/rust/crates/msg-storage/src/postgres/authority.rs +++ b/rust/crates/msg-storage/src/postgres/authority.rs @@ -5,7 +5,7 @@ use msg_core::{Error, Json, Result}; use msg_identity::{ authorization::{AuthorizationStore, DirectConversation, LegacyShare, ShareSource}, models::{decode, SubjectKind, Timestamp}, - oauth::OAuthState, + sessions::CredentialState, store::{AuthorityStore, Record, RecoveryDelivery}, }; use std::collections::BTreeSet; @@ -84,13 +84,13 @@ impl AuthorityStore for PgSession { } Ok(Some(result)) } - fn oauth_state(&self, id: &str) -> Result> { + fn credential_state(&self, id: &str) -> Result> { self.one( - "SELECT expires,substring(body,1,1048577) FROM oauth_states WHERE id=$1", + "SELECT expires,substring(body,1,1048577) FROM credential_states WHERE id=$1", &[id.into()], )? .map(|r| { - Ok(OAuthState { + Ok(CredentialState { expires_at: Timestamp::parse(r.text(0)?)?, body: r.json(1)?, }) diff --git a/rust/crates/msg-storage/src/postgres/mod.rs b/rust/crates/msg-storage/src/postgres/mod.rs index d0ddc3e6..de878511 100644 --- a/rust/crates/msg-storage/src/postgres/mod.rs +++ b/rust/crates/msg-storage/src/postgres/mod.rs @@ -399,7 +399,7 @@ impl PgSession { ), ("settings", vec!["key", "value"]), ("results", vec!["subject", "request_id", "digest", "body"]), - ("oauth_states", vec!["id", "kind", "expires", "body"]), + ("credential_states", vec!["id", "kind", "expires", "body"]), ("revisions", vec!["id", "resource_id", "created_at", "body"]), ( "relations", diff --git a/rust/crates/msg-storage/src/writer/tests.rs b/rust/crates/msg-storage/src/writer/tests.rs index a5a6dbc1..8c36381c 100644 --- a/rust/crates/msg-storage/src/writer/tests.rs +++ b/rust/crates/msg-storage/src/writer/tests.rs @@ -31,7 +31,7 @@ impl Fixture { CREATE TABLE resources(id TEXT PRIMARY KEY,type TEXT,name TEXT,parent TEXT REFERENCES resources(id) DEFERRABLE INITIALLY DEFERRED,owner TEXT,grp TEXT,mode INTEGER,generation INTEGER,revision TEXT,state TEXT,created_at TEXT,modified_at TEXT,body TEXT,UNIQUE(parent,name)); CREATE UNIQUE INDEX one_root ON resources((1)) WHERE parent IS NULL; CREATE TABLE results(subject TEXT,request_id TEXT,digest TEXT,body TEXT,PRIMARY KEY(subject,request_id)); - CREATE TABLE oauth_states(id TEXT PRIMARY KEY,expires TEXT,body TEXT); + CREATE TABLE credential_states(id TEXT PRIMARY KEY,kind TEXT,expires TEXT,body TEXT); CREATE TABLE custodial_vault(subject TEXT PRIMARY KEY,signing_key_id TEXT,status TEXT); CREATE TABLE token_deliveries(credential_id TEXT,subject TEXT,recovery_verifier TEXT,recovery_expires_at TEXT,consumed_at TEXT,request_id TEXT); CREATE TABLE resource_tags(resource_id TEXT REFERENCES resources(id) ON DELETE CASCADE,tag TEXT,PRIMARY KEY(tag,resource_id)); diff --git a/rust/tests/test_identity_parity.py b/rust/tests/test_identity_parity.py index 3a08f67c..6ffc9c83 100644 --- a/rust/tests/test_identity_parity.py +++ b/rust/tests/test_identity_parity.py @@ -18,6 +18,7 @@ from datetime import UTC, datetime, timedelta from pathlib import Path +from msg.browser_config import BrowserConfig from msg.core.codec import b64, canonical, digest, wire from msg.core.errors import Failure from msg.core.models import ( @@ -40,7 +41,6 @@ from msg.core.registry import Registry from msg.core.requests import payload_fields, signing_bytes from msg.core.schemas import NETWORK_CONSTRAINTS -from msg.oauth_config import SCOPES, OAuthClient, OAuthConfig from msg.security.authentication import AuthenticationService from msg.security.certificates import ( CertificateValidator, @@ -49,7 +49,6 @@ sign_certificate, ) from msg.security.crypto import Ed25519Signer, key_id, subject_id -from msg.security.oauth import resource_execution from msg.security.policy import allows, constraints_subset, scope_contains, scope_subset from msg.security.token_delivery import recovery_verifier from msg.storage.sqlite import SqliteMetadataStore @@ -310,10 +309,8 @@ def __init__(self, path: Path): lambda: self.primary, lambda: self.primary, ) - self.oauth = OAuthConfig( - enabled=True, clients=(OAuthClient('fixture-client', 'Fixture', scopes=SCOPES),) - ) - self.auth.oauth_config = self.oauth + self.browser = BrowserConfig(enabled=True) + self.auth.browser_config = self.browser def grant(self, name, operations, *, scope='r_root', descendants=True, constraints=None): return CapabilityGrant( @@ -371,7 +368,7 @@ def context(self): 'capabilities': wire(self.registry.capabilities()), 'primary_ceiling': wire(self.primary), 'temporary_ceiling': wire(self.primary), - 'oauth': wire(self.oauth), + 'browser': wire(self.browser), } def setting(self, key, value): @@ -445,6 +442,7 @@ def request( actor='alice', subject='default', token=False, + token_id='t_fixture', anonymous=False, expires=60, certificates=(), @@ -474,7 +472,7 @@ def request( request = replace(request, payload_digest=digest(payload_fields(request))) if not anonymous: proof = ( - TokenProof(credential_id='t_fixture', token=self.token) + TokenProof(credential_id=token_id, token=self.token) if token else SignatureProof( signature=self.signers[actor].sign(signing_bytes(request), purpose='request'), @@ -542,7 +540,6 @@ async def auth_case( *, entry='network', replay=False, - resource=None, ): command = { 'action': 'authenticate', @@ -550,12 +547,10 @@ async def auth_case( 'policy': policy, 'entry': entry, 'now': wire(self.fixture.now), - 'resource': resource, } async with self.fixture.metadata.transaction(write=False) as tx: try: - with resource_execution(resource): - principal = await self.fixture.auth.authenticate(request, tx, entry=entry) + principal = await self.fixture.auth.authenticate(request, tx, entry=entry) if replay: previous = await tx.request_result( principal.subject, request.request_id, request.payload_digest @@ -616,7 +611,7 @@ async def run(binary, report): 'failure_count': len(check.failures), 'fixture': 'real Python SqliteMetadataStore schema; deterministic public test keys', 'limitations': [ - 'OAuth issuance, refresh writes and token delivery are not part of this read-only slice', + 'Key-approved issuance and token delivery are not part of this read-only slice; retired OAuth authority is rejected', 'Replay admission returns only the stored result, not a fresh principal', 'Mode corpus counts each mode/class/permission decision separately', ], @@ -825,111 +820,133 @@ async def cases(f, check): 'ceiling': wire(f.primary), } with f.body_change('credentials', derived.id, derived): - for binding in ('browser:', 'api:'): - f.conn.execute( - 'INSERT INTO oauth_states VALUES (?,?,?,?)', - ( - binding + derived.id, - 'binding', - wire(NOW + timedelta(days=1)), - canonical(source).decode(), - ), - ) - await check.auth_case('oauth_bound_token', f.request(token=True), policy) - for field, value in [('revoked_at', NOW), ('expires_at', NOW), ('ceiling', ())]: - parent = f.credentials['alice'] - with f.body_change('credentials', parent.id, replace(parent, **{field: value})): - await check.auth_case('oauth_live_parent', f.request(token=True), policy) - with f.body_change( - 'identities', f.users['alice'], replace(f.subjects['alice'], auth_version=2) - ): - await check.auth_case('oauth_auth_version', f.request(token=True), policy) - nested = {**source, 'session': 'session:fixture'} - f.conn.execute( - 'UPDATE oauth_states SET body=? WHERE id=?', - (canonical(nested).decode(), binding + derived.id), - ) - f.conn.execute( - 'INSERT INTO oauth_states VALUES (?,?,?,?)', - ( - 'session:fixture', - 'session', - wire(NOW + timedelta(seconds=60)), - canonical(source).decode(), - ), - ) - await check.auth_case('oauth_session_source', f.request(token=True), policy) - f.conn.execute( - 'UPDATE oauth_states SET body=? WHERE id=?', - (canonical({**source, 'revoked': True}).decode(), 'session:fixture'), - ) - await check.auth_case('oauth_session_revoked', f.request(token=True), policy) - f.conn.execute('DELETE FROM oauth_states') - family = {**source, 'client_id': 'fixture-client', 'scopes': ['msg.read', 'msg.write']} - access = {'family': 'family:fixture'} f.conn.execute( - 'INSERT INTO oauth_states VALUES (?,?,?,?)', + 'INSERT INTO credential_states VALUES (?,?,?,?)', ( - 'access:' + derived.id, - 'access', + 'api:' + derived.id, + 'binding', wire(NOW + timedelta(days=1)), - canonical(access).decode(), + canonical(source).decode(), + ), + ) + await check.auth_case('key_approved_api_binding', f.request(token=True), policy) + for field, value in [('revoked_at', NOW), ('expires_at', NOW), ('ceiling', ())]: + parent = f.credentials['alice'] + with f.body_change('credentials', parent.id, replace(parent, **{field: value})): + await check.auth_case('binding_live_parent', f.request(token=True), policy) + with f.body_change( + 'identities', f.users['alice'], replace(f.subjects['alice'], auth_version=2) + ): + await check.auth_case('binding_auth_version', f.request(token=True), policy) + nested = {**source, 'session': 'session:fixture'} + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical(nested).decode(), 'api:' + derived.id), + ) + f.conn.execute( + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ( + 'session:fixture', + 'session', + wire(NOW + timedelta(seconds=60)), + canonical(source).decode(), ), ) + await check.auth_case('binding_nested_session', f.request(token=True), policy) + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical({**source, 'revoked': True}).decode(), 'session:fixture'), + ) + await check.auth_case('binding_session_revoked', f.request(token=True), policy) + f.conn.execute( + 'UPDATE credential_states SET body=?,expires=? WHERE id=?', + (canonical(source).decode(), wire(NOW), 'session:fixture'), + ) + await check.auth_case('binding_session_expired', f.request(token=True), policy) for field, value in [ - (None, None), - ('revoked', True), - ('client_id', 'missing-client'), - ('scopes', ['unknown.scope']), + ('login_binding', True), + ('custodial', True), ('ceiling', []), - ('auth_version', 9), + ('subject', f.users['bob']), ]: - body = family if field is None else {**family, field: value} f.conn.execute( - 'INSERT OR REPLACE INTO oauth_states VALUES (?,?,?,?)', - ( - 'family:fixture', - 'family', - wire(NOW + timedelta(seconds=60)), - canonical(body).decode(), - ), + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical({**source, field: value}).decode(), 'api:' + derived.id), ) - await check.auth_case('oauth_family', f.request(token=True), policy) + await check.auth_case('binding_unapproved_source', f.request(token=True), policy) + f.conn.execute('DELETE FROM credential_states') + + # Positive browser case must have a genuine current session ID, never a + # restored OAuth access credential with a new-looking binding attached. + browser = replace(derived, id='t_session_fixture') + f.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (browser.id, browser.subject_id, canonical(browser).decode()), + ) + try: f.conn.execute( - 'UPDATE oauth_states SET expires=? WHERE id=?', (wire(NOW), 'family:fixture') + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ( + 'browser:' + browser.id, + 'binding', + wire(NOW + timedelta(days=1)), + canonical(source).decode(), + ), ) - await check.auth_case('oauth_family_expired', f.request(token=True), policy) - mcp_grants = (f.grant('resource.basic', {READ + '@1'}),) - mcp_token = replace(derived, ceiling=mcp_grants) - audience = SERVICE + '/-/mcp' - mcp_family = { - **source, - 'client_id': 'fixture-client', - 'resource': audience, - 'scopes': ['msg.mcp.read'], - 'ceiling': wire(mcp_grants), - } - f.conn.execute( - 'UPDATE oauth_states SET expires=?,body=? WHERE id=?', - (wire(NOW + timedelta(seconds=60)), canonical(mcp_family).decode(), 'family:fixture'), + await check.auth_case( + 'key_approved_browser_binding', f.request(token=True, token_id=browser.id), policy ) - with f.body_change('credentials', derived.id, mcp_token): - for resource in (None, audience, SERVICE + '/other'): - await check.auth_case( - 'mcp_audience', f.request(token=True), policy, resource=resource - ) - for scopes in (['msg.read', 'msg.mcp.read'], ['msg.mcp.post'], ['msg.mcp.read']): - f.conn.execute( - 'UPDATE oauth_states SET body=? WHERE id=?', - (canonical({**mcp_family, 'scopes': scopes}).decode(), 'family:fixture'), - ) - await check.auth_case( - 'mcp_scope_ceiling', f.request(token=True), policy, resource=audience - ) + for body in [ + {**source, 'login_binding': True}, + {**source, 'custodial': True}, + {**source, 'parent': f.keys['bob']}, + {**source, 'ceiling': []}, + ]: + f.conn.execute( + 'UPDATE credential_states SET body=? WHERE id=?', + (canonical(body).decode(), 'browser:' + browser.id), + ) + await check.auth_case( + 'browser_binding_mismatch', f.request(token=True, token_id=browser.id), policy + ) + f.conn.execute('DELETE FROM credential_states') await check.auth_case( - 'mcp_operation_denied', f.request(token=True), policy, resource=audience + 'browser_binding_missing', f.request(token=True, token_id=browser.id), policy ) - f.conn.execute('DELETE FROM oauth_states') + finally: + f.conn.execute('DELETE FROM credentials WHERE id=?', (browser.id,)) + + # Even old authority rows restored beside a valid live parent are rejected. + f.conn.execute( + 'CREATE TABLE oauth_states(id TEXT PRIMARY KEY,kind TEXT,expires TEXT,body TEXT)' + ) + try: + for id in ('t_oauth_fixture', 't_browser_fixture'): + legacy = replace(derived, id=id) + f.conn.execute( + 'INSERT INTO credentials VALUES (?,?,?)', + (id, legacy.subject_id, canonical(legacy).decode()), + ) + f.conn.execute( + 'INSERT INTO oauth_states VALUES (?,?,?,?)', + ( + 'access:' + id, + 'access', + wire(NOW + timedelta(days=1)), + canonical(source).decode(), + ), + ) + f.conn.execute( + 'INSERT INTO credential_states VALUES (?,?,?,?)', + ('api:' + id, 'binding', wire(NOW + timedelta(days=1)), canonical(source).decode()), + ) + await check.auth_case( + 'restored_legacy_authority_denied', f.request(token=True, token_id=id), policy + ) + f.conn.execute('DELETE FROM credentials WHERE id=?', (id,)) + f.conn.execute('DELETE FROM credential_states') + finally: + f.conn.execute('DROP TABLE oauth_states') for actor, kind in [('alice', 'custodial'), ('alice', 'registered')]: subject = f.subjects[actor]