//! Version manifest: schema v1, validation, and the verified form the installer uses //! (the Python module `odian_packs.manifest`). use std::collections::HashSet; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; use crate::canonical::canonical_bytes; use crate::error::{Error, Result}; use crate::hashing::hex; pub const SCHEMA: u64 = 1; pub const CHUNK_BYTES: u64 = 256 * 1024 * 1024; pub const BLOCK_SHIFT: u32 = 6; pub const DB_SECTIONS: [&str; 4] = ["pre-data", "data", "sequences", "post-data"]; const WINDOWS_FORBIDDEN: &[char] = &['<', '>', ':', '"', '|', '?', '*']; /// `MAJOR.MINOR.PATCH` with no leading zeros (ASCII digits only). pub fn parse_version(text: &str) -> Result<(u64, u64, u64)> { let bad = || Error::manifest(format!("version must be MAJOR.MINOR.PATCH, got {text:?}")); let parts: Vec<&str> = text.split('.').collect(); if parts.len() != 3 { return Err(bad()); } let mut out = [0u64; 3]; for (slot, part) in out.iter_mut().zip(&parts) { let ok = !part.is_empty() && part.bytes().all(|b| b.is_ascii_digit()) && (*part == "0" || !part.starts_with('0')); if !ok { return Err(bad()); } *slot = part.parse().map_err(|_| bad())?; } Ok((out[0], out[1], out[2])) } /// A path that stays inside the directory it is joined to, and that Windows can create as /// a plain file. Same rules and order of checks as Python's `safe_relpath`. pub fn safe_relpath(path: &str) -> Result<&str> { if path.is_empty() { return Err(Error::manifest("empty path")); } if path.contains('\\') || path.starts_with('/') { return Err(Error::manifest(format!("path must be relative with '/' separators: {path:?}"))); } let parts: Vec<&str> = path.split('/').collect(); if parts.iter().any(|p| matches!(*p, "" | "." | "..")) { return Err(Error::manifest(format!("path must not contain empty, '.' or '..' parts: {path:?}"))); } for part in parts { if part.chars().any(|c| (c as u32) < 32 || WINDOWS_FORBIDDEN.contains(&c)) { return Err(Error::manifest(format!( "path part {part:?} holds a control character or one of <>:\"|?*: {path:?}" ))); } if part.ends_with('.') || part.ends_with(' ') { return Err(Error::manifest(format!("path part {part:?} ends with a dot or space: {path:?}"))); } if is_windows_reserved(part) { return Err(Error::manifest(format!("path part {part:?} is a reserved Windows device name: {path:?}"))); } } Ok(path) } /// `^(con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³])(\..*)?$`, case-insensitive. fn is_windows_reserved(part: &str) -> bool { let base = match part.find('.') { Some(i) => &part[..i], None => part, }; let lower = base.to_ascii_lowercase(); if matches!(lower.as_str(), "con" | "prn" | "aux" | "nul") { return true; } for prefix in ["com", "lpt"] { if let Some(rest) = lower.strip_prefix(prefix) { let mut chars = rest.chars(); if let (Some(c), None) = (chars.next(), chars.next()) { if c.is_ascii_digit() || matches!(c, '¹' | '²' | '³') { return true; } } } } false } pub fn chunk_count(size: u64, chunk_bytes: u64) -> u64 { size.div_ceil(chunk_bytes) } /// sha256 over the concatenated chunk hashes (hex text): a source file's identity. pub fn chunks_digest>(chunks: &[S]) -> String { let mut digest = Sha256::new(); for chunk in chunks { digest.update(chunk.as_ref().as_bytes()); } hex(&digest.finalize()) } /// Where a version keeps a tile set's frozen index, relative to `versions//`. pub fn tile_index_name(style: &str, dataset: &str) -> String { format!("tiles/{style}.{dataset}.index.json") } /// What the Ed25519 signature covers: canonical JSON of the manifest minus `signature`. pub fn signing_payload(manifest: &Value) -> Result> { let object = manifest.as_object().ok_or_else(|| Error::manifest("manifest must be a JSON object"))?; let mut copy = Map::new(); for (k, v) in object { if k != "signature" { copy.insert(k.clone(), v.clone()); } } canonical_bytes(&Value::Object(copy)) } // ---- validation ------------------------------------------------------------- fn require(cond: bool, message: impl FnOnce() -> String) -> Result<()> { if cond { Ok(()) } else { Err(Error::Manifest(message())) } } fn is_count(value: Option<&Value>) -> bool { value.and_then(Value::as_u64).is_some() } fn is_sha256(value: Option<&Value>) -> bool { matches!(value.and_then(Value::as_str), Some(s) if s.len() == 64 && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f'))) } fn check_sha(value: Option<&Value>, place: &str) -> Result<()> { require(is_sha256(value), || format!("{place}: sha256 must be 64 lowercase hex chars")) } fn is_name(s: &str) -> bool { (1..=128).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-')) } /// Lower-case hyphenated uuid, as Postgres prints gen_random_uuid(). fn is_uuid(s: &str) -> bool { s.len() == 36 && s.bytes().enumerate().all(|(i, b)| match i { 8 | 13 | 18 | 23 => b == b'-', _ => b.is_ascii_digit() || (b'a'..=b'f').contains(&b), }) } /// A Postgres xid8 in decimal. fn is_xid(s: &str) -> bool { (1..=20).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_digit()) } fn is_snapshot(s: &str) -> bool { let b = s.as_bytes(); b.len() == 16 && b[..8].iter().all(u8::is_ascii_digit) && b[8] == b'T' && b[9..15].iter().all(u8::is_ascii_digit) && b[15] == b'Z' } fn relpath_value(value: Option<&Value>) -> Result<&str> { match value.and_then(Value::as_str) { Some(s) => safe_relpath(s), None => Err(Error::manifest("empty path")), } } fn version_value(value: Option<&Value>) -> Result<()> { match value { None => parse_version("").map(|_| ()), Some(Value::String(s)) => parse_version(s).map(|_| ()), Some(other) => Err(Error::manifest(format!("version must be MAJOR.MINOR.PATCH, got {other}"))), } } fn entries<'a>(value: Option<&'a Value>, place: &str) -> Result>> { let list = value.and_then(Value::as_array).ok_or_else(|| Error::manifest(format!("{place} must be a list")))?; list.iter() .enumerate() .map(|(i, entry)| entry.as_object().ok_or_else(|| Error::manifest(format!("{place}[{i}] must be an object")))) .collect() } fn object<'a>(value: &'a Value, message: &str) -> Result<&'a Map> { value.as_object().ok_or_else(|| Error::manifest(message)) } /// Error unless `manifest` is a well-formed schema-1 manifest. Structure only; the /// signature is checked by [`crate::signing::verify`]. pub fn validate(manifest: &Value) -> Result<()> { let m = object(manifest, "manifest must be a JSON object")?; require(m.get("schema").and_then(Value::as_u64) == Some(SCHEMA), || format!("schema must be {SCHEMA}"))?; require(m.get("product").and_then(Value::as_str) == Some("odian"), || "product must be 'odian'".into())?; version_value(m.get("version"))?; require(m.get("released_at").is_some_and(Value::is_string), || "released_at must be an ISO-8601 string".into())?; if let Some(v) = m.get("min_launcher") { version_value(Some(v))?; } if let Some(v) = m.get("notes") { relpath_value(Some(v))?; } let app_files = match m.get("app") { None => Vec::new(), Some(app) => entries(object(app, "app must be an object")?.get("files"), "app.files")?, }; let mut seen: HashSet = HashSet::new(); for (i, entry) in app_files.iter().enumerate() { let place = format!("app.files[{i}]"); let path = relpath_value(entry.get("path"))?; require(seen.insert(path.to_lowercase()), || format!("{place}: duplicate path {path:?}"))?; require(is_count(entry.get("size")), || format!("{place}: size must be a non-negative integer"))?; check_sha(entry.get("sha256"), &place)?; } if let Some(db) = m.get("db") { let db = object(db, "db must be an object")?; require(is_count(db.get("schema_version")), || "db.schema_version must be a non-negative integer".into())?; let snapshot = db .get("snapshot") .and_then(Value::as_str) .filter(|s| is_snapshot(s)) .ok_or_else(|| Error::manifest("db.snapshot must look like 20260915T101500Z"))?; let files = entries(db.get("files"), "db.files")?; require(!files.is_empty(), || "db.files must not be empty".into())?; if let Some(sync) = db.get("sync") { let db_id = sync.get("db_id").and_then(Value::as_str).is_some_and(is_uuid); let horizon = sync.get("horizon").and_then(Value::as_str).is_some_and(is_xid); require(db_id && horizon, || { "db.sync must hold the DHP's db_id (a uuid) and a horizon (a transaction id)".into() })?; } let prefix = format!("db/{snapshot}/"); let mut seen: HashSet = HashSet::new(); for (i, entry) in files.iter().enumerate() { let place = format!("db.files[{i}]"); let section = entry.get("section").and_then(Value::as_str).filter(|s| DB_SECTIONS.contains(s)); let section = section.ok_or_else(|| Error::manifest(format!("{place}: section must be one of {DB_SECTIONS:?}")))?; if section == "data" { require(entry.get("table").and_then(Value::as_str).is_some_and(|t| !t.is_empty()), || { format!("{place}: data dumps name their table") })?; } let path = relpath_value(entry.get("path"))?; require(path.starts_with(&prefix), || format!("{place}: path must live under {prefix}"))?; require(seen.insert(path.to_lowercase()), || format!("{place}: duplicate path {path:?}"))?; require(is_count(entry.get("size")), || format!("{place}: size must be a non-negative integer"))?; check_sha(entry.get("sha256"), &place)?; } } if let Some(tiles) = m.get("tiles") { let tiles = object(tiles, "tiles must be an object")?; require(tiles.get("block_shift").and_then(Value::as_u64) == Some(BLOCK_SHIFT as u64), || { format!("tiles.block_shift must be {BLOCK_SHIFT}") })?; let mut sets_seen: HashSet<(String, String)> = HashSet::new(); for (i, entry) in entries(tiles.get("sets"), "tiles.sets")?.iter().enumerate() { let place = format!("tiles.sets[{i}]"); for key in ["style", "dataset"] { require(entry.get(key).and_then(Value::as_str).is_some_and(is_name), || format!("{place}: bad {key}"))?; } let style = entry["style"].as_str().unwrap_or_default().to_string(); let dataset = entry["dataset"].as_str().unwrap_or_default().to_string(); require(!sets_seen.contains(&(style.clone(), dataset.clone())), || { format!("{place}: duplicate tile set {style}/{dataset}") })?; for count in ["journal_seq", "blocks", "files", "bytes"] { require(is_count(entry.get(count)), || format!("{place}: {count} must be a non-negative integer"))?; } check_sha(entry.get("index_sha256"), &place)?; let expected = tile_index_name(&style, &dataset); require(entry.get("index").and_then(Value::as_str) == Some(expected.as_str()), || { format!("{place}: index must be {expected:?}") })?; sets_seen.insert((style, dataset)); } } if let Some(src) = m.get("src") { let src = object(src, "src must be an object")?; require(src.get("enabled").is_some_and(Value::is_boolean), || "src.enabled must be true or false".into())?; let chunk_bytes = src.get("chunk_bytes").and_then(Value::as_u64).filter(|n| *n > 0); let chunk_bytes = chunk_bytes.ok_or_else(|| Error::manifest("src.chunk_bytes must be a positive integer"))?; let mut seen: HashSet = HashSet::new(); for (i, entry) in entries(src.get("files"), "src.files")?.iter().enumerate() { let place = format!("src.files[{i}]"); let path = relpath_value(entry.get("path"))?; require(seen.insert(path.to_lowercase()), || format!("{place}: duplicate path {path:?}"))?; let size = entry.get("size").and_then(Value::as_u64); let size = size.ok_or_else(|| Error::manifest(format!("{place}: size must be a non-negative integer")))?; let chunks = entry.get("chunks").and_then(Value::as_array); let chunks = chunks.ok_or_else(|| Error::manifest(format!("{place}: chunks must be a list")))?; require(chunks.len() as u64 == chunk_count(size, chunk_bytes), || { format!("{place}: {} chunks for {size} bytes at {chunk_bytes} per chunk", chunks.len()) })?; let mut hashes = Vec::with_capacity(chunks.len()); for chunk in chunks { check_sha(Some(chunk), &place)?; hashes.push(chunk.as_str().unwrap_or_default()); } require(entry.get("chunks_sha256").and_then(Value::as_str) == Some(chunks_digest(&hashes).as_str()), || { format!("{place}: chunks_sha256 does not match chunks") })?; } } if let Some(signature) = m.get("signature") { require(signature.as_str().is_some_and(|s| s.starts_with("ed25519:")), || { "signature must look like 'ed25519:'".into() })?; } Ok(()) } // ---- typed view ------------------------------------------------------------- #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AppFile { pub path: String, pub size: u64, pub sha256: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] pub struct AppSection { pub files: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DbFile { pub section: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub table: Option, pub path: String, pub size: u64, pub sha256: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DbSection { pub schema_version: u64, pub snapshot: String, pub files: Vec, /// Present when the snapshot comes from a DHP running operator-data sync. #[serde(default, skip_serializing_if = "Option::is_none")] pub sync: Option, } /// Where an install restored from this snapshot starts syncing. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct DbSync { /// The DHP database's sync id: an install only ever syncs with the database it came from. pub db_id: String, /// Oldest transaction still running when the snapshot was taken; the first pull starts there. pub horizon: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TileSet { pub style: String, pub dataset: String, pub journal_seq: u64, pub blocks: u64, pub files: u64, pub bytes: u64, pub index: String, pub index_sha256: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct TilesSection { pub block_shift: u64, pub sets: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SrcFile { pub path: String, pub size: u64, pub chunks: Vec, pub chunks_sha256: String, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SrcSection { pub enabled: bool, pub chunk_bytes: u64, pub files: Vec, } /// A validated manifest's fields (unknown keys are kept only in the raw bytes). #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Manifest { pub schema: u64, pub product: String, pub version: String, pub released_at: String, #[serde(default)] pub min_launcher: Option, #[serde(default)] pub notes: Option, #[serde(default)] pub app: AppSection, #[serde(default)] pub db: Option, #[serde(default)] pub tiles: Option, #[serde(default)] pub src: Option, #[serde(default)] pub signature: Option, } /// A manifest that parsed, validated and carries a good signature by the given key. /// Only [`VerifiedManifest::from_bytes`] builds one. #[derive(Debug, Clone)] pub struct VerifiedManifest { raw: Vec, value: Value, manifest: Manifest, } impl VerifiedManifest { /// Parse, validate and signature-check manifest bytes (e.g. an installed /// `versions//manifest.json`, re-checked offline). pub fn from_bytes(raw: Vec, public_key: &str) -> Result { let value: Value = serde_json::from_slice(&raw) .map_err(|e| Error::manifest(format!("manifest is not valid UTF-8 JSON: {e}")))?; // Refuse non-integer numbers anywhere, before anything is hashed. canonical_bytes(&value)?; validate(&value)?; crate::signing::verify(&value, public_key)?; let manifest: Manifest = serde_json::from_value(value.clone()) .map_err(|e| Error::manifest(format!("validated manifest does not map to its fields: {e}")))?; Ok(VerifiedManifest { raw, value, manifest }) } /// The bytes exactly as served; written verbatim to `versions//manifest.json`. pub fn raw(&self) -> &[u8] { &self.raw } pub fn value(&self) -> &Value { &self.value } pub fn manifest(&self) -> &Manifest { &self.manifest } pub fn version(&self) -> &str { &self.manifest.version } } #[cfg(test)] mod tests { use super::*; #[test] fn relpaths_like_python_tests() { assert!(safe_relpath("Odian/x.exe").is_ok()); for bad in ["../a", "a/../b", "/a", "C:/a", "a\\b", "a//b", "", "./a", "Odian/Odian.exe:hidden", "a/b.", "a/b ", "CON", "a/aux.txt", "a/Lpt1.log", "a/b\u{0}c", "a/b\nc", "a/b?c", "a/b*c", "a/b\"c", "a/b|c", "a/com¹"] { assert!(safe_relpath(bad).is_err(), "{bad:?}"); } for fine in ["a/console.txt", "a/auxiliary", "a/com10", "a/.hidden", "a/b c.txt", "a/comA"] { assert!(safe_relpath(fine).is_ok(), "{fine:?}"); } } #[test] fn versions() { assert_eq!(parse_version("0.51.0").unwrap(), (0, 51, 0)); for bad in ["1.0", "01.2.3", "0.51.0-rc1", "0.51.0\n", "", "1..2", "a.b.c"] { assert!(parse_version(bad).is_err(), "{bad:?}"); } } }