Kelorastus
488 lines
19 KiB
Rust
488 lines
19 KiB
Rust
//! Version manifest: schema v1, validation, and the verified form the installer uses
|
|
//! (the Python module `odian_packs.manifest`).
|
|
|
|
use std::collections::HashSet;
|
|
|
|
use serde::{Deserialize, Serialize};
|
|
use serde_json::{Map, Value};
|
|
use sha2::{Digest, Sha256};
|
|
|
|
use crate::canonical::canonical_bytes;
|
|
use crate::error::{Error, Result};
|
|
use crate::hashing::hex;
|
|
|
|
pub const SCHEMA: u64 = 1;
|
|
pub const CHUNK_BYTES: u64 = 256 * 1024 * 1024;
|
|
pub const BLOCK_SHIFT: u32 = 6;
|
|
pub const DB_SECTIONS: [&str; 4] = ["pre-data", "data", "sequences", "post-data"];
|
|
|
|
const WINDOWS_FORBIDDEN: &[char] = &['<', '>', ':', '"', '|', '?', '*'];
|
|
|
|
/// `MAJOR.MINOR.PATCH` with no leading zeros (ASCII digits only).
|
|
pub fn parse_version(text: &str) -> Result<(u64, u64, u64)> {
|
|
let bad = || Error::manifest(format!("version must be MAJOR.MINOR.PATCH, got {text:?}"));
|
|
let parts: Vec<&str> = text.split('.').collect();
|
|
if parts.len() != 3 {
|
|
return Err(bad());
|
|
}
|
|
let mut out = [0u64; 3];
|
|
for (slot, part) in out.iter_mut().zip(&parts) {
|
|
let ok = !part.is_empty()
|
|
&& part.bytes().all(|b| b.is_ascii_digit())
|
|
&& (*part == "0" || !part.starts_with('0'));
|
|
if !ok {
|
|
return Err(bad());
|
|
}
|
|
*slot = part.parse().map_err(|_| bad())?;
|
|
}
|
|
Ok((out[0], out[1], out[2]))
|
|
}
|
|
|
|
/// A path that stays inside the directory it is joined to, and that Windows can create as
|
|
/// a plain file. Same rules and order of checks as Python's `safe_relpath`.
|
|
pub fn safe_relpath(path: &str) -> Result<&str> {
|
|
if path.is_empty() {
|
|
return Err(Error::manifest("empty path"));
|
|
}
|
|
if path.contains('\\') || path.starts_with('/') {
|
|
return Err(Error::manifest(format!("path must be relative with '/' separators: {path:?}")));
|
|
}
|
|
let parts: Vec<&str> = path.split('/').collect();
|
|
if parts.iter().any(|p| matches!(*p, "" | "." | "..")) {
|
|
return Err(Error::manifest(format!("path must not contain empty, '.' or '..' parts: {path:?}")));
|
|
}
|
|
for part in parts {
|
|
if part.chars().any(|c| (c as u32) < 32 || WINDOWS_FORBIDDEN.contains(&c)) {
|
|
return Err(Error::manifest(format!(
|
|
"path part {part:?} holds a control character or one of <>:\"|?*: {path:?}"
|
|
)));
|
|
}
|
|
if part.ends_with('.') || part.ends_with(' ') {
|
|
return Err(Error::manifest(format!("path part {part:?} ends with a dot or space: {path:?}")));
|
|
}
|
|
if is_windows_reserved(part) {
|
|
return Err(Error::manifest(format!("path part {part:?} is a reserved Windows device name: {path:?}")));
|
|
}
|
|
}
|
|
Ok(path)
|
|
}
|
|
|
|
/// `^(con|prn|aux|nul|com[0-9¹²³]|lpt[0-9¹²³])(\..*)?$`, case-insensitive.
|
|
fn is_windows_reserved(part: &str) -> bool {
|
|
let base = match part.find('.') {
|
|
Some(i) => &part[..i],
|
|
None => part,
|
|
};
|
|
let lower = base.to_ascii_lowercase();
|
|
if matches!(lower.as_str(), "con" | "prn" | "aux" | "nul") {
|
|
return true;
|
|
}
|
|
for prefix in ["com", "lpt"] {
|
|
if let Some(rest) = lower.strip_prefix(prefix) {
|
|
let mut chars = rest.chars();
|
|
if let (Some(c), None) = (chars.next(), chars.next()) {
|
|
if c.is_ascii_digit() || matches!(c, '¹' | '²' | '³') {
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
false
|
|
}
|
|
|
|
pub fn chunk_count(size: u64, chunk_bytes: u64) -> u64 {
|
|
size.div_ceil(chunk_bytes)
|
|
}
|
|
|
|
/// sha256 over the concatenated chunk hashes (hex text): a source file's identity.
|
|
pub fn chunks_digest<S: AsRef<str>>(chunks: &[S]) -> String {
|
|
let mut digest = Sha256::new();
|
|
for chunk in chunks {
|
|
digest.update(chunk.as_ref().as_bytes());
|
|
}
|
|
hex(&digest.finalize())
|
|
}
|
|
|
|
/// Where a version keeps a tile set's frozen index, relative to `versions/<v>/`.
|
|
pub fn tile_index_name(style: &str, dataset: &str) -> String {
|
|
format!("tiles/{style}.{dataset}.index.json")
|
|
}
|
|
|
|
/// What the Ed25519 signature covers: canonical JSON of the manifest minus `signature`.
|
|
pub fn signing_payload(manifest: &Value) -> Result<Vec<u8>> {
|
|
let object = manifest.as_object().ok_or_else(|| Error::manifest("manifest must be a JSON object"))?;
|
|
let mut copy = Map::new();
|
|
for (k, v) in object {
|
|
if k != "signature" {
|
|
copy.insert(k.clone(), v.clone());
|
|
}
|
|
}
|
|
canonical_bytes(&Value::Object(copy))
|
|
}
|
|
|
|
// ---- validation -------------------------------------------------------------
|
|
|
|
fn require(cond: bool, message: impl FnOnce() -> String) -> Result<()> {
|
|
if cond { Ok(()) } else { Err(Error::Manifest(message())) }
|
|
}
|
|
|
|
fn is_count(value: Option<&Value>) -> bool {
|
|
value.and_then(Value::as_u64).is_some()
|
|
}
|
|
|
|
fn is_sha256(value: Option<&Value>) -> bool {
|
|
matches!(value.and_then(Value::as_str), Some(s) if s.len() == 64 && s.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')))
|
|
}
|
|
|
|
fn check_sha(value: Option<&Value>, place: &str) -> Result<()> {
|
|
require(is_sha256(value), || format!("{place}: sha256 must be 64 lowercase hex chars"))
|
|
}
|
|
|
|
fn is_name(s: &str) -> bool {
|
|
(1..=128).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'-'))
|
|
}
|
|
|
|
/// Lower-case hyphenated uuid, as Postgres prints gen_random_uuid().
|
|
fn is_uuid(s: &str) -> bool {
|
|
s.len() == 36
|
|
&& s.bytes().enumerate().all(|(i, b)| match i {
|
|
8 | 13 | 18 | 23 => b == b'-',
|
|
_ => b.is_ascii_digit() || (b'a'..=b'f').contains(&b),
|
|
})
|
|
}
|
|
|
|
/// A Postgres xid8 in decimal.
|
|
fn is_xid(s: &str) -> bool {
|
|
(1..=20).contains(&s.len()) && s.bytes().all(|b| b.is_ascii_digit())
|
|
}
|
|
|
|
fn is_snapshot(s: &str) -> bool {
|
|
let b = s.as_bytes();
|
|
b.len() == 16
|
|
&& b[..8].iter().all(u8::is_ascii_digit)
|
|
&& b[8] == b'T'
|
|
&& b[9..15].iter().all(u8::is_ascii_digit)
|
|
&& b[15] == b'Z'
|
|
}
|
|
|
|
fn relpath_value(value: Option<&Value>) -> Result<&str> {
|
|
match value.and_then(Value::as_str) {
|
|
Some(s) => safe_relpath(s),
|
|
None => Err(Error::manifest("empty path")),
|
|
}
|
|
}
|
|
|
|
fn version_value(value: Option<&Value>) -> Result<()> {
|
|
match value {
|
|
None => parse_version("").map(|_| ()),
|
|
Some(Value::String(s)) => parse_version(s).map(|_| ()),
|
|
Some(other) => Err(Error::manifest(format!("version must be MAJOR.MINOR.PATCH, got {other}"))),
|
|
}
|
|
}
|
|
|
|
fn entries<'a>(value: Option<&'a Value>, place: &str) -> Result<Vec<&'a Map<String, Value>>> {
|
|
let list = value.and_then(Value::as_array).ok_or_else(|| Error::manifest(format!("{place} must be a list")))?;
|
|
list.iter()
|
|
.enumerate()
|
|
.map(|(i, entry)| entry.as_object().ok_or_else(|| Error::manifest(format!("{place}[{i}] must be an object"))))
|
|
.collect()
|
|
}
|
|
|
|
fn object<'a>(value: &'a Value, message: &str) -> Result<&'a Map<String, Value>> {
|
|
value.as_object().ok_or_else(|| Error::manifest(message))
|
|
}
|
|
|
|
/// Error unless `manifest` is a well-formed schema-1 manifest. Structure only; the
|
|
/// signature is checked by [`crate::signing::verify`].
|
|
pub fn validate(manifest: &Value) -> Result<()> {
|
|
let m = object(manifest, "manifest must be a JSON object")?;
|
|
require(m.get("schema").and_then(Value::as_u64) == Some(SCHEMA), || format!("schema must be {SCHEMA}"))?;
|
|
require(m.get("product").and_then(Value::as_str) == Some("odian"), || "product must be 'odian'".into())?;
|
|
version_value(m.get("version"))?;
|
|
require(m.get("released_at").is_some_and(Value::is_string), || "released_at must be an ISO-8601 string".into())?;
|
|
if let Some(v) = m.get("min_launcher") {
|
|
version_value(Some(v))?;
|
|
}
|
|
if let Some(v) = m.get("notes") {
|
|
relpath_value(Some(v))?;
|
|
}
|
|
|
|
let app_files = match m.get("app") {
|
|
None => Vec::new(),
|
|
Some(app) => entries(object(app, "app must be an object")?.get("files"), "app.files")?,
|
|
};
|
|
let mut seen: HashSet<String> = HashSet::new();
|
|
for (i, entry) in app_files.iter().enumerate() {
|
|
let place = format!("app.files[{i}]");
|
|
let path = relpath_value(entry.get("path"))?;
|
|
require(seen.insert(path.to_lowercase()), || format!("{place}: duplicate path {path:?}"))?;
|
|
require(is_count(entry.get("size")), || format!("{place}: size must be a non-negative integer"))?;
|
|
check_sha(entry.get("sha256"), &place)?;
|
|
}
|
|
|
|
if let Some(db) = m.get("db") {
|
|
let db = object(db, "db must be an object")?;
|
|
require(is_count(db.get("schema_version")), || "db.schema_version must be a non-negative integer".into())?;
|
|
let snapshot = db
|
|
.get("snapshot")
|
|
.and_then(Value::as_str)
|
|
.filter(|s| is_snapshot(s))
|
|
.ok_or_else(|| Error::manifest("db.snapshot must look like 20260915T101500Z"))?;
|
|
let files = entries(db.get("files"), "db.files")?;
|
|
require(!files.is_empty(), || "db.files must not be empty".into())?;
|
|
if let Some(sync) = db.get("sync") {
|
|
let db_id = sync.get("db_id").and_then(Value::as_str).is_some_and(is_uuid);
|
|
let horizon = sync.get("horizon").and_then(Value::as_str).is_some_and(is_xid);
|
|
require(db_id && horizon, || {
|
|
"db.sync must hold the DHP's db_id (a uuid) and a horizon (a transaction id)".into()
|
|
})?;
|
|
}
|
|
let prefix = format!("db/{snapshot}/");
|
|
let mut seen: HashSet<String> = HashSet::new();
|
|
for (i, entry) in files.iter().enumerate() {
|
|
let place = format!("db.files[{i}]");
|
|
let section = entry.get("section").and_then(Value::as_str).filter(|s| DB_SECTIONS.contains(s));
|
|
let section = section.ok_or_else(|| Error::manifest(format!("{place}: section must be one of {DB_SECTIONS:?}")))?;
|
|
if section == "data" {
|
|
require(entry.get("table").and_then(Value::as_str).is_some_and(|t| !t.is_empty()), || {
|
|
format!("{place}: data dumps name their table")
|
|
})?;
|
|
}
|
|
let path = relpath_value(entry.get("path"))?;
|
|
require(path.starts_with(&prefix), || format!("{place}: path must live under {prefix}"))?;
|
|
require(seen.insert(path.to_lowercase()), || format!("{place}: duplicate path {path:?}"))?;
|
|
require(is_count(entry.get("size")), || format!("{place}: size must be a non-negative integer"))?;
|
|
check_sha(entry.get("sha256"), &place)?;
|
|
}
|
|
}
|
|
|
|
if let Some(tiles) = m.get("tiles") {
|
|
let tiles = object(tiles, "tiles must be an object")?;
|
|
require(tiles.get("block_shift").and_then(Value::as_u64) == Some(BLOCK_SHIFT as u64), || {
|
|
format!("tiles.block_shift must be {BLOCK_SHIFT}")
|
|
})?;
|
|
let mut sets_seen: HashSet<(String, String)> = HashSet::new();
|
|
for (i, entry) in entries(tiles.get("sets"), "tiles.sets")?.iter().enumerate() {
|
|
let place = format!("tiles.sets[{i}]");
|
|
for key in ["style", "dataset"] {
|
|
require(entry.get(key).and_then(Value::as_str).is_some_and(is_name), || format!("{place}: bad {key}"))?;
|
|
}
|
|
let style = entry["style"].as_str().unwrap_or_default().to_string();
|
|
let dataset = entry["dataset"].as_str().unwrap_or_default().to_string();
|
|
require(!sets_seen.contains(&(style.clone(), dataset.clone())), || {
|
|
format!("{place}: duplicate tile set {style}/{dataset}")
|
|
})?;
|
|
for count in ["journal_seq", "blocks", "files", "bytes"] {
|
|
require(is_count(entry.get(count)), || format!("{place}: {count} must be a non-negative integer"))?;
|
|
}
|
|
check_sha(entry.get("index_sha256"), &place)?;
|
|
let expected = tile_index_name(&style, &dataset);
|
|
require(entry.get("index").and_then(Value::as_str) == Some(expected.as_str()), || {
|
|
format!("{place}: index must be {expected:?}")
|
|
})?;
|
|
sets_seen.insert((style, dataset));
|
|
}
|
|
}
|
|
|
|
if let Some(src) = m.get("src") {
|
|
let src = object(src, "src must be an object")?;
|
|
require(src.get("enabled").is_some_and(Value::is_boolean), || "src.enabled must be true or false".into())?;
|
|
let chunk_bytes = src.get("chunk_bytes").and_then(Value::as_u64).filter(|n| *n > 0);
|
|
let chunk_bytes = chunk_bytes.ok_or_else(|| Error::manifest("src.chunk_bytes must be a positive integer"))?;
|
|
let mut seen: HashSet<String> = HashSet::new();
|
|
for (i, entry) in entries(src.get("files"), "src.files")?.iter().enumerate() {
|
|
let place = format!("src.files[{i}]");
|
|
let path = relpath_value(entry.get("path"))?;
|
|
require(seen.insert(path.to_lowercase()), || format!("{place}: duplicate path {path:?}"))?;
|
|
let size = entry.get("size").and_then(Value::as_u64);
|
|
let size = size.ok_or_else(|| Error::manifest(format!("{place}: size must be a non-negative integer")))?;
|
|
let chunks = entry.get("chunks").and_then(Value::as_array);
|
|
let chunks = chunks.ok_or_else(|| Error::manifest(format!("{place}: chunks must be a list")))?;
|
|
require(chunks.len() as u64 == chunk_count(size, chunk_bytes), || {
|
|
format!("{place}: {} chunks for {size} bytes at {chunk_bytes} per chunk", chunks.len())
|
|
})?;
|
|
let mut hashes = Vec::with_capacity(chunks.len());
|
|
for chunk in chunks {
|
|
check_sha(Some(chunk), &place)?;
|
|
hashes.push(chunk.as_str().unwrap_or_default());
|
|
}
|
|
require(entry.get("chunks_sha256").and_then(Value::as_str) == Some(chunks_digest(&hashes).as_str()), || {
|
|
format!("{place}: chunks_sha256 does not match chunks")
|
|
})?;
|
|
}
|
|
}
|
|
|
|
if let Some(signature) = m.get("signature") {
|
|
require(signature.as_str().is_some_and(|s| s.starts_with("ed25519:")), || {
|
|
"signature must look like 'ed25519:<base64>'".into()
|
|
})?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
// ---- typed view -------------------------------------------------------------
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct AppFile {
|
|
pub path: String,
|
|
pub size: u64,
|
|
pub sha256: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
|
|
pub struct AppSection {
|
|
pub files: Vec<AppFile>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct DbFile {
|
|
pub section: String,
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub table: Option<String>,
|
|
pub path: String,
|
|
pub size: u64,
|
|
pub sha256: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct DbSection {
|
|
pub schema_version: u64,
|
|
pub snapshot: String,
|
|
pub files: Vec<DbFile>,
|
|
/// Present when the snapshot comes from a DHP running operator-data sync.
|
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
pub sync: Option<DbSync>,
|
|
}
|
|
|
|
/// Where an install restored from this snapshot starts syncing.
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct DbSync {
|
|
/// The DHP database's sync id: an install only ever syncs with the database it came from.
|
|
pub db_id: String,
|
|
/// Oldest transaction still running when the snapshot was taken; the first pull starts there.
|
|
pub horizon: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct TileSet {
|
|
pub style: String,
|
|
pub dataset: String,
|
|
pub journal_seq: u64,
|
|
pub blocks: u64,
|
|
pub files: u64,
|
|
pub bytes: u64,
|
|
pub index: String,
|
|
pub index_sha256: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct TilesSection {
|
|
pub block_shift: u64,
|
|
pub sets: Vec<TileSet>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct SrcFile {
|
|
pub path: String,
|
|
pub size: u64,
|
|
pub chunks: Vec<String>,
|
|
pub chunks_sha256: String,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct SrcSection {
|
|
pub enabled: bool,
|
|
pub chunk_bytes: u64,
|
|
pub files: Vec<SrcFile>,
|
|
}
|
|
|
|
/// A validated manifest's fields (unknown keys are kept only in the raw bytes).
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
|
pub struct Manifest {
|
|
pub schema: u64,
|
|
pub product: String,
|
|
pub version: String,
|
|
pub released_at: String,
|
|
#[serde(default)]
|
|
pub min_launcher: Option<String>,
|
|
#[serde(default)]
|
|
pub notes: Option<String>,
|
|
#[serde(default)]
|
|
pub app: AppSection,
|
|
#[serde(default)]
|
|
pub db: Option<DbSection>,
|
|
#[serde(default)]
|
|
pub tiles: Option<TilesSection>,
|
|
#[serde(default)]
|
|
pub src: Option<SrcSection>,
|
|
#[serde(default)]
|
|
pub signature: Option<String>,
|
|
}
|
|
|
|
/// A manifest that parsed, validated and carries a good signature by the given key.
|
|
/// Only [`VerifiedManifest::from_bytes`] builds one.
|
|
#[derive(Debug, Clone)]
|
|
pub struct VerifiedManifest {
|
|
raw: Vec<u8>,
|
|
value: Value,
|
|
manifest: Manifest,
|
|
}
|
|
|
|
impl VerifiedManifest {
|
|
/// Parse, validate and signature-check manifest bytes (e.g. an installed
|
|
/// `versions/<v>/manifest.json`, re-checked offline).
|
|
pub fn from_bytes(raw: Vec<u8>, public_key: &str) -> Result<Self> {
|
|
let value: Value = serde_json::from_slice(&raw)
|
|
.map_err(|e| Error::manifest(format!("manifest is not valid UTF-8 JSON: {e}")))?;
|
|
// Refuse non-integer numbers anywhere, before anything is hashed.
|
|
canonical_bytes(&value)?;
|
|
validate(&value)?;
|
|
crate::signing::verify(&value, public_key)?;
|
|
let manifest: Manifest = serde_json::from_value(value.clone())
|
|
.map_err(|e| Error::manifest(format!("validated manifest does not map to its fields: {e}")))?;
|
|
Ok(VerifiedManifest { raw, value, manifest })
|
|
}
|
|
|
|
/// The bytes exactly as served; written verbatim to `versions/<v>/manifest.json`.
|
|
pub fn raw(&self) -> &[u8] {
|
|
&self.raw
|
|
}
|
|
|
|
pub fn value(&self) -> &Value {
|
|
&self.value
|
|
}
|
|
|
|
pub fn manifest(&self) -> &Manifest {
|
|
&self.manifest
|
|
}
|
|
|
|
pub fn version(&self) -> &str {
|
|
&self.manifest.version
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn relpaths_like_python_tests() {
|
|
assert!(safe_relpath("Odian/x.exe").is_ok());
|
|
for bad in ["../a", "a/../b", "/a", "C:/a", "a\\b", "a//b", "", "./a", "Odian/Odian.exe:hidden", "a/b.", "a/b ",
|
|
"CON", "a/aux.txt", "a/Lpt1.log", "a/b\u{0}c", "a/b\nc", "a/b?c", "a/b*c", "a/b\"c", "a/b|c", "a/com¹"] {
|
|
assert!(safe_relpath(bad).is_err(), "{bad:?}");
|
|
}
|
|
for fine in ["a/console.txt", "a/auxiliary", "a/com10", "a/.hidden", "a/b c.txt", "a/comA"] {
|
|
assert!(safe_relpath(fine).is_ok(), "{fine:?}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn versions() {
|
|
assert_eq!(parse_version("0.51.0").unwrap(), (0, 51, 0));
|
|
for bad in ["1.0", "01.2.3", "0.51.0-rc1", "0.51.0\n", "", "1..2", "a.b.c"] {
|
|
assert!(parse_version(bad).is_err(), "{bad:?}");
|
|
}
|
|
}
|
|
}
|