Kelorastus
737 lines
36 KiB
Rust
737 lines
36 KiB
Rust
//! End to end against a local stub site: `tests/stub_site.py` builds a small signed pack
|
|
//! store with the Python reference code (odian_packs) and serves it with the real update
|
|
//! service (`odian_update.api.create_app`) plus nginx-like Range routes. The same store is
|
|
//! also installed as an ODIAN drive.
|
|
//!
|
|
//! These tests spawn Python, so they are `#[ignore]`d by default. They need Python 3 with
|
|
//! fastapi, uvicorn, httpx and cryptography, and the ODIAN_MCP_Server sources:
|
|
//!
|
|
//! ```text
|
|
//! ODIAN_PACKS_PYTHONPATH=C:/Users/JFEXP_D/Documents/Gitea/ODIAN_MCP_Server/src \
|
|
//! cargo test -p odian-packs --test e2e_site -- --ignored
|
|
//! ```
|
|
//!
|
|
//! `ODIAN_PACKS_PYTHON` picks the interpreter (default `python`).
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::io::{BufRead, BufReader};
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use odian_packs::hashing::sha256_hex;
|
|
use odian_packs::{
|
|
bundled_public_key, current_manifest, fetch_notes, install, version_manifest, Error, InstallOptions, IntegrityKind,
|
|
Progress, Source, Stage, Summary,
|
|
};
|
|
use serde_json::Value;
|
|
use tokio_util::sync::CancellationToken;
|
|
|
|
const CHUNK: usize = 1024 * 1024;
|
|
const NOTES: &str = "# ODIAN 0.51.0\n\nPremier pack signé.\n";
|
|
|
|
fn python() -> String {
|
|
std::env::var("ODIAN_PACKS_PYTHON").unwrap_or_else(|_| "python".into())
|
|
}
|
|
|
|
fn pythonpath() -> String {
|
|
std::env::var("ODIAN_PACKS_PYTHONPATH")
|
|
.expect("ODIAN_PACKS_PYTHONPATH must point at ODIAN_MCP_Server/src (see the header of tests/e2e_site.rs)")
|
|
}
|
|
|
|
struct Stub {
|
|
child: Child,
|
|
stdin: Option<ChildStdin>,
|
|
_stdout: BufReader<ChildStdout>,
|
|
info: Value,
|
|
dir: tempfile::TempDir,
|
|
}
|
|
|
|
impl Stub {
|
|
fn start(args: &[&str]) -> Stub {
|
|
let dir = tempfile::Builder::new().prefix("odian-packs-e2e-").tempdir().unwrap();
|
|
let log_path = dir.path().join("stub.log");
|
|
let log = std::fs::File::create(&log_path).unwrap();
|
|
let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("stub_site.py");
|
|
let mut child = Command::new(python())
|
|
.arg(script)
|
|
.arg("--root")
|
|
.arg(dir.path().join("store"))
|
|
.args(args)
|
|
.env("PYTHONPATH", pythonpath())
|
|
.env("PYTHONDONTWRITEBYTECODE", "1")
|
|
.env("PYTHONIOENCODING", "utf-8")
|
|
.stdin(Stdio::piped())
|
|
.stdout(Stdio::piped())
|
|
.stderr(log)
|
|
.spawn()
|
|
.expect("spawn python");
|
|
let stdin = child.stdin.take();
|
|
let mut stdout = BufReader::new(child.stdout.take().unwrap());
|
|
let mut line = String::new();
|
|
let _ = stdout.read_line(&mut line);
|
|
if line.trim().is_empty() {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
panic!("stub site did not start:\n{}", std::fs::read_to_string(&log_path).unwrap_or_default());
|
|
}
|
|
let info: Value = serde_json::from_str(&line).unwrap();
|
|
Stub { child, stdin, _stdout: stdout, info, dir }
|
|
}
|
|
|
|
fn base(&self) -> String {
|
|
self.info["base_url"].as_str().unwrap().to_string()
|
|
}
|
|
|
|
fn root(&self) -> PathBuf {
|
|
PathBuf::from(self.info["root"].as_str().unwrap())
|
|
}
|
|
|
|
fn key(&self) -> String {
|
|
self.info["public_key"].as_str().unwrap().to_string()
|
|
}
|
|
|
|
fn site(&self) -> Source {
|
|
Source::Site { base_url: self.base(), token: "good".into() }
|
|
}
|
|
|
|
fn drive(&self) -> Source {
|
|
Source::Drive { root: self.root() }
|
|
}
|
|
|
|
fn dest(&self, name: &str) -> PathBuf {
|
|
self.dir.path().join(name)
|
|
}
|
|
|
|
fn store_manifest_path(&self) -> PathBuf {
|
|
self.root().join("packs").join("versions").join("0.51.0").join("manifest.json")
|
|
}
|
|
|
|
fn store_manifest(&self) -> Value {
|
|
serde_json::from_slice(&std::fs::read(self.store_manifest_path()).unwrap()).unwrap()
|
|
}
|
|
|
|
fn app_sha(&self, install_path: &str) -> String {
|
|
let m = self.store_manifest();
|
|
let entry = m["app"]["files"].as_array().unwrap().iter().find(|f| f["path"] == install_path).unwrap();
|
|
entry["sha256"].as_str().unwrap().to_string()
|
|
}
|
|
|
|
fn app_blob(&self, install_path: &str) -> PathBuf {
|
|
self.root().join("packs").join("app").join(self.app_sha(install_path))
|
|
}
|
|
|
|
async fn ctl(&self, path: &str) -> Value {
|
|
let text = reqwest::get(format!("{}{}", self.base(), path)).await.unwrap().text().await.unwrap();
|
|
serde_json::from_str(&text).unwrap()
|
|
}
|
|
|
|
async fn requests(&self) -> Vec<(String, Option<String>)> {
|
|
let log = self.ctl("/_ctl/log").await;
|
|
log["requests"]
|
|
.as_array()
|
|
.unwrap()
|
|
.iter()
|
|
.map(|r| (r["path"].as_str().unwrap().to_string(), r["range"].as_str().map(str::to_string)))
|
|
.collect()
|
|
}
|
|
}
|
|
|
|
impl Drop for Stub {
|
|
fn drop(&mut self) {
|
|
drop(self.stdin.take()); // the stub exits when its stdin closes
|
|
let _ = self.child.kill();
|
|
let _ = self.child.wait();
|
|
}
|
|
}
|
|
|
|
fn only(stages: &[Stage]) -> InstallOptions {
|
|
InstallOptions { only: stages.to_vec(), ..InstallOptions::default() }
|
|
}
|
|
|
|
async fn install_from(source: &Source, dest: &Path, key: &str, opts: InstallOptions) -> Result<Summary, Error> {
|
|
let manifest = current_manifest(source, key).await?;
|
|
install(source, dest, &manifest, opts, |_| {}, CancellationToken::new()).await
|
|
}
|
|
|
|
/// Every file under `root`: relative path -> sha256.
|
|
fn files(root: &Path) -> BTreeMap<String, String> {
|
|
fn walk(dir: &Path, prefix: &str, out: &mut BTreeMap<String, String>) {
|
|
for entry in std::fs::read_dir(dir).unwrap() {
|
|
let entry = entry.unwrap();
|
|
let name = entry.file_name().to_string_lossy().into_owned();
|
|
let rel = if prefix.is_empty() { name } else { format!("{prefix}/{name}") };
|
|
if entry.file_type().unwrap().is_dir() {
|
|
walk(&entry.path(), &rel, out);
|
|
} else {
|
|
out.insert(rel, sha256_hex(&std::fs::read(entry.path()).unwrap()));
|
|
}
|
|
}
|
|
}
|
|
let mut out = BTreeMap::new();
|
|
walk(root, "", &mut out);
|
|
out
|
|
}
|
|
|
|
fn flip(path: &Path, offset: usize) {
|
|
let mut data = std::fs::read(path).unwrap();
|
|
data[offset] ^= 0xFF;
|
|
std::fs::write(path, data).unwrap();
|
|
}
|
|
|
|
fn integrity(result: Result<Summary, Error>, expected: IntegrityKind) -> String {
|
|
match result {
|
|
Err(Error::Integrity { kind, what, detail }) if kind == expected => format!("{what}: {detail}"),
|
|
other => panic!("expected an {expected:?} integrity error, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
fn is_payload(path: &str) -> bool {
|
|
path.starts_with("/packs/") || path.starts_with("/src/") || path.contains("/block/")
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn site_install_verifies_everything_then_rerun_downloads_nothing() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
let manifest = current_manifest(&site, &key).await.unwrap();
|
|
assert_eq!(manifest.raw(), std::fs::read(stub.store_manifest_path()).unwrap().as_slice());
|
|
|
|
let events: Arc<Mutex<Vec<Progress>>> = Arc::default();
|
|
let sink = events.clone();
|
|
let summary = install(&site, &dest, &manifest, InstallOptions::default(), move |p| sink.lock().unwrap().push(p), CancellationToken::new())
|
|
.await
|
|
.unwrap();
|
|
|
|
let app = summary.app.clone().unwrap();
|
|
assert_eq!((app.files, app.downloaded, app.skipped), (5, 5, 0));
|
|
let db = summary.db.clone().unwrap();
|
|
assert_eq!((db.files, db.downloaded, db.skipped), (3, 3, 0));
|
|
let src: Vec<(String, usize)> = summary.src.clone().unwrap().into_iter().map(|s| (s.path, s.chunks)).collect();
|
|
assert_eq!(src, vec![
|
|
("osm_source/world_aeroway_v1.sqlite".to_string(), 3),
|
|
("osm_source/small.bin".to_string(), 1),
|
|
("osm_source/empty.bin".to_string(), 0),
|
|
]);
|
|
let tiles = summary.tiles.clone().unwrap();
|
|
assert_eq!((tiles[0].blocks, tiles[0].downloaded, tiles[0].files), (4, 4, 8));
|
|
assert!(summary.transferred_bytes > 3 * CHUNK as u64);
|
|
|
|
// Every byte in place, in the fetch.py layout.
|
|
let store = stub.root();
|
|
let m = stub.store_manifest();
|
|
let version_dir = dest.join("versions").join("0.51.0");
|
|
for f in m["app"]["files"].as_array().unwrap() {
|
|
let path = version_dir.join(f["path"].as_str().unwrap());
|
|
assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap(), "{}", path.display());
|
|
}
|
|
assert_eq!(std::fs::metadata(version_dir.join("Odian/assets/empty.conf")).unwrap().len(), 0);
|
|
for f in m["db"]["files"].as_array().unwrap() {
|
|
let path = dest.join("data").join(f["path"].as_str().unwrap());
|
|
assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap());
|
|
}
|
|
for f in m["src"]["files"].as_array().unwrap() {
|
|
let rel = f["path"].as_str().unwrap();
|
|
let path = dest.join("data").join("src").join(rel);
|
|
assert_eq!(std::fs::read(&path).unwrap(), std::fs::read(store.join("src").join(rel)).unwrap());
|
|
let marker = std::fs::read_to_string(format!("{}.ok", path.display())).unwrap();
|
|
assert_eq!(marker, f["chunks_sha256"].as_str().unwrap());
|
|
assert!(!Path::new(&format!("{}.chunks.json", path.display())).exists());
|
|
}
|
|
assert_eq!(std::fs::read(version_dir.join("manifest.json")).unwrap(), manifest.raw());
|
|
let set_dir = dest.join("data/tiles/dark/test_ds");
|
|
let mut tile_files = files(&set_dir);
|
|
let done = tile_files.remove(".blocks-done.json").map(|_| std::fs::read_to_string(set_dir.join(".blocks-done.json")).unwrap()).unwrap();
|
|
let expected_tiles = ["0/0/0.png", "3/1/2.png", "7/1/2.png", "7/127/63.png", "7/3/2.jpg", "7/3/2.png", "7/63/63.pbf", "7/64/5.png"];
|
|
assert_eq!(tile_files.keys().map(String::as_str).collect::<Vec<_>>(), expected_tiles, "the late tile 7/5/5.png must not be installed");
|
|
for rel in expected_tiles {
|
|
assert_eq!(tile_files[rel], sha256_hex(&std::fs::read(store.join("tiles/dark/test_ds").join(rel)).unwrap()));
|
|
}
|
|
assert!(done.starts_with("{\"index_sha256\": \""), "Python's json.dumps layout: {done}");
|
|
let done: Value = serde_json::from_str(&done).unwrap();
|
|
assert_eq!(done["index_sha256"], m["tiles"]["sets"][0]["index_sha256"]);
|
|
assert_eq!(done["blocks"].as_array().unwrap().len(), 4);
|
|
|
|
// No request for the zero-byte file: it never needs the network.
|
|
let log = stub.requests().await;
|
|
assert!(!log.iter().any(|(p, _)| p.ends_with(&stub.app_sha("Odian/assets/empty.conf"))));
|
|
|
|
// Progress: stages in order, ending at their totals, contract-shaped JSON.
|
|
let events = events.lock().unwrap().clone();
|
|
let mut stages: Vec<Stage> = events.iter().map(|p| p.stage).collect();
|
|
stages.dedup();
|
|
assert_eq!(stages, vec![Stage::Manifest, Stage::App, Stage::Db, Stage::Src, Stage::Tiles]);
|
|
assert!(events.iter().all(|p| p.stage_count == 5));
|
|
for stage in [Stage::App, Stage::Db, Stage::Src, Stage::Tiles] {
|
|
let last = events.iter().rev().find(|p| p.stage == stage).unwrap();
|
|
assert_eq!((last.done_bytes, last.done_items), (last.total_bytes, last.total_items), "{stage:?}");
|
|
}
|
|
assert!(events.iter().any(|p| p.message.as_deref() == Some("tiles dark/test_ds: 4/4 blocks")));
|
|
let json = serde_json::to_value(&events[0]).unwrap();
|
|
assert_eq!(json["stage"], "manifest");
|
|
for field in ["stage_index", "stage_count", "done_bytes", "total_bytes", "done_items", "total_items", "bytes_per_sec", "message"] {
|
|
assert!(json.get(field).is_some(), "{field}");
|
|
}
|
|
|
|
// Re-run: nothing is downloaded again.
|
|
stub.ctl("/_ctl/reset").await;
|
|
let again = install(&site, &dest, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
|
assert_eq!(again.transferred_bytes, 0);
|
|
assert_eq!(again.app.unwrap().downloaded, 0);
|
|
assert_eq!(again.db.unwrap().downloaded, 0);
|
|
assert!(again.src.unwrap().iter().all(|s| s.chunks == 0));
|
|
assert_eq!(again.tiles.unwrap()[0].downloaded, 0);
|
|
let log = stub.requests().await;
|
|
assert!(log.iter().all(|(p, _)| !is_payload(p)), "{log:?}");
|
|
|
|
assert_eq!(fetch_notes(&site, "0.51.0").await.unwrap(), NOTES);
|
|
assert_eq!(version_manifest(&site, "0.51.0", &key).await.unwrap().raw(), manifest.raw());
|
|
assert!(matches!(version_manifest(&site, "9.9.9", &key).await, Err(Error::NotFound { .. })));
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn an_update_reuses_the_installed_files_and_fetches_only_the_db_files_asked_for() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
install_from(&site, &dest, &key, only(&[Stage::App])).await.unwrap();
|
|
// What is installed becomes an older version: installing the current one is an update.
|
|
let old = dest.join("versions").join("0.50.0");
|
|
std::fs::rename(dest.join("versions").join("0.51.0"), &old).unwrap();
|
|
// One old file went bad on disk: taken from there anyway, it must be caught.
|
|
let m = stub.store_manifest();
|
|
let bad = m["app"]["files"].as_array().unwrap().iter().find(|f| f["size"].as_u64().unwrap() > 0).unwrap();
|
|
let bad_path = bad["path"].as_str().unwrap().to_string();
|
|
flip(&old.join(&bad_path), 0);
|
|
let bad_old = std::fs::read(old.join(&bad_path)).unwrap();
|
|
|
|
stub.ctl("/_ctl/reset").await;
|
|
let first_db = m["db"]["files"][0]["path"].as_str().unwrap().to_string();
|
|
let opts = InstallOptions {
|
|
only: vec![Stage::App, Stage::Db],
|
|
db_paths: Some([first_db.clone()].into_iter().collect()),
|
|
..InstallOptions::default()
|
|
};
|
|
let summary = install_from(&site, &dest, &key, opts).await.unwrap();
|
|
|
|
let app = summary.app.unwrap();
|
|
assert_eq!((app.files, app.downloaded, app.reused, app.skipped), (5, 1, 4, 0), "{app:?}");
|
|
let app_requests: Vec<String> =
|
|
stub.requests().await.into_iter().map(|(p, _)| p).filter(|p| p.starts_with("/packs/app/")).collect();
|
|
assert_eq!(app_requests, vec![format!("/packs/app/{}", stub.app_sha(&bad_path))], "only the bad file is downloaded");
|
|
let new = dest.join("versions").join("0.51.0");
|
|
for f in m["app"]["files"].as_array().unwrap() {
|
|
let path = new.join(f["path"].as_str().unwrap());
|
|
assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap(), "{}", path.display());
|
|
}
|
|
// The download replaced the link; it never wrote through it into the old version.
|
|
assert_eq!(std::fs::read(old.join(&bad_path)).unwrap(), bad_old);
|
|
|
|
let db = summary.db.unwrap();
|
|
assert_eq!((db.files, db.downloaded), (1, 1));
|
|
for f in m["db"]["files"].as_array().unwrap() {
|
|
let path = f["path"].as_str().unwrap();
|
|
assert_eq!(dest.join("data").join(path).is_file(), path == first_db, "{path}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn only_the_chosen_tile_sets_and_source_files_are_installed() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
let opts = InstallOptions {
|
|
only: vec![Stage::Src, Stage::Tiles],
|
|
tile_sets: Some(BTreeSet::new()),
|
|
src_paths: Some(["osm_source/small.bin".to_string()].into_iter().collect()),
|
|
..InstallOptions::default()
|
|
};
|
|
let summary = install_from(&site, &dest, &key, opts).await.unwrap();
|
|
let src: Vec<String> = summary.src.unwrap().into_iter().map(|s| s.path).collect();
|
|
assert_eq!(src, ["osm_source/small.bin"]);
|
|
assert!(summary.tiles.unwrap().is_empty());
|
|
assert!(dest.join("data/src/osm_source/small.bin").is_file());
|
|
assert!(!dest.join("data/src/osm_source/world_aeroway_v1.sqlite").exists());
|
|
assert!(!dest.join("data/tiles/dark/test_ds").exists(), "no tile set was chosen");
|
|
let log = stub.requests().await;
|
|
assert!(!log.iter().any(|(p, _)| p.contains("/block/")), "no tile block fetched: {log:?}");
|
|
|
|
// Choosing the tile set later adds it; what is installed stays as it is.
|
|
let opts = InstallOptions {
|
|
only: vec![Stage::Src, Stage::Tiles],
|
|
tile_sets: Some(["dark/test_ds".to_string()].into_iter().collect()),
|
|
src_paths: Some(["osm_source/small.bin".to_string()].into_iter().collect()),
|
|
..InstallOptions::default()
|
|
};
|
|
let summary = install_from(&site, &dest, &key, opts).await.unwrap();
|
|
assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
|
|
assert_eq!(summary.src.unwrap()[0].chunks, 0, "the source file was already there");
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn a_pc_installs_by_copying_another_install_and_checks_every_file() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key) = (stub.site(), stub.key());
|
|
let first = stub.dest("first");
|
|
install_from(&site, &first, &key, InstallOptions::default()).await.unwrap();
|
|
|
|
// The drive an operator carries holds that install, not a store: the second PC copies it.
|
|
stub.ctl("/_ctl/reset").await;
|
|
let drive = Source::Install { root: first.clone() };
|
|
let second = stub.dest("second");
|
|
let summary = install_from(&drive, &second, &key, InstallOptions::default()).await.unwrap();
|
|
assert!(stub.requests().await.is_empty(), "nothing was asked of the site");
|
|
assert_eq!(summary.version, "0.51.0");
|
|
assert_eq!(files(&second.join("versions").join("0.51.0")), files(&first.join("versions").join("0.51.0")));
|
|
assert_eq!(files(&second.join("data")), files(&first.join("data")), "same data, tile by tile");
|
|
let done = second.join("data/tiles/dark/test_ds/.blocks-done.json");
|
|
assert!(done.is_file(), "the blocks the source holds travel with the tiles");
|
|
|
|
// Running it again copies nothing: what is already there is recognised.
|
|
let again = install_from(&drive, &second, &key, InstallOptions::default()).await.unwrap();
|
|
assert_eq!(again.app.unwrap().downloaded, 0);
|
|
assert_eq!(again.tiles.unwrap().iter().map(|t| t.downloaded).sum::<usize>(), 0);
|
|
|
|
// A set this PC already recorded as complete is not read again: the marker names this
|
|
// index and every block of it, so what lies here IS the set. Hundreds of thousands of
|
|
// files are not hashed twice to conclude "identical", and a tile that changes on the
|
|
// drive afterwards stays there, as a site update leaves a complete set alone too.
|
|
let on_drive = first.join("data/tiles/dark/test_ds/0/0/0.png");
|
|
let here = second.join("data/tiles/dark/test_ds/0/0/0.png");
|
|
let kept = std::fs::read(&here).unwrap();
|
|
flip(&on_drive, 3);
|
|
install_from(&drive, &second, &key, only(&[Stage::Tiles])).await.unwrap();
|
|
assert_eq!(std::fs::read(&here).unwrap(), kept, "the set was recorded complete: it was left alone");
|
|
flip(&on_drive, 3);
|
|
|
|
// An app file that went bad on the drive is caught: the manifest says what it should be.
|
|
let third = stub.dest("third");
|
|
flip(&first.join("versions").join("0.51.0").join("Odian").join("Odian.exe"), 3);
|
|
let what = integrity(install_from(&drive, &third, &key, InstallOptions::default()).await, IntegrityKind::FileHash);
|
|
assert!(what.contains("Odian.exe"), "{what}");
|
|
|
|
// A TILE cannot be checked that way: only a site's index carries per-tile hashes, and an
|
|
// install keeps none. What is checked is the copy - the bytes that land are the bytes read.
|
|
let fourth = stub.dest("fourth");
|
|
let tile = first.join("data/tiles/dark/test_ds/0/0/0.png");
|
|
flip(&tile, 3);
|
|
install_from(&drive, &fourth, &key, only(&[Stage::Tiles])).await.unwrap();
|
|
assert_eq!(
|
|
std::fs::read(fourth.join("data/tiles/dark/test_ds/0/0/0.png")).unwrap(),
|
|
std::fs::read(&tile).unwrap(),
|
|
"faithful to the drive, whatever the drive holds"
|
|
);
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn wrong_token_is_unauthorized_and_edited_manifest_is_a_signature_error() {
|
|
let stub = Stub::start(&[]);
|
|
let key = stub.key();
|
|
let bad = Source::Site { base_url: stub.base(), token: "bad".into() };
|
|
assert!(matches!(current_manifest(&bad, &key).await, Err(Error::Unauthorized { status: 401, .. })));
|
|
|
|
let manifest = current_manifest(&stub.site(), &key).await.unwrap();
|
|
let dest = stub.dest("install");
|
|
let result = install(&bad, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await;
|
|
assert!(matches!(result, Err(Error::Unauthorized { .. })), "{result:?}");
|
|
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
|
|
|
// The stub's manifest is signed by the test key, not the release key.
|
|
let release = current_manifest(&stub.site(), bundled_public_key()).await;
|
|
assert!(matches!(release, Err(Error::Integrity { kind: IntegrityKind::Signature, .. })));
|
|
|
|
let path = stub.store_manifest_path();
|
|
let text = std::fs::read_to_string(&path).unwrap();
|
|
let edited = text.replacen("\"released_at\": \"2026-09-15T12:00:00Z\"", "\"released_at\": \"2026-09-15T12:00:01Z\"", 1);
|
|
assert_ne!(text, edited);
|
|
std::fs::write(&path, edited).unwrap();
|
|
let result = current_manifest(&stub.site(), &key).await;
|
|
assert!(matches!(result, Err(Error::Integrity { kind: IntegrityKind::Signature, .. })), "{result:?}");
|
|
|
|
std::fs::remove_file(stub.root().join("packs/versions/CURRENT")).unwrap();
|
|
assert!(matches!(current_manifest(&stub.site(), &key).await, Err(Error::NotFound { .. })));
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn flipped_app_byte_is_an_integrity_error() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
let manifest = current_manifest(&site, &key).await.unwrap();
|
|
let sha = stub.app_sha("Odian/Odian.exe");
|
|
flip(&stub.app_blob("Odian/Odian.exe"), 1234);
|
|
stub.ctl("/_ctl/reset").await;
|
|
|
|
let detail = integrity(install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await, IntegrityKind::FileHash);
|
|
assert!(detail.contains(&sha), "{detail}");
|
|
let gets = stub.requests().await.iter().filter(|(p, _)| p.ends_with(&sha)).count();
|
|
assert_eq!(gets, 2, "one clean retry, then give up");
|
|
let target = dest.join("versions/0.51.0/Odian/Odian.exe");
|
|
assert!(!target.exists() && !Path::new(&format!("{}.part", target.display())).exists());
|
|
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn flipped_tile_byte_is_an_integrity_error() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
flip(&stub.root().join("tiles/dark/test_ds/7/63/63.pbf"), 100);
|
|
|
|
let detail = integrity(install_from(&site, &dest, &key, only(&[Stage::Tiles])).await, IntegrityKind::BlockDigest);
|
|
assert!(detail.contains("block/7/0/0.tar"), "{detail}");
|
|
let set_dir = dest.join("data/tiles/dark/test_ds");
|
|
assert!(!set_dir.join("7/63/63.pbf").exists() && !set_dir.join("7/1/2.png").exists(), "nothing of a bad block moves in");
|
|
if let Ok(done) = std::fs::read_to_string(set_dir.join(".blocks-done.json")) {
|
|
assert!(!done.contains("\"7/0/0:"), "{done}");
|
|
}
|
|
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn flipped_source_chunk_byte_is_an_integrity_error_and_the_rest_resumes() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
let original = stub.root().join("src/osm_source/world_aeroway_v1.sqlite");
|
|
flip(&original, CHUNK + 5);
|
|
|
|
let detail = integrity(install_from(&site, &dest, &key, only(&[Stage::Src])).await, IntegrityKind::ChunkHash);
|
|
assert!(detail.contains("chunk 1"), "{detail}");
|
|
let target = dest.join("data/src/osm_source/world_aeroway_v1.sqlite");
|
|
assert!(!Path::new(&format!("{}.ok", target.display())).exists());
|
|
let progress: Value = serde_json::from_str(&std::fs::read_to_string(format!("{}.chunks.json", target.display())).unwrap()).unwrap();
|
|
let verified: Vec<u64> = progress["verified"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect();
|
|
assert!(!verified.contains(&1), "{verified:?}");
|
|
|
|
flip(&original, CHUNK + 5); // the site is fixed
|
|
stub.ctl("/_ctl/reset").await;
|
|
let summary = install_from(&site, &dest, &key, only(&[Stage::Src])).await.unwrap();
|
|
let world = &summary.src.unwrap()[0];
|
|
assert_eq!(world.chunks + world.skipped, 3);
|
|
assert_eq!(world.skipped, verified.len());
|
|
let ranges: Vec<String> = stub
|
|
.requests()
|
|
.await
|
|
.into_iter()
|
|
.filter(|(p, _)| p.ends_with("world_aeroway_v1.sqlite"))
|
|
.map(|(_, r)| r.unwrap())
|
|
.collect();
|
|
assert_eq!(ranges.len(), world.chunks);
|
|
assert!(ranges.contains(&format!("bytes={}-{}", CHUNK, 2 * CHUNK - 1)), "{ranges:?}");
|
|
assert_eq!(std::fs::read(&target).unwrap(), std::fs::read(&original).unwrap());
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn interrupted_part_resumes_with_a_range_request() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key) = (stub.site(), stub.key());
|
|
let manifest = current_manifest(&site, &key).await.unwrap();
|
|
let sha = stub.app_sha("Odian/data/big.bin");
|
|
let big = std::fs::read(stub.app_blob("Odian/data/big.bin")).unwrap();
|
|
let app_total: u64 = manifest.manifest().app.files.iter().map(|f| f.size).sum();
|
|
|
|
// A .part left by an earlier run (Python or Rust).
|
|
let dest = stub.dest("seeded");
|
|
let target = dest.join("versions/0.51.0/Odian/data/big.bin");
|
|
let part = PathBuf::from(format!("{}.part", target.display()));
|
|
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
|
|
let have = CHUNK + 1000;
|
|
std::fs::write(&part, &big[..have]).unwrap();
|
|
stub.ctl("/_ctl/reset").await;
|
|
let summary = install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await.unwrap();
|
|
assert_eq!(summary.app.unwrap().downloaded, 5);
|
|
assert_eq!(summary.transferred_bytes, app_total - have as u64);
|
|
let log = stub.requests().await;
|
|
let big_requests: Vec<_> = log.iter().filter(|(p, _)| p.ends_with(&sha)).collect();
|
|
assert_eq!(big_requests.len(), 1);
|
|
assert_eq!(big_requests[0].1.as_deref(), Some(format!("bytes={have}-").as_str()));
|
|
assert_eq!(std::fs::read(&target).unwrap(), big);
|
|
assert!(!part.exists());
|
|
|
|
// A connection that really drops mid-file.
|
|
let dest = stub.dest("cut");
|
|
let target = dest.join("versions/0.51.0/Odian/data/big.bin");
|
|
let part = PathBuf::from(format!("{}.part", target.display()));
|
|
stub.ctl(&format!("/_ctl/cut?prefix=/packs/app/{sha}&after=700000&count=1")).await;
|
|
let result = install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await;
|
|
assert!(matches!(result, Err(Error::Http { .. })), "{result:?}");
|
|
let have = std::fs::metadata(&part).unwrap().len();
|
|
assert!(have > 0 && have <= 700_000, "{have}");
|
|
stub.ctl("/_ctl/reset").await;
|
|
install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await.unwrap();
|
|
let log = stub.requests().await;
|
|
let big_requests: Vec<_> = log.iter().filter(|(p, _)| p.ends_with(&sha)).collect();
|
|
assert_eq!(big_requests.len(), 1);
|
|
assert_eq!(big_requests[0].1.as_deref(), Some(format!("bytes={have}-").as_str()));
|
|
assert_eq!(std::fs::read(&target).unwrap(), big);
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn busy_site_is_retried_with_retry_after() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key) = (stub.site(), stub.key());
|
|
stub.ctl("/_ctl/reset").await;
|
|
stub.ctl("/_ctl/fail?prefix=/v1/versions/current&status=503&count=2&retry_after=0").await;
|
|
current_manifest(&site, &key).await.unwrap();
|
|
let tries = stub.requests().await.iter().filter(|(p, _)| p == "/v1/versions/current").count();
|
|
assert_eq!(tries, 3);
|
|
|
|
stub.ctl("/_ctl/fail?prefix=/v1/versions/current&status=503&count=4&retry_after=0").await;
|
|
let result = current_manifest(&site, &key).await;
|
|
assert!(matches!(result, Err(Error::SiteBusy { attempts: 4, .. })), "{result:?}");
|
|
|
|
// A 502 on a tile block and a 504 on an app file are retried too.
|
|
stub.ctl("/_ctl/fail?prefix=/v1/versions/0.51.0/tiles/dark/test_ds/block/&status=502&count=1&retry_after=0").await;
|
|
stub.ctl("/_ctl/fail?prefix=/packs/app/&status=504&count=1&retry_after=0").await;
|
|
let summary = install_from(&site, &stub.dest("install"), &key, only(&[Stage::App, Stage::Tiles])).await.unwrap();
|
|
assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn cancel_stops_and_keeps_verified_work() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
|
let manifest = current_manifest(&site, &key).await.unwrap();
|
|
let cancel = CancellationToken::new();
|
|
let trigger = cancel.clone();
|
|
let result = install(&site, &dest, &manifest, InstallOptions::default(), move |p| {
|
|
if p.stage == Stage::App && p.done_items >= 1 {
|
|
trigger.cancel();
|
|
}
|
|
}, cancel)
|
|
.await;
|
|
assert!(matches!(result, Err(Error::Cancelled)), "{result:?}");
|
|
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
|
|
|
let summary = install(&site, &dest, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
|
assert!(summary.app.unwrap().skipped >= 1);
|
|
assert!(dest.join("versions/0.51.0/manifest.json").exists());
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn a_copy_stopped_halfway_leaves_the_set_partial() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, key) = (stub.site(), stub.key());
|
|
let drive_install = stub.dest("drive");
|
|
install_from(&site, &drive_install, &key, InstallOptions::default()).await.unwrap();
|
|
let marker = ".blocks-done.json";
|
|
let carried = drive_install.join("data/tiles/dark/test_ds");
|
|
assert!(carried.join(marker).is_file(), "the drive holds the whole set");
|
|
// A set of a few tiles copies faster than the first progress report: give the drive enough
|
|
// of them (tiles drawn beyond the index, which travel too) to be stopped in the middle.
|
|
for x in 0..40 {
|
|
let dir = carried.join("9").join(x.to_string());
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
for y in 0..10 {
|
|
std::fs::write(dir.join(format!("{y}.png")), vec![b't'; 4096]).unwrap();
|
|
}
|
|
}
|
|
|
|
// Stop the copy in the middle of the tiles.
|
|
let drive = Source::Install { root: drive_install.clone() };
|
|
let manifest = current_manifest(&drive, &key).await.unwrap();
|
|
let dest = stub.dest("half");
|
|
let cancel = CancellationToken::new();
|
|
let trigger = cancel.clone();
|
|
let result = install(&drive, &dest, &manifest, only(&[Stage::Tiles]), move |p| {
|
|
if p.stage == Stage::Tiles && p.done_items >= 1 {
|
|
trigger.cancel();
|
|
}
|
|
}, cancel)
|
|
.await;
|
|
assert!(matches!(result, Err(Error::Cancelled)), "{result:?}");
|
|
let set_dir = dest.join("data/tiles/dark/test_ds");
|
|
// Tiles landed, but nothing here claims the set is whole: the next run copies the rest
|
|
// instead of trusting a record written before the files it speaks for.
|
|
assert!(!set_dir.join(marker).exists(), "a half-copied set must not read as complete");
|
|
|
|
let summary = install(&drive, &dest, &manifest, only(&[Stage::Tiles]), |_| {}, CancellationToken::new()).await.unwrap();
|
|
assert!(set_dir.join(marker).is_file(), "finished, so it is recorded");
|
|
assert!(summary.tiles.unwrap().iter().map(|t| t.downloaded).sum::<usize>() > 0, "the rest was copied");
|
|
assert_eq!(files(&set_dir), files(&carried), "same set, tile by tile");
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn drive_install_equals_site_install() {
|
|
let stub = Stub::start(&[]);
|
|
let (site, drive, key) = (stub.site(), stub.drive(), stub.key());
|
|
let from_site = stub.dest("from-site");
|
|
install_from(&site, &from_site, &key, InstallOptions::default()).await.unwrap();
|
|
|
|
let manifest = current_manifest(&drive, &key).await.unwrap();
|
|
assert_eq!(manifest.raw(), std::fs::read(stub.store_manifest_path()).unwrap().as_slice());
|
|
let from_drive = stub.dest("from-drive");
|
|
let summary = install(&drive, &from_drive, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
|
assert_eq!(summary.app.unwrap().downloaded, 5);
|
|
assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
|
|
assert_eq!(files(&from_site), files(&from_drive), "site and drive installs must be identical, file by file");
|
|
|
|
let again = install(&drive, &from_drive, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
|
assert_eq!(again.transferred_bytes, 0);
|
|
assert_eq!(fetch_notes(&drive, "0.51.0").await.unwrap(), NOTES);
|
|
assert_eq!(version_manifest(&drive, "0.51.0", &key).await.unwrap().raw(), manifest.raw());
|
|
|
|
// The same checks guard a drive.
|
|
let tile = stub.root().join("tiles/dark/test_ds/7/64/5.png");
|
|
flip(&tile, 3);
|
|
let detail = integrity(install_from(&drive, &stub.dest("drive-tile"), &key, only(&[Stage::Tiles])).await, IntegrityKind::BlockDigest);
|
|
assert!(detail.contains("7/1/0"), "{detail}");
|
|
flip(&tile, 3);
|
|
flip(&stub.app_blob("api/odian_api.exe"), 7);
|
|
integrity(install_from(&drive, &stub.dest("drive-app"), &key, only(&[Stage::App])).await, IntegrityKind::FileHash);
|
|
}
|
|
|
|
fn python_fetch(stub: &Stub, dest: &Path) -> Value {
|
|
let code = "import json, sys\n\
|
|
from odian_packs.fetch import fetch\n\
|
|
summary = fetch(sys.argv[1], 'good', sys.argv[2], public_key=sys.argv[3], log=lambda m: None)\n\
|
|
print(json.dumps(summary))\n";
|
|
let output = Command::new(python())
|
|
.arg("-c")
|
|
.arg(code)
|
|
.arg(stub.base())
|
|
.arg(dest)
|
|
.arg(stub.key())
|
|
.env("PYTHONPATH", pythonpath())
|
|
.env("PYTHONDONTWRITEBYTECODE", "1")
|
|
.output()
|
|
.unwrap();
|
|
assert!(output.status.success(), "python fetch failed:\n{}", String::from_utf8_lossy(&output.stderr));
|
|
let stdout = String::from_utf8(output.stdout).unwrap();
|
|
serde_json::from_str(stdout.lines().last().unwrap()).unwrap()
|
|
}
|
|
|
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
|
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
|
async fn python_and_rust_installs_are_interchangeable() {
|
|
// fetch.py cannot install a zero-byte source file (see the report), so leave it out here.
|
|
let stub = Stub::start(&["--no-empty-src"]);
|
|
let (site, key) = (stub.site(), stub.key());
|
|
|
|
let python_first = stub.dest("python-first");
|
|
let py = tokio::task::block_in_place(|| python_fetch(&stub, &python_first));
|
|
assert_eq!(py["app"]["downloaded"], 5);
|
|
let rust = install_from(&site, &python_first, &key, InstallOptions::default()).await.unwrap();
|
|
assert_eq!(rust.transferred_bytes, 0, "Rust re-verifies a Python install and downloads nothing: {rust:?}");
|
|
assert_eq!(rust.app.unwrap().skipped, 5);
|
|
|
|
let rust_first = stub.dest("rust-first");
|
|
install_from(&site, &rust_first, &key, InstallOptions::default()).await.unwrap();
|
|
let py = tokio::task::block_in_place(|| python_fetch(&stub, &rust_first));
|
|
assert_eq!(py["app"]["downloaded"], 0, "{py}");
|
|
assert_eq!(py["db"]["downloaded"], 0, "{py}");
|
|
assert!(py["src"].as_array().unwrap().iter().all(|s| s["chunks"] == 0), "{py}");
|
|
assert_eq!(py["tiles"][0]["downloaded"], 0, "{py}");
|
|
|
|
assert_eq!(files(&python_first), files(&rust_first));
|
|
}
|