ODIAN Launcher: hub, installer, local stack and remembered sign-in (M2)
- Tauri 2 + Svelte 5 hub in ODIAN dark: patch notes and roadmap, install and update progress, local stack status, settings with DHP and MCP addresses. - crates/odian-packs: Rust installer for pack format v1 (signed manifests, resumable verified downloads from a DHP or an ODIAN drive), byte-compatible with ODIAN_MCP_Server's Python odian_packs. - Local stack on 127.0.0.1: Postgres -> Engine API -> tile server -> Odian.exe, one Job Object, clean shutdown; database snapshot restore on a fresh PC only. - Keycloak sign-in through the DHP's MCP, remembered until Sign out with a launcher session that renews the Engine token silently. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Kelorastus
@@ -0,0 +1,557 @@
|
||||
//! End to end against a local stub site: `tests/stub_site.py` builds a small signed pack
|
||||
//! store with the Python reference code (odian_packs) and serves it with the real update
|
||||
//! service (`odian_update.api.create_app`) plus nginx-like Range routes. The same store is
|
||||
//! also installed as an ODIAN drive.
|
||||
//!
|
||||
//! These tests spawn Python, so they are `#[ignore]`d by default. They need Python 3 with
|
||||
//! fastapi, uvicorn, httpx and cryptography, and the ODIAN_MCP_Server sources:
|
||||
//!
|
||||
//! ```text
|
||||
//! ODIAN_PACKS_PYTHONPATH=C:/Users/JFEXP_D/Documents/Gitea/ODIAN_MCP_Server/src \
|
||||
//! cargo test -p odian-packs --test e2e_site -- --ignored
|
||||
//! ```
|
||||
//!
|
||||
//! `ODIAN_PACKS_PYTHON` picks the interpreter (default `python`).
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io::{BufRead, BufReader};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use odian_packs::hashing::sha256_hex;
|
||||
use odian_packs::{
|
||||
bundled_public_key, current_manifest, fetch_notes, install, version_manifest, Error, InstallOptions, IntegrityKind,
|
||||
Progress, Source, Stage, Summary,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const CHUNK: usize = 1024 * 1024;
|
||||
const NOTES: &str = "# ODIAN 0.51.0\n\nPremier pack signé.\n";
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("ODIAN_PACKS_PYTHON").unwrap_or_else(|_| "python".into())
|
||||
}
|
||||
|
||||
fn pythonpath() -> String {
|
||||
std::env::var("ODIAN_PACKS_PYTHONPATH")
|
||||
.expect("ODIAN_PACKS_PYTHONPATH must point at ODIAN_MCP_Server/src (see the header of tests/e2e_site.rs)")
|
||||
}
|
||||
|
||||
struct Stub {
|
||||
child: Child,
|
||||
stdin: Option<ChildStdin>,
|
||||
_stdout: BufReader<ChildStdout>,
|
||||
info: Value,
|
||||
dir: tempfile::TempDir,
|
||||
}
|
||||
|
||||
impl Stub {
|
||||
fn start(args: &[&str]) -> Stub {
|
||||
let dir = tempfile::Builder::new().prefix("odian-packs-e2e-").tempdir().unwrap();
|
||||
let log_path = dir.path().join("stub.log");
|
||||
let log = std::fs::File::create(&log_path).unwrap();
|
||||
let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("stub_site.py");
|
||||
let mut child = Command::new(python())
|
||||
.arg(script)
|
||||
.arg("--root")
|
||||
.arg(dir.path().join("store"))
|
||||
.args(args)
|
||||
.env("PYTHONPATH", pythonpath())
|
||||
.env("PYTHONDONTWRITEBYTECODE", "1")
|
||||
.env("PYTHONIOENCODING", "utf-8")
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(log)
|
||||
.spawn()
|
||||
.expect("spawn python");
|
||||
let stdin = child.stdin.take();
|
||||
let mut stdout = BufReader::new(child.stdout.take().unwrap());
|
||||
let mut line = String::new();
|
||||
let _ = stdout.read_line(&mut line);
|
||||
if line.trim().is_empty() {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
panic!("stub site did not start:\n{}", std::fs::read_to_string(&log_path).unwrap_or_default());
|
||||
}
|
||||
let info: Value = serde_json::from_str(&line).unwrap();
|
||||
Stub { child, stdin, _stdout: stdout, info, dir }
|
||||
}
|
||||
|
||||
fn base(&self) -> String {
|
||||
self.info["base_url"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
fn root(&self) -> PathBuf {
|
||||
PathBuf::from(self.info["root"].as_str().unwrap())
|
||||
}
|
||||
|
||||
fn key(&self) -> String {
|
||||
self.info["public_key"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
fn site(&self) -> Source {
|
||||
Source::Site { base_url: self.base(), token: "good".into() }
|
||||
}
|
||||
|
||||
fn drive(&self) -> Source {
|
||||
Source::Drive { root: self.root() }
|
||||
}
|
||||
|
||||
fn dest(&self, name: &str) -> PathBuf {
|
||||
self.dir.path().join(name)
|
||||
}
|
||||
|
||||
fn store_manifest_path(&self) -> PathBuf {
|
||||
self.root().join("packs").join("versions").join("0.51.0").join("manifest.json")
|
||||
}
|
||||
|
||||
fn store_manifest(&self) -> Value {
|
||||
serde_json::from_slice(&std::fs::read(self.store_manifest_path()).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn app_sha(&self, install_path: &str) -> String {
|
||||
let m = self.store_manifest();
|
||||
let entry = m["app"]["files"].as_array().unwrap().iter().find(|f| f["path"] == install_path).unwrap();
|
||||
entry["sha256"].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
fn app_blob(&self, install_path: &str) -> PathBuf {
|
||||
self.root().join("packs").join("app").join(self.app_sha(install_path))
|
||||
}
|
||||
|
||||
async fn ctl(&self, path: &str) -> Value {
|
||||
let text = reqwest::get(format!("{}{}", self.base(), path)).await.unwrap().text().await.unwrap();
|
||||
serde_json::from_str(&text).unwrap()
|
||||
}
|
||||
|
||||
async fn requests(&self) -> Vec<(String, Option<String>)> {
|
||||
let log = self.ctl("/_ctl/log").await;
|
||||
log["requests"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|r| (r["path"].as_str().unwrap().to_string(), r["range"].as_str().map(str::to_string)))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Stub {
|
||||
fn drop(&mut self) {
|
||||
drop(self.stdin.take()); // the stub exits when its stdin closes
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn only(stages: &[Stage]) -> InstallOptions {
|
||||
InstallOptions { only: stages.to_vec(), ..InstallOptions::default() }
|
||||
}
|
||||
|
||||
async fn install_from(source: &Source, dest: &Path, key: &str, opts: InstallOptions) -> Result<Summary, Error> {
|
||||
let manifest = current_manifest(source, key).await?;
|
||||
install(source, dest, &manifest, opts, |_| {}, CancellationToken::new()).await
|
||||
}
|
||||
|
||||
/// Every file under `root`: relative path -> sha256.
|
||||
fn files(root: &Path) -> BTreeMap<String, String> {
|
||||
fn walk(dir: &Path, prefix: &str, out: &mut BTreeMap<String, String>) {
|
||||
for entry in std::fs::read_dir(dir).unwrap() {
|
||||
let entry = entry.unwrap();
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
let rel = if prefix.is_empty() { name } else { format!("{prefix}/{name}") };
|
||||
if entry.file_type().unwrap().is_dir() {
|
||||
walk(&entry.path(), &rel, out);
|
||||
} else {
|
||||
out.insert(rel, sha256_hex(&std::fs::read(entry.path()).unwrap()));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = BTreeMap::new();
|
||||
walk(root, "", &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn flip(path: &Path, offset: usize) {
|
||||
let mut data = std::fs::read(path).unwrap();
|
||||
data[offset] ^= 0xFF;
|
||||
std::fs::write(path, data).unwrap();
|
||||
}
|
||||
|
||||
fn integrity(result: Result<Summary, Error>, expected: IntegrityKind) -> String {
|
||||
match result {
|
||||
Err(Error::Integrity { kind, what, detail }) if kind == expected => format!("{what}: {detail}"),
|
||||
other => panic!("expected an {expected:?} integrity error, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_payload(path: &str) -> bool {
|
||||
path.starts_with("/packs/") || path.starts_with("/src/") || path.contains("/block/")
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn site_install_verifies_everything_then_rerun_downloads_nothing() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
||||
let manifest = current_manifest(&site, &key).await.unwrap();
|
||||
assert_eq!(manifest.raw(), std::fs::read(stub.store_manifest_path()).unwrap().as_slice());
|
||||
|
||||
let events: Arc<Mutex<Vec<Progress>>> = Arc::default();
|
||||
let sink = events.clone();
|
||||
let summary = install(&site, &dest, &manifest, InstallOptions::default(), move |p| sink.lock().unwrap().push(p), CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let app = summary.app.clone().unwrap();
|
||||
assert_eq!((app.files, app.downloaded, app.skipped), (5, 5, 0));
|
||||
let db = summary.db.clone().unwrap();
|
||||
assert_eq!((db.files, db.downloaded, db.skipped), (3, 3, 0));
|
||||
let src: Vec<(String, usize)> = summary.src.clone().unwrap().into_iter().map(|s| (s.path, s.chunks)).collect();
|
||||
assert_eq!(src, vec![
|
||||
("osm_source/world_aeroway_v1.sqlite".to_string(), 3),
|
||||
("osm_source/small.bin".to_string(), 1),
|
||||
("osm_source/empty.bin".to_string(), 0),
|
||||
]);
|
||||
let tiles = summary.tiles.clone().unwrap();
|
||||
assert_eq!((tiles[0].blocks, tiles[0].downloaded, tiles[0].files), (4, 4, 8));
|
||||
assert!(summary.transferred_bytes > 3 * CHUNK as u64);
|
||||
|
||||
// Every byte in place, in the fetch.py layout.
|
||||
let store = stub.root();
|
||||
let m = stub.store_manifest();
|
||||
let version_dir = dest.join("versions").join("0.51.0");
|
||||
for f in m["app"]["files"].as_array().unwrap() {
|
||||
let path = version_dir.join(f["path"].as_str().unwrap());
|
||||
assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap(), "{}", path.display());
|
||||
}
|
||||
assert_eq!(std::fs::metadata(version_dir.join("Odian/assets/empty.conf")).unwrap().len(), 0);
|
||||
for f in m["db"]["files"].as_array().unwrap() {
|
||||
let path = dest.join("data").join(f["path"].as_str().unwrap());
|
||||
assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap());
|
||||
}
|
||||
for f in m["src"]["files"].as_array().unwrap() {
|
||||
let rel = f["path"].as_str().unwrap();
|
||||
let path = dest.join("data").join("src").join(rel);
|
||||
assert_eq!(std::fs::read(&path).unwrap(), std::fs::read(store.join("src").join(rel)).unwrap());
|
||||
let marker = std::fs::read_to_string(format!("{}.ok", path.display())).unwrap();
|
||||
assert_eq!(marker, f["chunks_sha256"].as_str().unwrap());
|
||||
assert!(!Path::new(&format!("{}.chunks.json", path.display())).exists());
|
||||
}
|
||||
assert_eq!(std::fs::read(version_dir.join("manifest.json")).unwrap(), manifest.raw());
|
||||
let set_dir = dest.join("data/tiles/dark/test_ds");
|
||||
let mut tile_files = files(&set_dir);
|
||||
let done = tile_files.remove(".blocks-done.json").map(|_| std::fs::read_to_string(set_dir.join(".blocks-done.json")).unwrap()).unwrap();
|
||||
let expected_tiles = ["0/0/0.png", "3/1/2.png", "7/1/2.png", "7/127/63.png", "7/3/2.jpg", "7/3/2.png", "7/63/63.pbf", "7/64/5.png"];
|
||||
assert_eq!(tile_files.keys().map(String::as_str).collect::<Vec<_>>(), expected_tiles, "the late tile 7/5/5.png must not be installed");
|
||||
for rel in expected_tiles {
|
||||
assert_eq!(tile_files[rel], sha256_hex(&std::fs::read(store.join("tiles/dark/test_ds").join(rel)).unwrap()));
|
||||
}
|
||||
assert!(done.starts_with("{\"index_sha256\": \""), "Python's json.dumps layout: {done}");
|
||||
let done: Value = serde_json::from_str(&done).unwrap();
|
||||
assert_eq!(done["index_sha256"], m["tiles"]["sets"][0]["index_sha256"]);
|
||||
assert_eq!(done["blocks"].as_array().unwrap().len(), 4);
|
||||
|
||||
// No request for the zero-byte file: it never needs the network.
|
||||
let log = stub.requests().await;
|
||||
assert!(!log.iter().any(|(p, _)| p.ends_with(&stub.app_sha("Odian/assets/empty.conf"))));
|
||||
|
||||
// Progress: stages in order, ending at their totals, contract-shaped JSON.
|
||||
let events = events.lock().unwrap().clone();
|
||||
let mut stages: Vec<Stage> = events.iter().map(|p| p.stage).collect();
|
||||
stages.dedup();
|
||||
assert_eq!(stages, vec![Stage::Manifest, Stage::App, Stage::Db, Stage::Src, Stage::Tiles]);
|
||||
assert!(events.iter().all(|p| p.stage_count == 5));
|
||||
for stage in [Stage::App, Stage::Db, Stage::Src, Stage::Tiles] {
|
||||
let last = events.iter().rev().find(|p| p.stage == stage).unwrap();
|
||||
assert_eq!((last.done_bytes, last.done_items), (last.total_bytes, last.total_items), "{stage:?}");
|
||||
}
|
||||
assert!(events.iter().any(|p| p.message.as_deref() == Some("tiles dark/test_ds: 4/4 blocks")));
|
||||
let json = serde_json::to_value(&events[0]).unwrap();
|
||||
assert_eq!(json["stage"], "manifest");
|
||||
for field in ["stage_index", "stage_count", "done_bytes", "total_bytes", "done_items", "total_items", "bytes_per_sec", "message"] {
|
||||
assert!(json.get(field).is_some(), "{field}");
|
||||
}
|
||||
|
||||
// Re-run: nothing is downloaded again.
|
||||
stub.ctl("/_ctl/reset").await;
|
||||
let again = install(&site, &dest, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(again.transferred_bytes, 0);
|
||||
assert_eq!(again.app.unwrap().downloaded, 0);
|
||||
assert_eq!(again.db.unwrap().downloaded, 0);
|
||||
assert!(again.src.unwrap().iter().all(|s| s.chunks == 0));
|
||||
assert_eq!(again.tiles.unwrap()[0].downloaded, 0);
|
||||
let log = stub.requests().await;
|
||||
assert!(log.iter().all(|(p, _)| !is_payload(p)), "{log:?}");
|
||||
|
||||
assert_eq!(fetch_notes(&site, "0.51.0").await.unwrap(), NOTES);
|
||||
assert_eq!(version_manifest(&site, "0.51.0", &key).await.unwrap().raw(), manifest.raw());
|
||||
assert!(matches!(version_manifest(&site, "9.9.9", &key).await, Err(Error::NotFound { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn wrong_token_is_unauthorized_and_edited_manifest_is_a_signature_error() {
|
||||
let stub = Stub::start(&[]);
|
||||
let key = stub.key();
|
||||
let bad = Source::Site { base_url: stub.base(), token: "bad".into() };
|
||||
assert!(matches!(current_manifest(&bad, &key).await, Err(Error::Unauthorized { status: 401, .. })));
|
||||
|
||||
let manifest = current_manifest(&stub.site(), &key).await.unwrap();
|
||||
let dest = stub.dest("install");
|
||||
let result = install(&bad, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await;
|
||||
assert!(matches!(result, Err(Error::Unauthorized { .. })), "{result:?}");
|
||||
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
||||
|
||||
// The stub's manifest is signed by the test key, not the release key.
|
||||
let release = current_manifest(&stub.site(), bundled_public_key()).await;
|
||||
assert!(matches!(release, Err(Error::Integrity { kind: IntegrityKind::Signature, .. })));
|
||||
|
||||
let path = stub.store_manifest_path();
|
||||
let text = std::fs::read_to_string(&path).unwrap();
|
||||
let edited = text.replacen("\"released_at\": \"2026-09-15T12:00:00Z\"", "\"released_at\": \"2026-09-15T12:00:01Z\"", 1);
|
||||
assert_ne!(text, edited);
|
||||
std::fs::write(&path, edited).unwrap();
|
||||
let result = current_manifest(&stub.site(), &key).await;
|
||||
assert!(matches!(result, Err(Error::Integrity { kind: IntegrityKind::Signature, .. })), "{result:?}");
|
||||
|
||||
std::fs::remove_file(stub.root().join("packs/versions/CURRENT")).unwrap();
|
||||
assert!(matches!(current_manifest(&stub.site(), &key).await, Err(Error::NotFound { .. })));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn flipped_app_byte_is_an_integrity_error() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
||||
let manifest = current_manifest(&site, &key).await.unwrap();
|
||||
let sha = stub.app_sha("Odian/Odian.exe");
|
||||
flip(&stub.app_blob("Odian/Odian.exe"), 1234);
|
||||
stub.ctl("/_ctl/reset").await;
|
||||
|
||||
let detail = integrity(install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await, IntegrityKind::FileHash);
|
||||
assert!(detail.contains(&sha), "{detail}");
|
||||
let gets = stub.requests().await.iter().filter(|(p, _)| p.ends_with(&sha)).count();
|
||||
assert_eq!(gets, 2, "one clean retry, then give up");
|
||||
let target = dest.join("versions/0.51.0/Odian/Odian.exe");
|
||||
assert!(!target.exists() && !Path::new(&format!("{}.part", target.display())).exists());
|
||||
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn flipped_tile_byte_is_an_integrity_error() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
||||
flip(&stub.root().join("tiles/dark/test_ds/7/63/63.pbf"), 100);
|
||||
|
||||
let detail = integrity(install_from(&site, &dest, &key, only(&[Stage::Tiles])).await, IntegrityKind::BlockDigest);
|
||||
assert!(detail.contains("block/7/0/0.tar"), "{detail}");
|
||||
let set_dir = dest.join("data/tiles/dark/test_ds");
|
||||
assert!(!set_dir.join("7/63/63.pbf").exists() && !set_dir.join("7/1/2.png").exists(), "nothing of a bad block moves in");
|
||||
if let Ok(done) = std::fs::read_to_string(set_dir.join(".blocks-done.json")) {
|
||||
assert!(!done.contains("\"7/0/0:"), "{done}");
|
||||
}
|
||||
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn flipped_source_chunk_byte_is_an_integrity_error_and_the_rest_resumes() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
||||
let original = stub.root().join("src/osm_source/world_aeroway_v1.sqlite");
|
||||
flip(&original, CHUNK + 5);
|
||||
|
||||
let detail = integrity(install_from(&site, &dest, &key, only(&[Stage::Src])).await, IntegrityKind::ChunkHash);
|
||||
assert!(detail.contains("chunk 1"), "{detail}");
|
||||
let target = dest.join("data/src/osm_source/world_aeroway_v1.sqlite");
|
||||
assert!(!Path::new(&format!("{}.ok", target.display())).exists());
|
||||
let progress: Value = serde_json::from_str(&std::fs::read_to_string(format!("{}.chunks.json", target.display())).unwrap()).unwrap();
|
||||
let verified: Vec<u64> = progress["verified"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect();
|
||||
assert!(!verified.contains(&1), "{verified:?}");
|
||||
|
||||
flip(&original, CHUNK + 5); // the site is fixed
|
||||
stub.ctl("/_ctl/reset").await;
|
||||
let summary = install_from(&site, &dest, &key, only(&[Stage::Src])).await.unwrap();
|
||||
let world = &summary.src.unwrap()[0];
|
||||
assert_eq!(world.chunks + world.skipped, 3);
|
||||
assert_eq!(world.skipped, verified.len());
|
||||
let ranges: Vec<String> = stub
|
||||
.requests()
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|(p, _)| p.ends_with("world_aeroway_v1.sqlite"))
|
||||
.map(|(_, r)| r.unwrap())
|
||||
.collect();
|
||||
assert_eq!(ranges.len(), world.chunks);
|
||||
assert!(ranges.contains(&format!("bytes={}-{}", CHUNK, 2 * CHUNK - 1)), "{ranges:?}");
|
||||
assert_eq!(std::fs::read(&target).unwrap(), std::fs::read(&original).unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn interrupted_part_resumes_with_a_range_request() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key) = (stub.site(), stub.key());
|
||||
let manifest = current_manifest(&site, &key).await.unwrap();
|
||||
let sha = stub.app_sha("Odian/data/big.bin");
|
||||
let big = std::fs::read(stub.app_blob("Odian/data/big.bin")).unwrap();
|
||||
let app_total: u64 = manifest.manifest().app.files.iter().map(|f| f.size).sum();
|
||||
|
||||
// A .part left by an earlier run (Python or Rust).
|
||||
let dest = stub.dest("seeded");
|
||||
let target = dest.join("versions/0.51.0/Odian/data/big.bin");
|
||||
let part = PathBuf::from(format!("{}.part", target.display()));
|
||||
std::fs::create_dir_all(target.parent().unwrap()).unwrap();
|
||||
let have = CHUNK + 1000;
|
||||
std::fs::write(&part, &big[..have]).unwrap();
|
||||
stub.ctl("/_ctl/reset").await;
|
||||
let summary = install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(summary.app.unwrap().downloaded, 5);
|
||||
assert_eq!(summary.transferred_bytes, app_total - have as u64);
|
||||
let log = stub.requests().await;
|
||||
let big_requests: Vec<_> = log.iter().filter(|(p, _)| p.ends_with(&sha)).collect();
|
||||
assert_eq!(big_requests.len(), 1);
|
||||
assert_eq!(big_requests[0].1.as_deref(), Some(format!("bytes={have}-").as_str()));
|
||||
assert_eq!(std::fs::read(&target).unwrap(), big);
|
||||
assert!(!part.exists());
|
||||
|
||||
// A connection that really drops mid-file.
|
||||
let dest = stub.dest("cut");
|
||||
let target = dest.join("versions/0.51.0/Odian/data/big.bin");
|
||||
let part = PathBuf::from(format!("{}.part", target.display()));
|
||||
stub.ctl(&format!("/_ctl/cut?prefix=/packs/app/{sha}&after=700000&count=1")).await;
|
||||
let result = install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await;
|
||||
assert!(matches!(result, Err(Error::Http { .. })), "{result:?}");
|
||||
let have = std::fs::metadata(&part).unwrap().len();
|
||||
assert!(have > 0 && have <= 700_000, "{have}");
|
||||
stub.ctl("/_ctl/reset").await;
|
||||
install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await.unwrap();
|
||||
let log = stub.requests().await;
|
||||
let big_requests: Vec<_> = log.iter().filter(|(p, _)| p.ends_with(&sha)).collect();
|
||||
assert_eq!(big_requests.len(), 1);
|
||||
assert_eq!(big_requests[0].1.as_deref(), Some(format!("bytes={have}-").as_str()));
|
||||
assert_eq!(std::fs::read(&target).unwrap(), big);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn busy_site_is_retried_with_retry_after() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key) = (stub.site(), stub.key());
|
||||
stub.ctl("/_ctl/reset").await;
|
||||
stub.ctl("/_ctl/fail?prefix=/v1/versions/current&status=503&count=2&retry_after=0").await;
|
||||
current_manifest(&site, &key).await.unwrap();
|
||||
let tries = stub.requests().await.iter().filter(|(p, _)| p == "/v1/versions/current").count();
|
||||
assert_eq!(tries, 3);
|
||||
|
||||
stub.ctl("/_ctl/fail?prefix=/v1/versions/current&status=503&count=4&retry_after=0").await;
|
||||
let result = current_manifest(&site, &key).await;
|
||||
assert!(matches!(result, Err(Error::SiteBusy { attempts: 4, .. })), "{result:?}");
|
||||
|
||||
// A 502 on a tile block and a 504 on an app file are retried too.
|
||||
stub.ctl("/_ctl/fail?prefix=/v1/versions/0.51.0/tiles/dark/test_ds/block/&status=502&count=1&retry_after=0").await;
|
||||
stub.ctl("/_ctl/fail?prefix=/packs/app/&status=504&count=1&retry_after=0").await;
|
||||
let summary = install_from(&site, &stub.dest("install"), &key, only(&[Stage::App, Stage::Tiles])).await.unwrap();
|
||||
assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn cancel_stops_and_keeps_verified_work() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
|
||||
let manifest = current_manifest(&site, &key).await.unwrap();
|
||||
let cancel = CancellationToken::new();
|
||||
let trigger = cancel.clone();
|
||||
let result = install(&site, &dest, &manifest, InstallOptions::default(), move |p| {
|
||||
if p.stage == Stage::App && p.done_items >= 1 {
|
||||
trigger.cancel();
|
||||
}
|
||||
}, cancel)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::Cancelled)), "{result:?}");
|
||||
assert!(!dest.join("versions/0.51.0/manifest.json").exists());
|
||||
|
||||
let summary = install(&site, &dest, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
||||
assert!(summary.app.unwrap().skipped >= 1);
|
||||
assert!(dest.join("versions/0.51.0/manifest.json").exists());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn drive_install_equals_site_install() {
|
||||
let stub = Stub::start(&[]);
|
||||
let (site, drive, key) = (stub.site(), stub.drive(), stub.key());
|
||||
let from_site = stub.dest("from-site");
|
||||
install_from(&site, &from_site, &key, InstallOptions::default()).await.unwrap();
|
||||
|
||||
let manifest = current_manifest(&drive, &key).await.unwrap();
|
||||
assert_eq!(manifest.raw(), std::fs::read(stub.store_manifest_path()).unwrap().as_slice());
|
||||
let from_drive = stub.dest("from-drive");
|
||||
let summary = install(&drive, &from_drive, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(summary.app.unwrap().downloaded, 5);
|
||||
assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
|
||||
assert_eq!(files(&from_site), files(&from_drive), "site and drive installs must be identical, file by file");
|
||||
|
||||
let again = install(&drive, &from_drive, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(again.transferred_bytes, 0);
|
||||
assert_eq!(fetch_notes(&drive, "0.51.0").await.unwrap(), NOTES);
|
||||
assert_eq!(version_manifest(&drive, "0.51.0", &key).await.unwrap().raw(), manifest.raw());
|
||||
|
||||
// The same checks guard a drive.
|
||||
let tile = stub.root().join("tiles/dark/test_ds/7/64/5.png");
|
||||
flip(&tile, 3);
|
||||
let detail = integrity(install_from(&drive, &stub.dest("drive-tile"), &key, only(&[Stage::Tiles])).await, IntegrityKind::BlockDigest);
|
||||
assert!(detail.contains("7/1/0"), "{detail}");
|
||||
flip(&tile, 3);
|
||||
flip(&stub.app_blob("api/odian_api.exe"), 7);
|
||||
integrity(install_from(&drive, &stub.dest("drive-app"), &key, only(&[Stage::App])).await, IntegrityKind::FileHash);
|
||||
}
|
||||
|
||||
fn python_fetch(stub: &Stub, dest: &Path) -> Value {
|
||||
let code = "import json, sys\n\
|
||||
from odian_packs.fetch import fetch\n\
|
||||
summary = fetch(sys.argv[1], 'good', sys.argv[2], public_key=sys.argv[3], log=lambda m: None)\n\
|
||||
print(json.dumps(summary))\n";
|
||||
let output = Command::new(python())
|
||||
.arg("-c")
|
||||
.arg(code)
|
||||
.arg(stub.base())
|
||||
.arg(dest)
|
||||
.arg(stub.key())
|
||||
.env("PYTHONPATH", pythonpath())
|
||||
.env("PYTHONDONTWRITEBYTECODE", "1")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(output.status.success(), "python fetch failed:\n{}", String::from_utf8_lossy(&output.stderr));
|
||||
let stdout = String::from_utf8(output.stdout).unwrap();
|
||||
serde_json::from_str(stdout.lines().last().unwrap()).unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
|
||||
async fn python_and_rust_installs_are_interchangeable() {
|
||||
// fetch.py cannot install a zero-byte source file (see the report), so leave it out here.
|
||||
let stub = Stub::start(&["--no-empty-src"]);
|
||||
let (site, key) = (stub.site(), stub.key());
|
||||
|
||||
let python_first = stub.dest("python-first");
|
||||
let py = tokio::task::block_in_place(|| python_fetch(&stub, &python_first));
|
||||
assert_eq!(py["app"]["downloaded"], 5);
|
||||
let rust = install_from(&site, &python_first, &key, InstallOptions::default()).await.unwrap();
|
||||
assert_eq!(rust.transferred_bytes, 0, "Rust re-verifies a Python install and downloads nothing: {rust:?}");
|
||||
assert_eq!(rust.app.unwrap().skipped, 5);
|
||||
|
||||
let rust_first = stub.dest("rust-first");
|
||||
install_from(&site, &rust_first, &key, InstallOptions::default()).await.unwrap();
|
||||
let py = tokio::task::block_in_place(|| python_fetch(&stub, &rust_first));
|
||||
assert_eq!(py["app"]["downloaded"], 0, "{py}");
|
||||
assert_eq!(py["db"]["downloaded"], 0, "{py}");
|
||||
assert!(py["src"].as_array().unwrap().iter().all(|s| s["chunks"] == 0), "{py}");
|
||||
assert_eq!(py["tiles"][0]["downloaded"], 0, "{py}");
|
||||
|
||||
assert_eq!(files(&python_first), files(&rust_first));
|
||||
}
|
||||
Reference in New Issue
Block a user