//! End to end against a local stub site: `tests/stub_site.py` builds a small signed pack
//! store with the Python reference code (odian_packs) and serves it with the real update
//! service (`odian_update.api.create_app`) plus nginx-like Range routes. The same store is
//! also installed as an ODIAN drive.
//!
//! These tests spawn Python, so they are `#[ignore]`d by default. They need Python 3 with
//! fastapi, uvicorn, httpx and cryptography, and the ODIAN_MCP_Server sources:
//!
//! ```text
//! ODIAN_PACKS_PYTHONPATH=C:/Users/JFEXP_D/Documents/Gitea/ODIAN_MCP_Server/src \
//!     cargo test -p odian-packs --test e2e_site -- --ignored
//! ```
//!
//! `ODIAN_PACKS_PYTHON` picks the interpreter (default `python`).

use std::collections::{BTreeMap, BTreeSet};
use std::io::{BufRead, BufReader};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::{Arc, Mutex};

use odian_packs::hashing::sha256_hex;
use odian_packs::{
    bundled_public_key, current_manifest, fetch_notes, install, version_manifest, Error, InstallOptions, IntegrityKind,
    Progress, Source, Stage, Summary,
};
use serde_json::Value;
use tokio_util::sync::CancellationToken;

const CHUNK: usize = 1024 * 1024;
const NOTES: &str = "# ODIAN 0.51.0\n\nPremier pack signé.\n";

fn python() -> String {
    std::env::var("ODIAN_PACKS_PYTHON").unwrap_or_else(|_| "python".into())
}

fn pythonpath() -> String {
    std::env::var("ODIAN_PACKS_PYTHONPATH")
        .expect("ODIAN_PACKS_PYTHONPATH must point at ODIAN_MCP_Server/src (see the header of tests/e2e_site.rs)")
}

struct Stub {
    child: Child,
    stdin: Option<ChildStdin>,
    _stdout: BufReader<ChildStdout>,
    info: Value,
    dir: tempfile::TempDir,
}

impl Stub {
    fn start(args: &[&str]) -> Stub {
        let dir = tempfile::Builder::new().prefix("odian-packs-e2e-").tempdir().unwrap();
        let log_path = dir.path().join("stub.log");
        let log = std::fs::File::create(&log_path).unwrap();
        let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests").join("stub_site.py");
        let mut child = Command::new(python())
            .arg(script)
            .arg("--root")
            .arg(dir.path().join("store"))
            .args(args)
            .env("PYTHONPATH", pythonpath())
            .env("PYTHONDONTWRITEBYTECODE", "1")
            .env("PYTHONIOENCODING", "utf-8")
            .stdin(Stdio::piped())
            .stdout(Stdio::piped())
            .stderr(log)
            .spawn()
            .expect("spawn python");
        let stdin = child.stdin.take();
        let mut stdout = BufReader::new(child.stdout.take().unwrap());
        let mut line = String::new();
        let _ = stdout.read_line(&mut line);
        if line.trim().is_empty() {
            let _ = child.kill();
            let _ = child.wait();
            panic!("stub site did not start:\n{}", std::fs::read_to_string(&log_path).unwrap_or_default());
        }
        let info: Value = serde_json::from_str(&line).unwrap();
        Stub { child, stdin, _stdout: stdout, info, dir }
    }

    fn base(&self) -> String {
        self.info["base_url"].as_str().unwrap().to_string()
    }

    fn root(&self) -> PathBuf {
        PathBuf::from(self.info["root"].as_str().unwrap())
    }

    fn key(&self) -> String {
        self.info["public_key"].as_str().unwrap().to_string()
    }

    fn site(&self) -> Source {
        Source::Site { base_url: self.base(), token: "good".into() }
    }

    fn drive(&self) -> Source {
        Source::Drive { root: self.root() }
    }

    fn dest(&self, name: &str) -> PathBuf {
        self.dir.path().join(name)
    }

    fn store_manifest_path(&self) -> PathBuf {
        self.root().join("packs").join("versions").join("0.51.0").join("manifest.json")
    }

    fn store_manifest(&self) -> Value {
        serde_json::from_slice(&std::fs::read(self.store_manifest_path()).unwrap()).unwrap()
    }

    fn app_sha(&self, install_path: &str) -> String {
        let m = self.store_manifest();
        let entry = m["app"]["files"].as_array().unwrap().iter().find(|f| f["path"] == install_path).unwrap();
        entry["sha256"].as_str().unwrap().to_string()
    }

    fn app_blob(&self, install_path: &str) -> PathBuf {
        self.root().join("packs").join("app").join(self.app_sha(install_path))
    }

    async fn ctl(&self, path: &str) -> Value {
        let text = reqwest::get(format!("{}{}", self.base(), path)).await.unwrap().text().await.unwrap();
        serde_json::from_str(&text).unwrap()
    }

    async fn requests(&self) -> Vec<(String, Option<String>)> {
        let log = self.ctl("/_ctl/log").await;
        log["requests"]
            .as_array()
            .unwrap()
            .iter()
            .map(|r| (r["path"].as_str().unwrap().to_string(), r["range"].as_str().map(str::to_string)))
            .collect()
    }
}

impl Drop for Stub {
    fn drop(&mut self) {
        drop(self.stdin.take()); // the stub exits when its stdin closes
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

fn only(stages: &[Stage]) -> InstallOptions {
    InstallOptions { only: stages.to_vec(), ..InstallOptions::default() }
}

async fn install_from(source: &Source, dest: &Path, key: &str, opts: InstallOptions) -> Result<Summary, Error> {
    let manifest = current_manifest(source, key).await?;
    install(source, dest, &manifest, opts, |_| {}, CancellationToken::new()).await
}

/// Every file under `root`: relative path -> sha256.
fn files(root: &Path) -> BTreeMap<String, String> {
    fn walk(dir: &Path, prefix: &str, out: &mut BTreeMap<String, String>) {
        for entry in std::fs::read_dir(dir).unwrap() {
            let entry = entry.unwrap();
            let name = entry.file_name().to_string_lossy().into_owned();
            let rel = if prefix.is_empty() { name } else { format!("{prefix}/{name}") };
            if entry.file_type().unwrap().is_dir() {
                walk(&entry.path(), &rel, out);
            } else {
                out.insert(rel, sha256_hex(&std::fs::read(entry.path()).unwrap()));
            }
        }
    }
    let mut out = BTreeMap::new();
    walk(root, "", &mut out);
    out
}

fn flip(path: &Path, offset: usize) {
    let mut data = std::fs::read(path).unwrap();
    data[offset] ^= 0xFF;
    std::fs::write(path, data).unwrap();
}

fn integrity(result: Result<Summary, Error>, expected: IntegrityKind) -> String {
    match result {
        Err(Error::Integrity { kind, what, detail }) if kind == expected => format!("{what}: {detail}"),
        other => panic!("expected an {expected:?} integrity error, got {other:?}"),
    }
}

fn is_payload(path: &str) -> bool {
    path.starts_with("/packs/") || path.starts_with("/src/") || path.contains("/block/")
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn site_install_verifies_everything_then_rerun_downloads_nothing() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    let manifest = current_manifest(&site, &key).await.unwrap();
    assert_eq!(manifest.raw(), std::fs::read(stub.store_manifest_path()).unwrap().as_slice());

    let events: Arc<Mutex<Vec<Progress>>> = Arc::default();
    let sink = events.clone();
    let summary = install(&site, &dest, &manifest, InstallOptions::default(), move |p| sink.lock().unwrap().push(p), CancellationToken::new())
        .await
        .unwrap();

    let app = summary.app.clone().unwrap();
    assert_eq!((app.files, app.downloaded, app.skipped), (5, 5, 0));
    let db = summary.db.clone().unwrap();
    assert_eq!((db.files, db.downloaded, db.skipped), (3, 3, 0));
    let src: Vec<(String, usize)> = summary.src.clone().unwrap().into_iter().map(|s| (s.path, s.chunks)).collect();
    assert_eq!(src, vec![
        ("osm_source/world_aeroway_v1.sqlite".to_string(), 3),
        ("osm_source/small.bin".to_string(), 1),
        ("osm_source/empty.bin".to_string(), 0),
    ]);
    let tiles = summary.tiles.clone().unwrap();
    assert_eq!((tiles[0].blocks, tiles[0].downloaded, tiles[0].files), (4, 4, 8));
    assert!(summary.transferred_bytes > 3 * CHUNK as u64);

    // Every byte in place, in the fetch.py layout.
    let store = stub.root();
    let m = stub.store_manifest();
    let version_dir = dest.join("versions").join("0.51.0");
    for f in m["app"]["files"].as_array().unwrap() {
        let path = version_dir.join(f["path"].as_str().unwrap());
        assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap(), "{}", path.display());
    }
    assert_eq!(std::fs::metadata(version_dir.join("Odian/assets/empty.conf")).unwrap().len(), 0);
    for f in m["db"]["files"].as_array().unwrap() {
        let path = dest.join("data").join(f["path"].as_str().unwrap());
        assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap());
    }
    for f in m["src"]["files"].as_array().unwrap() {
        let rel = f["path"].as_str().unwrap();
        let path = dest.join("data").join("src").join(rel);
        assert_eq!(std::fs::read(&path).unwrap(), std::fs::read(store.join("src").join(rel)).unwrap());
        let marker = std::fs::read_to_string(format!("{}.ok", path.display())).unwrap();
        assert_eq!(marker, f["chunks_sha256"].as_str().unwrap());
        assert!(!Path::new(&format!("{}.chunks.json", path.display())).exists());
    }
    assert_eq!(std::fs::read(version_dir.join("manifest.json")).unwrap(), manifest.raw());
    let set_dir = dest.join("data/tiles/dark/test_ds");
    let mut tile_files = files(&set_dir);
    let done = tile_files.remove(".blocks-done.json").map(|_| std::fs::read_to_string(set_dir.join(".blocks-done.json")).unwrap()).unwrap();
    let expected_tiles = ["0/0/0.png", "3/1/2.png", "7/1/2.png", "7/127/63.png", "7/3/2.jpg", "7/3/2.png", "7/63/63.pbf", "7/64/5.png"];
    assert_eq!(tile_files.keys().map(String::as_str).collect::<Vec<_>>(), expected_tiles, "the late tile 7/5/5.png must not be installed");
    for rel in expected_tiles {
        assert_eq!(tile_files[rel], sha256_hex(&std::fs::read(store.join("tiles/dark/test_ds").join(rel)).unwrap()));
    }
    assert!(done.starts_with("{\"index_sha256\": \""), "Python's json.dumps layout: {done}");
    let done: Value = serde_json::from_str(&done).unwrap();
    assert_eq!(done["index_sha256"], m["tiles"]["sets"][0]["index_sha256"]);
    assert_eq!(done["blocks"].as_array().unwrap().len(), 4);

    // No request for the zero-byte file: it never needs the network.
    let log = stub.requests().await;
    assert!(!log.iter().any(|(p, _)| p.ends_with(&stub.app_sha("Odian/assets/empty.conf"))));

    // Progress: stages in order, ending at their totals, contract-shaped JSON.
    let events = events.lock().unwrap().clone();
    let mut stages: Vec<Stage> = events.iter().map(|p| p.stage).collect();
    stages.dedup();
    assert_eq!(stages, vec![Stage::Manifest, Stage::App, Stage::Db, Stage::Src, Stage::Tiles]);
    assert!(events.iter().all(|p| p.stage_count == 5));
    for stage in [Stage::App, Stage::Db, Stage::Src, Stage::Tiles] {
        let last = events.iter().rev().find(|p| p.stage == stage).unwrap();
        assert_eq!((last.done_bytes, last.done_items), (last.total_bytes, last.total_items), "{stage:?}");
    }
    assert!(events.iter().any(|p| p.message.as_deref() == Some("tiles dark/test_ds: 4/4 blocks")));
    let json = serde_json::to_value(&events[0]).unwrap();
    assert_eq!(json["stage"], "manifest");
    for field in ["stage_index", "stage_count", "done_bytes", "total_bytes", "done_items", "total_items", "bytes_per_sec", "message"] {
        assert!(json.get(field).is_some(), "{field}");
    }

    // Re-run: nothing is downloaded again.
    stub.ctl("/_ctl/reset").await;
    let again = install(&site, &dest, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
    assert_eq!(again.transferred_bytes, 0);
    assert_eq!(again.app.unwrap().downloaded, 0);
    assert_eq!(again.db.unwrap().downloaded, 0);
    assert!(again.src.unwrap().iter().all(|s| s.chunks == 0));
    assert_eq!(again.tiles.unwrap()[0].downloaded, 0);
    let log = stub.requests().await;
    assert!(log.iter().all(|(p, _)| !is_payload(p)), "{log:?}");

    assert_eq!(fetch_notes(&site, "0.51.0").await.unwrap(), NOTES);
    assert_eq!(version_manifest(&site, "0.51.0", &key).await.unwrap().raw(), manifest.raw());
    assert!(matches!(version_manifest(&site, "9.9.9", &key).await, Err(Error::NotFound { .. })));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn an_update_reuses_the_installed_files_and_fetches_only_the_db_files_asked_for() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    install_from(&site, &dest, &key, only(&[Stage::App])).await.unwrap();
    // What is installed becomes an older version: installing the current one is an update.
    let old = dest.join("versions").join("0.50.0");
    std::fs::rename(dest.join("versions").join("0.51.0"), &old).unwrap();
    // One old file went bad on disk: taken from there anyway, it must be caught.
    let m = stub.store_manifest();
    let bad = m["app"]["files"].as_array().unwrap().iter().find(|f| f["size"].as_u64().unwrap() > 0).unwrap();
    let bad_path = bad["path"].as_str().unwrap().to_string();
    flip(&old.join(&bad_path), 0);
    let bad_old = std::fs::read(old.join(&bad_path)).unwrap();

    stub.ctl("/_ctl/reset").await;
    let first_db = m["db"]["files"][0]["path"].as_str().unwrap().to_string();
    let opts = InstallOptions {
        only: vec![Stage::App, Stage::Db],
        db_paths: Some([first_db.clone()].into_iter().collect()),
        ..InstallOptions::default()
    };
    let summary = install_from(&site, &dest, &key, opts).await.unwrap();

    let app = summary.app.unwrap();
    assert_eq!((app.files, app.downloaded, app.reused, app.skipped), (5, 1, 4, 0), "{app:?}");
    let app_requests: Vec<String> =
        stub.requests().await.into_iter().map(|(p, _)| p).filter(|p| p.starts_with("/packs/app/")).collect();
    assert_eq!(app_requests, vec![format!("/packs/app/{}", stub.app_sha(&bad_path))], "only the bad file is downloaded");
    let new = dest.join("versions").join("0.51.0");
    for f in m["app"]["files"].as_array().unwrap() {
        let path = new.join(f["path"].as_str().unwrap());
        assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap(), "{}", path.display());
    }
    // The download replaced the link; it never wrote through it into the old version.
    assert_eq!(std::fs::read(old.join(&bad_path)).unwrap(), bad_old);

    let db = summary.db.unwrap();
    assert_eq!((db.files, db.downloaded), (1, 1));
    for f in m["db"]["files"].as_array().unwrap() {
        let path = f["path"].as_str().unwrap();
        assert_eq!(dest.join("data").join(path).is_file(), path == first_db, "{path}");
    }
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn only_the_chosen_tile_sets_and_source_files_are_installed() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    let opts = InstallOptions {
        only: vec![Stage::Src, Stage::Tiles],
        tile_sets: Some(BTreeSet::new()),
        src_paths: Some(["osm_source/small.bin".to_string()].into_iter().collect()),
        ..InstallOptions::default()
    };
    let summary = install_from(&site, &dest, &key, opts).await.unwrap();
    let src: Vec<String> = summary.src.unwrap().into_iter().map(|s| s.path).collect();
    assert_eq!(src, ["osm_source/small.bin"]);
    assert!(summary.tiles.unwrap().is_empty());
    assert!(dest.join("data/src/osm_source/small.bin").is_file());
    assert!(!dest.join("data/src/osm_source/world_aeroway_v1.sqlite").exists());
    assert!(!dest.join("data/tiles/dark/test_ds").exists(), "no tile set was chosen");
    let log = stub.requests().await;
    assert!(!log.iter().any(|(p, _)| p.contains("/block/")), "no tile block fetched: {log:?}");

    // Choosing the tile set later adds it; what is installed stays as it is.
    let opts = InstallOptions {
        only: vec![Stage::Src, Stage::Tiles],
        tile_sets: Some(["dark/test_ds".to_string()].into_iter().collect()),
        src_paths: Some(["osm_source/small.bin".to_string()].into_iter().collect()),
        ..InstallOptions::default()
    };
    let summary = install_from(&site, &dest, &key, opts).await.unwrap();
    assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
    assert_eq!(summary.src.unwrap()[0].chunks, 0, "the source file was already there");
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn a_pc_installs_by_copying_another_install_and_checks_every_file() {
    let stub = Stub::start(&[]);
    let (site, key) = (stub.site(), stub.key());
    let first = stub.dest("first");
    install_from(&site, &first, &key, InstallOptions::default()).await.unwrap();

    // The drive an operator carries holds that install, not a store: the second PC copies it.
    stub.ctl("/_ctl/reset").await;
    let drive = Source::Install { root: first.clone() };
    let second = stub.dest("second");
    let summary = install_from(&drive, &second, &key, InstallOptions::default()).await.unwrap();
    assert!(stub.requests().await.is_empty(), "nothing was asked of the site");
    assert_eq!(summary.version, "0.51.0");
    assert_eq!(files(&second.join("versions").join("0.51.0")), files(&first.join("versions").join("0.51.0")));
    assert_eq!(files(&second.join("data")), files(&first.join("data")), "same data, tile by tile");
    let done = second.join("data/tiles/dark/test_ds/.blocks-done.json");
    assert!(done.is_file(), "the blocks the source holds travel with the tiles");

    // Running it again copies nothing: what is already there is recognised.
    let again = install_from(&drive, &second, &key, InstallOptions::default()).await.unwrap();
    assert_eq!(again.app.unwrap().downloaded, 0);
    assert_eq!(again.tiles.unwrap().iter().map(|t| t.downloaded).sum::<usize>(), 0);

    // A set this PC already recorded as complete is not read again: the marker names this
    // index and every block of it, so what lies here IS the set. Hundreds of thousands of
    // files are not hashed twice to conclude "identical", and a tile that changes on the
    // drive afterwards stays there, as a site update leaves a complete set alone too.
    let on_drive = first.join("data/tiles/dark/test_ds/0/0/0.png");
    let here = second.join("data/tiles/dark/test_ds/0/0/0.png");
    let kept = std::fs::read(&here).unwrap();
    flip(&on_drive, 3);
    install_from(&drive, &second, &key, only(&[Stage::Tiles])).await.unwrap();
    assert_eq!(std::fs::read(&here).unwrap(), kept, "the set was recorded complete: it was left alone");
    flip(&on_drive, 3);

    // An app file that went bad on the drive is caught: the manifest says what it should be.
    let third = stub.dest("third");
    flip(&first.join("versions").join("0.51.0").join("Odian").join("Odian.exe"), 3);
    let what = integrity(install_from(&drive, &third, &key, InstallOptions::default()).await, IntegrityKind::FileHash);
    assert!(what.contains("Odian.exe"), "{what}");

    // A TILE cannot be checked that way: only a site's index carries per-tile hashes, and an
    // install keeps none. What is checked is the copy - the bytes that land are the bytes read.
    let fourth = stub.dest("fourth");
    let tile = first.join("data/tiles/dark/test_ds/0/0/0.png");
    flip(&tile, 3);
    install_from(&drive, &fourth, &key, only(&[Stage::Tiles])).await.unwrap();
    assert_eq!(
        std::fs::read(fourth.join("data/tiles/dark/test_ds/0/0/0.png")).unwrap(),
        std::fs::read(&tile).unwrap(),
        "faithful to the drive, whatever the drive holds"
    );
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn wrong_token_is_unauthorized_and_edited_manifest_is_a_signature_error() {
    let stub = Stub::start(&[]);
    let key = stub.key();
    let bad = Source::Site { base_url: stub.base(), token: "bad".into() };
    assert!(matches!(current_manifest(&bad, &key).await, Err(Error::Unauthorized { status: 401, .. })));

    let manifest = current_manifest(&stub.site(), &key).await.unwrap();
    let dest = stub.dest("install");
    let result = install(&bad, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await;
    assert!(matches!(result, Err(Error::Unauthorized { .. })), "{result:?}");
    assert!(!dest.join("versions/0.51.0/manifest.json").exists());

    // The stub's manifest is signed by the test key, not the release key.
    let release = current_manifest(&stub.site(), bundled_public_key()).await;
    assert!(matches!(release, Err(Error::Integrity { kind: IntegrityKind::Signature, .. })));

    let path = stub.store_manifest_path();
    let text = std::fs::read_to_string(&path).unwrap();
    let edited = text.replacen("\"released_at\": \"2026-09-15T12:00:00Z\"", "\"released_at\": \"2026-09-15T12:00:01Z\"", 1);
    assert_ne!(text, edited);
    std::fs::write(&path, edited).unwrap();
    let result = current_manifest(&stub.site(), &key).await;
    assert!(matches!(result, Err(Error::Integrity { kind: IntegrityKind::Signature, .. })), "{result:?}");

    std::fs::remove_file(stub.root().join("packs/versions/CURRENT")).unwrap();
    assert!(matches!(current_manifest(&stub.site(), &key).await, Err(Error::NotFound { .. })));
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn flipped_app_byte_is_an_integrity_error() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    let manifest = current_manifest(&site, &key).await.unwrap();
    let sha = stub.app_sha("Odian/Odian.exe");
    flip(&stub.app_blob("Odian/Odian.exe"), 1234);
    stub.ctl("/_ctl/reset").await;

    let detail = integrity(install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await, IntegrityKind::FileHash);
    assert!(detail.contains(&sha), "{detail}");
    let gets = stub.requests().await.iter().filter(|(p, _)| p.ends_with(&sha)).count();
    assert_eq!(gets, 2, "one clean retry, then give up");
    let target = dest.join("versions/0.51.0/Odian/Odian.exe");
    assert!(!target.exists() && !Path::new(&format!("{}.part", target.display())).exists());
    assert!(!dest.join("versions/0.51.0/manifest.json").exists());
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn flipped_tile_byte_is_an_integrity_error() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    flip(&stub.root().join("tiles/dark/test_ds/7/63/63.pbf"), 100);

    let detail = integrity(install_from(&site, &dest, &key, only(&[Stage::Tiles])).await, IntegrityKind::BlockDigest);
    assert!(detail.contains("block/7/0/0.tar"), "{detail}");
    let set_dir = dest.join("data/tiles/dark/test_ds");
    assert!(!set_dir.join("7/63/63.pbf").exists() && !set_dir.join("7/1/2.png").exists(), "nothing of a bad block moves in");
    if let Ok(done) = std::fs::read_to_string(set_dir.join(".blocks-done.json")) {
        assert!(!done.contains("\"7/0/0:"), "{done}");
    }
    assert!(!dest.join("versions/0.51.0/manifest.json").exists());
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn flipped_source_chunk_byte_is_an_integrity_error_and_the_rest_resumes() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    let original = stub.root().join("src/osm_source/world_aeroway_v1.sqlite");
    flip(&original, CHUNK + 5);

    let detail = integrity(install_from(&site, &dest, &key, only(&[Stage::Src])).await, IntegrityKind::ChunkHash);
    assert!(detail.contains("chunk 1"), "{detail}");
    let target = dest.join("data/src/osm_source/world_aeroway_v1.sqlite");
    assert!(!Path::new(&format!("{}.ok", target.display())).exists());
    let progress: Value = serde_json::from_str(&std::fs::read_to_string(format!("{}.chunks.json", target.display())).unwrap()).unwrap();
    let verified: Vec<u64> = progress["verified"].as_array().unwrap().iter().map(|v| v.as_u64().unwrap()).collect();
    assert!(!verified.contains(&1), "{verified:?}");

    flip(&original, CHUNK + 5); // the site is fixed
    stub.ctl("/_ctl/reset").await;
    let summary = install_from(&site, &dest, &key, only(&[Stage::Src])).await.unwrap();
    let world = &summary.src.unwrap()[0];
    assert_eq!(world.chunks + world.skipped, 3);
    assert_eq!(world.skipped, verified.len());
    let ranges: Vec<String> = stub
        .requests()
        .await
        .into_iter()
        .filter(|(p, _)| p.ends_with("world_aeroway_v1.sqlite"))
        .map(|(_, r)| r.unwrap())
        .collect();
    assert_eq!(ranges.len(), world.chunks);
    assert!(ranges.contains(&format!("bytes={}-{}", CHUNK, 2 * CHUNK - 1)), "{ranges:?}");
    assert_eq!(std::fs::read(&target).unwrap(), std::fs::read(&original).unwrap());
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn interrupted_part_resumes_with_a_range_request() {
    let stub = Stub::start(&[]);
    let (site, key) = (stub.site(), stub.key());
    let manifest = current_manifest(&site, &key).await.unwrap();
    let sha = stub.app_sha("Odian/data/big.bin");
    let big = std::fs::read(stub.app_blob("Odian/data/big.bin")).unwrap();
    let app_total: u64 = manifest.manifest().app.files.iter().map(|f| f.size).sum();

    // A .part left by an earlier run (Python or Rust).
    let dest = stub.dest("seeded");
    let target = dest.join("versions/0.51.0/Odian/data/big.bin");
    let part = PathBuf::from(format!("{}.part", target.display()));
    std::fs::create_dir_all(target.parent().unwrap()).unwrap();
    let have = CHUNK + 1000;
    std::fs::write(&part, &big[..have]).unwrap();
    stub.ctl("/_ctl/reset").await;
    let summary = install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await.unwrap();
    assert_eq!(summary.app.unwrap().downloaded, 5);
    assert_eq!(summary.transferred_bytes, app_total - have as u64);
    let log = stub.requests().await;
    let big_requests: Vec<_> = log.iter().filter(|(p, _)| p.ends_with(&sha)).collect();
    assert_eq!(big_requests.len(), 1);
    assert_eq!(big_requests[0].1.as_deref(), Some(format!("bytes={have}-").as_str()));
    assert_eq!(std::fs::read(&target).unwrap(), big);
    assert!(!part.exists());

    // A connection that really drops mid-file.
    let dest = stub.dest("cut");
    let target = dest.join("versions/0.51.0/Odian/data/big.bin");
    let part = PathBuf::from(format!("{}.part", target.display()));
    stub.ctl(&format!("/_ctl/cut?prefix=/packs/app/{sha}&after=700000&count=1")).await;
    let result = install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await;
    assert!(matches!(result, Err(Error::Http { .. })), "{result:?}");
    let have = std::fs::metadata(&part).unwrap().len();
    assert!(have > 0 && have <= 700_000, "{have}");
    stub.ctl("/_ctl/reset").await;
    install(&site, &dest, &manifest, only(&[Stage::App]), |_| {}, CancellationToken::new()).await.unwrap();
    let log = stub.requests().await;
    let big_requests: Vec<_> = log.iter().filter(|(p, _)| p.ends_with(&sha)).collect();
    assert_eq!(big_requests.len(), 1);
    assert_eq!(big_requests[0].1.as_deref(), Some(format!("bytes={have}-").as_str()));
    assert_eq!(std::fs::read(&target).unwrap(), big);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn busy_site_is_retried_with_retry_after() {
    let stub = Stub::start(&[]);
    let (site, key) = (stub.site(), stub.key());
    stub.ctl("/_ctl/reset").await;
    stub.ctl("/_ctl/fail?prefix=/v1/versions/current&status=503&count=2&retry_after=0").await;
    current_manifest(&site, &key).await.unwrap();
    let tries = stub.requests().await.iter().filter(|(p, _)| p == "/v1/versions/current").count();
    assert_eq!(tries, 3);

    stub.ctl("/_ctl/fail?prefix=/v1/versions/current&status=503&count=4&retry_after=0").await;
    let result = current_manifest(&site, &key).await;
    assert!(matches!(result, Err(Error::SiteBusy { attempts: 4, .. })), "{result:?}");

    // A 502 on a tile block and a 504 on an app file are retried too.
    stub.ctl("/_ctl/fail?prefix=/v1/versions/0.51.0/tiles/dark/test_ds/block/&status=502&count=1&retry_after=0").await;
    stub.ctl("/_ctl/fail?prefix=/packs/app/&status=504&count=1&retry_after=0").await;
    let summary = install_from(&site, &stub.dest("install"), &key, only(&[Stage::App, Stage::Tiles])).await.unwrap();
    assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn cancel_stops_and_keeps_verified_work() {
    let stub = Stub::start(&[]);
    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
    let manifest = current_manifest(&site, &key).await.unwrap();
    let cancel = CancellationToken::new();
    let trigger = cancel.clone();
    let result = install(&site, &dest, &manifest, InstallOptions::default(), move |p| {
        if p.stage == Stage::App && p.done_items >= 1 {
            trigger.cancel();
        }
    }, cancel)
    .await;
    assert!(matches!(result, Err(Error::Cancelled)), "{result:?}");
    assert!(!dest.join("versions/0.51.0/manifest.json").exists());

    let summary = install(&site, &dest, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
    assert!(summary.app.unwrap().skipped >= 1);
    assert!(dest.join("versions/0.51.0/manifest.json").exists());
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn a_copy_stopped_halfway_leaves_the_set_partial() {
    let stub = Stub::start(&[]);
    let (site, key) = (stub.site(), stub.key());
    let drive_install = stub.dest("drive");
    install_from(&site, &drive_install, &key, InstallOptions::default()).await.unwrap();
    let marker = ".blocks-done.json";
    let carried = drive_install.join("data/tiles/dark/test_ds");
    assert!(carried.join(marker).is_file(), "the drive holds the whole set");
    // A set of a few tiles copies faster than the first progress report: give the drive enough
    // of them (tiles drawn beyond the index, which travel too) to be stopped in the middle.
    for x in 0..40 {
        let dir = carried.join("9").join(x.to_string());
        std::fs::create_dir_all(&dir).unwrap();
        for y in 0..10 {
            std::fs::write(dir.join(format!("{y}.png")), vec![b't'; 4096]).unwrap();
        }
    }

    // Stop the copy in the middle of the tiles.
    let drive = Source::Install { root: drive_install.clone() };
    let manifest = current_manifest(&drive, &key).await.unwrap();
    let dest = stub.dest("half");
    let cancel = CancellationToken::new();
    let trigger = cancel.clone();
    let result = install(&drive, &dest, &manifest, only(&[Stage::Tiles]), move |p| {
        if p.stage == Stage::Tiles && p.done_items >= 1 {
            trigger.cancel();
        }
    }, cancel)
    .await;
    assert!(matches!(result, Err(Error::Cancelled)), "{result:?}");
    let set_dir = dest.join("data/tiles/dark/test_ds");
    // Tiles landed, but nothing here claims the set is whole: the next run copies the rest
    // instead of trusting a record written before the files it speaks for.
    assert!(!set_dir.join(marker).exists(), "a half-copied set must not read as complete");

    let summary = install(&drive, &dest, &manifest, only(&[Stage::Tiles]), |_| {}, CancellationToken::new()).await.unwrap();
    assert!(set_dir.join(marker).is_file(), "finished, so it is recorded");
    assert!(summary.tiles.unwrap().iter().map(|t| t.downloaded).sum::<usize>() > 0, "the rest was copied");
    assert_eq!(files(&set_dir), files(&carried), "same set, tile by tile");
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn drive_install_equals_site_install() {
    let stub = Stub::start(&[]);
    let (site, drive, key) = (stub.site(), stub.drive(), stub.key());
    let from_site = stub.dest("from-site");
    install_from(&site, &from_site, &key, InstallOptions::default()).await.unwrap();

    let manifest = current_manifest(&drive, &key).await.unwrap();
    assert_eq!(manifest.raw(), std::fs::read(stub.store_manifest_path()).unwrap().as_slice());
    let from_drive = stub.dest("from-drive");
    let summary = install(&drive, &from_drive, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
    assert_eq!(summary.app.unwrap().downloaded, 5);
    assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
    assert_eq!(files(&from_site), files(&from_drive), "site and drive installs must be identical, file by file");

    let again = install(&drive, &from_drive, &manifest, InstallOptions::default(), |_| {}, CancellationToken::new()).await.unwrap();
    assert_eq!(again.transferred_bytes, 0);
    assert_eq!(fetch_notes(&drive, "0.51.0").await.unwrap(), NOTES);
    assert_eq!(version_manifest(&drive, "0.51.0", &key).await.unwrap().raw(), manifest.raw());

    // The same checks guard a drive.
    let tile = stub.root().join("tiles/dark/test_ds/7/64/5.png");
    flip(&tile, 3);
    let detail = integrity(install_from(&drive, &stub.dest("drive-tile"), &key, only(&[Stage::Tiles])).await, IntegrityKind::BlockDigest);
    assert!(detail.contains("7/1/0"), "{detail}");
    flip(&tile, 3);
    flip(&stub.app_blob("api/odian_api.exe"), 7);
    integrity(install_from(&drive, &stub.dest("drive-app"), &key, only(&[Stage::App])).await, IntegrityKind::FileHash);
}

fn python_fetch(stub: &Stub, dest: &Path) -> Value {
    let code = "import json, sys\n\
                from odian_packs.fetch import fetch\n\
                summary = fetch(sys.argv[1], 'good', sys.argv[2], public_key=sys.argv[3], log=lambda m: None)\n\
                print(json.dumps(summary))\n";
    let output = Command::new(python())
        .arg("-c")
        .arg(code)
        .arg(stub.base())
        .arg(dest)
        .arg(stub.key())
        .env("PYTHONPATH", pythonpath())
        .env("PYTHONDONTWRITEBYTECODE", "1")
        .output()
        .unwrap();
    assert!(output.status.success(), "python fetch failed:\n{}", String::from_utf8_lossy(&output.stderr));
    let stdout = String::from_utf8(output.stdout).unwrap();
    serde_json::from_str(stdout.lines().last().unwrap()).unwrap()
}

#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
async fn python_and_rust_installs_are_interchangeable() {
    // fetch.py cannot install a zero-byte source file (see the report), so leave it out here.
    let stub = Stub::start(&["--no-empty-src"]);
    let (site, key) = (stub.site(), stub.key());

    let python_first = stub.dest("python-first");
    let py = tokio::task::block_in_place(|| python_fetch(&stub, &python_first));
    assert_eq!(py["app"]["downloaded"], 5);
    let rust = install_from(&site, &python_first, &key, InstallOptions::default()).await.unwrap();
    assert_eq!(rust.transferred_bytes, 0, "Rust re-verifies a Python install and downloads nothing: {rust:?}");
    assert_eq!(rust.app.unwrap().skipped, 5);

    let rust_first = stub.dest("rust-first");
    install_from(&site, &rust_first, &key, InstallOptions::default()).await.unwrap();
    let py = tokio::task::block_in_place(|| python_fetch(&stub, &rust_first));
    assert_eq!(py["app"]["downloaded"], 0, "{py}");
    assert_eq!(py["db"]["downloaded"], 0, "{py}");
    assert!(py["src"].as_array().unwrap().iter().all(|s| s["chunks"] == 0), "{py}");
    assert_eq!(py["tiles"][0]["downloaded"], 0, "{py}");

    assert_eq!(files(&python_first), files(&rust_first));
}