update

crates/odian-packs/tests/e2e_site.rs

@@ -13,7 +13,7 @@
 //!
 //! `ODIAN_PACKS_PYTHON` picks the interpreter (default `python`).
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::io::{BufRead, BufReader};
 use std::path::{Path, PathBuf};
 use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
@@ -290,6 +290,141 @@ async fn site_install_verifies_everything_then_rerun_downloads_nothing() {
     assert!(matches!(version_manifest(&site, "9.9.9", &key).await, Err(Error::NotFound { .. })));
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
+async fn an_update_reuses_the_installed_files_and_fetches_only_the_db_files_asked_for() {
+    let stub = Stub::start(&[]);
+    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
+    install_from(&site, &dest, &key, only(&[Stage::App])).await.unwrap();
+    // What is installed becomes an older version: installing the current one is an update.
+    let old = dest.join("versions").join("0.50.0");
+    std::fs::rename(dest.join("versions").join("0.51.0"), &old).unwrap();
+    // One old file went bad on disk: taken from there anyway, it must be caught.
+    let m = stub.store_manifest();
+    let bad = m["app"]["files"].as_array().unwrap().iter().find(|f| f["size"].as_u64().unwrap() > 0).unwrap();
+    let bad_path = bad["path"].as_str().unwrap().to_string();
+    flip(&old.join(&bad_path), 0);
+    let bad_old = std::fs::read(old.join(&bad_path)).unwrap();
+
+    stub.ctl("/_ctl/reset").await;
+    let first_db = m["db"]["files"][0]["path"].as_str().unwrap().to_string();
+    let opts = InstallOptions {
+        only: vec![Stage::App, Stage::Db],
+        db_paths: Some([first_db.clone()].into_iter().collect()),
+        ..InstallOptions::default()
+    };
+    let summary = install_from(&site, &dest, &key, opts).await.unwrap();
+
+    let app = summary.app.unwrap();
+    assert_eq!((app.files, app.downloaded, app.reused, app.skipped), (5, 1, 4, 0), "{app:?}");
+    let app_requests: Vec<String> =
+        stub.requests().await.into_iter().map(|(p, _)| p).filter(|p| p.starts_with("/packs/app/")).collect();
+    assert_eq!(app_requests, vec![format!("/packs/app/{}", stub.app_sha(&bad_path))], "only the bad file is downloaded");
+    let new = dest.join("versions").join("0.51.0");
+    for f in m["app"]["files"].as_array().unwrap() {
+        let path = new.join(f["path"].as_str().unwrap());
+        assert_eq!(sha256_hex(&std::fs::read(&path).unwrap()), f["sha256"].as_str().unwrap(), "{}", path.display());
+    }
+    // The download replaced the link; it never wrote through it into the old version.
+    assert_eq!(std::fs::read(old.join(&bad_path)).unwrap(), bad_old);
+
+    let db = summary.db.unwrap();
+    assert_eq!((db.files, db.downloaded), (1, 1));
+    for f in m["db"]["files"].as_array().unwrap() {
+        let path = f["path"].as_str().unwrap();
+        assert_eq!(dest.join("data").join(path).is_file(), path == first_db, "{path}");
+    }
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
+async fn only_the_chosen_tile_sets_and_source_files_are_installed() {
+    let stub = Stub::start(&[]);
+    let (site, key, dest) = (stub.site(), stub.key(), stub.dest("install"));
+    let opts = InstallOptions {
+        only: vec![Stage::Src, Stage::Tiles],
+        tile_sets: Some(BTreeSet::new()),
+        src_paths: Some(["osm_source/small.bin".to_string()].into_iter().collect()),
+        ..InstallOptions::default()
+    };
+    let summary = install_from(&site, &dest, &key, opts).await.unwrap();
+    let src: Vec<String> = summary.src.unwrap().into_iter().map(|s| s.path).collect();
+    assert_eq!(src, ["osm_source/small.bin"]);
+    assert!(summary.tiles.unwrap().is_empty());
+    assert!(dest.join("data/src/osm_source/small.bin").is_file());
+    assert!(!dest.join("data/src/osm_source/world_aeroway_v1.sqlite").exists());
+    assert!(!dest.join("data/tiles/dark/test_ds").exists(), "no tile set was chosen");
+    let log = stub.requests().await;
+    assert!(!log.iter().any(|(p, _)| p.contains("/block/")), "no tile block fetched: {log:?}");
+
+    // Choosing the tile set later adds it; what is installed stays as it is.
+    let opts = InstallOptions {
+        only: vec![Stage::Src, Stage::Tiles],
+        tile_sets: Some(["dark/test_ds".to_string()].into_iter().collect()),
+        src_paths: Some(["osm_source/small.bin".to_string()].into_iter().collect()),
+        ..InstallOptions::default()
+    };
+    let summary = install_from(&site, &dest, &key, opts).await.unwrap();
+    assert_eq!(summary.tiles.unwrap()[0].downloaded, 4);
+    assert_eq!(summary.src.unwrap()[0].chunks, 0, "the source file was already there");
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
+async fn a_pc_installs_by_copying_another_install_and_checks_every_file() {
+    let stub = Stub::start(&[]);
+    let (site, key) = (stub.site(), stub.key());
+    let first = stub.dest("first");
+    install_from(&site, &first, &key, InstallOptions::default()).await.unwrap();
+
+    // The drive an operator carries holds that install, not a store: the second PC copies it.
+    stub.ctl("/_ctl/reset").await;
+    let drive = Source::Install { root: first.clone() };
+    let second = stub.dest("second");
+    let summary = install_from(&drive, &second, &key, InstallOptions::default()).await.unwrap();
+    assert!(stub.requests().await.is_empty(), "nothing was asked of the site");
+    assert_eq!(summary.version, "0.51.0");
+    assert_eq!(files(&second.join("versions").join("0.51.0")), files(&first.join("versions").join("0.51.0")));
+    assert_eq!(files(&second.join("data")), files(&first.join("data")), "same data, tile by tile");
+    let done = second.join("data/tiles/dark/test_ds/.blocks-done.json");
+    assert!(done.is_file(), "the blocks the source holds travel with the tiles");
+
+    // Running it again copies nothing: what is already there is recognised.
+    let again = install_from(&drive, &second, &key, InstallOptions::default()).await.unwrap();
+    assert_eq!(again.app.unwrap().downloaded, 0);
+    assert_eq!(again.tiles.unwrap().iter().map(|t| t.downloaded).sum::<usize>(), 0);
+
+    // A set this PC already recorded as complete is not read again: the marker names this
+    // index and every block of it, so what lies here IS the set. Hundreds of thousands of
+    // files are not hashed twice to conclude "identical", and a tile that changes on the
+    // drive afterwards stays there, as a site update leaves a complete set alone too.
+    let on_drive = first.join("data/tiles/dark/test_ds/0/0/0.png");
+    let here = second.join("data/tiles/dark/test_ds/0/0/0.png");
+    let kept = std::fs::read(&here).unwrap();
+    flip(&on_drive, 3);
+    install_from(&drive, &second, &key, only(&[Stage::Tiles])).await.unwrap();
+    assert_eq!(std::fs::read(&here).unwrap(), kept, "the set was recorded complete: it was left alone");
+    flip(&on_drive, 3);
+
+    // An app file that went bad on the drive is caught: the manifest says what it should be.
+    let third = stub.dest("third");
+    flip(&first.join("versions").join("0.51.0").join("Odian").join("Odian.exe"), 3);
+    let what = integrity(install_from(&drive, &third, &key, InstallOptions::default()).await, IntegrityKind::FileHash);
+    assert!(what.contains("Odian.exe"), "{what}");
+
+    // A TILE cannot be checked that way: only a site's index carries per-tile hashes, and an
+    // install keeps none. What is checked is the copy - the bytes that land are the bytes read.
+    let fourth = stub.dest("fourth");
+    let tile = first.join("data/tiles/dark/test_ds/0/0/0.png");
+    flip(&tile, 3);
+    install_from(&drive, &fourth, &key, only(&[Stage::Tiles])).await.unwrap();
+    assert_eq!(
+        std::fs::read(fourth.join("data/tiles/dark/test_ds/0/0/0.png")).unwrap(),
+        std::fs::read(&tile).unwrap(),
+        "faithful to the drive, whatever the drive holds"
+    );
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 #[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
 async fn wrong_token_is_unauthorized_and_edited_manifest_is_a_signature_error() {
@@ -480,6 +615,50 @@ async fn cancel_stops_and_keeps_verified_work() {
     assert!(dest.join("versions/0.51.0/manifest.json").exists());
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
+#[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
+async fn a_copy_stopped_halfway_leaves_the_set_partial() {
+    let stub = Stub::start(&[]);
+    let (site, key) = (stub.site(), stub.key());
+    let drive_install = stub.dest("drive");
+    install_from(&site, &drive_install, &key, InstallOptions::default()).await.unwrap();
+    let marker = ".blocks-done.json";
+    let carried = drive_install.join("data/tiles/dark/test_ds");
+    assert!(carried.join(marker).is_file(), "the drive holds the whole set");
+    // A set of a few tiles copies faster than the first progress report: give the drive enough
+    // of them (tiles drawn beyond the index, which travel too) to be stopped in the middle.
+    for x in 0..40 {
+        let dir = carried.join("9").join(x.to_string());
+        std::fs::create_dir_all(&dir).unwrap();
+        for y in 0..10 {
+            std::fs::write(dir.join(format!("{y}.png")), vec![b't'; 4096]).unwrap();
+        }
+    }
+
+    // Stop the copy in the middle of the tiles.
+    let drive = Source::Install { root: drive_install.clone() };
+    let manifest = current_manifest(&drive, &key).await.unwrap();
+    let dest = stub.dest("half");
+    let cancel = CancellationToken::new();
+    let trigger = cancel.clone();
+    let result = install(&drive, &dest, &manifest, only(&[Stage::Tiles]), move |p| {
+        if p.stage == Stage::Tiles && p.done_items >= 1 {
+            trigger.cancel();
+        }
+    }, cancel)
+    .await;
+    assert!(matches!(result, Err(Error::Cancelled)), "{result:?}");
+    let set_dir = dest.join("data/tiles/dark/test_ds");
+    // Tiles landed, but nothing here claims the set is whole: the next run copies the rest
+    // instead of trusting a record written before the files it speaks for.
+    assert!(!set_dir.join(marker).exists(), "a half-copied set must not read as complete");
+
+    let summary = install(&drive, &dest, &manifest, only(&[Stage::Tiles]), |_| {}, CancellationToken::new()).await.unwrap();
+    assert!(set_dir.join(marker).is_file(), "finished, so it is recorded");
+    assert!(summary.tiles.unwrap().iter().map(|t| t.downloaded).sum::<usize>() > 0, "the rest was copied");
+    assert_eq!(files(&set_dir), files(&carried), "same set, tile by tile");
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
 #[ignore = "spawns Python: set ODIAN_PACKS_PYTHONPATH and run with --ignored (see the file header)"]
 async fn drive_install_equals_site_install() {