updtae

src-tauri/src/market.rs

@@ -39,14 +39,22 @@ pub struct Item {
   pub description: String,
   /// What installing it downloads, in full.
   pub bytes: u64,
+  /// How many files it is made of: what a removal counts down.
+  pub files: u64,
   /// Part of every install: shown, never offered for removal.
   pub required: bool,
   pub presence: Presence,
   /// Chosen: installed with the version and kept by updates (required items always are).
   pub selected: bool,
+  /// False when the DHP does not carry this item's bytes: it is listed so an operator knows
+  /// it exists, but only an ODIAN drive can install it.
+  pub on_dhp: bool,
 }
 
-/// The map ODIAN opens on, kept required by `catalog.json`.
+/// The map a version from BEFORE the marketplace opens on. Those versions flag nothing, so
+/// the catalog's rule is applied to them here - and what they shipped as their one certain
+/// basemap was `dark`. Current versions say it themselves (`catalog.json` now keeps
+/// `dark_v2` required instead), and this is not read for them.
 const ALWAYS_INSTALLED_STYLE: &str = "dark";
 
 /// A manifest from before the marketplace (pack 0.61.0) flags nothing optional, so everything
@@ -106,6 +114,55 @@ fn tile_set_presence(data_dir: &Path, set: &TileSet) -> Presence {
   }
 }
 
+/// One item weighed against what the version says it holds.
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub struct Check {
+  pub key: String,
+  pub found: u64,
+  pub expected: u64,
+  /// True when at least everything the version names is on disk. MORE than that is normal
+  /// and not a fault: a PC bakes tiles of its own into the same folders, and the tile sync
+  /// brings in what other operators baked.
+  pub whole: bool,
+}
+
+/// What a tile set's folder really holds. Dot-files are the installer's own markers
+/// (`.blocks-done.json`) and the tile server's negative ones (`.missing`), never tiles.
+fn count_tiles(dir: &Path) -> u64 {
+  let Ok(entries) = std::fs::read_dir(dir) else { return 0 };
+  let mut found = 0;
+  for entry in entries.flatten() {
+    let name = entry.file_name();
+    if name.to_string_lossy().starts_with('.') {
+      continue;
+    }
+    match entry.file_type() {
+      Ok(kind) if kind.is_dir() => found += count_tiles(&entry.path()),
+      Ok(kind) if kind.is_file() => found += 1,
+      _ => {}
+    }
+  }
+  found
+}
+
+/// Count what is on disk for every tile set a marker calls installed.
+///
+/// This is the slow, true answer - a walk of hundreds of thousands of files - so it is
+/// never on the path that opens the marketplace. The window asks for it after, and moves
+/// the badge when it comes back.
+pub fn check_installed(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> Vec<Check> {
+  let mut out = Vec::new();
+  for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) {
+    if tile_set_presence(data_dir, set) != Presence::Installed {
+      continue;
+    }
+    let found = count_tiles(&tile_set_dir(data_dir, set));
+    out.push(Check { key: tiles_key(set), found, expected: set.files, whole: found >= set.files });
+  }
+  let _ = extras;
+  out
+}
+
 /// A source file is complete when its `.ok` marker names its chunk list and the size is right
 /// (the installer's own test); partial when the file exists.
 fn src_presence(data_dir: &Path, file: &SrcFile) -> Presence {
@@ -151,13 +208,16 @@ fn all_items(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) ->
   for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) {
     let key = tiles_key(set);
     let optional = tiles_optional(set, old);
+    let presence = tile_set_presence(data_dir, set);
     out.push(Item {
       selected: !optional || extras.contains(&key),
       title: set.title.clone().unwrap_or_else(|| format!("{} map ({})", set.style, set.dataset)),
       description: set.description.clone().unwrap_or_default(),
       bytes: set.bytes,
+      files: set.files,
       required: !optional,
-      presence: tile_set_presence(data_dir, set),
+      presence,
+      on_dhp: set.on_dhp,
       kind: Kind::Tiles,
       key,
     });
@@ -165,13 +225,17 @@ fn all_items(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) ->
   for file in manifest.src.iter().filter(|src| src.enabled).flat_map(|src| &src.files) {
     let key = src_key(file);
     let optional = src_optional(file, old);
+    let presence = src_presence(data_dir, file);
     out.push(Item {
       selected: !optional || extras.contains(&key),
       title: file.title.clone().unwrap_or_else(|| file.path.clone()),
       description: file.description.clone().unwrap_or_default(),
       bytes: file.size,
+      // A source is one file (plus its small markers, not worth counting).
+      files: 1,
       required: !optional,
-      presence: src_presence(data_dir, file),
+      presence,
+      on_dhp: file.on_dhp,
       kind: Kind::Src,
       key,
     });
@@ -182,12 +246,24 @@ fn all_items(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) ->
 
 /// What installing `manifest` fetches: every required item and the chosen optional ones, as
 /// the installer's `tile_sets` (`style/dataset`) and `src_paths` filters.
-pub fn install_filter(manifest: &Manifest, extras: &BTreeSet<String>) -> (BTreeSet<String>, BTreeSet<String>) {
+///
+/// `from_dhp` drops what the store does not carry. The same choice is honoured when the
+/// bytes come from a drive - that is where such an item comes from - but asked of the DHP
+/// it would be a 404 per chunk and a failed install, for an item nobody could have got that
+/// way. It is dropped here rather than refused at the checkbox because the choice outlives
+/// the version it was made against: a PC that took the height map off a drive keeps it in
+/// `extras`, and its next update from the DHP must simply leave it alone.
+pub fn install_filter(
+  manifest: &Manifest,
+  extras: &BTreeSet<String>,
+  from_dhp: bool,
+) -> (BTreeSet<String>, BTreeSet<String>) {
   let old = pre_catalog(manifest);
   let tile_sets = manifest
     .tiles
     .iter()
     .flat_map(|tiles| &tiles.sets)
+    .filter(|set| set.on_dhp || !from_dhp)
     .filter(|set| !tiles_optional(set, old) || extras.contains(&tiles_key(set)))
     .map(|set| format!("{}/{}", set.style, set.dataset))
     .collect();
@@ -195,6 +271,7 @@ pub fn install_filter(manifest: &Manifest, extras: &BTreeSet<String>) -> (BTreeS
     .src
     .iter()
     .flat_map(|src| &src.files)
+    .filter(|file| file.on_dhp || !from_dhp)
     .filter(|file| !src_optional(file, old) || extras.contains(&src_key(file)))
     .map(|file| file.path.clone())
     .collect();
@@ -227,20 +304,67 @@ pub fn to_remove(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>
     .collect()
 }
 
+/// What this PC has settled on: what the operator chose, plus whatever is actually here.
+///
+/// `extras` records answers about OPTIONAL items, so an item that is HERE and not in it was
+/// never a choice the operator could have made - required by the version that installed it
+/// (Dark, until Dark V2 took its place), off a drive, or not downloadable at all. Applying
+/// `extras` alone as a selection therefore reads as "remove every one of those".
+pub fn held_keys(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> BTreeSet<String> {
+  items(manifest, data_dir, extras)
+    .into_iter()
+    .filter(|item| !item.required && (item.selected || item.presence != Presence::Absent))
+    .map(|item| item.key)
+    .collect()
+}
+
+/// The chosen items that are not all here, weighed against a count when one has been taken.
+///
+/// `to_add` answers from the installers' markers alone, which is what the marketplace opens
+/// on. A set whose marker says "installed" while the disk holds half of it is exactly what
+/// `check_installed` is for, so anything it found short is counted as unfinished here too.
+pub fn unfinished(
+  manifest: &Manifest,
+  data_dir: &Path,
+  extras: &BTreeSet<String>,
+  checked: &[Check],
+) -> Vec<Item> {
+  let short: BTreeSet<&str> = checked.iter().filter(|c| !c.whole).map(|c| c.key.as_str()).collect();
+  items(manifest, data_dir, extras)
+    .into_iter()
+    .filter(|item| !item.required && item.selected)
+    .filter(|item| item.presence != Presence::Installed || short.contains(item.key.as_str()))
+    .collect()
+}
+
 /// Whether the chosen optional items still need downloading.
 pub fn to_add(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> Vec<Item> {
   items(manifest, data_dir, extras)
     .into_iter()
+    // An item the store does not carry is never downloaded, so it is never announced as
+    // one: `install_filter` drops it, and a line saying it is being fetched would be a
+    // promise nothing keeps.
+    .filter(|item| item.on_dhp)
     .filter(|item| !item.required && item.selected && item.presence != Presence::Installed)
     .collect()
 }
 
 /// Remove an item's files from the PC: a tile set's folder, or a source file with its markers.
 pub fn remove(manifest: &Manifest, data_dir: &Path, key: &str) -> Result<(), String> {
+  remove_counting(manifest, data_dir, key, &mut |_| {})
+}
+
+/// The same, calling `removed` with each file as it goes, so a caller can show progress.
+pub fn remove_counting(
+  manifest: &Manifest,
+  data_dir: &Path,
+  key: &str,
+  removed: &mut dyn FnMut(u64),
+) -> Result<(), String> {
   for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) {
     if tiles_key(set) == key {
       let dir = tile_set_dir(data_dir, set);
-      return if dir.exists() { crate::prune::remove_folder(&dir) } else { Ok(()) };
+      return if dir.exists() { crate::prune::remove_folder_counting(&dir, removed) } else { Ok(()) };
     }
   }
   for file in manifest.src.iter().flat_map(|src| &src.files) {
@@ -250,6 +374,9 @@ pub fn remove(manifest: &Manifest, data_dir: &Path, key: &str) -> Result<(), Str
         let target = with_suffix(&path, extra);
         if target.exists() {
           std::fs::remove_file(&target).map_err(|e| format!("cannot remove {}: {e}", target.display()))?;
+          if extra.is_empty() {
+            removed(1);
+          }
         }
       }
       return Ok(());
@@ -317,11 +444,85 @@ mod tests {
     assert!(listed.iter().all(|i| i.presence == Presence::Absent));
   }
 
+  #[test]
+  fn an_item_the_store_does_not_carry_is_listed_but_never_asked_of_it() {
+    // The world height map source is 705 GB and arrives on a drive. Leaving it out of the
+    // catalogue hid the one thing an operator needs to know - that it exists at all - and
+    // listing it without this flag would have the installer ask the DHP for 2629 chunks it
+    // has never had.
+    let mut carrying = manifest();
+    carrying.src.as_mut().unwrap().files.push(
+      serde_json::from_value(serde_json::json!({
+        "path": "planet_heightmap.pmtiles", "size": 8, "chunks": ["e".repeat(64)],
+        "chunks_sha256": "f".repeat(64), "optional": true, "on_dhp": false, "title": "Terrain source"
+      }))
+      .unwrap(),
+    );
+    let dir = std::env::temp_dir().join(format!("odian-market-{}", uuid::Uuid::new_v4().simple()));
+    let items = items(&carrying, &dir, &BTreeSet::new());
+    let terrain = items.iter().find(|item| item.key == "src:planet_heightmap.pmtiles").expect("it is listed");
+    assert!(!terrain.on_dhp, "and it says where it does NOT come from");
+    assert_eq!(terrain.title, "Terrain source");
+    assert!(items.iter().all(|item| item.key == "src:planet_heightmap.pmtiles" || item.on_dhp));
+
+    // Chosen - off a drive, say - it is still left alone when the DHP is the source.
+    let chosen: BTreeSet<String> = ["src:planet_heightmap.pmtiles".to_string()].into_iter().collect();
+    let (_, from_dhp) = install_filter(&carrying, &chosen, true);
+    assert!(!from_dhp.contains("planet_heightmap.pmtiles"), "the DHP is never asked for it");
+    let (_, from_drive) = install_filter(&carrying, &chosen, false);
+    assert!(from_drive.contains("planet_heightmap.pmtiles"), "a drive is exactly where it comes from");
+  }
+
+  #[test]
+  fn finishing_a_download_takes_nothing_away() {
+    // Dark V2 is installed and NOT in `extras`: it was required by the version that put it
+    // there, so no choice about it was ever recorded (which is exactly what happened when
+    // Dark V2 and Dark swapped places). Applying `extras` as a selection would read as
+    // "remove Dark V2", which is what Finish did before `held_keys`.
+    let data = data_dir();
+    complete_tile_set(&data, "dark_v2", 'b', 2);
+    let extras: BTreeSet<String> = ["src:osm/x.pmtiles".to_string()].into_iter().collect();
+    let keep = held_keys(&manifest(), &data, &extras);
+    let dropped = to_remove(&manifest(), &data, &keep);
+    std::fs::remove_dir_all(&data).unwrap();
+    assert!(keep.contains("tiles:dark_v2/ds"), "what is on disk stays chosen");
+    assert!(keep.contains("src:osm/x.pmtiles"), "and so does what was asked for");
+    assert!(dropped.is_empty(), "so finishing removes nothing: {dropped:?}");
+  }
+
+  #[test]
+  fn untick_still_means_remove_even_for_what_only_a_drive_replaces() {
+    // `selected` means "the operator asked for this", and `to_remove` finds what to delete
+    // by looking for present-and-not-selected. An item that always read as selected - which
+    // is what "installed counts as chosen" did to it - could never be removed at all.
+    let root = std::env::temp_dir().join(format!("odian-market-{}", uuid::Uuid::new_v4().simple()));
+    let src = root.join("src");
+    std::fs::create_dir_all(&src).unwrap();
+    std::fs::write(src.join("planet_heightmap.pmtiles"), b"12345678").unwrap();
+    std::fs::write(src.join("planet_heightmap.pmtiles.ok"), "f".repeat(64)).unwrap();
+
+    let mut carrying = manifest();
+    carrying.src.as_mut().unwrap().files.push(
+      serde_json::from_value(serde_json::json!({
+        "path": "planet_heightmap.pmtiles", "size": 8, "chunks": ["e".repeat(64)],
+        "chunks_sha256": "f".repeat(64), "optional": true, "on_dhp": false, "title": "Terrain source"
+      }))
+      .unwrap(),
+    );
+    let dropped = to_remove(&carrying, &root, &BTreeSet::new());
+    std::fs::remove_dir_all(&root).unwrap();
+    assert_eq!(
+      dropped.iter().map(|item| item.key.as_str()).collect::<Vec<_>>(),
+      ["src:planet_heightmap.pmtiles"],
+      "unticked and on disk: it goes"
+    );
+  }
+
   #[test]
   fn an_install_fetches_the_required_items_and_the_chosen_ones() {
-    let (sets, paths) = install_filter(&manifest(), &BTreeSet::new());
+    let (sets, paths) = install_filter(&manifest(), &BTreeSet::new(), true);
     assert_eq!((sets.into_iter().collect::<Vec<_>>(), paths.len()), (vec!["dark/ds".to_string()], 0));
-    let (sets, paths) = install_filter(&manifest(), &["tiles:dark_v2/ds".to_string()].into_iter().collect());
+    let (sets, paths) = install_filter(&manifest(), &["tiles:dark_v2/ds".to_string()].into_iter().collect(), true);
     assert_eq!(sets.len(), 2);
     assert!(paths.is_empty());
   }
@@ -388,7 +589,7 @@ mod tests {
         item.title
       );
     }
-    let (sets, paths) = install_filter(manifest, &BTreeSet::new());
+    let (sets, paths) = install_filter(manifest, &BTreeSet::new(), true);
     println!("a new PC installs: {sets:?} {paths:?}");
   }
 
@@ -407,12 +608,12 @@ mod tests {
     let offered: Vec<&str> = items.iter().filter(|item| !item.required).map(|item| item.key.as_str()).collect();
     assert_eq!(offered, vec!["tiles:dark_v2/ds", "src:osm/x.pmtiles"]);
     // Unchosen, an install of it brings the dark map alone.
-    let (tiles, src) = install_filter(&old_manifest(), &BTreeSet::new());
+    let (tiles, src) = install_filter(&old_manifest(), &BTreeSet::new(), true);
     assert_eq!(tiles, BTreeSet::from(["dark/ds".to_string()]));
     assert!(src.is_empty());
     // Chosen, they come along.
     let chosen = BTreeSet::from(["src:osm/x.pmtiles".to_string()]);
-    let (_, src) = install_filter(&old_manifest(), &chosen);
+    let (_, src) = install_filter(&old_manifest(), &chosen, true);
     assert_eq!(src, BTreeSet::from(["osm/x.pmtiles".to_string()]));
   }