This commit is contained in:
Kelorastus
2026-09-28 11:19:02 +02:00
parent 21d6e6bffb
commit 73a8f2c30c
35 changed files with 1615 additions and 86 deletions
+18
View File
@@ -404,6 +404,11 @@ pub struct TileSet {
pub title: Option<String>, pub title: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>, pub description: Option<String>,
/// False when the store does not hold this item's bytes and never will. It is still
/// listed and described - an operator has to know it exists - but it arrives on a
/// drive, not over the network. Absent means true: everything else is downloadable.
#[serde(default = "carried", skip_serializing_if = "is_carried")]
pub on_dhp: bool,
} }
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -425,6 +430,19 @@ pub struct SrcFile {
pub title: Option<String>, pub title: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>, pub description: Option<String>,
/// See `TileSet::on_dhp`. The world height map source is the case this exists for: 705 GB
/// that no DHP will ever serve over a link, and that an operator still needs to know about.
#[serde(default = "carried", skip_serializing_if = "is_carried")]
pub on_dhp: bool,
}
/// An item says nothing when the store carries it, which is nearly always.
fn carried() -> bool {
true
}
fn is_carried(on_dhp: &bool) -> bool {
*on_dhp
} }
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.7 KiB

After

Width:  |  Height:  |  Size: 7.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.3 KiB

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.5 KiB

After

Width:  |  Height:  |  Size: 5.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.3 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.9 KiB

After

Width:  |  Height:  |  Size: 8.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.6 KiB

After

Width:  |  Height:  |  Size: 4.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 4.6 KiB

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

Binary file not shown.
Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 32 KiB

+16
View File
@@ -5,6 +5,7 @@ use tauri_plugin_dialog::DialogExt;
use tauri_plugin_opener::OpenerExt; use tauri_plugin_opener::OpenerExt;
use crate::launcher::{Launcher, Market, Status}; use crate::launcher::{Launcher, Market, Status};
use crate::market;
use crate::settings::Settings; use crate::settings::Settings;
use crate::stack::StackEvent; use crate::stack::StackEvent;
use crate::sync::Conflict; use crate::sync::Conflict;
@@ -41,6 +42,16 @@ pub async fn set_market(
} }
/// Choose the optional items (their keys) and apply the choice; see `Launcher::apply_market`. /// Choose the optional items (their keys) and apply the choice; see `Launcher::apply_market`.
#[tauri::command]
pub async fn finish_market(app: AppHandle, launcher: State<'_, Launcher>) -> CmdResult<()> {
launcher.finish_market(&app).await
}
#[tauri::command]
pub async fn check_market(app: AppHandle, launcher: State<'_, Launcher>) -> CmdResult<Vec<market::Check>> {
launcher.check_market(&app).await
}
#[tauri::command] #[tauri::command]
pub async fn apply_market(app: AppHandle, launcher: State<'_, Launcher>, selected: Vec<String>) -> CmdResult<()> { pub async fn apply_market(app: AppHandle, launcher: State<'_, Launcher>, selected: Vec<String>) -> CmdResult<()> {
launcher.apply_market(&app, selected).await launcher.apply_market(&app, selected).await
@@ -115,6 +126,11 @@ pub async fn get_stack(launcher: State<'_, Launcher>) -> CmdResult<Vec<StackEven
Ok(launcher.stack_states()) Ok(launcher.stack_states())
} }
#[tauri::command]
pub async fn sync_tiles(app: AppHandle, launcher: State<'_, Launcher>) -> CmdResult<()> {
launcher.sync_tiles(&app).await
}
#[tauri::command] #[tauri::command]
pub async fn sync_now(app: AppHandle, launcher: State<'_, Launcher>) -> CmdResult<()> { pub async fn sync_now(app: AppHandle, launcher: State<'_, Launcher>) -> CmdResult<()> {
launcher.sync_now(&app).await launcher.sync_now(&app).await
+243 -7
View File
@@ -3,7 +3,7 @@
use std::collections::BTreeSet; use std::collections::BTreeSet;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
use std::sync::Arc; use std::sync::Arc;
use odian_packs::{InstallOptions, Progress, Source, Stage}; use odian_packs::{InstallOptions, Progress, Source, Stage};
@@ -21,6 +21,7 @@ use crate::site::{self, SessionRefresh, TokenCheck};
use crate::stack::{self, Layout, RunningStack, StackConfig, StackEvent}; use crate::stack::{self, Layout, RunningStack, StackConfig, StackEvent};
use crate::state::{self, LauncherState, SyncOrigin}; use crate::state::{self, LauncherState, SyncOrigin};
use crate::sync::{self, SyncError, SyncStatus}; use crate::sync::{self, SyncError, SyncStatus};
use crate::tilesync::{self, Waiting};
use crate::tokens::{self, SiteToken}; use crate::tokens::{self, SiteToken};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
@@ -33,6 +34,12 @@ pub enum Phase {
Restoring, Restoring,
/// Exchanging operator data with the DHP (ODIAN is not running meanwhile). /// Exchanging operator data with the DHP (ODIAN is not running meanwhile).
Syncing, Syncing,
/// Taking tiles the DHP baked while this PC was away. The map files are being written
/// under ODIAN, so it does not start meanwhile.
PullingTiles,
/// Deleting what the operator unticked in the marketplace. A tile set is hundreds of
/// thousands of files, so this is minutes of work with something to show for it.
Removing,
Launching, Launching,
Running, Running,
Error, Error,
@@ -72,6 +79,24 @@ pub struct Status {
pub map_from_site: Option<bool>, pub map_from_site: Option<bool>,
/// Why, in one line, when `map_from_site` is false. /// Why, in one line, when `map_from_site` is false.
pub map_problem: Option<String>, pub map_problem: Option<String>,
/// Tiles the DHP has baked for the sets this PC holds since it last synced: what a sync
/// would fetch. None offline, with nothing installed, or when the DHP's tile service is
/// too old to be asked. Zero tiles means this PC is level with the site.
pub tiles_waiting: Option<Waiting>,
/// Marketplace items the operator chose that are not all on this PC. None when nothing
/// is installed or nothing is missing: ODIAN still runs, with less map than was asked for.
pub unfinished: Option<Unfinished>,
}
/// What is missing of what was chosen.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
pub struct Unfinished {
pub items: u32,
/// What finishing them would download, in full - the whole item, not just the gap, which
/// is what the installer skips its way through.
pub bytes: u64,
/// The first few, to name in one line.
pub titles: Vec<String>,
} }
/// What the marketplace shows (`get_market`). /// What the marketplace shows (`get_market`).
@@ -87,6 +112,17 @@ pub struct Market {
pub free_bytes: Option<u64>, pub free_bytes: Option<u64>,
} }
/// Told to the window while unticked items are being deleted.
#[derive(Clone, Serialize)]
struct RemoveProgress {
title: String,
done: u64,
total: u64,
/// Which of them, and how many in all: one Apply can drop several.
index: u32,
count: u32,
}
#[derive(Clone, Serialize)] #[derive(Clone, Serialize)]
struct LogEvent { struct LogEvent {
level: &'static str, level: &'static str,
@@ -132,6 +168,10 @@ struct Inner {
live: Option<bool>, live: Option<bool>,
map_from_site: Option<bool>, map_from_site: Option<bool>,
map_problem: Option<String>, map_problem: Option<String>,
tiles_waiting: Option<Waiting>,
unfinished: Option<Unfinished>,
/// What the last count found on disk, kept for as long as nothing can have changed it.
market_checked: Option<Vec<market::Check>>,
/// Read once at start: this launcher ships on an ODIAN drive (`settings::portable_root`). /// Read once at start: this launcher ships on an ODIAN drive (`settings::portable_root`).
portable: bool, portable: bool,
/// A newer launcher is being checked for or downloaded (see `update_self`): nothing may /// A newer launcher is being checked for or downloaded (see `update_self`): nothing may
@@ -175,6 +215,8 @@ impl Inner {
live: self.live, live: self.live,
map_from_site: self.map_from_site, map_from_site: self.map_from_site,
map_problem: self.map_problem.clone(), map_problem: self.map_problem.clone(),
tiles_waiting: self.tiles_waiting.clone(),
unfinished: self.unfinished.clone(),
} }
} }
} }
@@ -296,7 +338,7 @@ impl Launcher {
.map_err(|e| format!("cannot start the HTTP client: {e}"))?; .map_err(|e| format!("cannot start the HTTP client: {e}"))?;
Ok(Launcher { Ok(Launcher {
http, http,
inner: Mutex::new(Inner { settings, state, online: false, available: None, token, phase: Phase::Checking, error: None, sync: None, engine_url: None, live: None, map_from_site: None, map_problem: None, portable: crate::settings::portable_drive().is_some(), updating_self: false, offline_reason: None, drive_extras: None }), inner: Mutex::new(Inner { settings, state, online: false, available: None, token, phase: Phase::Checking, error: None, sync: None, engine_url: None, live: None, map_from_site: None, map_problem: None, tiles_waiting: None, unfinished: None, market_checked: None, portable: crate::settings::portable_drive().is_some(), updating_self: false, offline_reason: None, drive_extras: None }),
stack: Mutex::new(None), stack: Mutex::new(None),
stack_states: std::sync::Mutex::new(Vec::new()), stack_states: std::sync::Mutex::new(Vec::new()),
sign_in_cancel: Notify::new(), sign_in_cancel: Notify::new(),
@@ -329,6 +371,9 @@ impl Launcher {
let mut inner = self.inner.lock().await; let mut inner = self.inner.lock().await;
inner.phase = if error.is_some() { Phase::Error } else { Phase::Idle }; inner.phase = if error.is_some() { Phase::Error } else { Phase::Idle };
inner.error = error; inner.error = error;
// Installing, removing and updating are the only things that change what is on disk,
// and all of them end here: the count is worth nothing now and is taken again.
inner.market_checked = None;
} }
self.emit(app).await; self.emit(app).await;
} }
@@ -476,6 +521,8 @@ impl Launcher {
let _ = self.renew(app).await; let _ = self.renew(app).await;
} }
} }
self.check_tiles(app).await;
self.read_unfinished(app).await;
{ {
let mut inner = self.inner.lock().await; let mut inner = self.inner.lock().await;
if inner.phase == Phase::Checking { if inner.phase == Phase::Checking {
@@ -485,6 +532,125 @@ impl Launcher {
self.emit(app).await self.emit(app).await
} }
/// What this PC needs to ask the site for: the sets it holds, its token, and how far
/// it has replayed. None when there is nothing to ask with or about.
async fn tile_sync_inputs(&self, app: &AppHandle) -> Option<(String, String, Vec<stack::TileSet>, i64, PathBuf)> {
let (site_url, version, install_dir) = {
let inner = self.inner.lock().await;
if !inner.online || inner.settings.site_url.is_empty() {
return None;
}
(inner.settings.site_url.clone(), inner.state.installed_version.clone()?, inner.settings.install_path())
};
let layout = Layout::new(&install_dir, &version);
// The sets this PC HOLDS, not the ones the version names: a set it never downloaded
// has no folder to fill, and asking for it would offer the operator a sync of tiles
// his tile server would not read.
let sets = stack::installed_tile_sets(&layout).ok()?;
if sets.is_empty() {
return None;
}
let token = self.site_access_token(app).await.ok()?;
let seq = self.inner.lock().await.state.tile_seq(&site_url);
Some((site_url, token, sets, seq, layout.data_dir()))
}
/// What the operator chose and has not fully got. Read from the installers' markers, so
/// it costs eight small files rather than a walk of the tile store.
async fn read_unfinished(&self, app: &AppHandle) {
let found = match self.market_manifest(app).await {
Ok((manifest, true, install_dir)) => {
let (extras, checked) = {
let inner = self.inner.lock().await;
(inner.state.extras.clone().unwrap_or_default(), inner.market_checked.clone().unwrap_or_default())
};
let data = install_dir.join("data");
let waiting = market::unfinished(&manifest, &data, &extras, &checked);
(!waiting.is_empty()).then(|| Unfinished {
items: waiting.len() as u32,
bytes: waiting.iter().map(|item| item.bytes).sum(),
titles: waiting.iter().take(3).map(|item| item.title.clone()).collect(),
})
}
_ => None,
};
self.inner.lock().await.unfinished = found;
}
/// Download what was chosen and never fully arrived, without changing the choice and
/// without removing anything: the selection it applies is everything this PC holds or
/// asked for, so `apply_market` finds nothing to delete and only fills the gaps.
pub async fn finish_market(&self, app: &AppHandle) -> Result<(), String> {
let (manifest, _, install_dir) = self.market_manifest(app).await?;
let extras = self.inner.lock().await.state.extras.clone().unwrap_or_default();
let keep = market::held_keys(&manifest, &install_dir.join("data"), &extras);
self.apply_market(app, keep.into_iter().collect()).await
}
/// Ask the site what it has baked since this PC last looked. Quiet on failure: a site
/// whose tile service predates the journal simply has nothing to say, and that is not
/// something to put in front of an operator.
async fn check_tiles(&self, app: &AppHandle) {
let waiting = match self.tile_sync_inputs(app).await {
Some((site_url, token, sets, seq, _)) => {
tilesync::waiting(&self.http, &site_url, &token, &sets, seq).await.ok()
}
None => None,
};
self.inner.lock().await.tiles_waiting = waiting;
}
/// Take them. The seq only moves over pages that landed whole, so stopping early -
/// cancelled, or a DHP that went away mid-pull - costs the tiles already written
/// nothing and resumes where it stopped.
pub async fn sync_tiles(&self, app: &AppHandle) -> Result<(), String> {
let Some((site_url, token, sets, seq, data_dir)) = self.tile_sync_inputs(app).await else {
return Err("Connect to your DHP first: tiles come from it.".to_string());
};
{
let mut inner = self.inner.lock().await;
if !matches!(inner.phase, Phase::Idle | Phase::Error) {
return Err("The launcher is busy: sync the tiles once it has finished.".to_string());
}
inner.phase = Phase::PullingTiles;
inner.error = None;
}
let cancel = CancellationToken::new();
*self.install_cancel.lock().unwrap() = Some(cancel.clone());
self.emit(app).await;
let handle = app.clone();
let pulled = tilesync::pull(&self.http, &site_url, &token, &sets, &data_dir, seq, &cancel, move |progress| {
let _ = handle.emit("tile-sync-progress", &progress);
})
.await;
*self.install_cancel.lock().unwrap() = None;
{
let mut inner = self.inner.lock().await;
let install_dir = inner.settings.install_path();
inner.state.tile_sync = Some(crate::state::TileSync { site: site_url, seq: pulled.seq });
if let Err(e) = inner.state.save(&install_dir) {
// The tiles are on disk either way; only the bookmark is lost, so the next sync
// offers them again. Worth saying, not worth failing over.
inner.tiles_waiting = None;
drop(inner);
self.log(app, "warn", format!("The tiles are here, but this PC could not remember how far it got: {e}"));
}
}
if pulled.tiles > 0 || pulled.gone > 0 {
self.log(
app,
"info",
format!("Took {} tiles ({}) from your DHP.", pulled.tiles, crate::prune::size_text(pulled.bytes)),
);
}
let problem = pulled.problem.clone();
self.check_tiles(app).await;
self.finish(app, problem.map(|why| format!("Not all the tiles arrived: {why}"))).await;
Ok(())
}
pub async fn sign_in(&self, app: &AppHandle) -> Result<String, String> { pub async fn sign_in(&self, app: &AppHandle) -> Result<String, String> {
let (mcp_url, site_url) = { let (mcp_url, site_url) = {
let mut inner = self.inner.lock().await; let mut inner = self.inner.lock().await;
@@ -905,7 +1071,8 @@ impl Launcher {
(Some(db), true) => self.db_files_needed(app, install_dir, db).await, (Some(db), true) => self.db_files_needed(app, install_dir, db).await,
_ => None, _ => None,
}; };
let (tile_sets, src_paths) = market::install_filter(manifest, &extras); // From a drive, an item the DHP does not carry is exactly what the drive is FOR.
let (tile_sets, src_paths) = market::install_filter(manifest, &extras, matches!(source, Source::Site { .. }));
let options = let options =
InstallOptions { db_paths, tile_sets: Some(tile_sets), src_paths: Some(src_paths), ..InstallOptions::default() }; InstallOptions { db_paths, tile_sets: Some(tile_sets), src_paths: Some(src_paths), ..InstallOptions::default() };
odian_packs::install(source, install_dir, &verified, options, progress, cancel.clone()).await?; odian_packs::install(source, install_dir, &verified, options, progress, cancel.clone()).await?;
@@ -1120,6 +1287,40 @@ impl Launcher {
/// Save the operator's choice of optional items (`selected`, item keys), then apply it to /// Save the operator's choice of optional items (`selected`, item keys), then apply it to
/// the installed version: the unchosen items leave this PC, the chosen missing ones are /// the installed version: the unchosen items leave this PC, the chosen missing ones are
/// downloaded. With nothing installed yet the choice only waits for Install. /// downloaded. With nothing installed yet the choice only waits for Install.
/// Count what is really on disk for the items the markers call installed.
///
/// Separate from `get_market` on purpose: this walks every tile of every installed set -
/// nearly two million files on a full install - and the marketplace has to open at once.
/// The answer is remembered per version, because the only things that change it are an
/// install and a removal, and both clear it.
pub async fn check_market(&self, app: &AppHandle) -> Result<Vec<market::Check>, String> {
let (manifest, _, install_dir) = self.market_manifest(app).await?;
let (extras, cached) = {
let inner = self.inner.lock().await;
(inner.state.extras.clone().unwrap_or_default(), inner.market_checked.clone())
};
if let Some(checks) = cached {
return Ok(checks);
}
let data = install_dir.join("data");
let checks =
tauri::async_runtime::spawn_blocking(move || market::check_installed(&manifest, &data, &extras))
.await
.map_err(|e| format!("could not check what is installed: {e}"))?;
for check in checks.iter().filter(|check| !check.whole) {
self.log(
app,
"warn",
format!("{} says it is installed but holds {} of {} files.", check.key, check.found, check.expected),
);
}
self.inner.lock().await.market_checked = Some(checks.clone());
// The hub said "Launch" on the markers' word; the count may have just contradicted it.
self.read_unfinished(app).await;
self.emit(app).await;
Ok(checks)
}
pub async fn apply_market(&self, app: &AppHandle, selected: Vec<String>) -> Result<(), String> { pub async fn apply_market(&self, app: &AppHandle, selected: Vec<String>) -> Result<(), String> {
let (manifest, installed, install_dir) = self.market_manifest(app).await?; let (manifest, installed, install_dir) = self.market_manifest(app).await?;
let extras = market::optional_keys(&manifest, selected); let extras = market::optional_keys(&manifest, selected);
@@ -1158,15 +1359,50 @@ impl Launcher {
}; };
let extras = self.inner.lock().await.state.extras.clone().unwrap_or_default(); let extras = self.inner.lock().await.state.extras.clone().unwrap_or_default();
let data = install_dir.join("data"); let data = install_dir.join("data");
for item in market::to_remove(&manifest, &data, &extras) { let going = market::to_remove(&manifest, &data, &extras);
if !going.is_empty() {
self.set_phase(app, Phase::Removing).await;
}
let count = going.len() as u32;
for (i, item) in going.iter().enumerate() {
let (m, d, key) = (manifest.clone(), data.clone(), item.key.clone()); let (m, d, key) = (manifest.clone(), data.clone(), item.key.clone());
match tauri::async_runtime::spawn_blocking(move || market::remove(&m, &d, &key)).await { // The blocking thread counts; this side reads that count on a timer and tells the
// window. Deleting 376 000 files is minutes, and a bar that does not move is
// indistinguishable from a launcher that has stopped.
let done = Arc::new(AtomicU64::new(0));
let ticker = {
let (handle, done, title, total) =
(app.clone(), done.clone(), item.title.clone(), item.files.max(1));
let (index, count) = (i as u32 + 1, count);
tauri::async_runtime::spawn(async move {
loop {
tokio::time::sleep(Duration::from_millis(250)).await;
let _ = handle.emit(
"remove-progress",
&RemoveProgress { title: title.clone(), done: done.load(Ordering::Relaxed), total, index, count },
);
}
})
};
let counted = done.clone();
let outcome =
tauri::async_runtime::spawn_blocking(move || market::remove_counting(&m, &d, &key, &mut |n| {
counted.fetch_add(n, Ordering::Relaxed);
}))
.await;
ticker.abort();
match outcome {
Ok(Ok(())) => self.log(app, "info", format!("Removed {} ({}).", item.title, prune::size_text(item.bytes))), Ok(Ok(())) => self.log(app, "info", format!("Removed {} ({}).", item.title, prune::size_text(item.bytes))),
Ok(Err(e)) => self.log(app, "warn", format!("Could not remove {}: {e}", item.title)), Ok(Err(e)) => self.log(app, "warn", format!("Could not remove {}: {e}", item.title)),
Err(e) => self.log(app, "warn", format!("Could not remove {}: {e}", item.title)), Err(e) => self.log(app, "warn", format!("Could not remove {}: {e}", item.title)),
} }
} }
!market::to_add(&manifest, &data, &extras).is_empty() let more = !market::to_add(&manifest, &data, &extras).is_empty();
if more {
// The download stages report themselves from here on.
self.set_phase(app, Phase::Installing).await;
}
more
} }
/// Download the chosen marketplace items of the installed version (tiles and source files /// Download the chosen marketplace items of the installed version (tiles and source files
@@ -1188,7 +1424,7 @@ impl Launcher {
let titles: Vec<String> = let titles: Vec<String> =
market::to_add(manifest, &install_dir.join("data"), &extras).into_iter().map(|item| item.title).collect(); market::to_add(manifest, &install_dir.join("data"), &extras).into_iter().map(|item| item.title).collect();
self.log(app, "info", format!("Downloading {}", titles.join(", "))); self.log(app, "info", format!("Downloading {}", titles.join(", ")));
let (tile_sets, src_paths) = market::install_filter(manifest, &extras); let (tile_sets, src_paths) = market::install_filter(manifest, &extras, true);
let options = InstallOptions { let options = InstallOptions {
only: vec![Stage::Src, Stage::Tiles], only: vec![Stage::Src, Stage::Tiles],
tile_sets: Some(tile_sets), tile_sets: Some(tile_sets),
+4
View File
@@ -13,6 +13,7 @@ mod site;
mod stack; mod stack;
mod state; mod state;
mod sync; mod sync;
mod tilesync;
mod tokens; mod tokens;
use tauri::Manager; use tauri::Manager;
@@ -52,6 +53,8 @@ pub fn run() {
commands::get_status, commands::get_status,
commands::get_market, commands::get_market,
commands::apply_market, commands::apply_market,
commands::check_market,
commands::finish_market,
commands::set_market, commands::set_market,
commands::refresh, commands::refresh,
commands::sign_in, commands::sign_in,
@@ -69,6 +72,7 @@ pub fn run() {
commands::set_settings, commands::set_settings,
commands::pick_folder, commands::pick_folder,
commands::sync_now, commands::sync_now,
commands::sync_tiles,
commands::get_sync_conflicts, commands::get_sync_conflicts,
commands::resolve_sync_conflict, commands::resolve_sync_conflict,
]) ])
+211 -10
View File
@@ -39,14 +39,22 @@ pub struct Item {
pub description: String, pub description: String,
/// What installing it downloads, in full. /// What installing it downloads, in full.
pub bytes: u64, pub bytes: u64,
/// How many files it is made of: what a removal counts down.
pub files: u64,
/// Part of every install: shown, never offered for removal. /// Part of every install: shown, never offered for removal.
pub required: bool, pub required: bool,
pub presence: Presence, pub presence: Presence,
/// Chosen: installed with the version and kept by updates (required items always are). /// Chosen: installed with the version and kept by updates (required items always are).
pub selected: bool, pub selected: bool,
/// False when the DHP does not carry this item's bytes: it is listed so an operator knows
/// it exists, but only an ODIAN drive can install it.
pub on_dhp: bool,
} }
/// The map ODIAN opens on, kept required by `catalog.json`. /// The map a version from BEFORE the marketplace opens on. Those versions flag nothing, so
/// the catalog's rule is applied to them here - and what they shipped as their one certain
/// basemap was `dark`. Current versions say it themselves (`catalog.json` now keeps
/// `dark_v2` required instead), and this is not read for them.
const ALWAYS_INSTALLED_STYLE: &str = "dark"; const ALWAYS_INSTALLED_STYLE: &str = "dark";
/// A manifest from before the marketplace (pack 0.61.0) flags nothing optional, so everything /// A manifest from before the marketplace (pack 0.61.0) flags nothing optional, so everything
@@ -106,6 +114,55 @@ fn tile_set_presence(data_dir: &Path, set: &TileSet) -> Presence {
} }
} }
/// One item weighed against what the version says it holds.
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
pub struct Check {
pub key: String,
pub found: u64,
pub expected: u64,
/// True when at least everything the version names is on disk. MORE than that is normal
/// and not a fault: a PC bakes tiles of its own into the same folders, and the tile sync
/// brings in what other operators baked.
pub whole: bool,
}
/// What a tile set's folder really holds. Dot-files are the installer's own markers
/// (`.blocks-done.json`) and the tile server's negative ones (`.missing`), never tiles.
fn count_tiles(dir: &Path) -> u64 {
let Ok(entries) = std::fs::read_dir(dir) else { return 0 };
let mut found = 0;
for entry in entries.flatten() {
let name = entry.file_name();
if name.to_string_lossy().starts_with('.') {
continue;
}
match entry.file_type() {
Ok(kind) if kind.is_dir() => found += count_tiles(&entry.path()),
Ok(kind) if kind.is_file() => found += 1,
_ => {}
}
}
found
}
/// Count what is on disk for every tile set a marker calls installed.
///
/// This is the slow, true answer - a walk of hundreds of thousands of files - so it is
/// never on the path that opens the marketplace. The window asks for it after, and moves
/// the badge when it comes back.
pub fn check_installed(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> Vec<Check> {
let mut out = Vec::new();
for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) {
if tile_set_presence(data_dir, set) != Presence::Installed {
continue;
}
let found = count_tiles(&tile_set_dir(data_dir, set));
out.push(Check { key: tiles_key(set), found, expected: set.files, whole: found >= set.files });
}
let _ = extras;
out
}
/// A source file is complete when its `.ok` marker names its chunk list and the size is right /// A source file is complete when its `.ok` marker names its chunk list and the size is right
/// (the installer's own test); partial when the file exists. /// (the installer's own test); partial when the file exists.
fn src_presence(data_dir: &Path, file: &SrcFile) -> Presence { fn src_presence(data_dir: &Path, file: &SrcFile) -> Presence {
@@ -151,13 +208,16 @@ fn all_items(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) ->
for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) { for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) {
let key = tiles_key(set); let key = tiles_key(set);
let optional = tiles_optional(set, old); let optional = tiles_optional(set, old);
let presence = tile_set_presence(data_dir, set);
out.push(Item { out.push(Item {
selected: !optional || extras.contains(&key), selected: !optional || extras.contains(&key),
title: set.title.clone().unwrap_or_else(|| format!("{} map ({})", set.style, set.dataset)), title: set.title.clone().unwrap_or_else(|| format!("{} map ({})", set.style, set.dataset)),
description: set.description.clone().unwrap_or_default(), description: set.description.clone().unwrap_or_default(),
bytes: set.bytes, bytes: set.bytes,
files: set.files,
required: !optional, required: !optional,
presence: tile_set_presence(data_dir, set), presence,
on_dhp: set.on_dhp,
kind: Kind::Tiles, kind: Kind::Tiles,
key, key,
}); });
@@ -165,13 +225,17 @@ fn all_items(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) ->
for file in manifest.src.iter().filter(|src| src.enabled).flat_map(|src| &src.files) { for file in manifest.src.iter().filter(|src| src.enabled).flat_map(|src| &src.files) {
let key = src_key(file); let key = src_key(file);
let optional = src_optional(file, old); let optional = src_optional(file, old);
let presence = src_presence(data_dir, file);
out.push(Item { out.push(Item {
selected: !optional || extras.contains(&key), selected: !optional || extras.contains(&key),
title: file.title.clone().unwrap_or_else(|| file.path.clone()), title: file.title.clone().unwrap_or_else(|| file.path.clone()),
description: file.description.clone().unwrap_or_default(), description: file.description.clone().unwrap_or_default(),
bytes: file.size, bytes: file.size,
// A source is one file (plus its small markers, not worth counting).
files: 1,
required: !optional, required: !optional,
presence: src_presence(data_dir, file), presence,
on_dhp: file.on_dhp,
kind: Kind::Src, kind: Kind::Src,
key, key,
}); });
@@ -182,12 +246,24 @@ fn all_items(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) ->
/// What installing `manifest` fetches: every required item and the chosen optional ones, as /// What installing `manifest` fetches: every required item and the chosen optional ones, as
/// the installer's `tile_sets` (`style/dataset`) and `src_paths` filters. /// the installer's `tile_sets` (`style/dataset`) and `src_paths` filters.
pub fn install_filter(manifest: &Manifest, extras: &BTreeSet<String>) -> (BTreeSet<String>, BTreeSet<String>) { ///
/// `from_dhp` drops what the store does not carry. The same choice is honoured when the
/// bytes come from a drive - that is where such an item comes from - but asked of the DHP
/// it would be a 404 per chunk and a failed install, for an item nobody could have got that
/// way. It is dropped here rather than refused at the checkbox because the choice outlives
/// the version it was made against: a PC that took the height map off a drive keeps it in
/// `extras`, and its next update from the DHP must simply leave it alone.
pub fn install_filter(
manifest: &Manifest,
extras: &BTreeSet<String>,
from_dhp: bool,
) -> (BTreeSet<String>, BTreeSet<String>) {
let old = pre_catalog(manifest); let old = pre_catalog(manifest);
let tile_sets = manifest let tile_sets = manifest
.tiles .tiles
.iter() .iter()
.flat_map(|tiles| &tiles.sets) .flat_map(|tiles| &tiles.sets)
.filter(|set| set.on_dhp || !from_dhp)
.filter(|set| !tiles_optional(set, old) || extras.contains(&tiles_key(set))) .filter(|set| !tiles_optional(set, old) || extras.contains(&tiles_key(set)))
.map(|set| format!("{}/{}", set.style, set.dataset)) .map(|set| format!("{}/{}", set.style, set.dataset))
.collect(); .collect();
@@ -195,6 +271,7 @@ pub fn install_filter(manifest: &Manifest, extras: &BTreeSet<String>) -> (BTreeS
.src .src
.iter() .iter()
.flat_map(|src| &src.files) .flat_map(|src| &src.files)
.filter(|file| file.on_dhp || !from_dhp)
.filter(|file| !src_optional(file, old) || extras.contains(&src_key(file))) .filter(|file| !src_optional(file, old) || extras.contains(&src_key(file)))
.map(|file| file.path.clone()) .map(|file| file.path.clone())
.collect(); .collect();
@@ -227,20 +304,67 @@ pub fn to_remove(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>
.collect() .collect()
} }
/// What this PC has settled on: what the operator chose, plus whatever is actually here.
///
/// `extras` records answers about OPTIONAL items, so an item that is HERE and not in it was
/// never a choice the operator could have made - required by the version that installed it
/// (Dark, until Dark V2 took its place), off a drive, or not downloadable at all. Applying
/// `extras` alone as a selection therefore reads as "remove every one of those".
pub fn held_keys(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> BTreeSet<String> {
items(manifest, data_dir, extras)
.into_iter()
.filter(|item| !item.required && (item.selected || item.presence != Presence::Absent))
.map(|item| item.key)
.collect()
}
/// The chosen items that are not all here, weighed against a count when one has been taken.
///
/// `to_add` answers from the installers' markers alone, which is what the marketplace opens
/// on. A set whose marker says "installed" while the disk holds half of it is exactly what
/// `check_installed` is for, so anything it found short is counted as unfinished here too.
pub fn unfinished(
manifest: &Manifest,
data_dir: &Path,
extras: &BTreeSet<String>,
checked: &[Check],
) -> Vec<Item> {
let short: BTreeSet<&str> = checked.iter().filter(|c| !c.whole).map(|c| c.key.as_str()).collect();
items(manifest, data_dir, extras)
.into_iter()
.filter(|item| !item.required && item.selected)
.filter(|item| item.presence != Presence::Installed || short.contains(item.key.as_str()))
.collect()
}
/// Whether the chosen optional items still need downloading. /// Whether the chosen optional items still need downloading.
pub fn to_add(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> Vec<Item> { pub fn to_add(manifest: &Manifest, data_dir: &Path, extras: &BTreeSet<String>) -> Vec<Item> {
items(manifest, data_dir, extras) items(manifest, data_dir, extras)
.into_iter() .into_iter()
// An item the store does not carry is never downloaded, so it is never announced as
// one: `install_filter` drops it, and a line saying it is being fetched would be a
// promise nothing keeps.
.filter(|item| item.on_dhp)
.filter(|item| !item.required && item.selected && item.presence != Presence::Installed) .filter(|item| !item.required && item.selected && item.presence != Presence::Installed)
.collect() .collect()
} }
/// Remove an item's files from the PC: a tile set's folder, or a source file with its markers. /// Remove an item's files from the PC: a tile set's folder, or a source file with its markers.
pub fn remove(manifest: &Manifest, data_dir: &Path, key: &str) -> Result<(), String> { pub fn remove(manifest: &Manifest, data_dir: &Path, key: &str) -> Result<(), String> {
remove_counting(manifest, data_dir, key, &mut |_| {})
}
/// The same, calling `removed` with each file as it goes, so a caller can show progress.
pub fn remove_counting(
manifest: &Manifest,
data_dir: &Path,
key: &str,
removed: &mut dyn FnMut(u64),
) -> Result<(), String> {
for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) { for set in manifest.tiles.iter().flat_map(|tiles| &tiles.sets) {
if tiles_key(set) == key { if tiles_key(set) == key {
let dir = tile_set_dir(data_dir, set); let dir = tile_set_dir(data_dir, set);
return if dir.exists() { crate::prune::remove_folder(&dir) } else { Ok(()) }; return if dir.exists() { crate::prune::remove_folder_counting(&dir, removed) } else { Ok(()) };
} }
} }
for file in manifest.src.iter().flat_map(|src| &src.files) { for file in manifest.src.iter().flat_map(|src| &src.files) {
@@ -250,6 +374,9 @@ pub fn remove(manifest: &Manifest, data_dir: &Path, key: &str) -> Result<(), Str
let target = with_suffix(&path, extra); let target = with_suffix(&path, extra);
if target.exists() { if target.exists() {
std::fs::remove_file(&target).map_err(|e| format!("cannot remove {}: {e}", target.display()))?; std::fs::remove_file(&target).map_err(|e| format!("cannot remove {}: {e}", target.display()))?;
if extra.is_empty() {
removed(1);
}
} }
} }
return Ok(()); return Ok(());
@@ -317,11 +444,85 @@ mod tests {
assert!(listed.iter().all(|i| i.presence == Presence::Absent)); assert!(listed.iter().all(|i| i.presence == Presence::Absent));
} }
#[test]
fn an_item_the_store_does_not_carry_is_listed_but_never_asked_of_it() {
// The world height map source is 705 GB and arrives on a drive. Leaving it out of the
// catalogue hid the one thing an operator needs to know - that it exists at all - and
// listing it without this flag would have the installer ask the DHP for 2629 chunks it
// has never had.
let mut carrying = manifest();
carrying.src.as_mut().unwrap().files.push(
serde_json::from_value(serde_json::json!({
"path": "planet_heightmap.pmtiles", "size": 8, "chunks": ["e".repeat(64)],
"chunks_sha256": "f".repeat(64), "optional": true, "on_dhp": false, "title": "Terrain source"
}))
.unwrap(),
);
let dir = std::env::temp_dir().join(format!("odian-market-{}", uuid::Uuid::new_v4().simple()));
let items = items(&carrying, &dir, &BTreeSet::new());
let terrain = items.iter().find(|item| item.key == "src:planet_heightmap.pmtiles").expect("it is listed");
assert!(!terrain.on_dhp, "and it says where it does NOT come from");
assert_eq!(terrain.title, "Terrain source");
assert!(items.iter().all(|item| item.key == "src:planet_heightmap.pmtiles" || item.on_dhp));
// Chosen - off a drive, say - it is still left alone when the DHP is the source.
let chosen: BTreeSet<String> = ["src:planet_heightmap.pmtiles".to_string()].into_iter().collect();
let (_, from_dhp) = install_filter(&carrying, &chosen, true);
assert!(!from_dhp.contains("planet_heightmap.pmtiles"), "the DHP is never asked for it");
let (_, from_drive) = install_filter(&carrying, &chosen, false);
assert!(from_drive.contains("planet_heightmap.pmtiles"), "a drive is exactly where it comes from");
}
#[test]
fn finishing_a_download_takes_nothing_away() {
// Dark V2 is installed and NOT in `extras`: it was required by the version that put it
// there, so no choice about it was ever recorded (which is exactly what happened when
// Dark V2 and Dark swapped places). Applying `extras` as a selection would read as
// "remove Dark V2", which is what Finish did before `held_keys`.
let data = data_dir();
complete_tile_set(&data, "dark_v2", 'b', 2);
let extras: BTreeSet<String> = ["src:osm/x.pmtiles".to_string()].into_iter().collect();
let keep = held_keys(&manifest(), &data, &extras);
let dropped = to_remove(&manifest(), &data, &keep);
std::fs::remove_dir_all(&data).unwrap();
assert!(keep.contains("tiles:dark_v2/ds"), "what is on disk stays chosen");
assert!(keep.contains("src:osm/x.pmtiles"), "and so does what was asked for");
assert!(dropped.is_empty(), "so finishing removes nothing: {dropped:?}");
}
#[test]
fn untick_still_means_remove_even_for_what_only_a_drive_replaces() {
// `selected` means "the operator asked for this", and `to_remove` finds what to delete
// by looking for present-and-not-selected. An item that always read as selected - which
// is what "installed counts as chosen" did to it - could never be removed at all.
let root = std::env::temp_dir().join(format!("odian-market-{}", uuid::Uuid::new_v4().simple()));
let src = root.join("src");
std::fs::create_dir_all(&src).unwrap();
std::fs::write(src.join("planet_heightmap.pmtiles"), b"12345678").unwrap();
std::fs::write(src.join("planet_heightmap.pmtiles.ok"), "f".repeat(64)).unwrap();
let mut carrying = manifest();
carrying.src.as_mut().unwrap().files.push(
serde_json::from_value(serde_json::json!({
"path": "planet_heightmap.pmtiles", "size": 8, "chunks": ["e".repeat(64)],
"chunks_sha256": "f".repeat(64), "optional": true, "on_dhp": false, "title": "Terrain source"
}))
.unwrap(),
);
let dropped = to_remove(&carrying, &root, &BTreeSet::new());
std::fs::remove_dir_all(&root).unwrap();
assert_eq!(
dropped.iter().map(|item| item.key.as_str()).collect::<Vec<_>>(),
["src:planet_heightmap.pmtiles"],
"unticked and on disk: it goes"
);
}
#[test] #[test]
fn an_install_fetches_the_required_items_and_the_chosen_ones() { fn an_install_fetches_the_required_items_and_the_chosen_ones() {
let (sets, paths) = install_filter(&manifest(), &BTreeSet::new()); let (sets, paths) = install_filter(&manifest(), &BTreeSet::new(), true);
assert_eq!((sets.into_iter().collect::<Vec<_>>(), paths.len()), (vec!["dark/ds".to_string()], 0)); assert_eq!((sets.into_iter().collect::<Vec<_>>(), paths.len()), (vec!["dark/ds".to_string()], 0));
let (sets, paths) = install_filter(&manifest(), &["tiles:dark_v2/ds".to_string()].into_iter().collect()); let (sets, paths) = install_filter(&manifest(), &["tiles:dark_v2/ds".to_string()].into_iter().collect(), true);
assert_eq!(sets.len(), 2); assert_eq!(sets.len(), 2);
assert!(paths.is_empty()); assert!(paths.is_empty());
} }
@@ -388,7 +589,7 @@ mod tests {
item.title item.title
); );
} }
let (sets, paths) = install_filter(manifest, &BTreeSet::new()); let (sets, paths) = install_filter(manifest, &BTreeSet::new(), true);
println!("a new PC installs: {sets:?} {paths:?}"); println!("a new PC installs: {sets:?} {paths:?}");
} }
@@ -407,12 +608,12 @@ mod tests {
let offered: Vec<&str> = items.iter().filter(|item| !item.required).map(|item| item.key.as_str()).collect(); let offered: Vec<&str> = items.iter().filter(|item| !item.required).map(|item| item.key.as_str()).collect();
assert_eq!(offered, vec!["tiles:dark_v2/ds", "src:osm/x.pmtiles"]); assert_eq!(offered, vec!["tiles:dark_v2/ds", "src:osm/x.pmtiles"]);
// Unchosen, an install of it brings the dark map alone. // Unchosen, an install of it brings the dark map alone.
let (tiles, src) = install_filter(&old_manifest(), &BTreeSet::new()); let (tiles, src) = install_filter(&old_manifest(), &BTreeSet::new(), true);
assert_eq!(tiles, BTreeSet::from(["dark/ds".to_string()])); assert_eq!(tiles, BTreeSet::from(["dark/ds".to_string()]));
assert!(src.is_empty()); assert!(src.is_empty());
// Chosen, they come along. // Chosen, they come along.
let chosen = BTreeSet::from(["src:osm/x.pmtiles".to_string()]); let chosen = BTreeSet::from(["src:osm/x.pmtiles".to_string()]);
let (_, src) = install_filter(&old_manifest(), &chosen); let (_, src) = install_filter(&old_manifest(), &chosen, true);
assert_eq!(src, BTreeSet::from(["osm/x.pmtiles".to_string()])); assert_eq!(src, BTreeSet::from(["osm/x.pmtiles".to_string()]));
} }
+11 -3
View File
@@ -127,6 +127,12 @@ fn entries(dir: &Path, folders: bool) -> Vec<(String, PathBuf)> {
/// Remove a folder the safe way (renamed aside, then emptied without following links). /// Remove a folder the safe way (renamed aside, then emptied without following links).
pub(crate) fn remove_folder(folder: &Path) -> Result<(), String> { pub(crate) fn remove_folder(folder: &Path) -> Result<(), String> {
remove_folder_counting(folder, &mut |_| {})
}
/// The same, calling `removed` once per file deleted. A tile set is hundreds of thousands
/// of them: without a count there is nothing to show but a spinner.
pub(crate) fn remove_folder_counting(folder: &Path, removed: &mut dyn FnMut(u64)) -> Result<(), String> {
let name = folder.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); let name = folder.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
let aside = if name.starts_with(TRASH_PREFIX) { let aside = if name.starts_with(TRASH_PREFIX) {
folder.to_path_buf() folder.to_path_buf()
@@ -136,10 +142,10 @@ pub(crate) fn remove_folder(folder: &Path) -> Result<(), String> {
std::fs::rename(folder, &aside).map_err(|e| format!("still in use ({e})"))?; std::fs::rename(folder, &aside).map_err(|e| format!("still in use ({e})"))?;
aside aside
}; };
empty_and_remove(&aside).map_err(|e| format!("partly removed ({e}); the rest goes next time")) empty_and_remove(&aside, removed).map_err(|e| format!("partly removed ({e}); the rest goes next time"))
} }
fn empty_and_remove(dir: &Path) -> std::io::Result<()> { fn empty_and_remove(dir: &Path, removed: &mut dyn FnMut(u64)) -> std::io::Result<()> {
for entry in std::fs::read_dir(dir)? { for entry in std::fs::read_dir(dir)? {
let path = entry?.path(); let path = entry?.path();
let meta = std::fs::symlink_metadata(&path)?; let meta = std::fs::symlink_metadata(&path)?;
@@ -150,8 +156,9 @@ fn empty_and_remove(dir: &Path) -> std::io::Result<()> {
} else { } else {
std::fs::remove_file(&path)?; std::fs::remove_file(&path)?;
} }
removed(1);
} else if meta.is_dir() { } else if meta.is_dir() {
empty_and_remove(&path)?; empty_and_remove(&path, removed)?;
} else { } else {
if meta.permissions().readonly() { if meta.permissions().readonly() {
let mut writable = meta.permissions(); let mut writable = meta.permissions();
@@ -160,6 +167,7 @@ fn empty_and_remove(dir: &Path) -> std::io::Result<()> {
std::fs::set_permissions(&path, writable)?; std::fs::set_permissions(&path, writable)?;
} }
std::fs::remove_file(&path)?; std::fs::remove_file(&path)?;
removed(1);
} }
} }
std::fs::remove_dir(dir) std::fs::remove_dir(dir)
+111 -25
View File
@@ -530,19 +530,36 @@ fn datasets(config: &StackConfig) -> serde_json::Value {
} }
entry entry
}); });
// A style the office fetches rather than draws must never be baked here: a baking
// server asked for a tile it lacks would render the vector map into a satellite
// photograph's place, or into OpenStreetMap's. Marked with its source, a miss is a
// clean 404 instead - this PC has no upstream of its own to ask.
let source = if !set.source.is_empty() {
set.source.clone()
} else if set.ext == ".jpg" || set.ext == ".jpeg" {
// Packs before the manifest said it: imagery was recognised by its name alone.
"wmts".to_string()
} else {
String::new()
};
// Imagery is .jpg, a drawn basemap .png: the tile server assumes .png unless told, // Imagery is .jpg, a drawn basemap .png: the tile server assumes .png unless told,
// and looking for the wrong name is a 404 for every tile of that style. // and looking for the wrong name is a 404 for every tile of that style.
if !set.ext.is_empty() { if !set.ext.is_empty() || !source.is_empty() || set.cache_only {
let styles = entry let styles = entry
.as_object_mut() .as_object_mut()
.expect("a dataset entry is an object") .expect("a dataset entry is an object")
.entry("styles") .entry("styles")
.or_insert_with(|| serde_json::Value::Object(serde_json::Map::new())); .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new()));
let mut style = serde_json::json!({ "tile_ext": set.ext, "media_type": media_type(&set.ext) }); let mut style = serde_json::json!({});
// Imagery is photographed, not drawn: a baking server must never render a vector map if !set.ext.is_empty() {
// into a satellite tile. Marked remote, a miss is a clean 404 (the PC has no upstream). style["tile_ext"] = serde_json::Value::String(set.ext.clone());
if set.ext == ".jpg" || set.ext == ".jpeg" { style["media_type"] = serde_json::Value::String(media_type(&set.ext).to_string());
style["source"] = serde_json::Value::String("wmts".to_string()); }
if !source.is_empty() {
style["source"] = serde_json::Value::String(source);
}
if set.cache_only {
style["cache_only"] = serde_json::Value::Bool(true);
} }
styles[&set.style] = style; styles[&set.style] = style;
} }
@@ -614,7 +631,7 @@ fn baker(config: &StackConfig) -> Option<PathBuf> {
/// The site's tile service, behind the DHP: `/tileserver` of the DHP's own address. /// The site's tile service, behind the DHP: `/tileserver` of the DHP's own address.
/// The Data PC that runs it is not on the internet, so this is the only way to it, /// The Data PC that runs it is not on the internet, so this is the only way to it,
/// and the DHP's token check covers it like every other route. /// and the DHP's token check covers it like every other route.
fn site_tiles_url(site_url: &str) -> String { pub(crate) fn site_tiles_url(site_url: &str) -> String {
format!("{}/tileserver", site_url.trim_end_matches('/')) format!("{}/tileserver", site_url.trim_end_matches('/'))
} }
@@ -676,10 +693,12 @@ fn client_env(config: &StackConfig) -> Vec<(&'static str, String)> {
} else { } else {
env.push(("ODIAN_MCP_API_URL", config.ai_url.clone())); env.push(("ODIAN_MCP_API_URL", config.ai_url.clone()));
} }
let basemap = config // Dark V2 is the map every install carries, so it is the one ODIAN opens on. Dark is
.tile_sets // the fallback for an install made before that was true, and anything that is not the
// height map for one that carries neither.
let basemap = ["dark_v2", "dark"]
.iter() .iter()
.find(|set| set.style == "dark") .find_map(|want| config.tile_sets.iter().find(|set| set.style == *want))
.or_else(|| config.tile_sets.iter().find(|set| set.style != "heightmap")); .or_else(|| config.tile_sets.iter().find(|set| set.style != "heightmap"));
if let Some(set) = basemap { if let Some(set) = basemap {
env.push(("ODIAN_TILE_STYLE", set.style.clone())); env.push(("ODIAN_TILE_STYLE", set.style.clone()));
@@ -717,6 +736,21 @@ pub struct TileSet {
/// for elevation. Empty when the pack predates this, and the tile server's own default /// for elevation. Empty when the pack predates this, and the tile server's own default
/// (.png) then applies, which is what those older packs carried anyway. /// (.png) then applies, which is what those older packs carried anyway.
pub ext: String, pub ext: String,
/// How the office GETS these tiles when they are ones it fetches rather than draws:
/// "wmts" for the satellite, Yandex and OpenStreetMap basemaps. Empty for a style
/// drawn from the vector archive (the dark basemaps) and for elevation.
///
/// It matters on a PC: this one has no upstream, so a fetched style must answer a
/// miss with a clean 404 rather than let the baker draw the vector map in its place.
/// Imagery used to be recognised by its .jpg name, which OpenStreetMap - a drawn map,
/// fetched from openstreetmap.org as .png - would have failed.
pub source: String,
/// True when NOTHING can make another tile of this set - not this PC, not the office.
/// The population basemap is the case: it was composited once from the world population
/// raster onto the dark tiles. A miss has to be a 404, because the alternative is what
/// used to happen - the baker has no preset for the style, draws a plain road map, and
/// it is cached as population density.
pub cache_only: bool,
} }
/// The installed version's database snapshot: its dump files and the extensions it needs. /// The installed version's database snapshot: its dump files and the extensions it needs.
@@ -748,6 +782,8 @@ pub fn version_tile_sets(layout: &Layout) -> Result<Vec<TileSet>, String> {
style: set.get("style")?.as_str()?.to_string(), style: set.get("style")?.as_str()?.to_string(),
dataset: set.get("dataset")?.as_str()?.to_string(), dataset: set.get("dataset")?.as_str()?.to_string(),
ext: set.get("tile_ext").and_then(|e| e.as_str()).unwrap_or_default().to_string(), ext: set.get("tile_ext").and_then(|e| e.as_str()).unwrap_or_default().to_string(),
source: set.get("source").and_then(|s| s.as_str()).unwrap_or_default().to_string(),
cache_only: set.get("cache_only").and_then(|c| c.as_bool()).unwrap_or(false),
}) })
}) })
.collect() .collect()
@@ -787,14 +823,14 @@ mod tests {
install_id: "install".to_string(), install_id: "install".to_string(),
tile_sets: tile_sets tile_sets: tile_sets
.iter() .iter()
.map(|(s, d)| TileSet { style: s.to_string(), dataset: d.to_string(), ext: String::new() }) .map(|(s, d)| TileSet { style: s.to_string(), dataset: d.to_string(), ext: String::new(), source: String::new(), cache_only: false })
.collect(), .collect(),
upstream_url: None, upstream_url: None,
ai_url: String::new(), ai_url: String::new(),
site_url: "https://dhp.example/dhp".to_string(), site_url: "https://dhp.example/dhp".to_string(),
site_tile_sets: tile_sets site_tile_sets: tile_sets
.iter() .iter()
.map(|(s, d)| TileSet { style: s.to_string(), dataset: d.to_string(), ext: String::new() }) .map(|(s, d)| TileSet { style: s.to_string(), dataset: d.to_string(), ext: String::new(), source: String::new(), cache_only: false })
.collect(), .collect(),
} }
} }
@@ -890,10 +926,16 @@ mod tests {
#[test] #[test]
fn datasets_point_every_set_at_the_shared_tile_store() { fn datasets_point_every_set_at_the_shared_tile_store() {
let doc = datasets(&config(&[("dark", "protomaps_20260423"), ("heightmap", "planet_heightmap")])); // An empty folder of its own, not the fixed `D:\ODIAN` the other tests name: whether an
// archive is on disk is the very thing this asserts, and on a PC with an ODIAN drive
// plugged in, D: really does hold planet_heightmap.pmtiles.
let root = std::env::temp_dir().join(format!("odian-launcher-test-{}", uuid::Uuid::new_v4().simple()));
let mut cfg = config(&[("dark", "protomaps_20260423"), ("heightmap", "planet_heightmap")]);
cfg.layout = Layout::new(&root, "0.52.0");
let doc = datasets(&cfg);
let object = doc.as_object().expect("an object"); let object = doc.as_object().expect("an object");
assert_eq!(object.len(), 2); assert_eq!(object.len(), 2);
assert_eq!(object["protomaps_20260423"]["cache_dir"], r"D:\ODIAN\data\tiles"); assert_eq!(object["protomaps_20260423"]["cache_dir"], text(cfg.layout.data_dir().join("tiles")));
assert!(object["planet_heightmap"].get("pmtiles").is_none(), "no pmtiles on disk, so none is named"); assert!(object["planet_heightmap"].get("pmtiles").is_none(), "no pmtiles on disk, so none is named");
} }
@@ -901,9 +943,9 @@ mod tests {
fn datasets_name_each_style_extension() { fn datasets_name_each_style_extension() {
let mut cfg = config(&[]); let mut cfg = config(&[]);
cfg.tile_sets = vec![ cfg.tile_sets = vec![
TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string() }, TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string(), source: String::new(), cache_only: false },
TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string() }, TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string(), source: String::new(), cache_only: false },
TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string() }, TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string(), source: String::new(), cache_only: false },
]; ];
let doc = datasets(&cfg); let doc = datasets(&cfg);
// Two styles share one dataset: both must be named under it, not overwrite each other. // Two styles share one dataset: both must be named under it, not overwrite each other.
@@ -914,17 +956,61 @@ mod tests {
assert_eq!(doc["planet_heightmap"]["styles"]["heightmap"]["tile_ext"], ".lerc"); assert_eq!(doc["planet_heightmap"]["styles"]["heightmap"]["tile_ext"], ".lerc");
} }
#[test]
fn a_fetched_style_is_never_drawn_from_the_vector_archive() {
// OpenStreetMap is a DRAWN map, but the office draws none of it: it asks
// openstreetmap.org and keeps the answer. Its tiles are .png like the dark basemap,
// so nothing but the manifest's own word distinguishes them - and without that word
// a PC holding the vector archive would render protomaps into OSM's place and call
// the result OpenStreetMap.
let (root, mut cfg) = baking_install(true);
cfg.tile_sets.push(TileSet {
style: "osm".to_string(),
dataset: "protomaps_20260423".to_string(),
ext: ".png".to_string(),
source: "wmts".to_string(),
cache_only: false,
});
let doc = datasets(&cfg);
std::fs::remove_dir_all(&root).unwrap();
let styles = &doc["protomaps_20260423"]["styles"];
assert_eq!(styles["osm"]["source"], "wmts", "the office fetches it, so this PC may not bake it");
assert_eq!(styles["osm"]["tile_ext"], ".png");
assert!(styles["dark_v2"].get("source").is_none(), "the dark basemap IS drawn here");
}
#[test]
fn a_set_nothing_can_reproduce_is_served_and_never_drawn() {
// Population density is data composited from a raster, not a map anyone draws.
// On a PC holding the vector archive the baker would happily draw SOMETHING for
// it - a plain road map - and the cache would keep it as a census.
let (root, mut cfg) = baking_install(true);
cfg.tile_sets.push(TileSet {
style: "popdark".to_string(),
dataset: "protomaps_20260423".to_string(),
ext: ".png".to_string(),
source: String::new(),
cache_only: true,
});
let doc = datasets(&cfg);
std::fs::remove_dir_all(&root).unwrap();
let styles = &doc["protomaps_20260423"]["styles"];
assert_eq!(styles["popdark"]["cache_only"], true);
assert!(styles["popdark"].get("source").is_none(), "it is not fetched either, just held");
assert!(doc["protomaps_20260423"].get("cache_only").is_none(), "the dataset still bakes the basemap");
}
#[test] #[test]
fn a_pc_that_downloaded_little_is_still_offered_the_sites_maps() { fn a_pc_that_downloaded_little_is_still_offered_the_sites_maps() {
// What this install actually holds: the one set every install gets. // What this install actually holds: the one set every install gets.
let mut cfg = config(&[]); let mut cfg = config(&[]);
cfg.tile_sets = cfg.tile_sets =
vec![TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string() }]; vec![TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string(), source: String::new(), cache_only: false }];
// What the version names, and the site can therefore draw. // What the version names, and the site can therefore draw.
cfg.site_tile_sets = vec![ cfg.site_tile_sets = vec![
TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string() }, TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string(), source: String::new(), cache_only: false },
TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string() }, TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string(), source: String::new(), cache_only: false },
TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string() }, TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string(), source: String::new(), cache_only: false },
]; ];
let env = client_env(&cfg); let env = client_env(&cfg);
@@ -1005,9 +1091,9 @@ mod tests {
let mut cfg = config(&[]); let mut cfg = config(&[]);
cfg.layout = Layout::new(&root, "0.58.0"); cfg.layout = Layout::new(&root, "0.58.0");
cfg.tile_sets = vec![ cfg.tile_sets = vec![
TileSet { style: "dark_v2".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string() }, TileSet { style: "dark_v2".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string(), source: String::new(), cache_only: false },
TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string() }, TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string(), source: String::new(), cache_only: false },
TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string() }, TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string(), source: String::new(), cache_only: false },
]; ];
let src = cfg.layout.data_dir().join("src"); let src = cfg.layout.data_dir().join("src");
std::fs::create_dir_all(&src).unwrap(); std::fs::create_dir_all(&src).unwrap();
@@ -1064,7 +1150,7 @@ mod tests {
std::fs::remove_dir_all(&root).unwrap(); std::fs::remove_dir_all(&root).unwrap();
assert_eq!( assert_eq!(
sets, sets,
vec![TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: String::new() }] vec![TileSet { style: "dark".to_string(), dataset: "protomaps_20260423".to_string(), ext: String::new(), source: String::new(), cache_only: false }]
); );
} }
} }
+22
View File
@@ -38,6 +38,28 @@ pub struct LauncherState {
/// an install from before the marketplace then counts what it has on disk as chosen. /// an install from before the marketplace then counts what it has on disk as chosen.
#[serde(default)] #[serde(default)]
pub extras: Option<std::collections::BTreeSet<String>>, pub extras: Option<std::collections::BTreeSet<String>>,
/// How far this PC has replayed its DHP's tile journal: the tiles other operators baked
/// while it was away. None until the first sync.
#[serde(default)]
pub tile_sync: Option<TileSync>,
}
/// Whose journal, and how far through it. The site is named because a PC moved to another
/// DHP is at seq 0 there: seq numbers mean nothing outside the journal that issued them.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct TileSync {
pub site: String,
pub seq: i64,
}
impl LauncherState {
/// Where this PC has replayed to on `site`, zero for a site it has never synced with.
pub fn tile_seq(&self, site: &str) -> i64 {
match &self.tile_sync {
Some(sync) if sync.site == site => sync.seq,
_ => 0,
}
}
} }
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
+425
View File
@@ -0,0 +1,425 @@
//! Tiles the site baked while this PC was away.
//!
//! An operator who is online already SEES every tile the office has ever baked: his tile
//! server asks his DHP, the DHP proxies to the Data PC that holds the cache, and the answer
//! is drawn. What he does not do is KEEP it - the answer goes to the screen, not to disk -
//! so the day he unplugs he has exactly what he installed and nothing that was baked since.
//!
//! This is the other half of that bridge, asked a different question: "what have you baked
//! since seq N?" The site's journal answers, this PC fetches those tiles over the same
//! proxied, token-checked path, and writes them where its own tile server reads.
//!
//! No signature and no version, deliberately. A signed manifest exists to make an OFFLINE
//! install verifiable from a drive that anyone could have handed over; keeping an install
//! that already exists level with the site it is talking to is a different job, and it is
//! the same trust as the tile that was drawn on screen a moment ago.
use std::path::{Path, PathBuf};
use std::time::Duration;
use serde::{Deserialize, Serialize};
use tokio_util::sync::CancellationToken;
use crate::fsutil::write_atomic;
use crate::stack::{site_tiles_url, TileSet};
/// Rows per page. The site caps its own answer at 5000; asking for a day of baking in one
/// call would only mean holding it all in memory before writing the first tile.
const PAGE: u32 = 2000;
/// Tiles fetched at once. The site is one PC serving every operator: this is a share, not
/// a race.
const LANES: usize = 6;
const PAGE_TIMEOUT: Duration = Duration::from_secs(20);
/// The "how much is there?" question rides on every check-in, so it waits a short while
/// and then gives up: an offer that is one refresh late costs nothing, a check-in that
/// hangs on a slow link is felt immediately.
const PROBE_TIMEOUT: Duration = Duration::from_secs(6);
const TILE_TIMEOUT: Duration = Duration::from_secs(30);
/// What a tile file may be called. A journal row names a path this PC will WRITE, so none
/// of it is taken on trust: the style and dataset must be ones this PC already holds (see
/// `wanted`), the coordinates are integers, and the extension comes from this list. Without
/// that, a row saying `style: "..\\..\\versions"` would be a way into the install.
const EXTS: [&str; 6] = [".png", ".jpg", ".jpeg", ".webp", ".lerc", ".tif"];
/// What the site would hand over right now.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Waiting {
pub tiles: u64,
pub bytes: u64,
/// The count is one page and the site holds more behind it: the operator is shown
/// "2000+" rather than a number that is quietly wrong.
pub more: bool,
}
/// How a pull ended. Always returned, even when it stopped early: `seq` is how far this PC
/// may honestly claim to have replayed, and a partial pull is a normal thing to resume.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Pulled {
pub tiles: u64,
pub bytes: u64,
/// Rows whose tile this PC already had, byte for byte. The first sync after an install
/// is mostly these: it replays the site's journal from the beginning, and the pack it
/// installed already carried most of what is in there.
pub held: u64,
/// Rows the site no longer has a tile for. Not an error: a tile can be removed.
pub gone: u64,
pub seq: i64,
pub problem: Option<String>,
}
/// Told to the window while a pull runs.
#[derive(Clone, Debug, Serialize)]
pub struct Progress {
pub tiles: u64,
pub bytes: u64,
/// What the current page is worth, so a bar can move within it.
pub page_tiles: u64,
pub page_done: u64,
}
#[derive(Clone, Debug, Deserialize)]
struct Row {
style: String,
dataset: String,
z: i64,
x: i64,
y: i64,
ext: String,
size: u64,
sha256: String,
}
#[derive(Clone, Debug, Deserialize)]
struct Page {
head: i64,
next: i64,
bytes: u64,
more: bool,
rows: Vec<Row>,
}
/// `style/dataset`, as the journal reader names a set.
fn key(set: &TileSet) -> String {
format!("{}/{}", set.style, set.dataset)
}
/// One page of "what is new", for the sets this PC holds.
async fn page(
http: &reqwest::Client,
site_url: &str,
token: &str,
sets: &[TileSet],
seq: i64,
limit: u32,
) -> Result<Page, String> {
let mut query: Vec<(&str, String)> = vec![("seq", seq.to_string()), ("limit", limit.to_string())];
query.extend(sets.iter().map(|set| ("set", key(set))));
let url = format!("{}/journal/after", site_tiles_url(site_url));
let response = http
.get(url)
.bearer_auth(token)
.query(&query)
.timeout(PAGE_TIMEOUT)
.send()
.await
.map_err(|e| format!("the DHP did not answer: {e}"))?;
if !response.status().is_success() {
// A site whose tile service predates the journal answers 404 here. That is not a
// failure to report at the operator: it simply has nothing to offer yet.
return Err(format!("the DHP answered {}", response.status()));
}
response.json::<Page>().await.map_err(|e| format!("the DHP's answer made no sense: {e}"))
}
/// Where a row's tile belongs on this PC, or None when the row is not one we asked for.
///
/// `wanted` is what this PC actually holds. Checking against it is what makes the path
/// safe: `style` and `dataset` are never used as text from the answer, they are matched
/// against strings this PC already had.
fn tile_path(data_dir: &Path, wanted: &[TileSet], row: &Row) -> Option<PathBuf> {
let set = wanted.iter().find(|set| set.style == row.style && set.dataset == row.dataset)?;
let ext = EXTS.iter().find(|allowed| row.ext.eq_ignore_ascii_case(allowed))?;
if row.z < 0 || row.x < 0 || row.y < 0 || row.sha256.len() != 64 || !row.sha256.bytes().all(|b| b.is_ascii_hexdigit()) {
return None;
}
Some(
data_dir
.join("tiles")
.join(&set.style)
.join(&set.dataset)
.join(row.z.to_string())
.join(row.x.to_string())
.join(format!("{}{}", row.y, ext)),
)
}
/// What the site has for this PC beyond what it holds: a count and a size, not the rows.
/// Asked on every check-in, so it is deliberately the cheap question - the site counts,
/// and sends nothing it would have to read off its own disk.
pub async fn waiting(
http: &reqwest::Client,
site_url: &str,
token: &str,
sets: &[TileSet],
seq: i64,
) -> Result<Waiting, String> {
if sets.is_empty() {
return Ok(Waiting::default());
}
let mut query: Vec<(&str, String)> = vec![("seq", seq.to_string())];
query.extend(sets.iter().map(|set| ("set", key(set))));
let url = format!("{}/journal/pending", site_tiles_url(site_url));
let response = http
.get(url)
.bearer_auth(token)
.query(&query)
.timeout(PROBE_TIMEOUT)
.send()
.await
.map_err(|e| format!("the DHP did not answer: {e}"))?;
if !response.status().is_success() {
// A tile service from before the journal answers 404. Nothing to offer, nothing to say.
return Err(format!("the DHP answered {}", response.status()));
}
response.json::<Waiting>().await.map_err(|e| format!("the DHP's answer made no sense: {e}"))
}
enum Fetched {
Written(u64),
/// Already on this PC, byte for byte.
Held,
/// The site has no tile there any more.
Gone,
}
/// Is this exact tile already here? Checked before asking the site, because a PC that has
/// just installed is at seq 0 and will be walked through the site's entire history - nearly
/// all of which its pack already carried. The size is read first and the bytes only when it
/// matches, so a tile that genuinely changed costs one `stat` to find out.
async fn already_held(path: &Path, size: u64, expect: &str) -> bool {
match tokio::fs::metadata(path).await {
Ok(meta) if meta.len() == size => match tokio::fs::read(path).await {
Ok(bytes) => odian_packs::hashing::sha256_hex(&bytes) == expect,
Err(_) => false,
},
_ => false,
}
}
async fn fetch(
http: reqwest::Client,
url: String,
token: String,
path: PathBuf,
expect: String,
size: u64,
cancel: CancellationToken,
) -> Result<Fetched, String> {
if cancel.is_cancelled() {
return Err("cancelled".to_string());
}
if already_held(&path, size, &expect).await {
return Ok(Fetched::Held);
}
let response = http
.get(&url)
.bearer_auth(&token)
.timeout(TILE_TIMEOUT)
.send()
.await
.map_err(|e| format!("{url}: {e}"))?;
if response.status() == reqwest::StatusCode::NOT_FOUND || response.status() == reqwest::StatusCode::GONE {
return Ok(Fetched::Gone);
}
if !response.status().is_success() {
return Err(format!("{url}: {}", response.status()));
}
let bytes = response.bytes().await.map_err(|e| format!("{url}: {e}"))?;
// The journal said what these bytes are. A tile that arrives truncated would otherwise
// be written, cached and drawn as a hole in the map that nothing ever re-fetches.
let got = odian_packs::hashing::sha256_hex(&bytes);
if got != expect {
return Err(format!("{url}: arrived as {got}, the DHP said {expect}"));
}
write_atomic(&path, &bytes)?;
Ok(Fetched::Written(bytes.len() as u64))
}
/// Replay the site's journal from `seq` into this PC's tiles.
///
/// Returns however far it got. A page is only counted as replayed when every row in it
/// landed, so an interrupted pull resumes from a seq this PC can honestly stand behind
/// rather than skipping the tiles it missed.
pub async fn pull(
http: &reqwest::Client,
site_url: &str,
token: &str,
sets: &[TileSet],
data_dir: &Path,
seq: i64,
cancel: &CancellationToken,
report: impl Fn(Progress),
) -> Pulled {
let mut out = Pulled { seq, ..Pulled::default() };
if sets.is_empty() {
return out;
}
let base = site_tiles_url(site_url);
loop {
if cancel.is_cancelled() {
return out;
}
let page = match page(http, site_url, token, sets, out.seq, PAGE).await {
Ok(page) => page,
Err(problem) => {
out.problem = Some(problem);
return out;
}
};
if page.rows.is_empty() {
// Nothing left: this PC is level with the site, including everything the site
// baked for sets this PC does not hold (it will never be offered those).
out.seq = page.head;
return out;
}
let page_tiles = page.rows.len() as u64;
let mut done = 0u64;
let mut work: Vec<(PathBuf, String, String, u64)> = Vec::with_capacity(page.rows.len());
for row in &page.rows {
match tile_path(data_dir, sets, row) {
Some(path) => {
let url = format!("{}/tiles/{}/{}/{}/{}/{}{}", base, row.style, row.dataset, row.z, row.x, row.y, row.ext);
work.push((path, url, row.sha256.clone(), row.size));
}
// A row for something this PC did not ask for. It cannot be placed, so it is
// passed over rather than guessed at - and the page still counts as replayed.
None => done += 1,
}
}
let mut lanes = tokio::task::JoinSet::new();
let mut queue = work.into_iter();
let mut failure: Option<String> = None;
for _ in 0..LANES {
let Some((path, url, sha, size)) = queue.next() else { break };
lanes.spawn(fetch(http.clone(), url, token.to_string(), path, sha, size, cancel.clone()));
}
while let Some(joined) = lanes.join_next().await {
match joined {
Ok(Ok(Fetched::Written(bytes))) => {
out.tiles += 1;
out.bytes += bytes;
}
Ok(Ok(Fetched::Held)) => out.held += 1,
Ok(Ok(Fetched::Gone)) => out.gone += 1,
Ok(Err(problem)) => {
failure.get_or_insert(problem);
}
Err(e) => {
failure.get_or_insert(format!("a tile could not be fetched: {e}"));
}
}
done += 1;
report(Progress { tiles: out.tiles, bytes: out.bytes, page_tiles, page_done: done });
if failure.is_some() || cancel.is_cancelled() {
lanes.abort_all();
break;
}
if let Some((path, url, sha, size)) = queue.next() {
lanes.spawn(fetch(http.clone(), url, token.to_string(), path, sha, size, cancel.clone()));
}
}
if let Some(problem) = failure {
// The seq stays where it was: the tiles that DID land are kept (they are correct,
// and hashed), and the page is simply replayed next time.
out.problem = Some(problem);
return out;
}
if cancel.is_cancelled() {
return out;
}
out.seq = page.next;
if !page.more {
return out;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn set(style: &str, dataset: &str) -> TileSet {
TileSet {
style: style.to_string(),
dataset: dataset.to_string(),
ext: ".png".to_string(),
source: String::new(),
cache_only: false,
}
}
fn row(style: &str, dataset: &str, ext: &str) -> Row {
Row {
style: style.to_string(),
dataset: dataset.to_string(),
z: 5,
x: 1,
y: 2,
ext: ext.to_string(),
size: 10,
sha256: "a".repeat(64),
}
}
#[test]
fn a_row_lands_where_the_tile_server_reads() {
let sets = [set("dark", "ds")];
let path = tile_path(Path::new(r"D:\ODIAN\data"), &sets, &row("dark", "ds", ".png")).unwrap();
assert_eq!(path, Path::new(r"D:\ODIAN\data\tiles\dark\ds\5\1\2.png"));
}
#[test]
fn nothing_in_a_row_is_taken_on_trust() {
// The row names a path this PC will write. Everything about it is checked against
// what this PC already holds, so an answer cannot reach outside the tile store.
let sets = [set("dark", "ds")];
let escape = row(r"..\..\versions", "ds", ".png");
assert!(tile_path(Path::new("data"), &sets, &escape).is_none(), "a style this PC does not hold");
assert!(tile_path(Path::new("data"), &sets, &row("dark", "../src", ".png")).is_none(), "nor a dataset");
assert!(tile_path(Path::new("data"), &sets, &row("dark", "ds", ".exe")).is_none(), "nor any file name");
let mut negative = row("dark", "ds", ".png");
negative.y = -1;
assert!(tile_path(Path::new("data"), &sets, &negative).is_none(), "nor a coordinate that climbs");
let mut bad_digest = row("dark", "ds", ".png");
bad_digest.sha256 = "nonsense".to_string();
assert!(tile_path(Path::new("data"), &sets, &bad_digest).is_none(), "a row with no usable digest");
}
#[tokio::test]
async fn a_tile_this_pc_already_has_is_not_asked_for_again() {
// The first sync after an install walks the site's whole journal, and the pack that
// was just installed carried most of it. Without this every one of those tiles would
// be downloaded a second time to land identical bytes on top of themselves.
let dir = std::env::temp_dir().join(format!("odian-tilesync-{}", uuid::Uuid::new_v4().simple()));
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("5.png");
std::fs::write(&path, b"a tile").unwrap();
let digest = odian_packs::hashing::sha256_hex(b"a tile");
assert!(already_held(&path, 6, &digest).await, "the same bytes");
assert!(!already_held(&path, 6, &"0".repeat(64)).await, "same size, different tile");
assert!(!already_held(&path, 99, &digest).await, "a size that does not match is not even read");
assert!(!already_held(&dir.join("nothing.png"), 6, &digest).await, "a tile this PC lacks");
std::fs::remove_dir_all(&dir).unwrap();
}
#[test]
fn a_set_this_pc_holds_is_named_the_way_the_site_names_it() {
assert_eq!(key(&set("popdark", "protomaps_20260423")), "popdark/protomaps_20260423");
}
}
+1 -1
View File
@@ -1,7 +1,7 @@
{ {
"$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
"productName": "ODIAN Launcher", "productName": "ODIAN Launcher",
"version": "0.1.12", "version": "0.1.24",
"identifier": "fr.odian.launcher", "identifier": "fr.odian.launcher",
"build": { "build": {
"frontendDist": "../dist", "frontendDist": "../dist",
Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.7 KiB

After

Width:  |  Height:  |  Size: 15 KiB

+32
View File
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="iso-8859-1"?>
<!-- Generator: Adobe Illustrator 29.5.1, SVG Export Plug-In . SVG Version: 9.03 Build 0) -->
<svg version="1.1" id="Calque_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
width="256px" height="256px" viewBox="0 0 256 256" enable-background="new 0 0 256 256" xml:space="preserve">
<g>
<g>
<g>
<path id="SVGID_00000111182432350397502420000010888337569888139697_" fill="#DFDFDF" d="M114.311,61.703l-0.025,7.156
c-0.004,1.106-0.745,2.058-1.804,2.376c-11.176,3.361-20.925,10.066-28.086,18.94c-0.424,0.52-0.835,1.052-1.234,1.597
c-0.112,0.136-0.224,0.285-0.324,0.433c-0.549,0.731-1.073,1.474-1.572,2.241c-3.14,4.714-5.594,9.924-7.227,15.477
c-0.315,1.073-1.278,1.822-2.396,1.818l-7.516-0.026c-1.378-0.005-2.479-1.126-2.474-2.491l0.05-14.61
c0.005-1.371,1.12-2.479,2.491-2.474l8.788,0.03c2.818,0.01,5.354-1.124,7.21-2.954c1.732-1.732,2.844-4.074,2.99-6.681
l0.072-20.941c0.005-1.371,1.12-2.479,2.491-2.474l26.092,0.089C113.208,59.216,114.315,60.331,114.311,61.703z"/>
</g>
</g>
<path fill="#DFDFDF" d="M114.328,180.693v7.515c0,1.371-1.112,2.483-2.483,2.483H85.741c-1.365,0-2.483-1.105-2.483-2.483v-18.396
c0-2.805-1.142-5.363-2.979-7.2c-1.85-1.837-4.394-2.979-7.212-2.979h-20.42c-1.371,0-2.483-1.112-2.483-2.483v-24.293
c-0.05-1.167-0.546-2.222-1.328-3.004c-0.695-0.708-1.626-1.179-2.669-1.303H40.32c-1.371,0-2.483-1.112-2.483-2.483v-7.349
c0-1.365,1.105-2.483,2.483-2.483h7.349c1.365,0,2.483,1.117,2.483,2.483l0.012,5.213c0,1.279,0.509,2.433,1.353,3.277
c0.782,0.794,1.837,1.303,3.029,1.353h15.31c1.236,0,2.303,0.907,2.457,2.133c1.107,8.833,4.231,17.044,8.909,24.158
c1.092,1.651,2.259,3.252,3.513,4.779c7.114,8.728,16.754,15.321,27.795,18.677C113.59,178.631,114.328,179.586,114.328,180.693z"
/>
<g>
<g>
<path id="SVGID_00000094603966042395717880000002472666982323682744_" fill="#2474E8" d="M207.917,124.705
c0,43.256-34.364,78.65-77.226,80.288c-1.69,0.065-3.106-1.265-3.106-2.956v-19.229c0-1.562,1.217-2.864,2.777-2.942
c29.166-1.452,52.438-25.64,52.438-55.161c0-29.52-23.272-53.695-52.438-55.146c-1.56-0.078-2.777-1.38-2.777-2.942V47.388
c0-1.691,1.417-3.021,3.106-2.956C173.553,46.069,207.917,81.449,207.917,124.705z"/>
</g>
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 2.2 KiB

+70
View File
@@ -62,6 +62,10 @@
return { label: "Restoring database", tone: "accent", pulse: true, meta: stage }; return { label: "Restoring database", tone: "accent", pulse: true, meta: stage };
case "syncing": case "syncing":
return { label: "Syncing with the DHP", tone: "accent", pulse: true, meta: version(s?.installed) }; return { label: "Syncing with the DHP", tone: "accent", pulse: true, meta: version(s?.installed) };
case "pulling_tiles":
return { label: "Taking map tiles", tone: "accent", pulse: true, meta: tileMeta };
case "removing":
return { label: "Removing", tone: "accent", pulse: true, meta: rp ? `${rp.index}/${rp.count}` : null };
case "update": case "update":
return { label: "Update required", tone: "warn", pulse: false, meta: `${s?.installed} → ${s?.available}` }; return { label: "Update required", tone: "warn", pulse: false, meta: `${s?.installed} → ${s?.available}` };
case "launch": case "launch":
@@ -76,6 +80,29 @@
} }
}); });
// What the DHP has baked, for the sets this PC holds, since it last looked. Zero means
// level with the site - which is worth saying nothing at all about.
// Chosen, and not all here. ODIAN runs without it, so this is said beside Launch rather
// than in place of it - but it IS said: the marketplace was the only place that knew.
const unfinished = $derived(s?.unfinished ?? null);
const unfinishedNames = $derived.by(() => {
const titles = unfinished?.titles ?? [];
if (titles.length === 0) return "";
const shown = titles.slice(0, 2).join(", ");
return (unfinished?.items ?? 0) > titles.length || titles.length > 2 ? `${shown}…` : shown;
});
const tiles = $derived(s?.tiles_waiting ?? null);
const tilesOffered = $derived(tiles !== null && tiles.tiles > 0);
const tilesCount = $derived(tiles === null ? "" : `${formatCount(tiles.tiles)}${tiles.more ? "+" : ""}`);
const rp = $derived(launcher.removeProgress);
const removeFraction = $derived(rp === null || rp.total === 0 ? null : Math.min(rp.done / rp.total, 1));
const tp = $derived(launcher.tileProgress);
const tileMeta = $derived(tp === null ? null : `${formatCount(tp.tiles)} tiles`);
const tileFraction = $derived(tp === null || tp.page_tiles === 0 ? null : tp.page_done / tp.page_tiles);
const showNotice = $derived(launcher.notice !== null && !(action === "error" && launcher.notice === s?.error)); const showNotice = $derived(launcher.notice !== null && !(action === "error" && launcher.notice === s?.error));
</script> </script>
@@ -182,6 +209,29 @@
<button class="btn-link" disabled={blocked} onclick={() => launcher.openSync()}>Sync</button> <button class="btn-link" disabled={blocked} onclick={() => launcher.openSync()}>Sync</button>
</p> </p>
{/if} {/if}
{#if unfinished}
<p class="secondary attention">
<Icon name="alert" size={14} />
<span>
<strong>{unfinished.items}</strong>
{unfinished.items === 1 ? "chosen map is" : "chosen maps are"} only partly downloaded
{#if unfinishedNames}<span class="dim">({unfinishedNames})</span>{/if}.
<button class="btn-link" disabled={blocked} onclick={() => launcher.finishMarket()}>Finish</button>
</span>
</p>
{/if}
{#if tilesOffered}
<!-- Ground another operator looked at while this PC was away. Online it is already
on screen, drawn from {site}; nothing keeps it, so unplugging loses a week of
other people's map. -->
<p class="secondary attention">
<Icon name="download" size={14} />
<span>
<strong>{tilesCount}</strong> new map tiles on {site} ({formatBytes(tiles?.bytes ?? 0)}).
<button class="btn-link" disabled={blocked} onclick={() => launcher.syncTiles()}>Take them</button>
</span>
</p>
{/if}
{:else if action === "installing"} {:else if action === "installing"}
<div class="progress"> <div class="progress">
<div class="stage-line"> <div class="stage-line">
@@ -236,6 +286,26 @@
<ProgressBar value={null} label="Syncing with the DHP" /> <ProgressBar value={null} label="Syncing with the DHP" />
</div> </div>
<p class="secondary">Sending this PC's changes to {site} and taking in the other operators' work.</p> <p class="secondary">Sending this PC's changes to {site} and taking in the other operators' work.</p>
{:else if action === "removing"}
<div class="progress">
<div class="stage-line">
<span class="stage-name">Removing {rp?.title ?? "what you unticked"}</span>
{#if removeFraction !== null}<span class="pct mono tabular">{Math.floor(removeFraction * 100)}%</span>{/if}
</div>
<ProgressBar value={removeFraction} label="Removing" />
{#if rp}<p class="message mono">{formatCount(rp.done)} / {formatCount(rp.total)} files</p>{/if}
</div>
<p class="secondary">Deleting a map is hundreds of thousands of files. Nothing is downloaded meanwhile.</p>
{:else if action === "pulling_tiles"}
<div class="progress">
<div class="stage-line">
<span class="stage-name">Taking map tiles from {site}</span>
{#if tp}<span class="pct mono tabular">{formatBytes(tp.bytes)}</span>{/if}
</div>
<ProgressBar value={tileFraction} label="Taking map tiles" />
</div>
<p class="secondary">Tiles the other operators baked while this PC was away. It can be stopped and resumed.</p>
<button class="btn wide" onclick={() => launcher.cancelInstall()}>Stop</button>
{:else if action === "stack"} {:else if action === "stack"}
<StackList stack={launcher.stack} phase={s?.phase ?? "launching"} /> <StackList stack={launcher.stack} phase={s?.phase ?? "launching"} />
{#if s?.map_from_site === false} {#if s?.map_from_site === false}
+205 -10
View File
@@ -1,9 +1,10 @@
<script lang="ts"> <script lang="ts">
import { untrack } from "svelte"; import { untrack } from "svelte";
import Icon from "./Icon.svelte"; import Icon from "./Icon.svelte";
import { backend } from "../lib/backend";
import { launcher } from "../lib/launcher.svelte"; import { launcher } from "../lib/launcher.svelte";
import { errorText, formatBytes } from "../lib/format"; import { errorText, formatBytes, formatCount } from "../lib/format";
import type { Market, MarketItem } from "../lib/types"; import type { Market, MarketCheck, MarketItem } from "../lib/types";
let dialog: HTMLDialogElement | undefined = $state(); let dialog: HTMLDialogElement | undefined = $state();
let market = $state<Market | null>(null); let market = $state<Market | null>(null);
@@ -11,20 +12,55 @@
let loadError = $state<string | null>(null); let loadError = $state<string | null>(null);
/** The optional items ticked here, by key: the draft the operator applies. */ /** The optional items ticked here, by key: the draft the operator applies. */
let chosen = $state<string[]>([]); let chosen = $state<string[]>([]);
/** What a count of the files on disk found, once it has been taken. */
let checks = $state<Record<string, MarketCheck> | null>(null);
let saved = $state(false); let saved = $state(false);
let applyError = $state<string | null>(null); let applyError = $state<string | null>(null);
/** Apply is waiting on a yes: it would delete something. */
let confirming = $state(false);
let cancelButton: HTMLButtonElement | undefined = $state();
const s = $derived(launcher.status); const s = $derived(launcher.status);
const locked = $derived(launcher.locked); const locked = $derived(launcher.locked);
/** Set while the panel describes a drive to install from, instead of this PC and its DHP. */ /** Set while the panel describes a drive to install from, instead of this PC and its DHP. */
const drive = $derived(launcher.marketDrive); const drive = $derived(launcher.marketDrive);
const siteName = $derived(launcher.siteName);
const applying = $derived(launcher.busy === "market"); const applying = $derived(launcher.busy === "market");
const busy = $derived(launcher.busy !== null); const busy = $derived(launcher.busy !== null);
const required = $derived(market?.items.filter((item) => item.required) ?? []); const required = $derived(market?.items.filter((item) => item.required) ?? []);
const optional = $derived(market?.items.filter((item) => !item.required) ?? []); const optional = $derived(market?.items.filter((item) => !item.required) ?? []);
const picked = (item: MarketItem) => chosen.includes(item.key); const picked = (item: MarketItem) => chosen.includes(item.key);
const dirty = $derived(optional.some((item) => picked(item) !== item.selected)); /**
* What is REALLY there, once counted.
*
* An item's badge comes from the installer's own marker - its note that it finished. Any
* emptying of the folder afterwards leaves that note behind, and the marketplace goes on
* calling it installed. The count is a walk of hundreds of thousands of files, far too
* slow to open on, so the drawer opens on the marker and corrects itself when the count
* comes back.
*/
const shortfall = (item: MarketItem): MarketCheck | null => {
const check = checks?.[item.key];
return check && !check.whole ? check : null;
};
const counting = $derived(checks === null && market !== null && drive === null);
/** What the badge should say, marker and disk together. */
const presenceOf = (item: MarketItem) =>
item.presence === "installed" && shortfall(item) !== null ? "partial" : item.presence;
/**
* What this PC has settled on: what the operator chose, PLUS whatever is actually here.
*
* `selected` answers only "did the operator ask for this", and an item can be installed
* without ever having been asked for - required by the version that installed it (Dark,
* until Dark V2 took its place), taken off a drive, or never downloadable at all. Opening
* on `selected` alone shows those unticked though installed, which this panel states as
* "removed from this PC, frees N GB" and acts on at the next Apply. Nobody asked for that.
*
* It is the OPENING position, not a veto: unticking works exactly as before.
*/
const held = (item: MarketItem) => item.selected || item.presence === "installed";
const dirty = $derived(optional.some((item) => picked(item) !== held(item)));
const adds = $derived(optional.filter((item) => picked(item) && item.presence !== "installed")); const adds = $derived(optional.filter((item) => picked(item) && item.presence !== "installed"));
const removes = $derived(optional.filter((item) => !picked(item) && item.presence !== "absent")); const removes = $derived(optional.filter((item) => !picked(item) && item.presence !== "absent"));
const installed = $derived((market?.installed ?? false) && drive === null); const installed = $derived((market?.installed ?? false) && drive === null);
@@ -49,6 +85,8 @@
const updateFirst = $derived(drive === null && installed && (s?.update_required ?? false)); const updateFirst = $derived(drive === null && installed && (s?.update_required ?? false));
/** A stopped download is work to do even when the choice did not change. */ /** A stopped download is work to do even when the choice did not change. */
const work = $derived(installed ? adds.length + removes.length > 0 : dirty); const work = $derived(installed ? adds.length + removes.length > 0 : dirty);
/** Removed items no download can bring back: only an ODIAN drive carries them. */
const gone = $derived(removes.filter((item) => !item.on_dhp));
// A copy from a drive starts here, so a disk that cannot hold it stops the button too. // A copy from a drive starts here, so a disk that cannot hold it stops the button too.
const blocked = $derived(locked || busy || offlineAdds || updateFirst || ((installed || drive !== null) && tooBig)); const blocked = $derived(locked || busy || offlineAdds || updateFirst || ((installed || drive !== null) && tooBig));
@@ -58,7 +96,9 @@
try { try {
const next = await launcher.loadMarket(drive ?? undefined); const next = await launcher.loadMarket(drive ?? undefined);
market = next; market = next;
chosen = next.items.filter((item) => !item.required && item.selected).map((item) => item.key); chosen = next.items.filter((item) => !item.required && held(item)).map((item) => item.key);
// Not awaited: the list is usable at once, and the badges settle when the count lands.
if (drive === null) void count();
} catch (e) { } catch (e) {
loadError = errorText(e); loadError = errorText(e);
} finally { } finally {
@@ -66,20 +106,69 @@
} }
} }
/** Weigh what the markers claim. Slow, so it runs behind the open list. */
async function count() {
checks = null;
try {
const found = await backend.checkMarket();
checks = Object.fromEntries(found.map((check) => [check.key, check]));
} catch {
// A count that cannot be taken leaves the markers' word standing, which is what the
// marketplace showed before there was a count at all.
checks = {};
}
}
function revert() { function revert() {
chosen = optional.filter((item) => item.selected).map((item) => item.key); chosen = optional.filter((item) => held(item)).map((item) => item.key);
saved = false; saved = false;
applyError = null; applyError = null;
confirming = false;
} }
function toggle(item: MarketItem) { function toggle(item: MarketItem) {
chosen = picked(item) ? chosen.filter((key) => key !== item.key) : [...chosen, item.key]; chosen = picked(item) ? chosen.filter((key) => key !== item.key) : [...chosen, item.key];
saved = false; saved = false;
applyError = null; applyError = null;
// The question was asked about a list that no longer holds.
confirming = false;
} }
// The question replaces the button that asked it, so focus would fall to the document
// and Enter would do nothing at all. It lands on Cancel: the safe answer, and the one a
// keyboard reaches first.
$effect(() => {
if (confirming) cancelButton?.focus();
});
// Escape while the question is up answers "no", and nothing more. Left alone it closes
// the whole drawer - which looks exactly like the removal having been accepted, on the
// one screen where that guess costs 705 GB. Taken on the KEYDOWN rather than the dialog's
// `cancel` event: closing on Escape is that key's default action, and `cancel` does not
// bubble, so a handler on it is the less certain of the two.
$effect(() => {
const el = dialog;
if (!el) return;
const onKey = (event: KeyboardEvent) => {
if (event.key !== "Escape" || !confirming) return;
event.preventDefault();
event.stopPropagation();
confirming = false;
};
el.addEventListener("keydown", onKey, true);
return () => el.removeEventListener("keydown", onKey, true);
});
async function apply() { async function apply() {
if (blocked) return; if (blocked) return;
// Deleting is the one thing here that cannot be taken back by pressing the button
// again: a re-download is hours, and for an item the DHP does not carry there is no
// re-download at all. Adding needs no ceremony, so only a removal is asked about.
if (drive === null && installed && removes.length > 0 && !confirming) {
confirming = true;
return;
}
confirming = false;
// From a drive: save the choice, then copy. The progress shows in the main panel. // From a drive: save the choice, then copy. The progress shows in the main panel.
if (drive !== null) { if (drive !== null) {
if (!(await launcher.setMarket([...chosen], drive))) { if (!(await launcher.setMarket([...chosen], drive))) {
@@ -107,17 +196,29 @@
/** True while the DHP would draw this item itself, so the PC lacking it shows nothing. */ /** True while the DHP would draw this item itself, so the PC lacking it shows nothing. */
const drawnBySite = (item: MarketItem) => const drawnBySite = (item: MarketItem) =>
item.presence === "absent" && drive === null && (s?.online ?? false); item.presence === "absent" && drive === null && (s?.online ?? false) && item.on_dhp;
// Listed, but the DHP has not got it: the terrain source is 705 GB and comes on a drive.
// Only true in the DHP's own marketplace - a drive that carries it is offering it - and
// only while this PC does NOT hold it. Once it is here, where it came from is history:
// saying "not on the DHP" beside "Installed" reads as a fault instead of a fact.
const offDhp = (item: MarketItem) => !item.on_dhp && drive === null && item.presence !== "installed";
function presenceLabel(item: MarketItem): string { function presenceLabel(item: MarketItem): string {
if (item.presence === "installed") return "Installed"; const presence = presenceOf(item);
if (item.presence === "partial") return "Partly downloaded"; if (presence === "installed") return counting ? "Installed, checking…" : "Installed";
if (presence === "partial") return "Partly downloaded";
// "Not installed" would read as missing when nothing is missing: while the DHP // "Not installed" would read as missing when nothing is missing: while the DHP
// answers it draws this itself. What this PC lacks is the copy that works offline. // answers it draws this itself. What this PC lacks is the copy that works offline.
return drawnBySite(item) ? "On the DHP" : "Not installed"; return drawnBySite(item) ? "On the DHP" : "Not installed";
} }
function presenceHelp(item: MarketItem): string | undefined { function presenceHelp(item: MarketItem): string | undefined {
const short = shortfall(item);
if (short) {
return `Its marker says it is installed, but only ${formatCount(short.found)} of ${formatCount(short.expected)} files are on this PC. Re-applying downloads the rest.`;
}
if (offDhp(item)) return "Your DHP does not carry this one. It is installed from an ODIAN drive.";
if (item.presence !== "absent" || drive !== null) return undefined; if (item.presence !== "absent" || drive !== null) return undefined;
return drawnBySite(item) return drawnBySite(item)
? "Your DHP draws this while you are connected. Download it to keep it when you are not." ? "Your DHP draws this while you are connected. Download it to keep it when you are not."
@@ -249,7 +350,7 @@
type="checkbox" type="checkbox"
class="check" class="check"
checked={picked(item)} checked={picked(item)}
disabled={locked || applying} disabled={locked || applying || offDhp(item)}
onchange={() => toggle(item)} onchange={() => toggle(item)}
/> />
<span class="body"> <span class="body">
@@ -259,9 +360,12 @@
</span> </span>
{#if item.description}<span class="desc">{item.description}</span>{/if} {#if item.description}<span class="desc">{item.description}</span>{/if}
<span class="meta"> <span class="meta">
<span class="pill {item.presence}" class:from-site={drawnBySite(item)} title={presenceHelp(item)}> <span class="pill {presenceOf(item)}" class:from-site={drawnBySite(item)} title={presenceHelp(item)}>
{presenceLabel(item)} {presenceLabel(item)}
</span> </span>
{#if offDhp(item)}
<span class="pill off-dhp" title={presenceHelp(item)}>Not on the DHP</span>
{/if}
{#if note}<span class="change" class:loss={!picked(item)}>{note}</span>{/if} {#if note}<span class="change" class:loss={!picked(item)}>{note}</span>{/if}
</span> </span>
</span> </span>
@@ -287,6 +391,39 @@
{/if} {/if}
</div> </div>
{#if confirming}
<section class="confirm" role="alert">
<h3 class="confirm-head">
<Icon name="alert" size={15} />
Remove {removes.length === 1 ? "this" : `these ${removes.length}`} from this PC?
</h3>
<ul class="confirm-list">
{#each removes as item (item.key)}
<li>
<span class="name">{item.title}</span>
<span class="size">{formatBytes(item.bytes)}</span>
{#if !item.on_dhp}<span class="pill off-dhp">Only from a drive</span>{/if}
</li>
{/each}
</ul>
<p class="confirm-note">
Their files are deleted from this PC, freeing {formatBytes(removeBytes)}.
{#if gone.length}
<strong>
{gone.length === 1 ? `${gone[0].title} is not on ${siteName}` : `${gone.length} of them are not on ${siteName}`}:
once removed, only an ODIAN drive can bring {gone.length === 1 ? "it" : "them"} back.
</strong>
{:else}
Taking {removes.length === 1 ? "it" : "them"} again later means downloading
{formatBytes(removeBytes)} from {siteName}.
{/if}
{#if adds.length}
&ensp;·&ensp;{adds.length === 1 ? "One item" : `${adds.length} items`} will be downloaded in the same pass.
{/if}
</p>
</section>
{/if}
<footer class="foot"> <footer class="foot">
<span class="summary" role="status"> <span class="summary" role="status">
{#if applyError} {#if applyError}
@@ -310,6 +447,18 @@
{/if} {/if}
{/if} {/if}
</span> </span>
{#if confirming}
<button
type="button"
class="btn"
bind:this={cancelButton}
disabled={applying}
onclick={() => (confirming = false)}>Cancel</button
>
<button type="button" class="btn btn-danger" disabled={blocked} onclick={apply}>
{applying ? "Removing…" : `Remove ${removes.length === 1 ? "it" : "them"}`}
</button>
{:else}
<button type="button" class="btn" disabled={!dirty || applying} onclick={revert}>Revert</button> <button type="button" class="btn" disabled={!dirty || applying} onclick={revert}>Revert</button>
<button <button
type="button" type="button"
@@ -319,6 +468,7 @@
> >
{#if applying}Applying…{:else if drive !== null}Install from drive{:else if installed}Apply changes{:else}Save choice{/if} {#if applying}Applying…{:else if drive !== null}Install from drive{:else if installed}Apply changes{:else}Save choice{/if}
</button> </button>
{/if}
</footer> </footer>
</div> </div>
</dialog> </dialog>
@@ -552,6 +702,11 @@
border-color: color-mix(in srgb, var(--accent) 45%, transparent); border-color: color-mix(in srgb, var(--accent) 45%, transparent);
color: var(--accent); color: var(--accent);
} }
/* The DHP has not got this one. Not a fault either - it comes on a drive - so it is
stated flatly rather than coloured like something that went wrong. */
.pill.off-dhp {
border-style: dashed;
}
.change { .change {
color: var(--accent-hover); color: var(--accent-hover);
font-size: var(--text-xs); font-size: var(--text-xs);
@@ -579,6 +734,46 @@
font-weight: 400; font-weight: 400;
} }
/* The question, shown where the answer is given: directly above the two buttons. */
.confirm {
padding: 14px 24px;
border-top: 1px solid color-mix(in srgb, var(--danger) 35%, transparent);
background: color-mix(in srgb, var(--danger) 8%, transparent);
}
.confirm-head {
display: flex;
align-items: center;
gap: 8px;
margin: 0 0 8px;
color: var(--danger);
font-size: var(--text-sm);
}
.confirm-list {
margin: 0 0 8px;
padding: 0;
list-style: none;
display: grid;
gap: 4px;
}
.confirm-list li {
display: flex;
align-items: center;
gap: 8px;
font-size: var(--text-sm);
}
.confirm-list .size {
color: var(--muted);
font-family: var(--font-mono);
}
.confirm-note {
margin: 0;
color: var(--muted);
font-size: var(--text-xs);
line-height: 1.5;
}
.confirm-note strong {
color: var(--danger);
}
.foot { .foot {
display: flex; display: flex;
align-items: center; align-items: center;
+8
View File
@@ -10,11 +10,14 @@ import type {
InstallProgress, InstallProgress,
LogEvent, LogEvent,
Market, Market,
MarketCheck,
Settings, Settings,
SignInResult, SignInResult,
StackEvent, StackEvent,
Status, Status,
RemoveProgress,
SyncConflict, SyncConflict,
TileSyncProgress,
Unlisten, Unlisten,
} from "./types"; } from "./types";
@@ -50,6 +53,8 @@ function createTauriBackend(): Backend {
startInstallFromDrive: (path) => invoke<void>("start_install_from_drive", { path }), startInstallFromDrive: (path) => invoke<void>("start_install_from_drive", { path }),
getMarket: (drive) => invoke<Market>("get_market", drive === undefined ? {} : { drive }), getMarket: (drive) => invoke<Market>("get_market", drive === undefined ? {} : { drive }),
applyMarket: (selected) => invoke<void>("apply_market", { selected }), applyMarket: (selected) => invoke<void>("apply_market", { selected }),
checkMarket: () => invoke<MarketCheck[]>("check_market"),
finishMarket: () => invoke<void>("finish_market"),
setMarket: (selected, drive) => invoke<void>("set_market", drive === undefined ? { selected } : { selected, drive }), setMarket: (selected, drive) => invoke<void>("set_market", drive === undefined ? { selected } : { selected, drive }),
cancelInstall: () => invoke<void>("cancel_install"), cancelInstall: () => invoke<void>("cancel_install"),
launch: () => invoke<void>("launch"), launch: () => invoke<void>("launch"),
@@ -58,12 +63,15 @@ function createTauriBackend(): Backend {
setSettings: (settings) => invoke<Status>("set_settings", { settings }), setSettings: (settings) => invoke<Status>("set_settings", { settings }),
pickFolder: () => invoke<string | null>("pick_folder"), pickFolder: () => invoke<string | null>("pick_folder"),
syncNow: () => invoke<void>("sync_now"), syncNow: () => invoke<void>("sync_now"),
syncTiles: () => invoke<void>("sync_tiles"),
getSyncConflicts: () => invoke<SyncConflict[]>("get_sync_conflicts"), getSyncConflicts: () => invoke<SyncConflict[]>("get_sync_conflicts"),
resolveSyncConflict: (tbl, key, choice, value) => resolveSyncConflict: (tbl, key, choice, value) =>
invoke<SyncConflict[]>("resolve_sync_conflict", { tbl, key, choice, value: value === undefined ? null : value }), invoke<SyncConflict[]>("resolve_sync_conflict", { tbl, key, choice, value: value === undefined ? null : value }),
onStatus: (handler) => subscribe<Status>("status", handler), onStatus: (handler) => subscribe<Status>("status", handler),
onInstallProgress: (handler) => subscribe<InstallProgress>("install-progress", handler), onInstallProgress: (handler) => subscribe<InstallProgress>("install-progress", handler),
onTileSyncProgress: (handler) => subscribe<TileSyncProgress>("tile-sync-progress", handler),
onRemoveProgress: (handler) => subscribe<RemoveProgress>("remove-progress", handler),
onStack: (handler) => subscribe<StackEvent>("stack", handler), onStack: (handler) => subscribe<StackEvent>("stack", handler),
onLog: (handler) => subscribe<LogEvent>("log", handler), onLog: (handler) => subscribe<LogEvent>("log", handler),
}; };
+34
View File
@@ -14,7 +14,9 @@ import type {
StackComponent, StackComponent,
StackEvent, StackEvent,
Status, Status,
RemoveProgress,
SyncConflict, SyncConflict,
TileSyncProgress,
Unlisten, Unlisten,
} from "./types"; } from "./types";
@@ -28,6 +30,8 @@ export type ActionKind =
| "installing" | "installing"
| "restoring" | "restoring"
| "syncing" | "syncing"
| "pulling_tiles"
| "removing"
| "update" | "update"
| "launch" | "launch"
| "stack" | "stack"
@@ -51,6 +55,10 @@ export function deriveAction(s: Status | null): ActionKind {
return "restoring"; return "restoring";
case "syncing": case "syncing":
return "syncing"; return "syncing";
case "pulling_tiles":
return "pulling_tiles";
case "removing":
return "removing";
case "launching": case "launching":
case "running": case "running":
return "stack"; return "stack";
@@ -98,6 +106,10 @@ class Launcher {
status = $state<Status | null>(null); status = $state<Status | null>(null);
settings = $state<Settings | null>(null); settings = $state<Settings | null>(null);
progress = $state<InstallProgress | null>(null); progress = $state<InstallProgress | null>(null);
/** While tiles are being taken from the DHP; cleared when the pull ends. */
tileProgress = $state<TileSyncProgress | null>(null);
/** While unticked items are being deleted; cleared when the phase moves on. */
removeProgress = $state<RemoveProgress | null>(null);
/** Smoothed bytes/sec for a steadier ETA. */ /** Smoothed bytes/sec for a steadier ETA. */
speed = $state(0); speed = $state(0);
stack = $state<StackMap>(emptyStack()); stack = $state<StackMap>(emptyStack());
@@ -161,6 +173,8 @@ class Launcher {
await Promise.all([ await Promise.all([
keep(backend.onStatus((s) => this.applyStatus(s))), keep(backend.onStatus((s) => this.applyStatus(s))),
keep(backend.onInstallProgress((p) => this.applyProgress(p))), keep(backend.onInstallProgress((p) => this.applyProgress(p))),
keep(backend.onTileSyncProgress((p) => (this.tileProgress = p))),
keep(backend.onRemoveProgress((p) => (this.removeProgress = p))),
keep(backend.onStack((e) => (this.stack[e.component] = e))), keep(backend.onStack((e) => (this.stack[e.component] = e))),
keep(backend.onLog((l) => this.pushLog(l))), keep(backend.onLog((l) => this.pushLog(l))),
]); ]);
@@ -389,6 +403,26 @@ class Launcher {
}); });
} }
/** Take the tiles other operators baked while this PC was away. */
/** Download what was chosen and never fully arrived. */
finishMarket() {
return this.run("market", async () => {
await backend.finishMarket();
});
}
syncTiles() {
this.tileProgress = null;
return this.run("tiles", async () => {
try {
await backend.syncTiles();
} finally {
// The bar belongs to one pull: left behind, it would describe the last one.
this.tileProgress = null;
}
});
}
syncNow() { syncNow() {
return this.run("sync", async () => { return this.run("sync", async () => {
await backend.syncNow(); await backend.syncNow();
+119 -17
View File
@@ -19,6 +19,7 @@ import {
type LogEvent, type LogEvent,
type LogLevel, type LogLevel,
type Market, type Market,
type MarketCheck,
type MarketItem, type MarketItem,
type MockControls, type MockControls,
type MockScenario, type MockScenario,
@@ -28,7 +29,9 @@ import {
type StackEvent, type StackEvent,
type Status, type Status,
type SyncConflict, type SyncConflict,
type RemoveProgress,
type SyncStatus, type SyncStatus,
type TileSyncProgress,
type Unlisten, type Unlisten,
} from "./types"; } from "./types";
@@ -175,6 +178,8 @@ const MOCK_CONFLICTS: SyncConflict[] = [
type Payloads = { type Payloads = {
status: Status; status: Status;
"install-progress": InstallProgress; "install-progress": InstallProgress;
"tile-sync-progress": TileSyncProgress;
"remove-progress": RemoveProgress;
stack: StackEvent; stack: StackEvent;
log: LogEvent; log: LogEvent;
}; };
@@ -186,20 +191,22 @@ const busy = "Busy: wait for the current task to finish.";
/** The 0.61.0 catalogue with real sizes: what the marketplace lists in the browser mock. */ /** The 0.61.0 catalogue with real sizes: what the marketplace lists in the browser mock. */
const MOCK_MARKET: Omit<MarketItem, "presence" | "selected">[] = [ const MOCK_MARKET: Omit<MarketItem, "presence" | "selected">[] = [
{ key: "tiles:dark/protomaps_20260423", kind: "tiles", title: "Dark map", bytes: 2_437_523_419, required: true, { key: "tiles:dark/protomaps_20260423", kind: "tiles", title: "Dark map", bytes: 2_437_523_419, required: false, on_dhp: true, files: 375_004,
description: "The standard ODIAN basemap: roads, water and borders in the dark style, as pre-drawn at the office. Every install carries it." }, description: "The standard ODIAN basemap: roads, water and borders in the dark style, as pre-drawn at the office. Every install carries it." },
{ key: "tiles:dark_v2/protomaps_20260423", kind: "tiles", title: "Dark V2 map", bytes: 1_649_512_829, required: false, { key: "tiles:dark_v2/protomaps_20260423", kind: "tiles", title: "Dark V2 map", bytes: 1_649_512_829, required: true, on_dhp: true, files: 253_771,
description: "The newer dark basemap, with buildings and forests. It is the Theatre of Operations' default ground: without it (or the Map source) the Theatre shows relief shading until Dark is picked." }, description: "The newer dark basemap, with buildings and forests. It is the Theatre of Operations' default ground: without it (or the Map source) the Theatre shows relief shading until Dark is picked." },
{ key: "tiles:satellite_hd/protomaps_20260423", kind: "tiles", title: "World satellite imagery", bytes: 5_660_999_147, required: false, { key: "tiles:satellite_hd/protomaps_20260423", kind: "tiles", title: "World satellite imagery", bytes: 5_660_999_147, required: false, on_dhp: true, files: 870_923,
description: "Photographic basemap of the whole world, for the map and the Theatre of Operations. It covers what the office pre-cached: this PC cannot fetch more imagery offline." }, description: "Photographic basemap of the whole world, for the map and the Theatre of Operations. It covers what the office pre-cached: this PC cannot fetch more imagery offline." },
{ key: "tiles:orthophotos/protomaps_20260423", kind: "tiles", title: "France orthophotos", bytes: 6_713_284_879, required: false, { key: "tiles:orthophotos/protomaps_20260423", kind: "tiles", title: "France orthophotos", bytes: 6_713_284_879, required: false, on_dhp: true, files: 1_032_813,
description: "High-resolution aerial photographs of France, much sharper than the world imagery over French territory." }, description: "High-resolution aerial photographs of France, much sharper than the world imagery over French territory." },
{ key: "tiles:heightmap/planet_heightmap", kind: "tiles", title: "Terrain elevation", bytes: 74_367_050_377, required: false, { key: "tiles:heightmap/planet_heightmap", kind: "tiles", title: "Terrain elevation", bytes: 74_367_050_377, required: false, on_dhp: true, files: 11_441_085,
description: "The world height map: real 3D relief for the globe and the Theatre of Operations, and ground elevation wherever ODIAN reads it. Without it the ground is flat." }, description: "The world height map: real 3D relief for the globe and the Theatre of Operations, and ground elevation wherever ODIAN reads it. Without it the ground is flat." },
{ key: "src:osm_source/world_aeroway_v1.sqlite", kind: "src", title: "Aeroway source data (OSM)", bytes: 884_244_480, required: false, { key: "src:osm_source/world_aeroway_v1.sqlite", kind: "src", title: "Aeroway source data (OSM)", bytes: 884_244_480, required: false, on_dhp: true, files: 1,
description: "Raw OpenStreetMap airports and runways. ODIAN reads aeroways from its own database, so this is only needed to rebuild that layer." }, description: "Raw OpenStreetMap airports and runways. ODIAN reads aeroways from its own database, so this is only needed to rebuild that layer." },
{ key: "src:protomaps_20260423.pmtiles", kind: "src", title: "Map source (world, vector)", bytes: 135_176_708_563, required: false, { key: "src:protomaps_20260423.pmtiles", kind: "src", title: "Map source (world, vector)", bytes: 135_176_708_563, required: false, on_dhp: true, files: 1,
description: "The vector map data the Dark and Dark V2 basemaps are drawn from. With it this PC draws any tile it lacks, anywhere and at any zoom: the detailed ground the Theatre of Operations needs offline. Without it only the pre-drawn tiles show." }, description: "The vector map data the Dark and Dark V2 basemaps are drawn from. With it this PC draws any tile it lacks, anywhere and at any zoom: the detailed ground the Theatre of Operations needs offline. Without it only the pre-drawn tiles show." },
{ key: "src:planet_heightmap.pmtiles", kind: "src", title: "Terrain source (world, 705 GB)", bytes: 705_591_064_874, required: false, on_dhp: false, files: 1,
description: "The elevation archive the terrain tiles are baked from. With it this PC makes relief for any ground at any zoom, instead of only where the office pre-baked. It is 705 GB and the DHP does not carry it: it arrives on an ODIAN drive, and the Terrain elevation tiles above are what an install gets over the network." },
]; ];
const MOCK_CORE_BYTES = 2_275_409_041 + 15_356_491_648; const MOCK_CORE_BYTES = 2_275_409_041 + 15_356_491_648;
@@ -238,6 +245,8 @@ function initialStatus(scenario: MockScenario): Status {
live: null, live: null,
map_from_site: null, map_from_site: null,
map_problem: null, map_problem: null,
tiles_waiting: null,
unfinished: null,
}; };
switch (scenario) { switch (scenario) {
case "first-run": case "first-run":
@@ -308,6 +317,8 @@ class MockBackend implements Backend {
private readonly listeners: Listeners = { private readonly listeners: Listeners = {
status: new Set(), status: new Set(),
"install-progress": new Set(), "install-progress": new Set(),
"tile-sync-progress": new Set(),
"remove-progress": new Set(),
stack: new Set(), stack: new Set(),
log: new Set(), log: new Set(),
}; };
@@ -327,7 +338,21 @@ class MockBackend implements Backend {
for (const item of MOCK_MARKET) if (item.required) this.marketPresent.set(item.key, "installed"); for (const item of MOCK_MARKET) if (item.required) this.marketPresent.set(item.key, "installed");
this.marketPresent.set("tiles:dark_v2/protomaps_20260423", "installed"); this.marketPresent.set("tiles:dark_v2/protomaps_20260423", "installed");
this.marketPresent.set("tiles:satellite_hd/protomaps_20260423", "partial"); this.marketPresent.set("tiles:satellite_hd/protomaps_20260423", "partial");
this.marketChosen = new Set(["tiles:dark_v2/protomaps_20260423", "tiles:satellite_hd/protomaps_20260423"]); this.marketPresent.set("src:planet_heightmap.pmtiles", "installed");
// Every PC installed before Dark V2 took Dark's place holds Dark without ever having
// chosen it. It must open ticked, or the marketplace offers to delete it.
this.marketPresent.set("tiles:dark/protomaps_20260423", "installed");
// Its marker says installed; the count finds under half the files. This is the state
// the badge used to report as "Installed" and say no more about.
this.marketPresent.set("tiles:orthophotos/protomaps_20260423", "installed");
this.marketChosen = new Set([
"tiles:dark_v2/protomaps_20260423",
"tiles:satellite_hd/protomaps_20260423",
"tiles:orthophotos/protomaps_20260423",
]);
// Satellite stopped half-way and the orthophotos' marker is a lie: two chosen maps
// the hub has to own up to, which is what an operator reported seeing as "Launch".
this.status = derive({ ...this.status, unfinished: this.unfinishedNow() });
} }
const self = this; const self = this;
@@ -361,6 +386,23 @@ class MockBackend implements Backend {
return Promise.resolve(() => void this.listeners[channel].delete(handler)); return Promise.resolve(() => void this.listeners[channel].delete(handler));
} }
/** What was chosen and is not all here, as the launcher works it out. */
private unfinishedNow(): Status["unfinished"] {
const short = new Set(["tiles:orthophotos/protomaps_20260423"]);
const waiting = MOCK_MARKET.filter(
(item) =>
!item.required &&
this.marketChosen.has(item.key) &&
(this.marketPresent.get(item.key) !== "installed" || short.has(item.key)),
);
if (waiting.length === 0) return null;
return {
items: waiting.length,
bytes: waiting.reduce((sum, item) => sum + item.bytes, 0),
titles: waiting.slice(0, 3).map((item) => item.title),
};
}
private commit(patch: Partial<Status>) { private commit(patch: Partial<Status>) {
this.status = derive({ ...this.status, ...patch }); this.status = derive({ ...this.status, ...patch });
this.emit("status", this.status); this.emit("status", this.status);
@@ -520,11 +562,20 @@ class MockBackend implements Backend {
version: this.status.installed ?? this.status.available ?? CURRENT, version: this.status.installed ?? this.status.available ?? CURRENT,
installed: this.status.installed !== null, installed: this.status.installed !== null,
core_bytes: MOCK_CORE_BYTES, core_bytes: MOCK_CORE_BYTES,
items: MOCK_MARKET.map((item) => ({ items: MOCK_MARKET.map((item) => {
const presence = this.marketPresent.get(item.key) ?? "absent";
return {
...item, ...item,
presence: this.marketPresent.get(item.key) ?? "absent", presence,
selected: item.required || this.marketChosen.has(item.key), // Mirrors market.rs `chosen`: an item the DHP cannot re-send counts as chosen
})), // while it is here, or an unticked box would offer to delete what only a drive
// could bring back.
selected:
item.required ||
this.marketChosen.has(item.key) ||
(!item.on_dhp && presence === "installed"),
};
}),
free_bytes: 297_100_000_000, free_bytes: 297_100_000_000,
}; };
} }
@@ -534,21 +585,62 @@ class MockBackend implements Backend {
this.marketChosen = new Set(selected.filter((key) => optional.has(key))); this.marketChosen = new Set(selected.filter((key) => optional.has(key)));
} }
async checkMarket(): Promise<MarketCheck[]> {
await this.wait(700);
// One set is deliberately short here: the badge said installed, the disk disagrees.
return MOCK_MARKET.filter((item) => item.kind === "tiles" && this.marketPresent.get(item.key) === "installed").map(
(item) => {
const expected = Math.max(1, Math.round(item.bytes / 6500));
const found = item.key === "tiles:orthophotos/protomaps_20260423" ? Math.round(expected * 0.42) : expected;
return { key: item.key, found, expected, whole: found >= expected };
},
);
}
async finishMarket(): Promise<void> {
// Mirrors market.rs `held_keys`: chosen OR already here, so nothing is proposed for
// deletion just because no choice about it was ever recorded.
const keep = new Set(this.marketChosen);
for (const item of MOCK_MARKET) {
if (!item.required && this.marketPresent.has(item.key)) keep.add(item.key);
}
await this.applyMarket([...keep]);
}
async applyMarket(selected: string[]): Promise<void> { async applyMarket(selected: string[]): Promise<void> {
const s = this.status; const s = this.status;
if (s.phase !== "idle" && s.phase !== "error") throw busy; if (s.phase !== "idle" && s.phase !== "error") throw busy;
const optional = new Set(MOCK_MARKET.filter((item) => !item.required).map((item) => item.key)); const optional = new Set(MOCK_MARKET.filter((item) => !item.required).map((item) => item.key));
this.marketChosen = new Set(selected.filter((key) => optional.has(key))); this.marketChosen = new Set(selected.filter((key) => optional.has(key)));
if (!s.installed) return; if (!s.installed) return;
for (const item of MOCK_MARKET) { const going = MOCK_MARKET.filter(
if (!item.required && !this.marketChosen.has(item.key) && this.marketPresent.has(item.key)) { (item) => !item.required && !this.marketChosen.has(item.key) && this.marketPresent.has(item.key),
);
const adds = MOCK_MARKET.filter((item) => this.marketChosen.has(item.key) && this.marketPresent.get(item.key) !== "installed");
if (going.length === 0 && adds.length === 0) return;
if (adds.length && !s.online) throw "Your DHP is unreachable: connect to download marketplace items.";
void this.runMarketChanges(going, adds);
}
/** Deleting a map is hundreds of thousands of files: the window watches it happen. */
private async runMarketChanges(going: typeof MOCK_MARKET, adds: typeof MOCK_MARKET) {
const job = ++this.job;
for (const [i, item] of going.entries()) {
this.commit({ phase: "removing", error: null });
// A tile set is ~2 KB a file at these sizes; enough to make the count look real.
const total = item.kind === "tiles" ? Math.max(1, Math.round(item.bytes / 6500)) : 1;
for (let done = 0; done <= total; done += Math.max(1, Math.round(total / 40))) {
if (job !== this.job) return;
this.emit("remove-progress", { title: item.title, done: Math.min(done, total), total, index: i + 1, count: going.length });
await this.wait(120);
}
this.marketPresent.delete(item.key); this.marketPresent.delete(item.key);
this.log("info", `Removed ${item.title}.`); this.log("info", `Removed ${item.title}.`);
} }
if (adds.length === 0) {
this.commit({ phase: "idle" });
return;
} }
const adds = MOCK_MARKET.filter((item) => this.marketChosen.has(item.key) && this.marketPresent.get(item.key) !== "installed");
if (adds.length === 0) return;
if (!s.online) throw "Your DHP is unreachable: connect to download marketplace items.";
void this.runMarketDownload(adds); void this.runMarketDownload(adds);
} }
@@ -640,6 +732,10 @@ class MockBackend implements Backend {
}; };
} }
async syncTiles(): Promise<void> {
// The mock has no DHP to take tiles from; the button is simply a no-op there.
}
async syncNow(): Promise<void> { async syncNow(): Promise<void> {
if (!this.status.online) throw "Your DHP is unreachable: changes stay on this PC until the next connected sync."; if (!this.status.online) throw "Your DHP is unreachable: changes stay on this PC until the next connected sync.";
if (this.status.phase !== "idle" && this.status.phase !== "error") throw "ODIAN is busy: sync once it has stopped."; if (this.status.phase !== "idle" && this.status.phase !== "error") throw "ODIAN is busy: sync once it has stopped.";
@@ -694,6 +790,12 @@ class MockBackend implements Backend {
onInstallProgress(handler: (progress: InstallProgress) => void) { onInstallProgress(handler: (progress: InstallProgress) => void) {
return this.listen("install-progress", handler); return this.listen("install-progress", handler);
} }
onTileSyncProgress(handler: (progress: TileSyncProgress) => void) {
return this.listen("tile-sync-progress", handler);
}
onRemoveProgress(handler: (progress: RemoveProgress) => void) {
return this.listen("remove-progress", handler);
}
onStack(handler: (event: StackEvent) => void) { onStack(handler: (event: StackEvent) => void) {
return this.listen("stack", handler); return this.listen("stack", handler);
} }
+72
View File
@@ -5,6 +5,8 @@ export type Phase =
| "idle" // nothing running | "idle" // nothing running
| "checking" // probing the site | "checking" // probing the site
| "signing_in" // browser sign-in in progress | "signing_in" // browser sign-in in progress
| "pulling_tiles" // taking tiles the DHP baked while this PC was away
| "removing" // deleting what was unticked in the marketplace
| "installing" // downloading + verifying a version | "installing" // downloading + verifying a version
| "restoring" // loading the database snapshot into local Postgres | "restoring" // loading the database snapshot into local Postgres
| "syncing" // exchanging operator data with the DHP (ODIAN is not running) | "syncing" // exchanging operator data with the DHP (ODIAN is not running)
@@ -57,6 +59,61 @@ export interface Status {
map_from_site: boolean | null; map_from_site: boolean | null;
/** Why, in one line, when `map_from_site` is false. */ /** Why, in one line, when `map_from_site` is false. */
map_problem: string | null; map_problem: string | null;
/**
* Tiles the DHP has baked for the sets this PC holds since it last synced. null when
* offline, with nothing installed, or when the DHP's tile service cannot say. Zero
* tiles means this PC is level with the site.
*/
tiles_waiting: TilesWaiting | null;
/**
* Marketplace items the operator chose that are not all on this PC. null when nothing is
* installed or nothing is missing. ODIAN still launches - with less map than was asked for.
*/
unfinished: Unfinished | null;
}
/** What is missing of what was chosen. */
export interface Unfinished {
items: number;
/** What finishing them downloads, in full; the installer skips what is already verified. */
bytes: number;
/** The first few, to name in one line. */
titles: string[];
}
/** What a tile sync would fetch from the DHP. */
export interface TilesWaiting {
tiles: number;
bytes: number;
/** One page's worth, with more behind it: shown as "2000+". */
more: boolean;
}
/** While unticked marketplace items are being deleted. */
export interface RemoveProgress {
title: string;
done: number;
total: number;
/** Which of them, and how many in all: one Apply can drop several. */
index: number;
count: number;
}
/** While a tile sync runs. */
export interface TileSyncProgress {
tiles: number;
bytes: number;
page_tiles: number;
page_done: number;
}
/** One item weighed against what the version says it holds. */
export interface MarketCheck {
key: string;
found: number;
expected: number;
/** At least everything the version names is on disk. More than that is normal. */
whole: boolean;
} }
/** A marketplace item: one tile set or source file of the version. */ /** A marketplace item: one tile set or source file of the version. */
@@ -68,12 +125,19 @@ export interface MarketItem {
description: string; description: string;
/** What installing it downloads, in full. */ /** What installing it downloads, in full. */
bytes: number; bytes: number;
/** How many files it is made of: what a removal counts down, and what a check weighs. */
files: number;
/** Part of every install: shown, never offered for removal. */ /** Part of every install: shown, never offered for removal. */
required: boolean; required: boolean;
/** installed: all of it is on this PC; partial: a download that stopped (installing resumes). */ /** installed: all of it is on this PC; partial: a download that stopped (installing resumes). */
presence: "installed" | "partial" | "absent"; presence: "installed" | "partial" | "absent";
/** Chosen: installed with the version and kept by updates (required items always are). */ /** Chosen: installed with the version and kept by updates (required items always are). */
selected: boolean; selected: boolean;
/**
* False when the DHP does not carry this item's bytes and never will (the 705 GB terrain
* source). It is listed so an operator knows it exists; it arrives on an ODIAN drive.
*/
on_dhp: boolean;
} }
export interface Market { export interface Market {
@@ -238,6 +302,10 @@ export interface Backend {
* ones download (install-progress follows). Before the first install it only saves the choice. * ones download (install-progress follows). Before the first install it only saves the choice.
*/ */
applyMarket(selected: string[]): Promise<void>; applyMarket(selected: string[]): Promise<void>;
/** Count what is really on disk for the items the markers call installed. Slow. */
checkMarket(): Promise<MarketCheck[]>;
/** Download what was chosen and never fully arrived, without changing the choice. */
finishMarket(): Promise<void>;
/** Choose the optional items an install about to run will bring; changes nothing installed. */ /** Choose the optional items an install about to run will bring; changes nothing installed. */
setMarket(selected: string[], drive?: string): Promise<void>; setMarket(selected: string[], drive?: string): Promise<void>;
cancelInstall(): Promise<void>; cancelInstall(): Promise<void>;
@@ -250,6 +318,8 @@ export interface Backend {
pickFolder(): Promise<string | null>; pickFolder(): Promise<string | null>;
/** Push and pull operator data now; only while ODIAN is stopped. */ /** Push and pull operator data now; only while ODIAN is stopped. */
syncNow(): Promise<void>; syncNow(): Promise<void>;
/** Take the tiles the DHP baked while this PC was away. */
syncTiles(): Promise<void>;
/** What the operator has to decide, oldest first. */ /** What the operator has to decide, oldest first. */
getSyncConflicts(): Promise<SyncConflict[]>; getSyncConflicts(): Promise<SyncConflict[]>;
/** Records one decision; resolves with what is left to decide. */ /** Records one decision; resolves with what is left to decide. */
@@ -262,6 +332,8 @@ export interface Backend {
onStatus(handler: (status: Status) => void): Promise<Unlisten>; onStatus(handler: (status: Status) => void): Promise<Unlisten>;
onInstallProgress(handler: (progress: InstallProgress) => void): Promise<Unlisten>; onInstallProgress(handler: (progress: InstallProgress) => void): Promise<Unlisten>;
onTileSyncProgress(handler: (progress: TileSyncProgress) => void): Promise<Unlisten>;
onRemoveProgress(handler: (progress: RemoveProgress) => void): Promise<Unlisten>;
onStack(handler: (event: StackEvent) => void): Promise<Unlisten>; onStack(handler: (event: StackEvent) => void): Promise<Unlisten>;
onLog(handler: (event: LogEvent) => void): Promise<Unlisten>; onLog(handler: (event: LogEvent) => void): Promise<Unlisten>;
} }