From 4a92cc0b63bedeb45c30b4c805cd3d40746373ff Mon Sep 17 00:00:00 2001 From: Kelorastus <108483685+Kelorastus@users.noreply.github.com> Date: Tue, 22 Sep 2026 13:37:29 +0200 Subject: [PATCH] update --- crates/odian-packs/src/manifest.rs | 4 + src-tauri/src/launcher.rs | 67 ++++- src-tauri/src/localdb.rs | 403 ++++++++++++++++++++++++++++- src-tauri/src/stack.rs | 157 ++++++++++- src-tauri/tauri.conf.json | 2 +- 5 files changed, 612 insertions(+), 21 deletions(-) diff --git a/crates/odian-packs/src/manifest.rs b/crates/odian-packs/src/manifest.rs index f5ab27a..ee5e641 100644 --- a/crates/odian-packs/src/manifest.rs +++ b/crates/odian-packs/src/manifest.rs @@ -352,6 +352,10 @@ pub struct DbSection { /// Present when the snapshot comes from a DHP running operator-data sync. #[serde(default, skip_serializing_if = "Option::is_none")] pub sync: Option, + /// Extensions the schema needs (`postgis`): pg_dump -n never emits CREATE EXTENSION, + /// so the restore creates them before the schema. Empty for older snapshots. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub extensions: Vec, } /// Where an install restored from this snapshot starts syncing. diff --git a/src-tauri/src/launcher.rs b/src-tauri/src/launcher.rs index 84196cf..1d002ad 100644 --- a/src-tauri/src/launcher.rs +++ b/src-tauri/src/launcher.rs @@ -658,8 +658,9 @@ impl Launcher { let files: Vec = db .files .iter() - .map(|f| localdb::DbFile { section: f.section.clone(), path: f.path.clone(), size: f.size }) + .map(|f| localdb::DbFile { section: f.section.clone(), table: f.table.clone(), path: f.path.clone(), size: f.size }) .collect(); + let extensions = db.extensions.clone(); let data_root = install_dir.join("data"); let reporter = app.clone(); let stop = cancel.clone(); @@ -684,7 +685,7 @@ impl Launcher { pg.init()?; } pg.start()?; - let result = pg.restore(&data_root, &files, 4, &report, &|| stop.is_cancelled()); + let result = pg.restore(&data_root, &files, &extensions, 4, &report, &|| stop.is_cancelled()); let stopped = pg.stop(); result.and(stopped) }) @@ -791,7 +792,69 @@ impl Launcher { Ok(()) } + /// Add the tables the installed snapshot has and this PC's database lacks. A PC keeps its + /// database across updates, so a table a later version brings (0.59.0: the PostGIS + /// overlays) would never reach it otherwise; its dump is already on disk. Checked on + /// every launch -- one query when nothing is missing -- so a PC that updated before this + /// existed catches up too. Never blocks the launch: ODIAN runs without those tables. + async fn add_missing_tables(&self, app: &AppHandle, config: &StackConfig) { + let Some(db) = stack::installed_db(&config.layout) else { return }; + let files: Vec = db + .files + .iter() + .map(|f| localdb::DbFile { section: f.section.clone(), table: f.table.clone(), path: f.path.clone(), size: f.size }) + .collect(); + let extensions = db.extensions.clone(); + let data_root = config.layout.data_dir(); + let pg = Pg::new(&config.layout, &config.pg_password); + let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::(); + let work = tauri::async_runtime::spawn_blocking(move || -> Result, String> { + let started = !pg.running(); + if started { + pg.start()?; + } + let result = (|| { + // An add cut short last time left tables that look present: drop them first. + pg.drop_interrupted_add()?; + let missing = pg.missing_tables(&files)?; + if !missing.is_empty() { + let _ = tx.send(format!( + "This version brings {} table(s) this PC's database lacks: {}. Adding them from the downloaded snapshot…", + missing.len(), + missing.join(", ") + )); + pg.add_tables(&data_root, &files, &extensions, &missing, &|step| { + let _ = tx.send(format!("Adding tables: {step}")); + })?; + } + Ok(missing) + })(); + if started { + let _ = pg.stop(); + } + result + }); + tokio::pin!(work); + let outcome = loop { + tokio::select! { + Some(line) = rx.recv() => { + self.set_phase(app, Phase::Restoring).await; + self.log(app, "info", line); + } + done = &mut work => break done, + } + }; + match outcome { + Ok(Ok(added)) if added.is_empty() => {} + Ok(Ok(added)) => self.log(app, "info", format!("Added {} table(s) to this PC's database.", added.len())), + Ok(Err(e)) => self.log(app, "warn", format!("Could not add the tables this version brings: {e}")), + Err(e) => self.log(app, "warn", format!("Could not add the tables this version brings: {e}")), + } + } + async fn run_stack(&self, app: &AppHandle, config: StackConfig) { + // A PC that installed before this version keeps its database: give it the new tables. + self.add_missing_tables(app, &config).await; // Take in the other operators' work before the API loads it. Never blocks the launch. self.set_phase(app, Phase::Syncing).await; self.sync_run(app, &config.layout, &config.pg_password, &config.install_id, true).await; diff --git a/src-tauri/src/localdb.rs b/src-tauri/src/localdb.rs index 3292c2a..4c619f0 100644 --- a/src-tauri/src/localdb.rs +++ b/src-tauri/src/localdb.rs @@ -19,6 +19,8 @@ pub const PG_DATABASE: &str = "odian"; #[derive(Clone, Debug)] pub struct DbFile { pub section: String, + /// `schema.table` for a data dump, as the snapshot recorded it. + pub table: Option, /// Relative to `data\`, e.g. `db/20260915T082529Z/data/public.db_ground_poi.dump`. pub path: String, pub size: u64, @@ -33,6 +35,8 @@ pub struct Pg { data: PathBuf, logs: PathBuf, password: String, + /// Always PG_PORT for ODIAN; tests move it so a PC running ODIAN can still run them. + port: u16, } impl Pg { @@ -42,9 +46,16 @@ impl Pg { data: layout.pg_data(), logs: layout.state_dir().join("pg"), password: password.to_string(), + port: PG_PORT, } } + #[cfg(test)] + fn on_port(mut self, port: u16) -> Pg { + self.port = port; + self + } + fn tool(&self, name: &str) -> Command { let mut cmd = command(&self.bin.join(format!("{name}.exe")), &self.logs, &[self.bin.clone()]); cmd.env("PGPASSWORD", &self.password).env("PGCONNECT_TIMEOUT", "5").env("PGAPPNAME", "odian-launcher"); @@ -108,7 +119,7 @@ impl Pg { return Ok(()); } // Another install's Postgres on the shared port would leave only "exit code 1" in the log. - if crate::stack::port_taken(PG_PORT) { + if crate::stack::port_taken(self.port) { return Err("Another ODIAN install is already running on this PC: it holds the database port. Stop it in the launcher window that started it, then try again." .to_string()); } @@ -120,7 +131,7 @@ impl Pg { .arg("-l") .arg(self.logs.join("postgres.log")) .args(["-w", "-t", "120", "-o"]) - .arg(format!("-p {PG_PORT} -c listen_addresses=127.0.0.1")); + .arg(format!("-p {} -c listen_addresses=127.0.0.1", self.port)); // The postmaster inherits pg_ctl's handles and outlives it, so a pipe would never // reach EOF: its output goes to a file. log_output(&mut cmd, &self.logs.join("pg_ctl.log"))?; @@ -148,7 +159,7 @@ impl Pg { } fn psql(&self, database: &str, sql: &str) -> Result { - let port = PG_PORT.to_string(); + let port = self.port.to_string(); let mut cmd = self.tool("psql"); cmd .args(["-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-h", "127.0.0.1", "-p", port.as_str()]) @@ -160,7 +171,7 @@ impl Pg { /// (unaligned, tuples only). For sync, whose JSON payloads are too large for a command line. pub fn script(&self, sql: &str) -> Result { use std::io::Write; - let port = PG_PORT.to_string(); + let port = self.port.to_string(); let mut cmd = self.tool("psql"); cmd .env("PGCLIENTENCODING", "UTF8") @@ -190,6 +201,7 @@ impl Pg { &self, data_root: &Path, files: &[DbFile], + extensions: &[String], workers: usize, progress: RestoreProgress<'_>, cancelled: &(dyn Fn() -> bool + Sync), @@ -197,6 +209,7 @@ impl Pg { self.ensure_logs()?; self.psql("postgres", &format!("DROP DATABASE IF EXISTS {PG_DATABASE}"))?; self.psql("postgres", &format!("CREATE DATABASE {PG_DATABASE} TEMPLATE template0 ENCODING 'UTF8'"))?; + self.create_extensions(extensions)?; let total_bytes: u64 = files.iter().map(|f| f.size).sum(); let done_bytes = AtomicU64::new(0); @@ -252,7 +265,7 @@ impl Pg { fn restore_one(&self, data_root: &Path, file: &DbFile, pre_data: bool) -> Result<(), String> { let dump = file.path.split('/').fold(data_root.to_path_buf(), |path, part| path.join(part)); - let port = PG_PORT.to_string(); + let port = self.port.to_string(); let mut cmd = self.tool("pg_restore"); cmd .args(["-h", "127.0.0.1", "-p", port.as_str(), "-U", PG_USER, "-d", PG_DATABASE]) @@ -281,4 +294,384 @@ impl Pg { std::fs::write(&path, kept).map_err(|e| format!("cannot write {}: {e}", path.display()))?; Ok(path) } + + /// Create the snapshot's extensions (`postgis`). A PostGIS table needs its types before + /// its CREATE TABLE, and pg_dump -n never emits CREATE EXTENSION. The names come from + /// the signed manifest; checked anyway, since they are spliced into SQL. + fn create_extensions(&self, extensions: &[String]) -> Result<(), String> { + for name in extensions { + if !is_plain_name(name) { + return Err(format!("the snapshot names an extension this PC will not create: {name:?}")); + } + self.psql(PG_DATABASE, &format!("CREATE EXTENSION IF NOT EXISTS {name}"))?; + } + Ok(()) + } + + /// The snapshot's tables this database lacks, as `schema.table`. + /// + /// A PC keeps its database when it updates -- that is where the operator's work lives -- + /// so a table a later snapshot adds (0.59.0: the PostGIS overlays) would never reach it. + /// Its dump is on disk regardless: an update downloads the whole snapshot. + pub fn missing_tables(&self, files: &[DbFile]) -> Result, String> { + let tables = candidate_tables(files); + if tables.is_empty() { + return Ok(Vec::new()); + } + let list = tables.iter().map(|t| format!("'{t}'")).collect::>().join(","); + let out = self.script(&format!( + "SELECT t FROM unnest(ARRAY[{list}]::text[]) AS t WHERE to_regclass(t) IS NULL ORDER BY t;" + ))?; + Ok(out.lines().map(str::trim).filter(|l| !l.is_empty()).map(str::to_string).collect()) + } + + /// Drop the tables an add was in the middle of when it was cut short (power loss, the + /// launcher killed): they would read as present and never be filled. The marker names + /// them; add_tables writes it before it creates anything and removes it when done. + pub fn drop_interrupted_add(&self) -> Result<(), String> { + let marker = self.logs.join(ADDING_MARKER); + let Ok(text) = std::fs::read_to_string(&marker) else { return Ok(()) }; + let tables: Vec<&str> = text.lines().map(str::trim).filter(|t| is_qualified_plain(t)).collect(); + if !tables.is_empty() { + self.script(&format!("DROP TABLE IF EXISTS {} CASCADE;", tables.join(", ")))?; + } + std::fs::remove_file(&marker).map_err(|e| format!("cannot remove {}: {e}", marker.display())) + } + + /// Add `tables` to this database from the snapshot on disk, touching nothing else: the + /// snapshot's extensions, then those tables' own schema statements, their rows, their + /// sequence positions, and last their indexes and constraints. + pub fn add_tables( + &self, + data_root: &Path, + files: &[DbFile], + extensions: &[String], + tables: &[String], + progress: &dyn Fn(String), + ) -> Result<(), String> { + self.ensure_logs()?; + self.create_extensions(extensions)?; + let marker = self.logs.join(ADDING_MARKER); + std::fs::write(&marker, tables.join("\n")).map_err(|e| format!("cannot write {}: {e}", marker.display()))?; + let result = self.add_tables_once(data_root, files, tables, progress); + if result.is_err() { + // A half-added table would read as present from now on and never be filled: drop + // what this run created, so the next launch starts again from nothing. Should even + // this fail, the marker stays and the next launch drops them first. + let list = tables.join(", "); + if self.script(&format!("DROP TABLE IF EXISTS {list} CASCADE;")).is_err() { + return result; + } + } + let _ = std::fs::remove_file(&marker); + result + } + + fn add_tables_once(&self, data_root: &Path, files: &[DbFile], tables: &[String], progress: &dyn Fn(String)) -> Result<(), String> { + let section = |name: &str| files.iter().find(|f| f.section == name); + + let pre = section("pre-data").ok_or("the snapshot has no schema dump")?; + let (preamble, blocks) = sql_blocks(&self.render(data_root, pre)?); + let schema = schema_blocks(&blocks, tables); + let sequences: Vec = schema.iter().filter(|b| b.kind == "SEQUENCE").map(|b| b.qualified()).collect(); + progress(format!("creating {} table(s)", tables.len())); + self.script(&join_blocks(&preamble, &schema))?; + + for file in files.iter().filter(|f| f.section == "data") { + if let Some(table) = table_of(file).filter(|t| tables.contains(t)) { + progress(format!("loading {table}")); + self.restore_one(data_root, file, false)?; + } + } + + if let Some(file) = section("sequences") { + let (preamble, blocks) = sql_blocks(&self.render(data_root, file)?); + let positions: Vec<&SqlBlock> = blocks.iter().filter(|b| sequences.contains(&b.qualified())).collect(); + if !positions.is_empty() { + self.script(&join_blocks(&preamble, &positions))?; + } + } + + if let Some(file) = section("post-data") { + progress("building their indexes".to_string()); + let (preamble, blocks) = sql_blocks(&self.render(data_root, file)?); + let indexes = index_blocks(&blocks, tables); + if !indexes.is_empty() { + self.script(&format!("SET maintenance_work_mem = '512MB';\n{}", join_blocks(&preamble, &indexes)))?; + } + } + Ok(()) + } + + /// A dump rendered as the SQL pg_restore would run. + fn render(&self, data_root: &Path, file: &DbFile) -> Result { + let dump = file.path.split('/').fold(data_root.to_path_buf(), |path, part| path.join(part)); + let mut cmd = self.tool("pg_restore"); + cmd.args(["--no-owner", "--no-privileges", "-f", "-"]).arg(&dump); + Self::run(cmd, &format!("pg_restore {}", file.path)) + } +} + +/// The file naming the tables an add has started on; see `drop_interrupted_add`. +const ADDING_MARKER: &str = "adding-tables.txt"; + +/// Tables that describe the database they live in, not data: never copied into another +/// one. `odian_schema_migrations` lists the migrations applied HERE -- the live database's +/// copy would make this PC's Engine API skip migrations its schema never had. +const OWN_DATABASE_ONLY: [&str; 1] = ["public.odian_schema_migrations"]; + +/// The snapshot's data tables an existing database may lack and be given. +fn candidate_tables(files: &[DbFile]) -> Vec { + files + .iter() + .filter(|f| f.section == "data") + .filter_map(table_of) + .filter(|t| !OWN_DATABASE_ONLY.contains(&t.as_str())) + .collect() +} + +/// `schema.table`, both plain names. +fn is_qualified_plain(name: &str) -> bool { + name.split_once('.').is_some_and(|(schema, table)| is_plain_name(schema) && is_plain_name(table)) +} + +/// Lower-case letters, digits and `_`: safe to splice into SQL unquoted. +fn is_plain_name(name: &str) -> bool { + !name.is_empty() && name.chars().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_') +} + +/// A data dump's table as `schema.table`, when it is a plain name. +fn table_of(file: &DbFile) -> Option { + let name = file.table.clone().or_else(|| { + let stem = file.path.rsplit('/').next()?.strip_suffix(".dump")?; + Some(stem.to_string()) + })?; + let (schema, table) = name.split_once('.')?; + (is_plain_name(schema) && is_plain_name(table)).then_some(name) +} + +/// One object from pg_restore's SQL: the `-- Name: x; Type: T; Schema: s` header pg_dump +/// writes before it, and the statements under it. +#[derive(Debug)] +struct SqlBlock { + name: String, + kind: String, + schema: String, + sql: String, +} + +impl SqlBlock { + fn qualified(&self) -> String { + format!("{}.{}", self.schema, self.name) + } +} + +/// Split pg_restore's SQL into its session settings and one block per object. Lines that +/// are psql meta-commands (`\restrict` and its pair) guard a whole dump; a partial replay +/// must leave them out or psql refuses to run it. +fn sql_blocks(text: &str) -> (String, Vec) { + let mut preamble = String::new(); + let mut blocks: Vec = Vec::new(); + for line in text.lines() { + if line.starts_with('\\') { + continue; + } + if let Some(header) = line.strip_prefix("-- Name: ") { + let field = |key: &str| header.split("; ").find_map(|p| p.strip_prefix(key)).unwrap_or_default().to_string(); + let name = header.split("; ").next().unwrap_or_default().to_string(); + blocks.push(SqlBlock { name, kind: field("Type: "), schema: field("Schema: "), sql: String::new() }); + } else if let Some(block) = blocks.last_mut() { + block.sql.push_str(line); + block.sql.push('\n'); + } else if !line.starts_with("--") { + preamble.push_str(line); + preamble.push('\n'); + } + } + (preamble, blocks) +} + +/// Does `sql` name `qualified` (`schema.table`) as a whole identifier, not as a prefix? +fn references(sql: &str, qualified: &str) -> bool { + let ident = |c: char| c.is_ascii_alphanumeric() || c == '_'; + sql.match_indices(qualified).any(|(at, found)| { + let before = sql[..at].chars().next_back(); + let after = sql[at + found.len()..].chars().next(); + !before.is_some_and(|c| ident(c) || c == '.' || c == '"') && !after.is_some_and(ident) + }) +} + +/// The schema statements that create `tables`: the tables themselves, their identity or +/// serial sequences, defaults and comments. Functions, views and the like are left alone: +/// they may already exist here, and CREATE without OR REPLACE would stop the whole script. +fn schema_blocks<'a>(blocks: &'a [SqlBlock], tables: &[String]) -> Vec<&'a SqlBlock> { + const TABLE_SCOPED: [&str; 5] = ["TABLE", "SEQUENCE", "SEQUENCE OWNED BY", "DEFAULT", "COMMENT"]; + let mut keep: Vec = blocks + .iter() + .map(|b| TABLE_SCOPED.contains(&b.kind.as_str()) && tables.iter().any(|t| references(&b.sql, t))) + .collect(); + // A serial column's CREATE SEQUENCE names no table; its OWNED BY and DEFAULT do. + let kept_sql: Vec<&str> = blocks.iter().zip(&keep).filter(|(_, k)| **k).map(|(b, _)| b.sql.as_str()).collect(); + for (i, block) in blocks.iter().enumerate() { + if !keep[i] && block.kind == "SEQUENCE" && kept_sql.iter().any(|sql| references(sql, &block.qualified())) { + keep[i] = true; + } + } + blocks.iter().zip(keep).filter(|(_, k)| *k).map(|(b, _)| b).collect() +} + +/// The indexes, constraints and triggers of `tables` -- never the sync marking triggers, +/// which call sync.mark() from a schema the launcher loads itself. +fn index_blocks<'a>(blocks: &'a [SqlBlock], tables: &[String]) -> Vec<&'a SqlBlock> { + const TABLE_SCOPED: [&str; 5] = ["INDEX", "CONSTRAINT", "FK CONSTRAINT", "TRIGGER", "INDEX ATTACH"]; + blocks + .iter() + .filter(|b| TABLE_SCOPED.contains(&b.kind.as_str())) + .filter(|b| !(b.kind == "TRIGGER" && b.name.starts_with("sync_mark_"))) + .filter(|b| tables.iter().any(|t| references(&b.sql, t))) + .collect() +} + +fn join_blocks(preamble: &str, blocks: &[&SqlBlock]) -> String { + blocks.iter().fold(preamble.to_string(), |mut sql, block| { + sql.push_str(&block.sql); + sql + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Abridged pg_restore output, shaped exactly like the 0.59.0 snapshot's. + const PRE: &str = "--\n-- PostgreSQL database dump\n--\n\n\\restrict KEY\n\nSET client_encoding = 'UTF8';\n\ +SELECT pg_catalog.set_config('search_path', '', false);\n\n\ +--\n-- Name: odb_note; Type: TABLE; Schema: public; Owner: -\n--\n\nCREATE TABLE public.odb_note (\n id integer\n);\n\n\ +--\n-- Name: postgis_hospital; Type: TABLE; Schema: public; Owner: -\n--\n\nCREATE TABLE public.postgis_hospital (\n id integer NOT NULL,\n geom public.geometry\n);\n\n\ +--\n-- Name: postgis_hospital_id_seq; Type: SEQUENCE; Schema: public; Owner: -\n--\n\n\ +ALTER TABLE public.postgis_hospital ALTER COLUMN id ADD GENERATED ALWAYS AS IDENTITY (\n SEQUENCE NAME public.postgis_hospital_id_seq\n);\n\n\ +--\n-- Name: postgis_hospital_x; Type: TABLE; Schema: public; Owner: -\n--\n\nCREATE TABLE public.postgis_hospital_x (\n id integer\n);\n\n\ +--\n-- Name: hospital_count; Type: FUNCTION; Schema: public; Owner: -\n--\n\n\ +CREATE FUNCTION public.hospital_count() RETURNS bigint AS $$ SELECT count(*) FROM public.postgis_hospital $$ LANGUAGE sql;\n\n\ +\\unrestrict KEY\n"; + + #[test] + fn blocks_keep_the_session_settings_and_drop_psql_guards() { + let (preamble, blocks) = sql_blocks(PRE); + assert!(preamble.contains("set_config('search_path', '', false)")); + assert!(!preamble.contains("restrict") && blocks.iter().all(|b| !b.sql.contains("restrict"))); + assert_eq!(blocks.len(), 5); + assert_eq!((blocks[1].name.as_str(), blocks[1].kind.as_str(), blocks[1].qualified()), + ("postgis_hospital", "TABLE", "public.postgis_hospital".to_string())); + } + + #[test] + fn a_table_is_referenced_whole_never_as_a_prefix() { + assert!(references("CREATE INDEX i ON public.postgis_hospital USING gist (geom);", "public.postgis_hospital")); + assert!(references("ALTER SEQUENCE s OWNED BY public.postgis_hospital.id;", "public.postgis_hospital")); + assert!(!references("CREATE TABLE public.postgis_hospital_x (", "public.postgis_hospital")); + assert!(!references("CREATE TABLE xpublic.postgis_hospital (", "public.postgis_hospital")); + } + + #[test] + fn only_the_missing_tables_own_statements_are_replayed() { + let (_, blocks) = sql_blocks(PRE); + let chosen: Vec = schema_blocks(&blocks, &["public.postgis_hospital".to_string()]) + .iter() + .map(|b| b.qualified()) + .collect(); + // Its table and identity sequence; not a look-alike table, not an existing one, and not a + // function that merely queries it (CREATE FUNCTION would fail on a PC that has it). + assert_eq!(chosen, ["public.postgis_hospital", "public.postgis_hospital_id_seq"]); + } + + #[test] + fn indexes_follow_their_table_but_sync_triggers_never_do() { + let post = "--\n-- Name: idx_postgis_hospital_geom; Type: INDEX; Schema: public; Owner: -\n--\n\n\ +CREATE INDEX idx_postgis_hospital_geom ON public.postgis_hospital USING gist (geom);\n\n\ +--\n-- Name: idx_note; Type: INDEX; Schema: public; Owner: -\n--\n\nCREATE INDEX idx_note ON public.odb_note USING btree (id);\n\n\ +--\n-- Name: sync_mark_ins; Type: TRIGGER; Schema: public; Owner: -\n--\n\n\ +CREATE TRIGGER sync_mark_ins AFTER INSERT ON public.postgis_hospital FOR EACH STATEMENT EXECUTE FUNCTION sync.mark();\n"; + let (_, blocks) = sql_blocks(post); + let chosen: Vec<&str> = index_blocks(&blocks, &["public.postgis_hospital".to_string()]) + .iter() + .map(|b| b.name.as_str()) + .collect(); + assert_eq!(chosen, ["idx_postgis_hospital_geom"]); + } + + /// The real thing on real snapshots: a database restored from the snapshot before PostGIS + /// (as every PC that installed then has it), then given the missing tables of a later one. + /// ODIAN_ADD_TABLES_ROOT: a layout with versions/test/runtime/pgsql (carrying PostGIS) and + /// both snapshots under data/db/. ODIAN_ADD_TABLES_OLD / _NEW: their snapshot.json files. + #[test] + #[ignore = "needs real snapshots and a PostGIS runtime: see the doc comment"] + fn adds_the_new_tables_to_a_database_restored_before_them() { + let env = |key: &str| std::env::var(key).unwrap_or_else(|_| panic!("set {key}")); + let root = PathBuf::from(env("ODIAN_ADD_TABLES_ROOT")); + let snapshot_files = |json: &str| -> Vec { + let doc: serde_json::Value = serde_json::from_slice(&std::fs::read(json).unwrap()).unwrap(); + doc["files"] + .as_array() + .unwrap() + .iter() + .map(|f| DbFile { + section: f["section"].as_str().unwrap().into(), + table: f["table"].as_str().map(str::to_string), + path: f["path"].as_str().unwrap().into(), + size: f["size"].as_u64().unwrap(), + }) + .collect() + }; + let old = snapshot_files(&env("ODIAN_ADD_TABLES_OLD")); + let new = snapshot_files(&env("ODIAN_ADD_TABLES_NEW")); + let data_root = root.join("data"); + let pg = Pg::new(&Layout::new(&root, "test"), "pw").on_port(55466); + if !pg.is_initialized() { + pg.init().unwrap(); + } + pg.start().unwrap(); + let never = || false; + let quiet = |_: u64, _: u64, _: usize, _: usize, _: &str| {}; + pg.restore(&data_root, &old, &[], 4, &quiet, &never).unwrap(); + let pois_before = pg.script("SELECT count(*) FROM public.db_ground_poi;").unwrap(); + + let missing = pg.missing_tables(&new).unwrap(); + eprintln!("missing before: {missing:?}"); + let added = pg.add_tables(&data_root, &new, &["postgis".to_string()], &missing, &|step| eprintln!(" {step}")); + let hospitals = pg.script("SELECT count(*) FROM public.postgis_hospital;"); + let spatial = pg.script( + "SELECT count(*) FROM pg_indexes WHERE tablename = 'postgis_hospital' AND indexdef LIKE '%USING gist%';", + ); + let pois_after = pg.script("SELECT count(*) FROM public.db_ground_poi;"); + let still_missing = pg.missing_tables(&new); + let _ = pg.stop(); + + added.unwrap(); + assert!(missing.contains(&"public.postgis_hospital".to_string()), "{missing:?}"); + assert!(hospitals.unwrap().trim().parse::().unwrap() > 100_000, "the rows arrived"); + assert_eq!(spatial.unwrap().trim(), "1", "with their spatial index"); + assert_eq!(pois_before, pois_after.unwrap(), "an existing table is not touched"); + assert!(still_missing.unwrap().is_empty(), "nothing is left to add"); + } + + #[test] + fn the_migrations_ledger_never_travels_between_databases() { + let file = |table: &str| DbFile { section: "data".into(), table: Some(table.into()), path: "x".into(), size: 0 }; + let files = [file("public.postgis_oil"), file("public.odian_schema_migrations"), file("public.osm_roads_import_runs")]; + assert_eq!(candidate_tables(&files), ["public.postgis_oil", "public.osm_roads_import_runs"]); + } + + #[test] + fn a_dump_names_its_table_even_without_the_manifest_field() { + let file = |table: Option<&str>, path: &str| DbFile { + section: "data".into(), + table: table.map(str::to_string), + path: path.into(), + size: 0, + }; + assert_eq!(table_of(&file(Some("public.postgis_oil"), "x")), Some("public.postgis_oil".into())); + assert_eq!(table_of(&file(None, "db/s/data/public.postgis_gas.dump")), Some("public.postgis_gas".into())); + assert_eq!(table_of(&file(None, "db/s/data/public.Weird Name.dump")), None, "never spliced into SQL"); + } } diff --git a/src-tauri/src/stack.rs b/src-tauri/src/stack.rs index e301f14..6e5483e 100644 --- a/src-tauri/src/stack.rs +++ b/src-tauri/src/stack.rs @@ -247,8 +247,16 @@ pub fn port_taken(port: u16) -> bool { /// Every install on this PC uses the same three loopback ports, so only one runs at a time. /// Said plainly here, rather than as "Postgres did not start" once the ports collide. -fn check_ports_free() -> Result<(), String> { +/// +/// `own_db_running`: this install's database is already up. pg_ctl starts Postgres detached, +/// so it outlives the launcher window that started it — close the launcher, or let it +/// restart for an update, and the next launch finds its own database on the port. That is +/// not another install: the stack adopts it (`Pg::start` returns early when it is running). +fn check_ports_free(own_db_running: bool) -> Result<(), String> { for (port, what) in [(PG_PORT, "the database"), (API_PORT, "the Engine API"), (TILES_PORT, "the tile server")] { + if port == PG_PORT && own_db_running { + continue; + } if port_taken(port) { return Err(format!( "Another ODIAN install is already running on this PC ({what} holds 127.0.0.1:{port}). Stop it in the launcher window that started it, then launch again." @@ -259,13 +267,12 @@ fn check_ports_free() -> Result<(), String> { } pub async fn start(http: &reqwest::Client, config: &StackConfig, emit: Emit<'_>) -> Result { - check_ports_free()?; + let pg = Pg::new(&config.layout, &config.pg_password); + // Asks pg_ctl about this install's own data directory, so a database on the port that + // belongs to another install still reads as someone else's. + check_ports_free(pg.running())?; prepare(config)?; - let mut running = RunningStack { - job: job::Job::new()?, - pg: Pg::new(&config.layout, &config.pg_password), - children: Vec::new(), - }; + let mut running = RunningStack { job: job::Job::new()?, pg, children: Vec::new() }; match start_all(http, config, emit, &mut running).await { Ok(()) => Ok(running), Err(message) => { @@ -432,6 +439,25 @@ fn pg_ini(config: &StackConfig) -> String { ) } +/// How the tile server reaches this PC's database (a libpq connection string). +const TILES_POSTGIS_DSN_ENV: &str = "ODIAN_TILE_POSTGIS_DSN"; + +/// The Theatre's POI layers -- hospitals, schools, places of worship, data centres. A site +/// serves them from the extractor's SQLite files; this PC ships none of those, but holds the +/// same tables in its own database, and the tile server reads them there instead. +const NEARBY_POSTGIS_SOURCES: [&str; 4] = + ["hospital_postgis", "school_postgis", "place_of_worship_postgis", "datacenter_postgis"]; + +fn pg_dsn(config: &StackConfig) -> String { + let quote = |value: &str| format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")); + format!( + "host=127.0.0.1 port={PG_PORT} dbname={} user={} password={} connect_timeout=5", + quote(crate::localdb::PG_DATABASE), + quote(crate::localdb::PG_USER), + quote(&config.pg_password) + ) +} + /// What a tile of this kind is served as. Anything else is bytes. fn media_type(ext: &str) -> &'static str { match ext { @@ -449,9 +475,14 @@ fn datasets(config: &StackConfig) -> serde_json::Value { for set in &config.tile_sets { let entry = doc.entry(set.dataset.clone()).or_insert_with(|| { let mut entry = serde_json::json!({ "cache_dir": text(tiles.clone()) }); - let pmtiles = config.layout.data_dir().join("src").join(format!("{}.pmtiles", set.dataset)); - if pmtiles.is_file() { - entry["pmtiles"] = serde_json::Value::String(text(pmtiles)); + match pmtiles_of(config, &set.dataset) { + Some(pmtiles) => entry["pmtiles"] = serde_json::Value::String(text(pmtiles)), + // No archive on this PC (the height map's is 705 GB): serve the tiles, bake none. + // Without the flag a baking server refuses every request for it, hits included. + None => entry["cache_only"] = serde_json::Value::Bool(true), + } + for key in NEARBY_POSTGIS_SOURCES { + entry[key] = serde_json::json!({ "dsn_env": TILES_POSTGIS_DSN_ENV }); } entry }); @@ -463,7 +494,13 @@ fn datasets(config: &StackConfig) -> serde_json::Value { .expect("a dataset entry is an object") .entry("styles") .or_insert_with(|| serde_json::Value::Object(serde_json::Map::new())); - styles[&set.style] = serde_json::json!({ "tile_ext": set.ext, "media_type": media_type(&set.ext) }); + let mut style = serde_json::json!({ "tile_ext": set.ext, "media_type": media_type(&set.ext) }); + // Imagery is photographed, not drawn: a baking server must never render a vector map + // into a satellite tile. Marked remote, a miss is a clean 404 (the PC has no upstream). + if set.ext == ".jpg" || set.ext == ".jpeg" { + style["source"] = serde_json::Value::String("wmts".to_string()); + } + styles[&set.style] = style; } } serde_json::Value::Object(doc) @@ -513,15 +550,35 @@ fn api_env(config: &StackConfig) -> Vec<(&'static str, String)> { env } +/// The archive a dataset's basemap can be baked from, when this install ships it. +fn pmtiles_of(config: &StackConfig, dataset: &str) -> Option { + let path = config.layout.data_dir().join("src").join(format!("{dataset}.pmtiles")); + path.is_file().then_some(path) +} + +/// The tile baker, when the version ships one AND some installed dataset has an archive +/// to bake from. Without both the tile server stays cache-only, exactly as before. +fn baker(config: &StackConfig) -> Option { + let exe = parent(&config.layout.app_file(TILE_SERVER_APP)).join("tile_baker.exe"); + let any_archive = config.tile_sets.iter().any(|set| pmtiles_of(config, &set.dataset).is_some()); + (any_archive && exe.is_file()).then_some(exe) +} + fn tiles_env(config: &StackConfig) -> Vec<(&'static str, String)> { let tiles = config.layout.data_dir().join("tiles"); + // With a baker the server draws any basemap tile the cache lacks -- the Theatre asks + // for far more than was ever cached. Everything it cannot draw stays cache-only per + // dataset or style (see `datasets`), and an offline PC never reaches a provider. + let baker = baker(config); vec![ ("ODIAN_TILE_DATASETS", text(config.layout.state_dir().join("tiles").join("datasets.json"))), - ("ODIAN_TILE_CACHE_ONLY", "1".to_string()), + ("ODIAN_TILE_CACHE_ONLY", if baker.is_some() { "0" } else { "1" }.to_string()), + ("ODIAN_TILE_NO_UPSTREAM", "1".to_string()), ("ODIAN_TILE_CACHE_DIR", text(tiles.clone())), ("ODIAN_TILE_JOURNAL", text(tiles.join("tiles.db"))), ("ODIAN_TILE_ORIGIN", config.install_id.clone()), - ("ODIAN_TILE_BAKER_EXE", String::new()), + ("ODIAN_TILE_BAKER_EXE", baker.map(text).unwrap_or_default()), + (TILES_POSTGIS_DSN_ENV, pg_dsn(config)), ("PYTHONDONTWRITEBYTECODE", "1".to_string()), ("PYTHONNOUSERSITE", "1".to_string()), ("PYTHONUTF8", "1".to_string()), @@ -585,6 +642,14 @@ pub struct TileSet { pub ext: String, } +/// The installed version's database snapshot: its dump files and the extensions it needs. +/// None for a version without one, or a manifest that cannot be read. +pub fn installed_db(layout: &Layout) -> Option { + let bytes = std::fs::read(layout.version_dir().join("manifest.json")).ok()?; + let manifest: serde_json::Value = serde_json::from_slice(&bytes).ok()?; + serde_json::from_value(manifest.get("db")?.clone()).ok() +} + /// The tile sets the installed version's manifest names. pub fn installed_tile_sets(layout: &Layout) -> Result, String> { let path = layout.version_dir().join("manifest.json"); @@ -736,6 +801,24 @@ mod tests { assert_eq!(doc["planet_heightmap"]["styles"]["heightmap"]["tile_ext"], ".lerc"); } + #[test] + fn the_theatre_pois_are_read_from_this_pcs_database() { + let mut cfg = config(&[("dark", "protomaps_20260423"), ("heightmap", "planet_heightmap")]); + let doc = datasets(&cfg); + for dataset in ["protomaps_20260423", "planet_heightmap"] { + for key in NEARBY_POSTGIS_SOURCES { + assert_eq!(doc[dataset][key]["dsn_env"], TILES_POSTGIS_DSN_ENV, "{dataset} {key}"); + } + } + let env = tiles_env(&cfg); + assert_eq!( + value(&env, TILES_POSTGIS_DSN_ENV), + Some("host=127.0.0.1 port=55432 dbname='odian' user='odian' password='pw' connect_timeout=5") + ); + cfg.pg_password = r"a'b\c d".to_string(); + assert!(pg_dsn(&cfg).contains(r"password='a\'b\\c d'"), "{}", pg_dsn(&cfg)); + } + #[test] fn a_pack_without_extensions_names_no_styles() { // Older packs said nothing, and the tile server's own default (.png) was right for them. @@ -743,6 +826,54 @@ mod tests { assert!(doc["protomaps_20260423"].get("styles").is_none()); } + /// An install on disk with the basemap archive and, optionally, the baker. + fn baking_install(with_baker: bool) -> (PathBuf, StackConfig) { + let root = std::env::temp_dir().join(format!("odian-launcher-test-{}", uuid::Uuid::new_v4().simple())); + let mut cfg = config(&[]); + cfg.layout = Layout::new(&root, "0.58.0"); + cfg.tile_sets = vec![ + TileSet { style: "dark_v2".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".png".to_string() }, + TileSet { style: "satellite_hd".to_string(), dataset: "protomaps_20260423".to_string(), ext: ".jpg".to_string() }, + TileSet { style: "heightmap".to_string(), dataset: "planet_heightmap".to_string(), ext: ".lerc".to_string() }, + ]; + let src = cfg.layout.data_dir().join("src"); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("protomaps_20260423.pmtiles"), b"PMTiles").unwrap(); + let server = parent(&cfg.layout.app_file(TILE_SERVER_APP)); + std::fs::create_dir_all(&server).unwrap(); + if with_baker { + std::fs::write(server.join("tile_baker.exe"), b"MZ").unwrap(); + } + (root, cfg) + } + + #[test] + fn with_an_archive_and_a_baker_the_basemap_is_baked() { + let (root, cfg) = baking_install(true); + let env = tiles_env(&cfg); + let doc = datasets(&cfg); + std::fs::remove_dir_all(&root).unwrap(); + assert_eq!(value(&env, "ODIAN_TILE_CACHE_ONLY"), Some("0")); + assert_eq!(value(&env, "ODIAN_TILE_NO_UPSTREAM"), Some("1"), "an offline PC never reaches a provider"); + assert!(value(&env, "ODIAN_TILE_BAKER_EXE").unwrap().ends_with("tile_baker.exe")); + assert!(doc["protomaps_20260423"]["pmtiles"].as_str().unwrap().ends_with("protomaps_20260423.pmtiles")); + assert!(doc["protomaps_20260423"].get("cache_only").is_none(), "the basemap bakes"); + // The height map ships as tiles alone: without this the baking server refuses it outright. + assert_eq!(doc["planet_heightmap"]["cache_only"], true); + // Imagery must never be drawn from vector data: marked remote, a miss is a clean 404. + assert_eq!(doc["protomaps_20260423"]["styles"]["satellite_hd"]["source"], "wmts"); + assert!(doc["protomaps_20260423"]["styles"]["dark_v2"].get("source").is_none()); + } + + #[test] + fn an_archive_without_a_baker_stays_cache_only() { + let (root, cfg) = baking_install(false); + let env = tiles_env(&cfg); + std::fs::remove_dir_all(&root).unwrap(); + assert_eq!(value(&env, "ODIAN_TILE_CACHE_ONLY"), Some("1")); + assert_eq!(value(&env, "ODIAN_TILE_BAKER_EXE"), Some("")); + } + #[test] fn tile_sets_come_from_the_installed_manifest() { let root = std::env::temp_dir().join(format!("odian-launcher-test-{}", uuid::Uuid::new_v4().simple())); diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 809c4e2..8d44c19 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "../node_modules/@tauri-apps/cli/config.schema.json", "productName": "ODIAN Launcher", - "version": "0.1.0", + "version": "0.1.1", "identifier": "fr.odian.launcher", "build": { "frontendDist": "../dist",