818 lines
32 KiB
Rust
818 lines
32 KiB
Rust
//! Cutting a release from the hub, in dev mode.
|
|
//!
|
|
//! Three artifacts ship from these repos and each has its own pipeline: the ODIAN pack
|
|
//! (client + API + tile server), this launcher, and the Engine API container on the
|
|
//! cluster. The scripts that do the work already exist and are the ones a person runs by
|
|
//! hand; nothing here reimplements them. This decides WHAT is behind and runs them.
|
|
//!
|
|
//! "Behind" is answered by the commit each published artifact records it was built from.
|
|
//! Comparing versions would not do: a version says what was cut, not from what.
|
|
use serde::Serialize;
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Command, Stdio};
|
|
|
|
/// What a release could do right now, and what stops it.
|
|
#[derive(Clone, Debug, Serialize)]
|
|
pub struct ReleasePlan {
|
|
/// Docker Desktop answers. Only the Engine API container needs it.
|
|
pub docker: bool,
|
|
/// The DHP answered its health check: nothing can be published without it.
|
|
pub online: bool,
|
|
/// The scripts that do the work are where they should be.
|
|
pub scripts: bool,
|
|
pub items: Vec<ReleaseItem>,
|
|
/// Why the whole thing cannot run, in one line; None when it can.
|
|
pub blocked: Option<String>,
|
|
/// What the pack carries but a release does not rebuild. None when nothing is installed
|
|
/// here, so there is no manifest to read it from.
|
|
pub data: Option<ReleaseData>,
|
|
}
|
|
|
|
#[derive(Clone, Debug, Serialize)]
|
|
pub struct ReleaseItem {
|
|
/// `pack`, `launcher` or `api`.
|
|
pub key: &'static str,
|
|
pub title: &'static str,
|
|
/// The version that is live, when it could be read.
|
|
pub live: Option<String>,
|
|
/// The repo's HEAD, short.
|
|
pub commit: Option<String>,
|
|
/// True when the repo has moved past what is published. None when nothing published
|
|
/// says which commit it came from - the honest answer until a release records one.
|
|
pub behind: Option<bool>,
|
|
/// Uncommitted changes in its repo: a release from those could not be rebuilt.
|
|
pub dirty: bool,
|
|
/// What the row says out loud.
|
|
pub note: String,
|
|
}
|
|
|
|
/// Is a Docker daemon answering? The Engine API image is built in one.
|
|
pub fn docker_running() -> bool {
|
|
run_ok(Command::new("docker").args(["info", "--format", "{{.ServerVersion}}"]))
|
|
}
|
|
|
|
fn run_ok(cmd: &mut Command) -> bool {
|
|
#[cfg(windows)]
|
|
{
|
|
use std::os::windows::process::CommandExt;
|
|
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW
|
|
}
|
|
cmd.output().map(|o| o.status.success()).unwrap_or(false)
|
|
}
|
|
|
|
fn git(repo: &Path, args: &[&str]) -> Option<String> {
|
|
let mut cmd = Command::new("git");
|
|
cmd.arg("-C").arg(repo).args(args);
|
|
#[cfg(windows)]
|
|
{
|
|
use std::os::windows::process::CommandExt;
|
|
cmd.creation_flags(0x0800_0000);
|
|
}
|
|
let out = cmd.output().ok()?;
|
|
out.status.success().then(|| String::from_utf8_lossy(&out.stdout).trim().to_string())
|
|
}
|
|
|
|
fn head(repo: &Path) -> Option<String> {
|
|
git(repo, &["rev-parse", "HEAD"])
|
|
}
|
|
|
|
/// `shipped` limits the question to the paths an artifact is actually built from. It matters
|
|
/// for the MCP: that repo also holds these release scripts, so an unscoped check let editing
|
|
/// a script lock out every other step - the pack and the launcher included.
|
|
fn dirty(repo: &Path, shipped: Option<&[&str]>) -> bool {
|
|
let mut args = vec!["status", "--porcelain"];
|
|
if let Some(paths) = shipped {
|
|
args.push("--");
|
|
args.extend_from_slice(paths);
|
|
}
|
|
git(repo, &args).is_some_and(|s| !s.trim().is_empty())
|
|
}
|
|
|
|
/// What lands in the MCP image: `Dockerfile` copies only these.
|
|
const MCP_SHIPPED: &[&str] = &["Dockerfile", "requirements.txt", "src"];
|
|
|
|
/// One artifact's row: is this repo past what is published?
|
|
fn item(
|
|
key: &'static str,
|
|
title: &'static str,
|
|
repo: PathBuf,
|
|
live: Option<String>,
|
|
published_commit: Option<String>,
|
|
shipped: Option<&[&str]>,
|
|
) -> ReleaseItem {
|
|
let exists = repo.is_dir();
|
|
let commit = exists.then(|| head(&repo)).flatten();
|
|
let dirty = exists && dirty(&repo, shipped);
|
|
let behind = match (&commit, &published_commit) {
|
|
(Some(now), Some(was)) => Some(now != was),
|
|
_ => None,
|
|
};
|
|
let note = match (exists, &commit, behind, dirty) {
|
|
(false, ..) => format!("{} is not in the checkout folder", repo.display()),
|
|
(_, None, ..) => "not a git checkout".to_string(),
|
|
(_, _, _, true) => "uncommitted changes: commit them before releasing".to_string(),
|
|
(_, _, Some(true), _) => "behind: the repo has moved past what is published".to_string(),
|
|
(_, _, Some(false), _) => "up to date".to_string(),
|
|
(_, _, None, _) => "what is published does not say which commit it came from".to_string(),
|
|
};
|
|
ReleaseItem { key, title, live, commit: commit.map(|c| c[..c.len().min(7)].to_string()), behind, dirty, note }
|
|
}
|
|
|
|
/// What a pack carries that a code release does NOT rebuild.
|
|
///
|
|
/// `cut_pack.py` copies the database seed, the source chunks and the tile sets forward from
|
|
/// the previous version every time, and asserts they did not change - which is what keeps a
|
|
/// code release from quietly altering data. The cost is that they age invisibly: the seed and
|
|
/// the tiles are whatever they were the day someone last built them. This puts a date on it.
|
|
#[derive(Clone, Debug, Serialize)]
|
|
pub struct ReleaseData {
|
|
/// The version whose manifest these numbers were read from.
|
|
pub version: String,
|
|
/// Whether that is also what the DHP serves, so the numbers describe the live pack.
|
|
pub is_live: bool,
|
|
/// The seed's snapshot stamp, e.g. `20260921T125618Z`.
|
|
pub db_snapshot: Option<String>,
|
|
pub db_age_days: Option<i64>,
|
|
pub db_bytes: u64,
|
|
pub tile_sets: usize,
|
|
pub tile_bytes: u64,
|
|
/// Every distinct dataset the tile sets name.
|
|
pub tile_datasets: Vec<String>,
|
|
/// The basemap's SOURCE vintage, as `YYYYMMDD`, taken from the dataset name
|
|
/// (`protomaps_20260423`). This is the date of the upstream extract the maps were drawn
|
|
/// from - NOT when anyone baked or froze them. Nothing in a tile index or a manifest
|
|
/// records a bake time, so that age simply does not exist to be shown, and calling this
|
|
/// one "old" read as if the pack were stale when it only means the world data is from
|
|
/// April. An undated name (`planet_heightmap`) is skipped: that set does not move.
|
|
pub basemap_vintage: Option<String>,
|
|
pub basemap_vintage_days: Option<i64>,
|
|
}
|
|
|
|
/// Days from 1970-01-01 to y-m-d, proleptic Gregorian. Hinnant's algorithm, which is exact
|
|
/// and needs no date crate - the alternative was a dependency to subtract two dates.
|
|
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
|
|
let y = if m <= 2 { y - 1 } else { y };
|
|
let era = if y >= 0 { y } else { y - 399 } / 400;
|
|
let yoe = y - era * 400;
|
|
let mp = (m + 9) % 12;
|
|
let doy = (153 * mp + 2) / 5 + d - 1;
|
|
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
|
era * 146_097 + doe - 719_468
|
|
}
|
|
|
|
/// How many days ago `YYYYMMDD` was, or None when those are not eight sane digits.
|
|
fn age_days(stamp: &str, now_secs: i64) -> Option<i64> {
|
|
let digits: String = stamp.chars().take_while(|c| c.is_ascii_digit()).collect();
|
|
if digits.len() < 8 {
|
|
return None;
|
|
}
|
|
let y: i64 = digits[0..4].parse().ok()?;
|
|
let m: i64 = digits[4..6].parse().ok()?;
|
|
let d: i64 = digits[6..8].parse().ok()?;
|
|
if !(1..=12).contains(&m) || !(1..=31).contains(&d) || !(2000..=2999).contains(&y) {
|
|
return None;
|
|
}
|
|
Some(now_secs / 86_400 - days_from_civil(y, m, d))
|
|
}
|
|
|
|
/// The date a dataset name carries, if any: `protomaps_20260423` -> `20260423`.
|
|
fn dataset_stamp(dataset: &str) -> Option<&str> {
|
|
let tail = dataset.rsplit('_').next()?;
|
|
(tail.len() == 8 && tail.chars().all(|c| c.is_ascii_digit())).then_some(tail)
|
|
}
|
|
|
|
/// Read the data payload out of an installed version's manifest.
|
|
pub fn data_from_manifest(m: &serde_json::Value, version: &str, is_live: bool, now_secs: i64) -> ReleaseData {
|
|
let db = m.get("db");
|
|
let db_snapshot = db.and_then(|d| d.get("snapshot")).and_then(|s| s.as_str()).map(str::to_string);
|
|
let db_bytes = db
|
|
.and_then(|d| d.get("files"))
|
|
.and_then(|f| f.as_array())
|
|
.map(|files| files.iter().filter_map(|f| f.get("size").and_then(serde_json::Value::as_u64)).sum())
|
|
.unwrap_or(0);
|
|
let sets = m.get("tiles").and_then(|t| t.get("sets")).and_then(|s| s.as_array());
|
|
let tile_bytes = sets
|
|
.map(|s| {
|
|
s.iter()
|
|
.filter_map(|set| set.get("bytes").or_else(|| set.get("size")).and_then(serde_json::Value::as_u64))
|
|
.sum()
|
|
})
|
|
.unwrap_or(0);
|
|
let mut tile_datasets: Vec<String> = Vec::new();
|
|
for set in sets.map(Vec::as_slice).unwrap_or(&[]) {
|
|
if let Some(d) = set.get("dataset").and_then(|d| d.as_str()) {
|
|
if !tile_datasets.iter().any(|have| have == d) {
|
|
tile_datasets.push(d.to_string());
|
|
}
|
|
}
|
|
}
|
|
// The oldest dated extract, since that is the one that limits what the maps show.
|
|
let basemap_vintage = tile_datasets
|
|
.iter()
|
|
.filter_map(|d| dataset_stamp(d))
|
|
.filter_map(|s| age_days(s, now_secs).map(|age| (age, s.to_string())))
|
|
.max_by_key(|(age, _)| *age)
|
|
.map(|(_, s)| s);
|
|
ReleaseData {
|
|
version: version.to_string(),
|
|
is_live,
|
|
db_age_days: db_snapshot.as_deref().and_then(|s| age_days(s, now_secs)),
|
|
db_snapshot,
|
|
db_bytes,
|
|
tile_sets: sets.map(|s| s.len()).unwrap_or(0),
|
|
tile_bytes,
|
|
tile_datasets,
|
|
basemap_vintage_days: basemap_vintage.as_deref().and_then(|s| age_days(s, now_secs)),
|
|
basemap_vintage,
|
|
}
|
|
}
|
|
|
|
/// Everything the Release panel shows.
|
|
pub fn plan(
|
|
dev_root: &Path,
|
|
mcp_repo: &Path,
|
|
online: bool,
|
|
pack_live: Option<String>,
|
|
pack_commit: Option<String>,
|
|
launcher_live: Option<String>,
|
|
launcher_commit: Option<String>,
|
|
api_live: Option<String>,
|
|
data: Option<ReleaseData>,
|
|
) -> ReleasePlan {
|
|
let docker = docker_running();
|
|
let scripts = mcp_repo.join("scripts").join("release_pack.ps1").is_file()
|
|
&& mcp_repo.join("scripts").join("package_launcher.ps1").is_file();
|
|
let items = vec![
|
|
item("pack", "ODIAN", dev_root.join("ODIAN_Engine"), pack_live, pack_commit, None),
|
|
item("launcher", "Launcher", dev_root.join("ODIAN_Launcher"), launcher_live, launcher_commit, None),
|
|
// The container is built from the same Engine checkout as the pack; only its pipeline
|
|
// differs, and only it needs Docker.
|
|
item("api", "Engine API", dev_root.join("ODIAN_Engine"), api_live, None, None),
|
|
item("mcp", "MCP server", dev_root.join("ODIAN_MCP_Server"), None, None, Some(MCP_SHIPPED)),
|
|
];
|
|
let blocked = if !online {
|
|
Some("The DHP is not answering: nothing can be published.".to_string())
|
|
} else if !scripts {
|
|
Some(format!("The release scripts are not in {}.", mcp_repo.display()))
|
|
} else if items.iter().any(|i| i.dirty) {
|
|
// Naming them matters: "pack" and "api" are the same checkout, so a bare message sent
|
|
// you hunting through three repos for one open file.
|
|
let mut who: Vec<&str> = Vec::new();
|
|
for i in items.iter().filter(|i| i.dirty) {
|
|
if !who.contains(&i.title) {
|
|
who.push(i.title);
|
|
}
|
|
}
|
|
Some(format!(
|
|
"Commit what is open first ({}): a release built from uncommitted work cannot be rebuilt.",
|
|
who.join(", ")
|
|
))
|
|
} else {
|
|
None
|
|
};
|
|
ReleasePlan { docker, online, scripts, items, blocked, data }
|
|
}
|
|
|
|
/// The Release console's buffer: filled by the running release, read by the window.
|
|
///
|
|
/// It sits behind a plain (not async) lock because the threads draining the child's pipes
|
|
/// have no async context to lock from.
|
|
#[derive(Default)]
|
|
pub struct Console {
|
|
lines: Vec<String>,
|
|
/// How many lines fell off the front, so indices stay stable while it trims.
|
|
dropped: usize,
|
|
running: bool,
|
|
exit: Option<i32>,
|
|
}
|
|
|
|
pub type Shared = std::sync::Arc<std::sync::Mutex<Console>>;
|
|
|
|
/// A release prints a great deal (cargo, npm, docker); keep the tail, not all of it.
|
|
const MAX_LINES: usize = 4000;
|
|
|
|
/// What the window asks for: everything it has not seen yet.
|
|
#[derive(Serialize, Clone)]
|
|
pub struct ConsoleTail {
|
|
/// Index of the first line returned.
|
|
pub from: usize,
|
|
pub lines: Vec<String>,
|
|
pub total: usize,
|
|
pub running: bool,
|
|
pub exit: Option<i32>,
|
|
}
|
|
|
|
impl Console {
|
|
fn start(&mut self) {
|
|
self.lines.clear();
|
|
self.dropped = 0;
|
|
self.running = true;
|
|
self.exit = None;
|
|
}
|
|
|
|
fn push(&mut self, line: String) {
|
|
self.lines.push(line);
|
|
if self.lines.len() > MAX_LINES {
|
|
let cut = self.lines.len() - MAX_LINES;
|
|
self.lines.drain(..cut);
|
|
self.dropped += cut;
|
|
}
|
|
}
|
|
|
|
fn finish(&mut self, exit: Option<i32>) {
|
|
self.running = false;
|
|
self.exit = exit;
|
|
}
|
|
|
|
pub fn is_running(&self) -> bool {
|
|
self.running
|
|
}
|
|
|
|
/// Everything from `since` on. A `since` already trimmed away starts at the oldest line
|
|
/// still held, rather than pretending the dropped ones are still coming.
|
|
pub fn tail(&self, since: usize) -> ConsoleTail {
|
|
let total = self.dropped + self.lines.len();
|
|
let from = since.clamp(self.dropped, total);
|
|
ConsoleTail {
|
|
from,
|
|
lines: self.lines[from - self.dropped..].to_vec(),
|
|
total,
|
|
running: self.running,
|
|
exit: self.exit,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Drain one of the child's streams into the console, a line at a time.
|
|
fn pump(stream: impl std::io::Read, console: Shared) {
|
|
use std::io::BufRead;
|
|
let mut reader = std::io::BufReader::new(stream);
|
|
let mut raw = Vec::new();
|
|
loop {
|
|
raw.clear();
|
|
match reader.read_until(b'\n', &mut raw) {
|
|
Ok(0) | Err(_) => return,
|
|
Ok(_) => {
|
|
let text = String::from_utf8_lossy(&raw);
|
|
let text = text.trim_end_matches('\n').trim_end_matches('\r');
|
|
// A terminal shows only what follows the last carriage return, so a progress line
|
|
// that kept rewriting itself does not become a thousand lines in the panel.
|
|
let shown = text.rsplit('\r').next().unwrap_or(text);
|
|
match console.lock() {
|
|
Ok(mut c) => c.push(shown.to_string()),
|
|
Err(_) => return,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Cut and publish, into the launcher's own console panel.
|
|
///
|
|
/// The three scripts are run as a person runs them, in order, and each decides for itself
|
|
/// whether it has anything to do: `release_pack.ps1` stops with "nothing changed" when the
|
|
/// repo matches the live manifest, and `deploy-api.ps1` skips the roll when the compiled
|
|
/// binary is identical. So "release whatever is behind" needs no prediction here - only
|
|
/// that they are run in the right order and that the first real failure stops the rest.
|
|
///
|
|
/// The node's sudo password goes in the ENVIRONMENT, not the command line: arguments are
|
|
/// readable by anyone who can list processes.
|
|
pub fn run_release(
|
|
mcp_repo: &Path,
|
|
work_dir: &Path,
|
|
pack_version: &str,
|
|
launcher_version: Option<&str>,
|
|
with_api: bool,
|
|
with_mcp: bool,
|
|
with_web: bool,
|
|
sudo: &str,
|
|
console: &Shared,
|
|
) -> Result<PathBuf, String> {
|
|
let scripts = mcp_repo.join("scripts");
|
|
let log = work_dir.join("release.log");
|
|
let mut body = String::new();
|
|
body.push_str(&format!(
|
|
"Write-Host '=== ODIAN pack {pack_version} ===' -ForegroundColor Cyan
|
|
& '{}' -Version '{pack_version}' -NodeSudoPw $pw
|
|
if ($LASTEXITCODE -eq 2) {{ Write-Host 'pack: nothing to release' -ForegroundColor Yellow }}
|
|
elseif ($LASTEXITCODE -ne 0) {{ throw 'the pack step failed' }}
|
|
",
|
|
scripts.join("release_pack.ps1").display()
|
|
));
|
|
if let Some(v) = launcher_version {
|
|
body.push_str(&format!(
|
|
"Write-Host '=== Launcher {v} ===' -ForegroundColor Cyan
|
|
& '{}' -Version '{v}' -SiteUrl 'https://engine.odian.fr/dhp' -McpUrl 'https://engine.odian.fr' -AiUrl 'https://engine.odian.fr' -SiteName 'ODIAN test DHP' -Publish
|
|
if ($LASTEXITCODE -ne 0) {{ throw 'the launcher step failed' }}
|
|
",
|
|
scripts.join("package_launcher.ps1").display()
|
|
));
|
|
}
|
|
if with_api {
|
|
// It ships with the release skill rather than the repo, so look there when the repo
|
|
// has no copy. A missing script used to fail the same way a broken one does.
|
|
let api = api_script(&scripts).ok_or(
|
|
format!(
|
|
"deploy-api.ps1 is not in {}. Pull ODIAN_MCP_Server on this PC, or untick the Engine API.",
|
|
scripts.display()
|
|
),
|
|
)?;
|
|
body.push_str(&format!(
|
|
"Write-Host '=== Engine API ===' -ForegroundColor Cyan
|
|
& '{}' -NodeSudoPw $pw -EngineRoot '{}'
|
|
if ($LASTEXITCODE -notin 0,3) {{ throw 'the API step failed' }}
|
|
",
|
|
api.display(),
|
|
// The checkout is the sibling of ODIAN_MCP_Server, which is what `mcp_repo` names.
|
|
// Passed explicitly so the script does not have to guess from where it was loaded -
|
|
// it can come from the repo or from the release skill, and only one of those is a
|
|
// sibling of the Engine.
|
|
mcp_repo.parent().unwrap_or(mcp_repo).join("ODIAN_Engine").display()
|
|
));
|
|
}
|
|
|
|
if with_web {
|
|
// From _WEB in the Engine checkout. Its Dockerfile runs npm inside the image, so this
|
|
// needs Docker and nothing else - no Node, no node_modules on this PC.
|
|
body.push_str(&format!(
|
|
"Write-Host '=== ODIAN WEB ===' -ForegroundColor Cyan
|
|
& '{}'
|
|
if ($LASTEXITCODE -eq 2) {{ Write-Host 'web: already serving this commit' -ForegroundColor Yellow }}
|
|
elseif ($LASTEXITCODE -ne 0) {{ throw 'the web step failed' }}
|
|
",
|
|
scripts.join("deploy_web.ps1").display()
|
|
));
|
|
}
|
|
|
|
if with_mcp {
|
|
// LAST, always. The MCP serves /download/* and proxies /dhp, so rolling it interrupts
|
|
// the very road the steps above publish over; anything still in flight would fail.
|
|
body.push_str(&format!(
|
|
"Write-Host '=== MCP server ===' -ForegroundColor Cyan
|
|
& '{}'
|
|
if ($LASTEXITCODE -eq 2) {{ Write-Host 'MCP: nothing to deploy' -ForegroundColor Yellow }}
|
|
elseif ($LASTEXITCODE -ne 0) {{ throw 'the MCP step failed' }}
|
|
",
|
|
scripts.join("deploy_mcp.ps1").display()
|
|
));
|
|
}
|
|
|
|
// Everything inside try/finally, and everything recorded. A step that throws used to
|
|
// take the window with it - the release simply vanished, with nothing to read.
|
|
//
|
|
// No `pause` here any more: this runs into the launcher's own panel, where there is no
|
|
// console to press Enter in. The exit code carries the verdict instead - swallowing a
|
|
// failure and exiting 0 would make a failed release look like a finished one.
|
|
let ps = console_script(&body, &log, "Release finished.");
|
|
|
|
spawn_console(work_dir, "release.ps1", &ps, &[("ODIAN_NODE_SUDO", sudo)], console)
|
|
}
|
|
|
|
/// Refresh the data a pack carries: the database seed, and optionally the tile indexes.
|
|
///
|
|
/// Deliberately NOT part of a release, and deliberately its own press. The seed is a
|
|
/// per-table dump of a database operators are using, and the tiles are indexed where the
|
|
/// tiles are. Neither should ride along with shipping a code change - but both should be one
|
|
/// click when refreshing the data IS the intent, which is what this is.
|
|
///
|
|
/// It only prepares the inputs. Cutting a version from them stays a separate step, so a
|
|
/// refresh can be looked at before anything is published.
|
|
pub fn run_refresh(
|
|
mcp_repo: &Path,
|
|
work_dir: &Path,
|
|
pg_password: &str,
|
|
tile_root: Option<&str>,
|
|
sudo: &str,
|
|
console: &Shared,
|
|
) -> Result<PathBuf, String> {
|
|
let script = mcp_repo.join("scripts").join("refresh_data.ps1");
|
|
if !script.is_file() {
|
|
return Err(format!("refresh_data.ps1 is not in {}. Pull ODIAN_MCP_Server on this PC.", mcp_repo.join("scripts").display()));
|
|
}
|
|
let log = work_dir.join("refresh.log");
|
|
let mut body = String::new();
|
|
if !pg_password.is_empty() {
|
|
body.push_str(&format!(
|
|
"Write-Host '=== Database seed ===' -ForegroundColor Cyan
|
|
& '{}' -Seed -PgPassword $pgpw -NodeSudoPw $pw
|
|
if ($LASTEXITCODE -ne 0) {{ throw 'the seed refresh failed' }}
|
|
",
|
|
script.display()
|
|
));
|
|
}
|
|
if let Some(root) = tile_root.filter(|r| !r.trim().is_empty()) {
|
|
body.push_str(&format!(
|
|
"Write-Host '=== Tile indexes ===' -ForegroundColor Cyan
|
|
& '{}' -Tiles -TileRoot '{}' -NodeSudoPw $pw
|
|
if ($LASTEXITCODE -ne 0) {{ throw 'the tile refresh failed' }}
|
|
",
|
|
script.display(),
|
|
root.trim()
|
|
));
|
|
}
|
|
if body.is_empty() {
|
|
return Err("Nothing to refresh: give the database password, a tile folder, or both.".to_string());
|
|
}
|
|
let ps = console_script(&body, &log, "Refresh finished. Nothing is published yet.");
|
|
spawn_console(work_dir, "refresh.ps1", &ps, &[("ODIAN_NODE_SUDO", sudo), ("ODIAN_PG_PW", pg_password)], console)
|
|
}
|
|
|
|
/// Wrap a body so a step that throws cannot take the window with it, and everything said is
|
|
/// also on disk. No `pause`: this runs into the launcher's own panel, where there is nothing
|
|
/// to press Enter with, and the exit code carries the verdict instead.
|
|
fn console_script(body: &str, log: &Path, done: &str) -> String {
|
|
format!(
|
|
"$ErrorActionPreference = 'Stop'
|
|
# The pipe gets UTF-8 or Windows' own French messages arrive as mojibake.
|
|
[Console]::OutputEncoding = New-Object System.Text.UTF8Encoding $false
|
|
$pw = $env:ODIAN_NODE_SUDO
|
|
$env:ODIAN_NODE_SUDO = $null
|
|
$pgpw = $env:ODIAN_PG_PW
|
|
$env:ODIAN_PG_PW = $null
|
|
Start-Transcript -Path '{}' -Force | Out-Null
|
|
$ok = $true
|
|
try {{
|
|
{}
|
|
Write-Host ''
|
|
Write-Host '{}'
|
|
}} catch {{
|
|
Write-Host ''
|
|
Write-Host \"FAILED: $_\"
|
|
$ok = $false
|
|
}} finally {{
|
|
Stop-Transcript | Out-Null
|
|
Write-Host ''
|
|
Write-Host 'Log: {}'
|
|
}}
|
|
if (-not $ok) {{ exit 1 }}
|
|
",
|
|
log.display(),
|
|
body,
|
|
done,
|
|
log.display()
|
|
)
|
|
}
|
|
|
|
/// Write `ps` and run it with its output going to the console panel.
|
|
///
|
|
/// Secrets go in the ENVIRONMENT, never the command line: arguments are readable by anyone
|
|
/// who can list processes.
|
|
fn spawn_console(
|
|
work_dir: &Path,
|
|
name: &str,
|
|
ps: &str,
|
|
env: &[(&str, &str)],
|
|
console: &Shared,
|
|
) -> Result<PathBuf, String> {
|
|
std::fs::create_dir_all(work_dir).map_err(|e| format!("cannot use {}: {e}", work_dir.display()))?;
|
|
let file = work_dir.join(name);
|
|
std::fs::write(&file, ps).map_err(|e| format!("cannot write {}: {e}", file.display()))?;
|
|
|
|
let mut cmd = Command::new("powershell");
|
|
cmd.args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-File"]).arg(&file);
|
|
for (k, v) in env {
|
|
cmd.env(k, v);
|
|
}
|
|
// Nothing can type into this, so stdin is closed: a script that ever asks a question
|
|
// then fails at once instead of hanging forever with no window to answer in.
|
|
cmd.stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::piped());
|
|
#[cfg(windows)]
|
|
{
|
|
use std::os::windows::process::CommandExt;
|
|
cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW: the output belongs in the panel
|
|
}
|
|
let mut child = cmd.spawn().map_err(|e| format!("cannot start the release: {e}"))?;
|
|
console.lock().map_err(|_| "the release console is unusable")?.start();
|
|
|
|
// Both streams, kept separate: merging them with `2>&1` inside PowerShell wraps every
|
|
// stderr line from a native exe in a NativeCommandError and makes a clean run look failed.
|
|
let mut pumps = Vec::new();
|
|
for stream in [child.stdout.take().map(Either::Out), child.stderr.take().map(Either::Err)] {
|
|
if let Some(s) = stream {
|
|
let c = console.clone();
|
|
pumps.push(std::thread::spawn(move || match s {
|
|
Either::Out(r) => pump(r, c),
|
|
Either::Err(r) => pump(r, c),
|
|
}));
|
|
}
|
|
}
|
|
let c = console.clone();
|
|
std::thread::spawn(move || {
|
|
let code = child.wait().ok().and_then(|s| s.code());
|
|
// Drain what is still sitting in the pipes BEFORE saying it finished: the window stops
|
|
// asking once running goes false, and the last lines are the ones that say why.
|
|
for p in pumps {
|
|
let _ = p.join();
|
|
}
|
|
if let Ok(mut c) = c.lock() {
|
|
c.finish(code);
|
|
}
|
|
});
|
|
Ok(file)
|
|
}
|
|
|
|
/// The two pipes have different types; this lets one thread body handle either.
|
|
enum Either {
|
|
Out(std::process::ChildStdout),
|
|
Err(std::process::ChildStderr),
|
|
}
|
|
|
|
/// Where the Engine API deploy script is: beside the other release scripts, or in the
|
|
/// release skill that carries it.
|
|
fn api_script(scripts: &Path) -> Option<PathBuf> {
|
|
let here = scripts.join("deploy-api.ps1");
|
|
if here.is_file() {
|
|
return Some(here);
|
|
}
|
|
let skill = dirs_home()?.join(".claude/skills/odian-api-release/scripts/deploy-api.ps1");
|
|
skill.is_file().then_some(skill)
|
|
}
|
|
|
|
fn dirs_home() -> Option<PathBuf> {
|
|
std::env::var_os("USERPROFILE").or_else(|| std::env::var_os("HOME")).map(PathBuf::from)
|
|
}
|
|
|
|
/// `0.71.0` -> `0.72.0`. A pack release is a minor; the patch place is not used.
|
|
pub fn next_pack(live: &str) -> Option<String> {
|
|
let p: Vec<u64> = live.split('.').map_while(|x| x.parse().ok()).collect();
|
|
(p.len() >= 2).then(|| format!("{}.{}.0", p[0], p[1] + 1))
|
|
}
|
|
|
|
/// `0.1.38` -> `0.1.39`.
|
|
pub fn next_launcher(live: &str) -> Option<String> {
|
|
let p: Vec<u64> = live.split('.').map_while(|x| x.parse().ok()).collect();
|
|
(p.len() >= 3).then(|| format!("{}.{}.{}", p[0], p[1], p[2] + 1))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn the_pack_says_how_old_the_data_it_carries_is() {
|
|
// 2026-10-01, the day this was written, as unix seconds.
|
|
let now = days_from_civil(2026, 10, 1) * 86_400;
|
|
assert_eq!(age_days("20261001T000000Z", now), Some(0), "today");
|
|
assert_eq!(age_days("20260921T125618Z", now), Some(10), "the real 0.72.0 seed");
|
|
assert_eq!(age_days("20260423", now), Some(161), "the real basemap extract");
|
|
// Leap day, and a year boundary: the arithmetic is the whole point of this test.
|
|
assert_eq!(age_days("20260228", now), Some(215));
|
|
assert_eq!(age_days("20251231", now), Some(274));
|
|
// Nonsense must not become a date.
|
|
assert_eq!(age_days("", now), None);
|
|
assert_eq!(age_days("2026", now), None);
|
|
assert_eq!(age_days("20261301", now), None, "month 13");
|
|
assert_eq!(age_days("latest", now), None);
|
|
|
|
assert_eq!(dataset_stamp("protomaps_20260423"), Some("20260423"));
|
|
assert_eq!(dataset_stamp("planet_heightmap"), None, "undated: it does not move");
|
|
|
|
// A manifest shaped like the real one.
|
|
let m: serde_json::Value = serde_json::json!({
|
|
"db": { "snapshot": "20260921T125618Z",
|
|
"files": [ {"size": 1000}, {"size": 2000} ] },
|
|
"tiles": { "sets": [
|
|
{"style": "dark", "dataset": "protomaps_20260423", "bytes": 10},
|
|
{"style": "dark_v2", "dataset": "protomaps_20260423", "bytes": 20},
|
|
{"style": "heightmap", "dataset": "planet_heightmap", "bytes": 70}
|
|
]}
|
|
});
|
|
let d = data_from_manifest(&m, "0.72.0", true, now);
|
|
assert_eq!(d.db_bytes, 3000);
|
|
assert_eq!(d.db_age_days, Some(10));
|
|
assert_eq!(d.tile_sets, 3);
|
|
assert_eq!(d.tile_bytes, 100);
|
|
// Deduplicated, and the undated set does not drag the age to None.
|
|
assert_eq!(d.tile_datasets, vec!["protomaps_20260423".to_string(), "planet_heightmap".to_string()]);
|
|
// The dataset name carries the upstream extract's date, not a bake time - there is no
|
|
// bake time anywhere in a tile index or a manifest, which is why none is shown.
|
|
assert_eq!(d.basemap_vintage.as_deref(), Some("20260423"));
|
|
assert_eq!(d.basemap_vintage_days, Some(161));
|
|
|
|
// A manifest with no tiles at all must not panic or invent numbers.
|
|
let empty = data_from_manifest(&serde_json::json!({}), "0.1.0", false, now);
|
|
assert_eq!(empty.tile_sets, 0);
|
|
assert_eq!(empty.db_bytes, 0);
|
|
assert_eq!(empty.db_age_days, None);
|
|
}
|
|
|
|
#[test]
|
|
fn the_next_version_moves_the_right_place() {
|
|
// A pack is a minor (0.71.0 -> 0.72.0); the launcher is a patch (0.1.38 -> 0.1.39).
|
|
assert_eq!(next_pack("0.71.0").as_deref(), Some("0.72.0"));
|
|
assert_eq!(next_pack("0.9.0").as_deref(), Some("0.10.0"), "ten follows nine");
|
|
assert_eq!(next_launcher("0.1.38").as_deref(), Some("0.1.39"));
|
|
assert_eq!(next_launcher("0.1.9").as_deref(), Some("0.1.10"));
|
|
// Nonsense from a feed must not become a version number.
|
|
assert_eq!(next_pack("").as_deref(), None);
|
|
assert_eq!(next_launcher("latest").as_deref(), None);
|
|
assert_eq!(next_launcher("0.1").as_deref(), None);
|
|
}
|
|
|
|
#[test]
|
|
fn a_repo_that_matches_what_is_published_is_not_behind() {
|
|
// The comparison is COMMIT against COMMIT. Versions cannot answer it: a version says
|
|
// what was cut, not from what - which is how a release gets cut twice from one tree.
|
|
let same = item("pack", "ODIAN", PathBuf::from("."), Some("0.71.0".into()), head(Path::new(".")), None);
|
|
if same.commit.is_some() {
|
|
assert_eq!(same.behind, Some(false), "same commit: nothing to release");
|
|
}
|
|
// Nothing published says where it came from: say so rather than guess.
|
|
let unknown = item("pack", "ODIAN", PathBuf::from("."), Some("0.71.0".into()), None, None);
|
|
assert_eq!(unknown.behind, None);
|
|
assert!(unknown.note.contains("does not say") || unknown.dirty || unknown.commit.is_none());
|
|
// A folder that is not there is named, not silently "up to date".
|
|
let missing = item("pack", "ODIAN", PathBuf::from(r"Z:\nothing\here"), None, None, None);
|
|
assert_eq!(missing.behind, None);
|
|
assert!(missing.note.contains("not in the checkout folder"));
|
|
}
|
|
|
|
/// The regression that adding the MCP step introduced: that repo also holds the release
|
|
/// scripts, so an unscoped `git status` made editing a script look like uncommitted
|
|
/// release content and locked out the pack and the launcher too.
|
|
#[test]
|
|
fn a_dirty_file_outside_the_shipped_paths_does_not_block() {
|
|
let dir = std::env::temp_dir().join(format!("odian-dirty-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
std::fs::create_dir_all(dir.join("src")).unwrap();
|
|
std::fs::create_dir_all(dir.join("scripts")).unwrap();
|
|
let ok = |args: &[&str]| git(&dir, args).is_some();
|
|
if !ok(&["init"]) {
|
|
return; // no git here; nothing to prove
|
|
}
|
|
let _ = git(&dir, &["config", "user.email", "t@t"]);
|
|
let _ = git(&dir, &["config", "user.name", "t"]);
|
|
std::fs::write(dir.join("src").join("app.py"), "x").unwrap();
|
|
std::fs::write(dir.join("requirements.txt"), "y").unwrap();
|
|
std::fs::write(dir.join("Dockerfile"), "FROM x").unwrap();
|
|
assert!(ok(&["add", "-A"]) && ok(&["commit", "-m", "base"]));
|
|
assert!(!dirty(&dir, Some(MCP_SHIPPED)), "clean tree");
|
|
|
|
// An untracked release script: tooling, never in the image.
|
|
std::fs::write(dir.join("scripts").join("deploy_mcp.ps1"), "# tool").unwrap();
|
|
assert!(dirty(&dir, None), "the repo really is dirty");
|
|
assert!(!dirty(&dir, Some(MCP_SHIPPED)), "but nothing that ships changed");
|
|
|
|
// Something that does ship: this must still block.
|
|
std::fs::write(dir.join("src").join("app.py"), "changed").unwrap();
|
|
assert!(dirty(&dir, Some(MCP_SHIPPED)), "shipped source changed: block");
|
|
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
}
|
|
|
|
#[test]
|
|
fn the_console_hands_over_only_what_the_window_has_not_seen() {
|
|
let mut c = Console::default();
|
|
c.start();
|
|
for i in 0..3 {
|
|
c.push(format!("line {i}"));
|
|
}
|
|
let first = c.tail(0);
|
|
assert_eq!(first.from, 0);
|
|
assert_eq!(first.total, 3);
|
|
assert_eq!(first.lines.len(), 3);
|
|
assert!(first.running, "a started console is running");
|
|
|
|
// Asking again from where it got to yields nothing, not the whole buffer again.
|
|
let again = c.tail(first.total);
|
|
assert!(again.lines.is_empty());
|
|
assert_eq!(again.total, 3);
|
|
|
|
c.push("line 3".into());
|
|
let more = c.tail(3);
|
|
assert_eq!(more.lines, vec!["line 3".to_string()]);
|
|
|
|
c.finish(Some(1));
|
|
let done = c.tail(4);
|
|
assert!(!done.running);
|
|
assert_eq!(done.exit, Some(1), "a failed release must not look finished");
|
|
}
|
|
|
|
#[test]
|
|
fn trimming_a_long_release_keeps_indices_honest() {
|
|
let mut c = Console::default();
|
|
c.start();
|
|
for i in 0..(MAX_LINES + 500) {
|
|
c.push(format!("line {i}"));
|
|
}
|
|
let t = c.tail(0);
|
|
assert_eq!(t.total, MAX_LINES + 500, "total counts every line ever printed");
|
|
assert_eq!(t.lines.len(), MAX_LINES, "only the tail is kept");
|
|
assert_eq!(t.from, 500, "it says where it really starts, not 0");
|
|
assert_eq!(t.lines[0], "line 500");
|
|
// A window that asked from beyond the end gets nothing rather than a panic.
|
|
assert!(c.tail(t.total + 99).lines.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_progress_line_that_rewrites_itself_stays_one_line() {
|
|
let stream: &[u8] = b"first\r\nbuilding 10%\rbuilding 90%\rbuilt\nplain\n";
|
|
let c: Shared = Default::default();
|
|
c.lock().unwrap().start();
|
|
pump(stream, c.clone());
|
|
let got = c.lock().unwrap().tail(0);
|
|
// CRLF is an ending, not content; a lone CR overwrites, exactly as a terminal shows it.
|
|
assert_eq!(got.lines, vec!["first".to_string(), "built".to_string(), "plain".to_string()]);
|
|
}
|
|
}
|