Kelorastus
b7f3061be3
- Tauri 2 + Svelte 5 hub in ODIAN dark: patch notes and roadmap, install and update progress, local stack status, settings with DHP and MCP addresses. - crates/odian-packs: Rust installer for pack format v1 (signed manifests, resumable verified downloads from a DHP or an ODIAN drive), byte-compatible with ODIAN_MCP_Server's Python odian_packs. - Local stack on 127.0.0.1: Postgres -> Engine API -> tile server -> Odian.exe, one Job Object, clean shutdown; database snapshot restore on a fresh PC only. - Keycloak sign-in through the DHP's MCP, remembered until Sign out with a launcher session that renews the Engine token silently. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
374 lines
19 KiB
Python
374 lines
19 KiB
Python
"""Generate the cross-language fixtures the odian-packs Rust tests read.
|
|
|
|
PYTHONPATH=C:/Users/JFEXP_D/Documents/Gitea/ODIAN_MCP_Server/src \
|
|
python crates/odian-packs/tests/fixtures/gen_fixtures.py [--real-manifest PATH]
|
|
|
|
Every expected value below is computed by the reference implementation (odian_packs);
|
|
the Rust tests must reproduce or verify each one. Outputs sit next to this script:
|
|
|
|
canonical.json canonical bytes of tricky JSON inputs (+ inputs Rust refuses)
|
|
signing.json test key, verdicts; signed/tampered manifests as separate files
|
|
relpaths.json safe_relpath verdicts
|
|
validate.json validate() verdicts, with the Python quirks Rust deliberately rejects
|
|
hashes.json block_of, block_digest, chunks_digest, chunk_count, index_sha256
|
|
real/0.51.0.* the real published 0.51.0 manifest and its Python hashes
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sys
|
|
|
|
sys.dont_write_bytecode = True
|
|
|
|
from odian_packs import manifest as mf # noqa: E402
|
|
from odian_packs import signing, tiles # noqa: E402
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
DEFAULT_REAL = (r"C:\Users\JFEXP_D\AppData\Local\Temp\claude\C--Users-JFEXP-D-Documents-Gitea"
|
|
r"\d92cce4d-3852-472e-a7e3-10137a35c218\scratchpad\m1\packs\versions\0.51.0\manifest.json")
|
|
|
|
|
|
def out_path(name):
|
|
path = os.path.join(HERE, *name.split("/"))
|
|
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
return path
|
|
|
|
|
|
def write_bytes(name, data):
|
|
with open(out_path(name), "wb") as f:
|
|
f.write(data)
|
|
|
|
|
|
def write_json(name, obj):
|
|
# ASCII-only fixture files: no editor or checkout can change their bytes.
|
|
write_bytes(name, (json.dumps(obj, indent=1, ensure_ascii=True) + "\n").encode("ascii"))
|
|
|
|
|
|
def sha(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def reversed_keys(obj):
|
|
if isinstance(obj, dict):
|
|
return {k: reversed_keys(obj[k]) for k in reversed(list(obj))}
|
|
if isinstance(obj, list):
|
|
return [reversed_keys(v) for v in obj]
|
|
return obj
|
|
|
|
|
|
def verdict(fn):
|
|
try:
|
|
fn()
|
|
return "ok", ""
|
|
except Exception as exc: # noqa: BLE001 - any exception is a refusal
|
|
return "error", f"{type(exc).__name__}: {exc}"
|
|
|
|
|
|
# ---- canonical JSON -------------------------------------------------------------
|
|
|
|
def minimal_manifest():
|
|
return {"schema": 1, "product": "odian", "version": "0.51.0", "released_at": "2026-09-15T12:00:00Z",
|
|
"app": {"files": [{"path": "Odian/Odian.exe", "size": 3, "sha256": sha(b"exe")}]}}
|
|
|
|
|
|
def canonical_fixture():
|
|
control = "".join(chr(i) for i in range(0x20)) + "\x7f\x80\x9f\u2028\u2029"
|
|
nested = "leaf"
|
|
for i in range(40):
|
|
nested = {f"k{i % 3}": [nested, i]} if i % 2 else [nested, {"i": i}]
|
|
values = [
|
|
("empty_object", {}),
|
|
("empty_array", []),
|
|
("empty_string", ""),
|
|
("null", None),
|
|
("true", True),
|
|
("false", False),
|
|
("empty_containers_nested", {"b": [], "a": {}, "c": [[], {}], "d": ""}),
|
|
("non_ascii", {"fr": "données été à Noël", "de": "Straße", "zh": "中文路径", "ar": "مرحبا", "ru": "Привет"}),
|
|
("emoji", {"emoji": "😀🚀 and 👨\u200d👩\u200d👧 zwj", "flag": "🇫🇷"}),
|
|
("control_chars", {"all": control}),
|
|
("quotes_backslashes", {"q": 'say "hi"', "b": "C:\\Odian\\x", "mixed": "\\\"\\n/<>&'"}),
|
|
("key_order_code_points", {"\ufb01": 1, "😀": 2, "a": 3, "B": 4, "": 5, "é": 6, "aa": 7, "a\u0000": 8,
|
|
"\uffff": 9, "Z": 10, "\u00ff": 11}),
|
|
("nested_sorting", {"z": {"y": [{"b": [], "a": {}}, [], {}], "x": None}, "a": [True, False, None, "", 0],
|
|
"m": {"k2": {"b": 1, "a": 2}, "k1": 3}}),
|
|
("big_ints", {"n": [0, -1, 1, 2**31, 2**32, 2**53, 2**53 + 1, 2**63 - 1, -2**63, 2**64 - 1, 2437523419]}),
|
|
("deep_nesting", nested),
|
|
("minimal_manifest", minimal_manifest()),
|
|
]
|
|
cases = []
|
|
for name, value in values:
|
|
for style, text in (
|
|
("ascii_escaped_reversed", json.dumps(reversed_keys(value), indent=2, ensure_ascii=True)),
|
|
("raw_utf8", json.dumps(value, ensure_ascii=False)),
|
|
):
|
|
assert json.loads(text) == value
|
|
cases.append({"name": f"{name}/{style}", "input": text,
|
|
"canonical_hex": mf.canonical_bytes(json.loads(text)).hex()})
|
|
rejects = []
|
|
for name, text in (("float", "1.5"), ("float_integral", "1.0"), ("exponent", "1e3"), ("negative_zero", "-0"),
|
|
("negative_zero_float", "-0.0"), ("above_u64", "18446744073709551616"),
|
|
("below_i64", "-9223372036854775809"), ("float_in_object", '{"a": 2.5}'),
|
|
("float_in_array", "[1, 0.1]")):
|
|
rejects.append({"name": name, "input": text,
|
|
"python_canonical_hex": mf.canonical_bytes(json.loads(text)).hex()})
|
|
write_json("canonical.json", {"cases": cases, "rust_rejects": rejects})
|
|
|
|
|
|
# ---- signing ---------------------------------------------------------------------
|
|
|
|
def full_manifest():
|
|
chunks = [sha(b"c0"), sha(b"c1")]
|
|
return {
|
|
"schema": 1, "product": "odian", "version": "0.51.0", "released_at": "2026-09-15T12:00:00Z",
|
|
"min_launcher": "0.1.0", "notes": "notes.md",
|
|
"app": {"files": [
|
|
{"path": "Odian/Odian.exe", "size": 3, "sha256": sha(b"exe")},
|
|
{"path": "Odian/assets/empty.conf", "size": 0, "sha256": sha(b"")},
|
|
{"path": "Odian/données/carte été.png", "size": 5, "sha256": sha(b"carte")},
|
|
]},
|
|
"db": {"schema_version": 0, "snapshot": "20260915T101500Z", "files": [
|
|
{"section": "pre-data", "path": "db/20260915T101500Z/pre-data.dump", "size": 1, "sha256": sha(b"p")},
|
|
{"section": "data", "table": "public.t1", "path": "db/20260915T101500Z/data/public.t1.dump",
|
|
"size": 1, "sha256": sha(b"d")},
|
|
]},
|
|
"tiles": {"block_shift": 6, "sets": [
|
|
{"style": "dark", "dataset": "ds", "journal_seq": 9, "blocks": 2, "files": 3, "bytes": 2437523419,
|
|
"index": "tiles/dark.ds.index.json", "index_sha256": "a" * 64},
|
|
]},
|
|
"src": {"enabled": True, "chunk_bytes": 10, "files": [
|
|
{"path": "osm/x.sqlite", "size": 15, "chunks": chunks, "chunks_sha256": mf.chunks_digest(chunks)},
|
|
]},
|
|
"extra_note": "unknown keys are signed too: \"quoted\" \\ \u0001 😀",
|
|
}
|
|
|
|
|
|
def signing_fixture():
|
|
raw_key = bytes(range(32))
|
|
key = signing.private_key_from_raw(raw_key)
|
|
public = signing.public_key_text(key.public_key())
|
|
other = signing.public_key_text(signing.private_key_from_raw(bytes(range(1, 33))).public_key())
|
|
signed = signing.sign(mf.validate(full_manifest()), key)
|
|
|
|
tampered = copy.deepcopy(signed)
|
|
tampered["app"]["files"][0]["size"] = 4
|
|
flipped = copy.deepcopy(signed)
|
|
sig = bytearray(base64.b64decode(signed["signature"][len("ed25519:"):]))
|
|
sig[0] ^= 0xFF
|
|
flipped["signature"] = "ed25519:" + base64.b64encode(bytes(sig)).decode("ascii")
|
|
unsigned = {k: v for k, v in signed.items() if k != "signature"}
|
|
|
|
files = {
|
|
"signed_manifest.utf8.json": (json.dumps(signed, indent=2, sort_keys=True, ensure_ascii=False) + "\n").encode("utf-8"),
|
|
"signed_manifest.ascii.json": (json.dumps(signed, indent=2, sort_keys=True) + "\n").encode("ascii"),
|
|
"tampered_manifest.json": (json.dumps(tampered, indent=2, sort_keys=True) + "\n").encode("ascii"),
|
|
"signature_flipped_manifest.json": (json.dumps(flipped, indent=2, sort_keys=True) + "\n").encode("ascii"),
|
|
"unsigned_manifest.json": (json.dumps(unsigned, indent=2, sort_keys=True) + "\n").encode("ascii"),
|
|
}
|
|
cases = []
|
|
for name, data in files.items():
|
|
write_bytes(name, data)
|
|
doc = json.loads(data.decode("utf-8"))
|
|
ok, error = verdict(lambda: (mf.validate(doc), signing.verify(doc, public)))
|
|
cases.append({"file": name, "python": ok, "python_error": error})
|
|
other_ok, _ = verdict(lambda: signing.verify(signed, other))
|
|
write_json("signing.json", {
|
|
"private_key_raw_hex": raw_key.hex(),
|
|
"public_key": public,
|
|
"other_public_key": other,
|
|
"signed_with_other_key_verdict": other_ok,
|
|
"signature": signed["signature"],
|
|
"payload_hex": mf.signing_payload(signed).hex(),
|
|
"cases": cases,
|
|
})
|
|
|
|
|
|
# ---- safe_relpath -----------------------------------------------------------------
|
|
|
|
RELPATHS = [
|
|
"Odian/x.exe", "db/20260915T082529Z/data/public.odb_air_timeline_2026_02.dump",
|
|
"../a", "a/../b", "/a", "C:/a", "a\\b", "a//b", "", "./a", "a/", "a/.", ".", "..", "a/..",
|
|
"Odian/Odian.exe:hidden", "a/b.", "a/b ", "CON", "con", "a/CoN.TxT", "a/aux.txt", "a/Lpt1.log", "NUL.tar.gz",
|
|
"a/nul..x", "a/prn", "a/prn_x", "a/com¹", "a/COM².txt", "a/lpt³", "a/lpt0", "a/com0", "a/comA", "a/com10",
|
|
"a/con .txt", "a/b\x00c", "a/b\nc", "a/b\x1fc", "a/b\x7fc", "a/b\u0085c", "a/b\u2028c", "a/b?c", "a/b*c",
|
|
'a/b"c', "a/b|c", "a/b<c", "a/b>c", "a/console.txt", "a/auxiliary", "a/.hidden", "a/b c.txt", " a/b",
|
|
"a/ b", "a/b.c.", "données/été.png", "中文/路径.txt", "a/😀.png", "a/b#c%20d", "a/aux\u00a0",
|
|
"a/com\u0660", "a/\u212aon",
|
|
]
|
|
|
|
|
|
def relpath_fixture():
|
|
cases = []
|
|
for path in RELPATHS:
|
|
ok, error = verdict(lambda: mf.safe_relpath(path))
|
|
cases.append({"path": path, "python": ok, "python_error": error})
|
|
write_json("relpaths.json", {"cases": cases})
|
|
|
|
|
|
# ---- validate ---------------------------------------------------------------------
|
|
|
|
def validate_fixture():
|
|
def app0(m):
|
|
return m["app"]["files"][0]
|
|
|
|
def setp(fn):
|
|
def mutate(m):
|
|
fn(m)
|
|
return m
|
|
return mutate
|
|
|
|
empty_src = {"path": "e.bin", "size": 0, "chunks": [], "chunks_sha256": mf.chunks_digest([])}
|
|
# (name, mutation, reason Rust deliberately differs from Python, or None)
|
|
mutations = [
|
|
("valid_full", lambda m: m, None),
|
|
("valid_minimal", lambda m: minimal_manifest(), None),
|
|
("no_app_section", setp(lambda m: m.pop("app")), None),
|
|
("app_null", setp(lambda m: m.update(app=None)), None),
|
|
("app_files_missing", setp(lambda m: m.update(app={})), None),
|
|
("app_entry_not_object", setp(lambda m: m.update(app={"files": [None]})), None),
|
|
("db_entry_not_object", setp(lambda m: m["db"].update(files=[1])), None),
|
|
("tiles_set_not_object", setp(lambda m: m["tiles"].update(sets=[1])), None),
|
|
("src_file_not_object", setp(lambda m: m["src"].update(files=["x"])), None),
|
|
("manifest_not_object", lambda m: [1], None),
|
|
("schema_2", setp(lambda m: m.update(schema=2)), None),
|
|
("schema_true", setp(lambda m: m.update(schema=True)),
|
|
"Python's `== 1` accepts True; Rust wants the integer 1"),
|
|
("schema_float", setp(lambda m: m.update(schema=1.0)),
|
|
"Python's `== 1` accepts 1.0; Rust refuses floats anywhere"),
|
|
("product_wrong", setp(lambda m: m.update(product="other")), None),
|
|
("version_two_parts", setp(lambda m: m.update(version="1.0")), None),
|
|
("version_leading_zero", setp(lambda m: m.update(version="01.2.3")), None),
|
|
("version_int", setp(lambda m: m.update(version=5)), None),
|
|
("version_missing", setp(lambda m: m.pop("version")), None),
|
|
("version_trailing_newline", setp(lambda m: m.update(version="0.51.0\n")),
|
|
"Python's regex `$` matches before a final newline; Rust refuses it"),
|
|
("version_unicode_digit", setp(lambda m: m.update(version="0.51.\u0663")),
|
|
"Python's `\\d` matches any Unicode digit; Rust takes ASCII digits only"),
|
|
("released_at_int", setp(lambda m: m.update(released_at=5)), None),
|
|
("min_launcher_bad", setp(lambda m: m.update(min_launcher="0.1")), None),
|
|
("notes_traversal", setp(lambda m: m.update(notes="../notes.md")), None),
|
|
("notes_int", setp(lambda m: m.update(notes=5)), None),
|
|
("sha_upper", setp(lambda m: app0(m).update(sha256="A" * 64)), None),
|
|
("sha_short", setp(lambda m: app0(m).update(sha256="a" * 63)), None),
|
|
("sha_trailing_newline", setp(lambda m: app0(m).update(sha256="a" * 64 + "\n")),
|
|
"Python's regex `$` matches before a final newline; Rust refuses it"),
|
|
("dup_app_path_case", setp(lambda m: m["app"]["files"].append({**app0(m), "path": "ODIAN/odian.EXE"})), None),
|
|
("dup_app_path_unicode_case", setp(lambda m: m["app"]["files"].extend([
|
|
{"path": "Odian/Ä.txt", "size": 1, "sha256": "b" * 64},
|
|
{"path": "odian/ä.txt", "size": 1, "sha256": "c" * 64}])), None),
|
|
("app_path_traversal", setp(lambda m: app0(m).update(path="../evil.exe")), None),
|
|
("size_negative", setp(lambda m: app0(m).update(size=-1)), None),
|
|
("size_bool", setp(lambda m: app0(m).update(size=True)), None),
|
|
("size_float", setp(lambda m: app0(m).update(size=3.0)), None),
|
|
("size_string", setp(lambda m: app0(m).update(size="3")), None),
|
|
("size_above_u64", setp(lambda m: app0(m).update(size=2**64)),
|
|
"Rust numbers are 64-bit; a size above u64::MAX is refused"),
|
|
("db_empty_files", setp(lambda m: m["db"].update(files=[])), None),
|
|
("db_bad_section", setp(lambda m: m["db"]["files"][0].update(section="schema")), None),
|
|
("db_data_without_table", setp(lambda m: m["db"]["files"][1].pop("table")), None),
|
|
("db_data_empty_table", setp(lambda m: m["db"]["files"][1].update(table="")), None),
|
|
("db_path_outside_snapshot", setp(lambda m: m["db"]["files"][0].update(path="db/20260101T000000Z/x.dump")), None),
|
|
("db_snapshot_bad", setp(lambda m: m["db"].update(snapshot="2026-09-15")), None),
|
|
("db_schema_version_bool", setp(lambda m: m["db"].update(schema_version=True)), None),
|
|
("tiles_block_shift_7", setp(lambda m: m["tiles"].update(block_shift=7)), None),
|
|
("tiles_block_shift_float", setp(lambda m: m["tiles"].update(block_shift=6.0)),
|
|
"Python's `== 6` accepts 6.0; Rust refuses floats anywhere"),
|
|
("tiles_bad_style", setp(lambda m: m["tiles"]["sets"][0].update(style="da rk")), None),
|
|
("tiles_style_trailing_newline", setp(lambda m: m["tiles"]["sets"][0].update(
|
|
style="dark\n", index="tiles/dark\n.ds.index.json")),
|
|
"Python's regex `$` matches before a final newline; Rust refuses it"),
|
|
("tiles_dup_set", setp(lambda m: m["tiles"]["sets"].append(dict(m["tiles"]["sets"][0]))), None),
|
|
("tiles_wrong_index_name", setp(lambda m: m["tiles"]["sets"][0].update(index="tiles/other.json")), None),
|
|
("tiles_count_negative", setp(lambda m: m["tiles"]["sets"][0].update(journal_seq=-1)), None),
|
|
("src_enabled_int", setp(lambda m: m["src"].update(enabled=1)), None),
|
|
("src_chunk_bytes_zero", setp(lambda m: m["src"].update(chunk_bytes=0)), None),
|
|
("src_chunk_count_wrong", setp(lambda m: m["src"]["files"][0].update(size=25)), None),
|
|
("src_chunks_sha_wrong", setp(lambda m: m["src"]["files"][0].update(chunks_sha256="c" * 64)), None),
|
|
("src_empty_file", setp(lambda m: m["src"]["files"].append(empty_src)), None),
|
|
("signature_not_ed25519", setp(lambda m: m.update(signature="rsa:xx")), None),
|
|
("signature_int", setp(lambda m: m.update(signature=5)), None),
|
|
("unknown_keys", setp(lambda m: m.update(future={"x": [1, "y"]})), None),
|
|
("unknown_float_key", setp(lambda m: m.update(ratio=0.5)),
|
|
"Rust refuses floats anywhere in a manifest, even under keys validate() ignores"),
|
|
]
|
|
cases = []
|
|
for name, mutate, quirk in mutations:
|
|
doc = mutate(copy.deepcopy(full_manifest()))
|
|
ok, error = verdict(lambda: mf.validate(doc))
|
|
if ok == "error":
|
|
quirk = None # Python refuses it as well: not a difference
|
|
rust = "error" if quirk else ok
|
|
cases.append({"name": name, "manifest_json": json.dumps(doc, ensure_ascii=True), "python": ok,
|
|
"python_error": error, "rust_expect": rust, "quirk": quirk})
|
|
write_json("validate.json", {"cases": cases})
|
|
|
|
|
|
# ---- hashes -----------------------------------------------------------------------
|
|
|
|
def hashes_fixture():
|
|
rows = [(7, 63, 5, ".png", 13, sha(b"b")), (7, 0, 0, ".png", 13, sha(b"a")), (7, 0, 0, ".jpg", 9, sha(b"c")),
|
|
(7, 10, 2, ".png", 700, sha(b"d")), (7, 2, 10, ".png", 1, sha(b"e"))]
|
|
chunks = [sha(b"x"), sha(b"y"), sha(b"z")]
|
|
index = {"format": 1, "style": "dark", "dataset": "ds", "block_shift": 6, "journal_seq": 376157, "files": 3,
|
|
"bytes": 2437523419, "blocks": [
|
|
{"z": 0, "bx": 0, "by": 0, "files": 1, "bytes": 5321, "digest": sha(b"b0")},
|
|
{"z": 7, "bx": 1, "by": 0, "files": 2, "bytes": 2437518098, "digest": sha(b"b1")}]}
|
|
write_json("hashes.json", {
|
|
"block_digest": {"rows": [list(r) for r in rows], "digest": tiles.block_digest(rows)},
|
|
"block_digest_empty": tiles.block_digest([]),
|
|
"tile_lines": [{"row": list(r), "line": tiles.tile_line(*r).decode("ascii")} for r in rows],
|
|
"block_of": [[z, x, y, *tiles.block_of(z, x, y)] for z, x, y in
|
|
((0, 0, 0), (7, 63, 63), (7, 64, 0), (20, 1048575, 65), (3, 7, 7))],
|
|
"chunks_digest": {"chunks": chunks, "digest": mf.chunks_digest(chunks)},
|
|
"chunks_digest_empty": mf.chunks_digest([]),
|
|
"chunk_count": [[size, cb, mf.chunk_count(size, cb)] for size, cb in
|
|
((0, 10), (1, 10), (10, 10), (11, 10), (884244480, mf.CHUNK_BYTES), (2**40, mf.CHUNK_BYTES))],
|
|
"index": {"text": json.dumps(index, indent=2, sort_keys=True) + "\n", "index_sha256": tiles.index_sha256(index)},
|
|
})
|
|
|
|
|
|
# ---- the real published manifest ---------------------------------------------------
|
|
|
|
def real_fixture(source):
|
|
target = out_path("real/0.51.0.manifest.json")
|
|
if source and os.path.isfile(source):
|
|
shutil.copyfile(source, target)
|
|
with open(target, "rb") as f:
|
|
raw = f.read()
|
|
doc = json.loads(raw.decode("utf-8"))
|
|
mf.validate(doc)
|
|
signing.verify(doc, signing.bundled_public_key())
|
|
canonical = mf.canonical_bytes(doc)
|
|
write_json("real/0.51.0.hashes.json", {
|
|
"version": doc["version"],
|
|
"raw_len": len(raw),
|
|
"raw_sha256": sha(raw),
|
|
"canonical_len": len(canonical),
|
|
"canonical_sha256": sha(canonical),
|
|
"payload_sha256": sha(mf.signing_payload(doc)),
|
|
"release_public_key": signing.bundled_public_key(),
|
|
"app_files": len(doc["app"]["files"]),
|
|
})
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--real-manifest", default=DEFAULT_REAL)
|
|
args = parser.parse_args()
|
|
canonical_fixture()
|
|
signing_fixture()
|
|
relpath_fixture()
|
|
validate_fixture()
|
|
hashes_fixture()
|
|
real_fixture(args.real_manifest)
|
|
print(f"fixtures written to {HERE}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|