Skip to content

fix(macos): gate boot on Secure Enclave session — stop TWO FEDERATION IDENTITIES (CIRISServer#380) #1046

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
emooreatx wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(macos): gate boot on Secure Enclave session — stop TWO FEDERATION IDENTITIES (CIRISServer#380) #1046

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions BUILD_INFO.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# Build Information
Code Hash: 1cbf00a4ba50
Build Time: 2026-08-15T11:19:53.240040
Git Commit: b3b522a88fa68ffa317cca987fb810c642aef995
Git Branch: release/ios-2.9.16
Code Hash: 41b9a0a4f6b9
Build Time: 2026-08-16T00:05:18.002897
Git Commit: b98b1066bf49dd7246e13d6f95ab8141b18cb615
Git Branch: fix/macos-se-session-gate

This hash is a SHA-256 of all Python source files in the repository.
It provides a deterministic version identifier based on the actual code content.
77 changes: 28 additions & 49 deletions ciris_engine/logic/persistence/db/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,8 @@ def _startup_lock_options() -> str:
"""
import os

lock_timeout = os.environ.get(
DB_INIT_LOCK_TIMEOUT_ENV, DB_INIT_LOCK_TIMEOUT_DEFAULT
).strip()
statement_timeout = os.environ.get(
DB_INIT_STATEMENT_TIMEOUT_ENV, DB_INIT_STATEMENT_TIMEOUT_DEFAULT
).strip()
lock_timeout = os.environ.get(DB_INIT_LOCK_TIMEOUT_ENV, DB_INIT_LOCK_TIMEOUT_DEFAULT).strip()
statement_timeout = os.environ.get(DB_INIT_STATEMENT_TIMEOUT_ENV, DB_INIT_STATEMENT_TIMEOUT_DEFAULT).strip()

parts = []
if lock_timeout and lock_timeout != "0":
Expand Down Expand Up @@ -252,17 +248,13 @@ def _blocker_hint(dsn: str) -> str:
conn.close()
if rows:
rendered = "; ".join(
f"pid={r[0]} state={r[1]} age={r[2]} blocked_by={r[3]} query={str(r[4])[:200]}"
for r in rows
f"pid={r[0]} state={r[1]} age={r[2]} blocked_by={r[3]} query={str(r[4])[:200]}" for r in rows
)
return f" Active backends: {rendered}."
except Exception as probe_err: # noqa: BLE001 - diagnostics are best-effort
logger.debug("pg_stat_activity blocker probe unavailable: %s", probe_err)

return (
" To identify the blocker, run against the same database: "
f"{PG_BLOCKER_DIAGNOSTIC_SQL}"
)
return " To identify the blocker, run against the same database: " f"{PG_BLOCKER_DIAGNOSTIC_SQL}"


def initialize_database(db_path: Optional[str] = None) -> None:
Expand Down Expand Up @@ -341,24 +333,16 @@ def _persist_dsn_and_sentinel(db_path: str) -> Tuple[str, Optional[Path]]:
# SQLAlchemy form: sqlite:///rel/path (3 slashes -> relative) or
# sqlite:////abs/path (4 slashes -> absolute). Splitting on
# 'sqlite:///' keeps the right leading-slash count for Path().
path_part = (
db_path.split("sqlite:///", 1)[-1] if "sqlite:///" in db_path else ""
)
sentinel = (
Path(path_part).resolve().parent
if path_part and path_part != ":memory:"
else None
)
path_part = db_path.split("sqlite:///", 1)[-1] if "sqlite:///" in db_path else ""
sentinel = Path(path_part).resolve().parent if path_part and path_part != ":memory:" else None
return db_path, sentinel
abs_path = Path(db_path).resolve()
# `sqlite:///{abs_path}` where abs_path begins with '/' yields
# 'sqlite:////absolute/path' — 4 slashes, absolute as required.
return f"sqlite:///{abs_path}", abs_path.parent


def _construct_engine_bounded(
construct: Callable[[], Any], dsn: str, bump_stack: bool = False
) -> Any:
def _construct_engine_bounded(construct: Callable[[], Any], dsn: str, bump_stack: bool = False) -> Any:
"""Run persist's blocking Engine constructor under a wall-clock bound.

Three properties #937 needs and the previous inline call lacked:
Expand Down Expand Up @@ -406,9 +390,7 @@ def _worker() -> None:
try:
if bump_stack:
threading.stack_size(8 * 1024 * 1024)
worker = threading.Thread(
target=_worker, name="persist-engine-init", daemon=True
)
worker = threading.Thread(target=_worker, name="persist-engine-init", daemon=True)
worker.start()
finally:
threading.stack_size(prev_stack)
Expand Down Expand Up @@ -482,13 +464,8 @@ def _bootstrap_persist_engine(db_path: Optional[str]) -> None:
# an _expected_dsn that actually matches and the idempotent-skip works.
_expected_dsn = _persist_dsn_and_sentinel(_resolved_db_path)[0]

if (
graph_persistence._engine is not None
and graph_persistence._engine_dsn == _expected_dsn
):
logger.debug(
"persist engine already wired to %s, skipping re-bootstrap", _expected_dsn
)
if graph_persistence._engine is not None and graph_persistence._engine_dsn == _expected_dsn:
logger.debug("persist engine already wired to %s, skipping re-bootstrap", _expected_dsn)
return

# Resolve the DSN. Postgres takes its own URL; SQLite uses
Expand Down Expand Up @@ -648,15 +625,26 @@ def _construct_engine() -> "Engine":
create_identity_if_missing=True,
)

# macOS Secure Enclave session gate (CIRISServer#380). The federation
# keystore is opened here (the Engine) and again by the node's compose in
# `serve_with_python_adapter`. On a macOS console session whose screen is
# LOCKED, the Secure Enclave is only intermittently reachable, so those two
# opens can seal DIFFERENT keys under one alias and the node refuses to boot
# with "TWO FEDERATION IDENTITIES IN ONE NODE". Block here until SE is
# deterministically reachable (unlocked session → use SE) or consistently
# unavailable (headless → software-only is deterministic). No-op off macOS,
# and on iOS (which reaches its Secure Enclave reliably in the foreground).
from ciris_engine.logic.runtime.se_session_gate import await_secure_enclave_session

await_secure_enclave_session()

try:
# #937 — ALWAYS construct on a worker thread, not just on iOS.
# The thread was originally an iOS stack-size workaround; it is now
# also the only way to bound a blocking FFI call from Python. The
# main thread joins with a deadline, logs progress while it waits,
# and raises DatabaseInitializationTimeout if the budget is spent.
engine = cast(
Any, _construct_engine_bounded(_construct_engine, dsn, bump_stack=_is_ios)
)
engine = cast(Any, _construct_engine_bounded(_construct_engine, dsn, bump_stack=_is_ios))
except DatabaseInitializationTimeout:
# #937 — MUST precede the stale-lock heuristic below. That heuristic
# is a substring match on "lock", and the timeout message says
Expand Down Expand Up @@ -698,16 +686,12 @@ def _construct_engine() -> "Engine":

reset_engine()
except Exception as reset_err:
logger.debug(
"reset_engine() before retry failed (non-fatal): %s", reset_err
)
logger.debug("reset_engine() before retry failed (non-fatal): %s", reset_err)
# Bounded on the retry too — a wedged retry is the same
# unbounded hang wearing a different hat (#937).
engine = cast(
Any,
_construct_engine_bounded(
lambda: Engine(dsn, signing_key_id), dsn, bump_stack=True
),
_construct_engine_bounded(lambda: Engine(dsn, signing_key_id), dsn, bump_stack=True),
)
else:
raise
Expand All @@ -728,15 +712,10 @@ def _construct_engine() -> "Engine":
try:
import json as _json

logger.info(
"A0a migration sentinel absent — running legacy graph migration"
)
logger.info("A0a migration sentinel absent — running legacy graph migration")
raw = engine.run_legacy_graph_migration(_json.dumps({"dry_run": False}))
stats = _json.loads(raw) if isinstance(raw, (bytes, str)) else raw
if (
stats.get("outcome") in ("ok", "partial")
and stats.get("errors", 0) == 0
):
if stats.get("outcome") in ("ok", "partial") and stats.get("errors", 0) == 0:
sentinel.write_text(
f'{{"nodes_written":{stats.get("nodes_written", 0)},'
f'"edges_written":{stats.get("edges_written", 0)}}}'
Expand Down
51 changes: 44 additions & 7 deletions ciris_engine/logic/runtime/node_fold.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

logger = logging.getLogger(__name__)


def _this_process_owns_port(port: int) -> Optional[bool]:
"""Does THIS process hold the listening socket on `port`?

Expand Down Expand Up @@ -139,7 +140,9 @@ def _surface_first_run_claim_pin() -> None:
return # pre-0.5.119 wheel — banner-only capture still applies
pin = accessor()
if pin:
line = f"Node fold: OWNERSHIP UNCLAIMED — one-time CLAIM PIN: {pin} (console-only; used by setup self-claim)"
line = (
f"Node fold: OWNERSHIP UNCLAIMED — one-time CLAIM PIN: {pin} (console-only; used by setup self-claim)"
)
print(line, flush=True) # → logcat python.stdout on Android; console on desktop
logger.info(line) # → <home>/logs/latest.log for the file-tail capture
except Exception as exc: # noqa: BLE001 — never let PIN surfacing break the boot
Expand All @@ -165,10 +168,15 @@ def _reprime_federation_delivery(path: str) -> None:

reprime = getattr(ciris_server, "reprime_federation_delivery", None)
if reprime is None:
logger.info("Node fold: reprime_federation_delivery unavailable (wheel <0.5.124) — canonical prime not re-driven (%s)", path)
logger.info(
"Node fold: reprime_federation_delivery unavailable (wheel <0.5.124) — canonical prime not re-driven (%s)",
path,
)
return
count = reprime()
logger.info("Node fold: reprime_federation_delivery(%s) → %s canonical delivery target(s) re-seeded", path, count)
logger.info(
"Node fold: reprime_federation_delivery(%s) → %s canonical delivery target(s) re-seeded", path, count
)
except Exception as exc: # noqa: BLE001
# Non-fatal: delivery re-prime failure must never take down the fold —
# the seal path still works; only the ship waits for the next prime.
Expand Down Expand Up @@ -391,6 +399,32 @@ def start_node_fold(brain_port: int, *, home: Optional[str] = None, key_id: Opti
resolved_key = key_id or _resolve_key_id()
adapter = BrainAdapter(upstream=f"http://127.0.0.1:{brain_port}")

# CIRISServer#380 "TWO FEDERATION IDENTITIES" RCA instrument: the node resolves
# its ONE federation identity from `<home>/identity` + alias, and the sealed
# keystore keys off (identity_dir, alias). When it refuses for a two-identity
# mismatch the substrate error does NOT name the on-disk artifacts, so a stray
# sealed blob, a bare `ed25519.seed`, a `.superseded-*` archive, or a second
# alias's key is invisible. Enumerate the dir here so the refusal is diagnosable
# from the agent log alone. Cheap, once per boot, never throws.
try:
from ciris_engine.logic.utils.path_resolution import get_identity_dir

_idir = get_identity_dir()
_entries = (
sorted(f"{p.name} ({p.stat().st_size}B)" for p in _idir.iterdir() if p.is_file())
if _idir.is_dir()
else ["<identity dir does not exist>"]
)
logger.info(
"Node fold: identity resolution — home=%s alias=%s identity_dir=%s\n contents: %s",
resolved_home,
resolved_key,
str(_idir),
"\n ".join(_entries) if _entries else "<empty>",
)
except Exception as exc: # noqa: BLE001 - diagnostic must never break boot
logger.warning("Node fold: identity-dir enumeration failed (non-fatal): %s", exc)

def _run() -> None:
global _node_error
try:
Expand Down Expand Up @@ -428,6 +462,7 @@ def _run() -> None:
_mobile = is_android() or is_ios()
except Exception: # noqa: BLE001
_mobile = False

# compose_status() (ciris-server ≥0.5.120, CIRISServer#279): in-process
# compose-progress snapshot — {"completed", "current": {phase, elapsed_s,
# stuck, ...} | null, "history": [{phase, ms}]}. Poll it during the bind
Expand All @@ -452,7 +487,9 @@ def _compose_phase() -> Optional[str]:
except Exception: # noqa: BLE001
return None

_attempts = 190 if _mobile else 115 # mobile ~100s (must sit UNDER the 120s Start Adapters step timeout), desktop ~60s
_attempts = (
190 if _mobile else 115
) # mobile ~100s (must sit UNDER the 120s Start Adapters step timeout), desktop ~60s
_last_phase: Optional[str] = None
for _i in range(_attempts):
if _node_error is not None:
Expand All @@ -477,7 +514,7 @@ def _compose_phase() -> Optional[str]:
f"(node-fails ⇒ agent-fails); compose phase at expiry: {_wedged or 'unknown (no compose_status — wheel <0.5.120?)'}"
)
logger.info("Node fold: node runtime started — substrate read-API LISTENING on 4243 ✅")

# Hand the node the deployment's OAuth providers now that it is serving.
#
# 2.9.14 moved /v1/auth/* onto the node but did not carry across the provider
Expand All @@ -486,10 +523,10 @@ def _compose_phase() -> Optional[str]:
# Best-effort and idempotent: a desktop install has no oauth.json and needs none.
try:
from ciris_engine.logic.runtime.oauth_provider_sync import sync_oauth_providers_to_node

sync_oauth_providers_to_node()
except Exception: # pragma: no cover - never block the boot on OAuth config
logger.exception('Node fold: OAuth provider sync failed (agent continues)')
logger.exception("Node fold: OAuth provider sync failed (agent continues)")
_reprime_federation_delivery("post-bind")
_author_federation_consent("post-bind")
# Surface the one-time first-run CLAIM PIN (minted during compose, stashed
Expand Down
Loading
Loading