diff --git a/src/services/http/cap_surface.lua b/src/services/http/cap_surface.lua index 63162f8b..048beb14 100644 --- a/src/services/http/cap_surface.lua +++ b/src/services/http/cap_surface.lua @@ -33,17 +33,18 @@ function M.retain_static(conn, id, status, stats) control_plane_only = true, compat = { response_parsers = { strict = true, ['legacy-http1-close'] = true } }, state = { stats = topics.state(id, 'stats') }, - observability = { stats = topics.obs_metric(id, 'stats') }, + observability = { status = topics.obs_metric(id, 'status') }, }) conn:retain(topics.status(id), status or { state = 'starting', available = false }) conn:retain(topics.state(id, 'stats'), stats or {}) - conn:retain(topics.obs_metric(id, 'stats'), stats or {}) + conn:retain(topics.obs_metric(id, 'status'), stats or {}) end function M.unretain_static(conn, id) conn:unretain(topics.meta(id)) conn:unretain(topics.status(id)) conn:unretain(topics.state(id, 'stats')) + conn:unretain(topics.obs_metric(id, 'status')) conn:unretain(topics.obs_metric(id, 'stats')) end diff --git a/src/services/http/config.lua b/src/services/http/config.lua index 2ee9e5a9..63fd5eec 100644 --- a/src/services/http/config.lua +++ b/src/services/http/config.lua @@ -18,6 +18,12 @@ local DEFAULTS = { max_response_body = 16 * 1024 * 1024, legacy_http1_close_max_response_bytes = 1024 * 1024, }, + observability = { + status_interval_s = 30, + request_trace = false, + success_events = false, + failure_rate_limit_s = 60, + }, } local ROOT_KEYS = { @@ -25,6 +31,14 @@ local ROOT_KEYS = { enabled = true, id = true, policy = true, + observability = true, +} + +local OBSERVABILITY_KEYS = { + status_interval_s = true, + request_trace = true, + success_events = true, + failure_rate_limit_s = true, } local POLICY_KEYS = { @@ -128,6 +142,35 @@ local function normalise_policy(raw) return out, nil end + +local function normalise_observability(raw) + if raw == nil then raw = {} end + if type(raw) ~= 'table' then return fail('observability must be a table') end + local ok, err = allowed(raw, OBSERVABILITY_KEYS, 'observability') + if not ok then return nil, err end + + local out = copy_plain(DEFAULTS.observability) + local v + + v, err = positive_number_or_nil(raw.status_interval_s, 'observability.status_interval_s') + if err then return nil, err end + if v ~= nil then out.status_interval_s = v end + + v, err = bool_or_nil(raw.request_trace, 'observability.request_trace') + if err then return nil, err end + if v ~= nil then out.request_trace = v end + + v, err = bool_or_nil(raw.success_events, 'observability.success_events') + if err then return nil, err end + if v ~= nil then out.success_events = v end + + v, err = positive_number_or_nil(raw.failure_rate_limit_s, 'observability.failure_rate_limit_s') + if err then return nil, err end + if v ~= nil then out.failure_rate_limit_s = v end + + return out, nil +end + function M.normalise(raw) if raw == nil then raw = {} end if type(raw) ~= 'table' then return fail('http config must be a table') end @@ -147,11 +190,15 @@ function M.normalise(raw) local policy, perr = normalise_policy(raw.policy) if not policy then return nil, perr end + local observability, oerr = normalise_observability(raw.observability) + if not observability then return nil, oerr end + return { schema = M.SCHEMA, enabled = enabled ~= false, id = id or DEFAULTS.id, policy = policy, + observability = observability, }, nil end diff --git a/src/services/http/service.lua b/src/services/http/service.lua index dbc5a442..49994960 100644 --- a/src/services/http/service.lua +++ b/src/services/http/service.lua @@ -79,6 +79,50 @@ local function count_where(t, pred) return n end +local function now() + return fibers.now and fibers.now() or os.clock() +end + +local function obs_config(self) + local cfg = self._state and self._state.config and self._state.config.observability + return cfg or config_mod.DEFAULTS.observability +end + +local function obs_status_key(snap) + return table.concat({ + tostring(snap.state), + tostring(snap.backend), + tostring(snap.ready), + tostring(snap.active_listeners), + tostring(snap.active_websockets), + tostring(snap.last_error), + tostring(snap.policy_generation), + }, '|') +end + +local function uri_summary(uri) + if type(uri) ~= 'string' then return {} end + local scheme, rest = uri:match('^([%w%+%-%.]+)://(.+)$') + if not scheme then return { uri = uri } end + local authority, path_query = rest:match('^([^/%?#]+)(.*)$') + path_query = path_query or '' + local host = authority or '' + host = host:gsub('^.-@', '') + local path, query = path_query:match('^([^%?]*)(%?.*)$') + if path == nil then path = path_query end + if path == '' then path = '/' end + local cmd = query and query:match('[%?&]cmd=([^&]+)') or nil + return { scheme = scheme, host = host, path = path, query = query, cmd = cmd } +end + +local function request_summary(req) + local p = req and req.payload or {} + local u = uri_summary(p.uri) + u.method = p.method or 'GET' + u.response_parser = p.response_parser + return u +end + function HttpService:_derive_snapshot() local st = self._state return { @@ -101,7 +145,100 @@ function HttpService:_derive_snapshot() } end -function HttpService:_publish_model() + +function HttpService:_publish_obs_log(level, payload) + payload = payload or {} + payload.service = 'http' + payload.http_id = self._id + payload.ts = now() + self._conn:publish(topics.obs_log(self._id, level), payload) + return true +end + +function HttpService:_should_publish_obs_status(snap, ev) + local cfg = obs_config(self) + local key = obs_status_key(snap) + local obs = self._obs or {} + if key ~= obs.last_status_key then return true, key end + local interval = cfg.status_interval_s or 30 + if interval > 0 and ((now() - (obs.last_status_emit_at or 0)) >= interval) then return true, key end + if ev and ev.kind == 'http_operation_done' and ev.status == 'ok' and cfg.success_events == true then return true, key end + return false, key +end + +function HttpService:_publish_obs_status(snap, ev) + local publish, key = self:_should_publish_obs_status(snap, ev) + if not publish then return true end + self._conn:retain(topics.obs_metric(self._id, 'status'), snap) + self._obs.last_status_key = key + self._obs.last_status_emit_at = now() + return true +end + +function HttpService:_record_http_failure(rec, ev) + local cfg = obs_config(self) + local limit_s = cfg.failure_rate_limit_s or 60 + local req = self._state.requests[rec.request_id] or {} + local target = req.target or {} + local reason = ev.primary or ev.status or 'operation_failed' + self._obs.had_failure = true + local key = table.concat({ + tostring(rec.operation), + tostring(reason), + tostring(target.method), + tostring(target.host), + tostring(target.path), + tostring(target.cmd or target.query), + }, '|') + local windows = self._obs.failure_windows + local win = windows[key] + local t = now() + if not win then + windows[key] = { first_at = t, last_at = t, suppressed = 0 } + return self:_publish_obs_log('warn', { + what = 'request_failed', + reason = reason, + operation = rec.operation, + method = target.method, + host = target.host, + path = target.path, + cmd = target.cmd, + response_parser = target.response_parser, + }) + end + win.last_at = t + win.suppressed = (win.suppressed or 0) + 1 + if limit_s <= 0 or (t - (win.first_at or t)) >= limit_s then + local suppressed = win.suppressed or 0 + win.first_at = t + win.suppressed = 0 + return self:_publish_obs_log('warn', { + what = 'request_failed_suppressed', + reason = reason, + operation = rec.operation, + method = target.method, + host = target.host, + path = target.path, + cmd = target.cmd, + response_parser = target.response_parser, + suppressed = suppressed, + window_s = limit_s, + }) + end + return true +end + +function HttpService:_record_http_recovery(rec) + if not self._obs.had_failure then return true end + self._obs.had_failure = false + self._obs.failure_windows = {} + return self:_publish_obs_log('info', { + what = 'request_recovered', + operation = rec and rec.operation, + }) +end + +function HttpService:_publish_model(ev) if self._closed and not self._publishing_after_close then return end local snap = self._model:snapshot() self._conn:retain(topics.status(self._id), { @@ -112,12 +249,12 @@ function HttpService:_publish_model() last_error = snap.last_error, }) self._conn:retain(topics.state(self._id, 'stats'), snap) - self._conn:retain(topics.obs_metric(self._id, 'stats'), snap) + self:_publish_obs_status(snap, ev) end -function HttpService:_refresh_model() +function HttpService:_refresh_model(ev) local changed = self._model:set_snapshot(self:_derive_snapshot()) - if changed ~= nil then self:_publish_model() end + if changed ~= nil then self:_publish_model(ev) end return true end @@ -442,6 +579,7 @@ function HttpService:_handle_cap_request(ev) generation = ev.generation or self._generation, verb = verb, owner = owner, + target = request_summary(req), state = 'received', } end @@ -483,10 +621,11 @@ function HttpService:_handle_operation_done(ev) rec.report = ev.report rec.result = ev.result rec.primary = ev.primary - if ev.status ~= 'ok' then self._state.last_error = ev.primary or ev.status end + if ev.status ~= 'ok' then self._state.last_error = ev.primary or ev.status; self._obs.had_failure = true end local owner = self._owned_requests[rec.request_id] local reqrec = self._state.requests[rec.request_id] + local operation_success = false if ev.status == 'ok' then local reply = ev.result or {} local ok, rerr = true, nil @@ -494,23 +633,41 @@ function HttpService:_handle_operation_done(ev) ok, rerr = owner:reply_once(reply) end if ok == true then + operation_success = true if reply.handle_id then self._registry:mark_transferred(reply.handle_id) end if reqrec then reqrec.state = 'resolved' end else if reply.handle_id then self:_terminate_handle(reply.handle_id, rerr or 'reply_failed') end if reqrec then reqrec.state = 'failed'; reqrec.reason = rerr or 'reply_failed' end self._state.last_error = rerr or 'reply_failed' + self._obs.had_failure = true + self:_record_http_failure(rec, { status = 'failed', primary = rerr or 'reply_failed' }) end else if owner and not owner:done() then owner:fail_once(ev.primary or ev.status or 'operation_failed') end if reqrec then reqrec.state = ev.status or 'failed'; reqrec.reason = ev.primary end + self:_record_http_failure(rec, ev) + end + if obs_config(self).request_trace == true then + local target = reqrec and reqrec.target or {} + self:_publish_obs_log('debug', { + what = 'request_completed', + operation = rec.operation, + status = ev.status, + reason = ev.primary, + method = target.method, + host = target.host, + path = target.path, + cmd = target.cmd, + response_parser = target.response_parser, + }) end + if operation_success == true then self:_record_http_recovery(rec) end self._owned_requests[rec.request_id] = nil self:_log_event(ev) return true end - function HttpService:_handle_config_changed(ev) local cfg, err = config_mod.normalise(ev.raw or {}) if not cfg then @@ -608,7 +765,7 @@ function HttpService:_handle_event(ev) local log_now = ev.kind ~= 'http_operation_done' if log_now then self:_log_event(ev) end local ok, err = self:_reduce_event(ev) - self:_refresh_model() + self:_refresh_model(ev) return ok, err end @@ -723,6 +880,12 @@ function M.open_handle(conn, opts) _events = {}, _event_seq = 0, _owned_requests = {}, + _obs = { + last_status_key = nil, + last_status_emit_at = nil, + failure_windows = {}, + had_failure = false, + }, _state = { service_state = 'starting', backend = 'starting', diff --git a/src/services/http/topics.lua b/src/services/http/topics.lua index feda59d8..2b5d87b1 100644 --- a/src/services/http/topics.lua +++ b/src/services/http/topics.lua @@ -15,6 +15,7 @@ function M.status(id) return topic.append(cap(id), 'status') end function M.state(id, key) return topic.append(cap(id), 'state', key) end function M.event(id, name) return topic.append(cap(id), 'event', name) end function M.obs_metric(id, name) return { 'obs', 'v1', 'http', 'metric', id or 'main', name } end +function M.obs_log(id, level) return { 'obs', 'v1', 'http', 'log', id or 'main', level or 'info' } end function M.rpc(id, verb) return topic.append(cap(id), 'rpc', verb) end return M diff --git a/src/services/ui/config.lua b/src/services/ui/config.lua index af1ced96..edec3cdf 100644 --- a/src/services/ui/config.lua +++ b/src/services/ui/config.lua @@ -32,6 +32,9 @@ local DEFAULTS = { sessions = { prune_interval = 60, }, + observability = { + status_interval_s = 30, + }, } local ROOT_KEYS = { @@ -42,6 +45,7 @@ local ROOT_KEYS = { sse = true, uploads = true, sessions = true, + observability = true, } local HTTP_KEYS = { @@ -59,6 +63,7 @@ local STATIC_KEYS = { root = true, index = true, chunk_size = true } local SSE_KEYS = { enabled = true, queue_len = true, max_replay = true, replay = true, pattern = true } local UPLOAD_KEYS = { enabled = true, max_bytes = true, require_auth = true } local SESSION_KEYS = { prune_interval = true } +local OBSERVABILITY_KEYS = { status_interval_s = true } local function fail(msg) return nil, msg end @@ -240,6 +245,22 @@ local function normalise_sessions(raw) return out, nil end + +local function normalise_observability(raw) + local err + raw, err = table_or_empty(raw, 'observability') + if not raw then return nil, err end + local ok + ok, err = allowed(raw, OBSERVABILITY_KEYS, 'observability') + if not ok then return nil, err end + local out = copy_plain(DEFAULTS.observability) + local v + v, err = positive_number_or_false_or_nil(raw.status_interval_s, 'observability.status_interval_s') + if err then return nil, err end + if v ~= nil then out.status_interval_s = v end + return out, nil +end + function M.normalise(raw) if raw == nil then raw = {} end if type(raw) ~= 'table' then return fail('ui config must be a table') end @@ -258,6 +279,7 @@ function M.normalise(raw) local sse; sse, err = normalise_sse(raw.sse); if not sse then return nil, err end local uploads; uploads, err = normalise_uploads(raw.uploads); if not uploads then return nil, err end local sessions; sessions, err = normalise_sessions(raw.sessions); if not sessions then return nil, err end + local observability; observability, err = normalise_observability(raw.observability); if not observability then return nil, err end return { schema = M.SCHEMA, @@ -267,6 +289,7 @@ function M.normalise(raw) sse = sse, uploads = uploads, sessions = sessions, + observability = observability, }, nil end diff --git a/src/services/ui/service.lua b/src/services/ui/service.lua index e96c6eb0..d5c01bdd 100644 --- a/src/services/ui/service.lua +++ b/src/services/ui/service.lua @@ -9,6 +9,7 @@ local mailbox = require 'fibers.mailbox' local service_base = require 'devicecode.service_base' local scoped_work = require 'devicecode.support.scoped_work' local sleep = require 'fibers.sleep' +local runtime = require 'fibers.runtime' local queue = require 'devicecode.support.queue' local read_model = require 'services.ui.read_model' local queries = require 'services.ui.queries' @@ -31,6 +32,56 @@ local function dependency_snapshot(state) return dep_slot.snapshot(state, 'http_deps') end +local function sorted_keys(t) + local keys = {} + for k in pairs(t or {}) do keys[#keys + 1] = k end + table.sort(keys, function (a, b) return tostring(a) < tostring(b) end) + return keys +end + +local function stable_status_value(v, key) + if key == 'at' or key == 'ts' or key == 'run_id' or key == 'updated_at' then return '' end + if type(v) ~= 'table' then return tostring(v) end + local parts = { '{' } + for _, k in ipairs(sorted_keys(v)) do + if k ~= 'at' and k ~= 'ts' and k ~= 'run_id' and k ~= 'updated_at' then + parts[#parts + 1] = tostring(k) + parts[#parts + 1] = '=' + parts[#parts + 1] = stable_status_value(v[k], k) + parts[#parts + 1] = ';' + end + end + parts[#parts + 1] = '}' + return table.concat(parts) +end + +local function lifecycle_status_key(service_state, payload) + return stable_status_value({ state = service_state, payload = payload }, nil) +end + +local function ui_observability(state) + local cfg = state and state.config and state.config.observability or nil + return cfg or config_mod.DEFAULTS.observability +end + +local function should_publish_lifecycle(state, key) + state.lifecycle_obs = state.lifecycle_obs or {} + local obs = state.lifecycle_obs + if obs.status_key ~= key then return true end + local interval = ui_observability(state).status_interval_s + if interval == false then return false end + interval = tonumber(interval) or 30 + if interval <= 0 then return false end + return (runtime.now() - (obs.last_status_emit_at or 0)) >= interval +end + +local function note_lifecycle_published(state, key) + state.lifecycle_obs = state.lifecycle_obs or {} + state.lifecycle_obs.status_key = key + state.lifecycle_obs.last_status_emit_at = runtime.now() +end + + local function component_summary(components) local out = {} for name, c in pairs(components or {}) do @@ -314,6 +365,9 @@ local function update_lifecycle(state) config_generation = state.config_generation, dependencies = dependency_snapshot(state), } + local key = lifecycle_status_key(service_state, payload) + if not should_publish_lifecycle(state, key) then return true, nil end + note_lifecycle_published(state, key) if service_state == 'disabled' then return lifecycle:status('disabled', payload) end if service_state == 'degraded' then return lifecycle:degraded(payload) end if service_state == 'failed' then return lifecycle:failed(reason or 'ui_failed', payload) end @@ -678,6 +732,7 @@ function M.run(scope, params) active_requests = 0, rejected_requests = 0, components = {}, + lifecycle_obs = {}, last_error = nil, } @@ -758,6 +813,8 @@ M._test = { apply_config = apply_config, lifecycle_readiness = lifecycle_readiness, update_lifecycle = update_lifecycle, + lifecycle_status_key = lifecycle_status_key, + should_publish_lifecycle = should_publish_lifecycle, } diff --git a/tests/unit/http/test_config.lua b/tests/unit/http/test_config.lua index 2396059b..8d0990c4 100644 --- a/tests/unit/http/test_config.lua +++ b/tests/unit/http/test_config.lua @@ -18,6 +18,10 @@ function M.test_normalise_supplies_safe_defaults() eq(cfg.policy.allowed_response_parsers.strict, true) eq(cfg.policy.allowed_response_parsers['legacy-http1-close'], nil) eq(cfg.policy.legacy_http1_close_max_response_bytes, 1024 * 1024) + eq(cfg.observability.status_interval_s, 30) + eq(cfg.observability.request_trace, false) + eq(cfg.observability.success_events, false) + eq(cfg.observability.failure_rate_limit_s, 60) end function M.test_policy_updates_are_copied_and_validated() @@ -41,6 +45,30 @@ function M.test_policy_updates_are_copied_and_validated() eq(cfg.policy.allowed_hosts['example.com'], true, 'normalised config must not alias raw tables') end + +function M.test_observability_updates_are_copied_and_validated() + local raw = { + schema = config.SCHEMA, + observability = { + status_interval_s = 10, + request_trace = true, + success_events = true, + failure_rate_limit_s = 20, + }, + } + local cfg = ok(config.normalise(raw)) + eq(cfg.observability.status_interval_s, 10) + eq(cfg.observability.request_trace, true) + eq(cfg.observability.success_events, true) + eq(cfg.observability.failure_rate_limit_s, 20) + raw.observability.status_interval_s = 99 + eq(cfg.observability.status_interval_s, 10, 'normalised config must not alias raw observability') + + local bad, err = config.normalise({ schema = config.SCHEMA, observability = { every_request = true } }) + eq(bad, nil) + ok(tostring(err):find('observability has unknown field', 1, true)) +end + function M.test_rejects_unknown_fields() local cfg, err = config.normalise({ schema = config.SCHEMA, backend = {} }) eq(cfg, nil) diff --git a/tests/unit/http/test_service.lua b/tests/unit/http/test_service.lua index f10b3551..2d993d8f 100644 --- a/tests/unit/http/test_service.lua +++ b/tests/unit/http/test_service.lua @@ -1,5 +1,6 @@ local fibers = require 'fibers' local runtime = require 'fibers.runtime' +local op = require 'fibers.op' local sleep = require 'fibers.sleep' local mailbox = require 'fibers.mailbox' local bus = require 'bus' @@ -36,6 +37,21 @@ local function yield_until(pred, msg) error(msg or 'condition was not reached', 2) end +local function recv_payload(sub, label) + local which, msg, err = fibers.perform(op.named_choice({ + msg = sub:recv_op(), + timeout = sleep.sleep_op(0.1), + })) + if which ~= 'msg' then error(label or 'timed out waiting for message', 2) end + ok(msg, err or label or 'expected message') + return msg.payload +end + +local function table_empty(t) + for _ in pairs(t or {}) do return false end + return true +end + local function fake_controller() local q = {} return { @@ -72,6 +88,59 @@ function M.test_service_retains_capability_metadata_and_status() end) end + +function M.test_http_failure_recovery_is_visible_and_resets_suppression() + fibers.run(function () + local topics = require 'services.http.topics' + local b = bus.new() + local root = b:connect({ origin_base = { kind = 'local' } }) + local warn_sub = root:subscribe(topics.obs_log('main', 'warn'), { queue_len = 10 }) + local info_sub = root:subscribe(topics.obs_log('main', 'info'), { queue_len = 10 }) + local svc = ok(http_service.open_handle(root, { driver = fake_driver(), id = 'main' })) + yield_many(4) + + local request_id = 'req-recovery' + svc._state.requests[request_id] = { + request_id = request_id, + target = { + method = 'GET', + host = '172.28.100.9', + path = '/cgi/get.cgi', + cmd = 'sys_cpumem', + response_parser = 'legacy-http1-close', + }, + } + local rec = { operation = 'exchange', request_id = request_id } + local ev = { status = 'failed', primary = 'timeout' } + + ok(svc:_record_http_failure(rec, ev)) + local first_failure = recv_payload(warn_sub, 'expected first failure log') + eq(first_failure.what, 'request_failed') + eq(first_failure.reason, 'timeout') + eq(first_failure.cmd, 'sys_cpumem') + + ok(svc:_record_http_failure(rec, ev)) + local saw_suppressed = false + for _, win in pairs(svc._obs.failure_windows) do + if (win.suppressed or 0) == 1 then saw_suppressed = true end + end + ok(saw_suppressed, 'second equivalent failure should be suppressed within the rate limit window') + + ok(svc:_record_http_recovery({ operation = 'exchange' })) + local recovery = recv_payload(info_sub, 'expected recovery log') + eq(recovery.what, 'request_recovered') + eq(recovery.operation, 'exchange') + ok(table_empty(svc._obs.failure_windows), 'recovery must clear failure suppression windows') + + ok(svc:_record_http_failure(rec, ev)) + local fresh_failure = recv_payload(warn_sub, 'expected post-recovery failure log') + eq(fresh_failure.what, 'request_failed') + eq(fresh_failure.reason, 'timeout') + + svc:terminate('done') + end) +end + function M.test_non_local_handle_returning_call_is_rejected_before_backend_work() fibers.run(function () local b = bus.new() diff --git a/tests/unit/ui/test_config.lua b/tests/unit/ui/test_config.lua index 801e519d..bf276558 100644 --- a/tests/unit/ui/test_config.lua +++ b/tests/unit/ui/test_config.lua @@ -17,6 +17,29 @@ function M.test_normalise_supplies_http_static_sse_and_upload_defaults() eq(cfg.sse.enabled, true) eq(cfg.uploads.enabled, true) eq(cfg.uploads.require_auth, false) + eq(cfg.observability.status_interval_s, 30) +end + + +function M.test_observability_status_interval_is_configurable() + local cfg = ok(config.normalise({ + schema = config.SCHEMA, + observability = { status_interval_s = 15 }, + })) + eq(cfg.observability.status_interval_s, 15) + + cfg = ok(config.normalise({ + schema = config.SCHEMA, + observability = { status_interval_s = false }, + })) + eq(cfg.observability.status_interval_s, false) + + local bad, err = config.normalise({ + schema = config.SCHEMA, + observability = { every_status = true }, + }) + eq(bad, nil) + ok(tostring(err):find('observability has unknown field', 1, true)) end function M.test_uploads_can_require_authentication() diff --git a/tests/unit/ui/test_service.lua b/tests/unit/ui/test_service.lua index 24083859..d9b2937b 100644 --- a/tests/unit/ui/test_service.lua +++ b/tests/unit/ui/test_service.lua @@ -191,6 +191,45 @@ function tests.test_cleanup_error_recording_is_explicit_and_non_throwing() end + +function tests.test_lifecycle_status_is_gated_by_semantic_change() + local calls = {} + local lifecycle = { + running = function (_, payload) calls[#calls + 1] = payload; return payload end, + } + local state = base_state() + state.lifecycle = lifecycle + state.config_status = 'ok' + state.config_generation = 1 + state.config = { enabled = true, http = { enabled = true }, observability = { status_interval_s = 30 } } + state.lifecycle_obs = {} + + service._test.update_lifecycle(state) + assert_eq(#calls, 1) + service._test.update_lifecycle(state) + assert_eq(#calls, 1, 'unchanged lifecycle status should not republish') + + state.config_generation = 2 + service._test.update_lifecycle(state) + assert_eq(#calls, 2, 'semantic lifecycle change should publish') +end + +function tests.test_lifecycle_status_key_ignores_volatile_dependency_timestamps() + local a = service._test.lifecycle_status_key('running', { + ready = true, + dependencies = { + http = { status = 'ready', available = true, updated_at = 1.0 }, + }, + }) + local b = service._test.lifecycle_status_key('running', { + ready = true, + dependencies = { + http = { status = 'ready', available = true, updated_at = 2.0 }, + }, + }) + assert_eq(a, b) +end + function tests.test_lifecycle_not_ready_until_config_and_listener_are_ready() local calls = {} local lifecycle = {