From 8af211a7a5a04d077afb8909dfc9ac6c2c8031d3 Mon Sep 17 00:00:00 2001 From: Mike Herak Date: Tue, 14 Jul 2026 15:01:07 -0400 Subject: [PATCH 1/5] fix(intake): _coord read Osiris [lat,lng] coords as GeoJSON [lng,lat] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The single coords branch applied the GeoJSON convention to both geometry.coordinates (correctly [lng,lat]) and the Osiris 'coords' field, which is [lat,lng] — see KEYWORD_COORDS in the news route ('iran': [32.427, 53.688]). Every located news event came out transposed: Iran stories plotted at 53.7N 32.4E (western Russia). Split the branches per source convention and add a range sanity guard (|lat|<=90, |lng|<=180). --- engine/osiris_intake.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/engine/osiris_intake.py b/engine/osiris_intake.py index 5ccf46e..02c660b 100644 --- a/engine/osiris_intake.py +++ b/engine/osiris_intake.py @@ -103,13 +103,21 @@ def _text(d: dict, *keys: str) -> str: def _coord(d: dict): lat = d.get("lat") or d.get("latitude") lng = d.get("lng") or d.get("lon") or d.get("longitude") - coords = d.get("coords") or (d.get("geometry") or {}).get("coordinates") - if (lat is None or lng is None) and isinstance(coords, (list, tuple)) and len(coords) >= 2: - lng, lat = coords[0], coords[1] # GeoJSON [lng, lat] + if lat is None or lng is None: + geo = (d.get("geometry") or {}).get("coordinates") + if isinstance(geo, (list, tuple)) and len(geo) >= 2: + lng, lat = geo[0], geo[1] # GeoJSON is [lng, lat] + if lat is None or lng is None: + coords = d.get("coords") + if isinstance(coords, (list, tuple)) and len(coords) >= 2: + lat, lng = coords[0], coords[1] # Osiris `coords` is [lat, lng] (see news route) try: - return (float(lat), float(lng)) if lat is not None and lng is not None else (None, None) + lat, lng = float(lat), float(lng) except (TypeError, ValueError): return (None, None) + if abs(lat) > 90 or abs(lng) > 180: + return (None, None) + return (lat, lng) def _salience(title: str, summary: str, raw: dict) -> float: From 8a9ac6e7a70b092c39e26ea4a5fdb614c7f3c5aa Mon Sep 17 00:00:00 2001 From: Mike Herak Date: Tue, 14 Jul 2026 15:01:38 -0400 Subject: [PATCH 2/5] fix(intake): _salience inverted the news 1-10 risk_score scale rs / (100 if rs > 1 else 1) assumed risk is 0-1 or 0-100, but the news route's scoreRisk() emits int 1-10. Result: risk 1 (noise) divided by 1 -> salience 1.0, risk 9 (missile attack) divided by 100 -> 0.09, floored to the 0.4 base. Ranking was upside down and min_salience filtering selected FOR spam. Normalize per scale: >10 -> /100, 1-10 int -> /10, 0-1 float as-is. Guard bool (isinstance(True, int) is True). --- engine/osiris_intake.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/engine/osiris_intake.py b/engine/osiris_intake.py index 02c660b..0b1b69a 100644 --- a/engine/osiris_intake.py +++ b/engine/osiris_intake.py @@ -126,10 +126,20 @@ def _salience(title: str, summary: str, raw: dict) -> float: for word, weight in _HOT.items(): if word in text: score = max(score, weight) - # honor an upstream risk score if Osiris provided one + # honor an upstream risk score if Osiris provided one. + # Scales in the wild: news scoreRisk() emits int 1-10 (starts at 1, +2 per + # keyword, capped 10); other feeds use 0-1 fractions or 0-100. The old + # `rs / (100 if rs > 1 else 1)` inverted the 1-10 scale: risk 1 (noise) + # scored 1.0 while risk 9 (missile attack) scored 0.09. rs = raw.get("risk_score") or raw.get("severity_score") - if isinstance(rs, (int, float)): - score = max(score, min(1.0, float(rs) / (100.0 if rs > 1 else 1.0))) + if isinstance(rs, (int, float)) and not isinstance(rs, bool) and rs > 0: + if rs > 10: # 0-100 scale + norm = float(rs) / 100.0 + elif rs > 1 or isinstance(rs, int): # 1-10 scale (int 1 is the scale floor) + norm = float(rs) / 10.0 + else: # 0-1 fraction + norm = float(rs) + score = max(score, min(1.0, norm)) return round(min(1.0, score), 2) From b2c0b5b0091cc6e5e0428c6cce7cbc50e3f0228e Mon Sep 17 00:00:00 2001 From: Mike Herak Date: Tue, 14 Jul 2026 15:02:24 -0400 Subject: [PATCH 3/5] fix(intake): stamp events with their real time, not fetch time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WorldEvent.ts always defaulted to now_ms() — the engine had no notion of when an event actually happened. Measured live: 20/29 news events served by /agent/events were >72h old (some 17 days) while stamped minutes-fresh. Consequences: the 'since' filter is meaningless for feeds with publish dates, and the swarm reasons over week-old posts as if they were breaking. Add _event_ts(): parse published/pubDate/date_added/updated/time (ISO 8601 or epoch s/ms) into ts, keeping now_ms() as the fallback for feeds that carry no time of their own. --- engine/osiris_intake.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/engine/osiris_intake.py b/engine/osiris_intake.py index 0b1b69a..5f8c4ae 100644 --- a/engine/osiris_intake.py +++ b/engine/osiris_intake.py @@ -9,6 +9,7 @@ import asyncio import logging import re +from datetime import datetime, timezone import httpx @@ -143,6 +144,34 @@ def _salience(title: str, summary: str, raw: dict) -> float: return round(min(1.0, score), 2) +def _event_ts(d: dict) -> int | None: + """The item's own event time in epoch ms, if the feed carries one. + Without this every event is stamped at fetch time, so week-old posts + surface as breaking and `since` filtering is meaningless.""" + for k in ("published", "pubDate", "pub_date", "date_added", "date", "datetime", "updated", "time"): + v = d.get(k) + if v is None or v == "": + continue + if isinstance(v, bool): + continue + if isinstance(v, (int, float)): # epoch seconds or ms + v = float(v) + if v > 1e12: + return int(v) + if v > 1e9: + return int(v * 1000) + continue + if isinstance(v, str): + try: + dt = datetime.fromisoformat(v.strip().replace("Z", "+00:00")) + except ValueError: + continue + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return int(dt.timestamp() * 1000) + return None + + def _to_event(d: dict, source: str, category: str) -> WorldEvent | None: title = _text(d, "title", "name", "headline", "question", "html", "place") mag = d.get("magnitude") or d.get("mag") @@ -163,7 +192,7 @@ def _to_event(d: dict, source: str, category: str) -> WorldEvent | None: except (TypeError, ValueError): pass lat, lng = _coord(d) - return WorldEvent( + evt = WorldEvent( title=title[:240], summary=summary[:2000], category=(category if source == "polymarket" else (_text(d, "category") or category)), @@ -174,6 +203,10 @@ def _to_event(d: dict, source: str, category: str) -> WorldEvent | None: salience=_salience(title, summary, d), raw={k: d[k] for k in list(d)[:25]}, ) + ts = _event_ts(d) + if ts: + evt.ts = ts + return evt def _markets_events(data: dict, source: str) -> list[WorldEvent]: From ec1817e59acdc49e8f90f1b1e6ced83fc393592a Mon Sep 17 00:00:00 2001 From: Mike Herak Date: Tue, 14 Jul 2026 16:30:41 -0400 Subject: [PATCH 4/5] fix(intake): declare risk scale per source instead of guessing it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix still guessed the scale from the value, which cannot work: 8 means 8/10 from the news route but 8/100 from country-risk. 'int 1-10 -> /10' therefore INVERTED any 0-100 feed below 10 — a country scoring 8/100 got salience 0.8 instead of 0.08. Same inversion class as the bug being fixed, moved to a different boundary; latent today only because country-risk currently emits no events (separate bug: its items have no title key, so _to_event drops all 20). Replace the heuristic with a source-keyed RISK_SCALE map. An unlisted source is ignored rather than guessed — dropping a signal is recoverable, inverting one is not. --- engine/osiris_intake.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/engine/osiris_intake.py b/engine/osiris_intake.py index 5f8c4ae..7f4c39b 100644 --- a/engine/osiris_intake.py +++ b/engine/osiris_intake.py @@ -121,26 +121,31 @@ def _coord(d: dict): return (lat, lng) -def _salience(title: str, summary: str, raw: dict) -> float: +# A feed's risk scale CANNOT be inferred from the value: 8 means 8/10 from the +# news route but 8/100 from country-risk, and guessing wrong INVERTS the ranking +# — which is the whole bug this map exists to kill. So each producer declares its +# own scale, keyed by source. An unlisted source is IGNORED rather than guessed: +# dropping a signal is recoverable, inverting one is not. +# news -> scoreRisk() in osiris news/route.ts: int, starts at 1, +2 per risk +# keyword, capped at 10. +# risk -> osiris country-risk/route.ts: 0-100 (floats occur, e.g. 66.6). +RISK_SCALE = { + "news": 10.0, + "risk": 100.0, +} + + +def _salience(title: str, summary: str, raw: dict, source: str = "") -> float: text = f"{title} {summary}".lower() score = 0.4 for word, weight in _HOT.items(): if word in text: score = max(score, weight) - # honor an upstream risk score if Osiris provided one. - # Scales in the wild: news scoreRisk() emits int 1-10 (starts at 1, +2 per - # keyword, capped 10); other feeds use 0-1 fractions or 0-100. The old - # `rs / (100 if rs > 1 else 1)` inverted the 1-10 scale: risk 1 (noise) - # scored 1.0 while risk 9 (missile attack) scored 0.09. + # Honor an upstream risk score ONLY when we know that source's scale. rs = raw.get("risk_score") or raw.get("severity_score") - if isinstance(rs, (int, float)) and not isinstance(rs, bool) and rs > 0: - if rs > 10: # 0-100 scale - norm = float(rs) / 100.0 - elif rs > 1 or isinstance(rs, int): # 1-10 scale (int 1 is the scale floor) - norm = float(rs) / 10.0 - else: # 0-1 fraction - norm = float(rs) - score = max(score, min(1.0, norm)) + scale = RISK_SCALE.get(source) + if scale and isinstance(rs, (int, float)) and not isinstance(rs, bool) and rs > 0: + score = max(score, min(1.0, float(rs) / scale)) return round(min(1.0, score), 2) @@ -200,7 +205,7 @@ def _to_event(d: dict, source: str, category: str) -> WorldEvent | None: lat=lat, lng=lng, url=_text(d, "url", "link", "feed_url"), - salience=_salience(title, summary, d), + salience=_salience(title, summary, d, source), raw={k: d[k] for k in list(d)[:25]}, ) ts = _event_ts(d) From 6e97d2a482647237a3ffffaaece647c27d71a603 Mon Sep 17 00:00:00 2001 From: Mike Herak Date: Tue, 14 Jul 2026 16:42:31 -0400 Subject: [PATCH 5/5] fix(intake): country-risk silently dropped all 20 countries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Osiris country-risk emits {code, risk_score, risk_level, tags} — no name, no coords — so _to_event found no title and returned None for every item. The feed was registered in FEEDS and contributed ZERO events, which is indistinguishable from a quiet feed. Nothing logged. Add _country_risk_events(): map the ISO-2 codes osiris actually serves to a name + approximate centroid so they become locatable events. Salience is capped at 0.5 and derived from the score alone — this is a hand-maintained baseline (osiris marks it static:true), background context for the oracle, never live signal. A 90/100 baseline is context; a missile landing is news. Drop 'risk' from RISK_SCALE: with a dedicated handler it never reaches _salience, so listing it would be dead config that reads as live. --- engine/osiris_intake.py | 59 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/engine/osiris_intake.py b/engine/osiris_intake.py index 7f4c39b..e9feecc 100644 --- a/engine/osiris_intake.py +++ b/engine/osiris_intake.py @@ -128,10 +128,12 @@ def _coord(d: dict): # dropping a signal is recoverable, inverting one is not. # news -> scoreRisk() in osiris news/route.ts: int, starts at 1, +2 per risk # keyword, capped at 10. -# risk -> osiris country-risk/route.ts: 0-100 (floats occur, e.g. 66.6). +# NOTE: country-risk ("risk", a 0-100 scale) is deliberately absent — it has a +# dedicated handler (_country_risk_events) that never reaches _salience. Listing +# it here would be dead config that reads as live. It was the feed that exposed +# the guessing bug: 8 means 8/10 from news but 8/100 from country-risk. RISK_SCALE = { "news": 10.0, - "risk": 100.0, } @@ -359,6 +361,57 @@ def _ioda_events(data: dict) -> list[WorldEvent]: return out +# Osiris country-risk emits {code, risk_score, risk_level, tags} — no name, no +# coords — so the generic path found no title and DROPPED all 20 silently. Map +# the ISO-2 codes it actually serves to a name and an approximate centroid so +# they become real, locatable events. Codes are from RISK_FACTORS in +# osiris/src/app/api/country-risk/route.ts; extend both together. +_COUNTRY = { + "UA": ("Ukraine", 49.0, 32.0), "RU": ("Russia", 61.5, 105.3), + "IL": ("Israel", 31.0, 34.9), "PS": ("Palestinian Territories", 31.9, 35.2), + "SY": ("Syria", 34.8, 39.0), "YE": ("Yemen", 15.6, 48.5), + "MM": ("Myanmar", 21.9, 95.9), "SD": ("Sudan", 12.9, 30.2), + "AF": ("Afghanistan", 33.9, 67.7), "KP": ("North Korea", 40.3, 127.5), + "IR": ("Iran", 32.4, 53.7), "CN": ("China", 35.9, 104.2), + "TW": ("Taiwan", 23.7, 121.0), "VE": ("Venezuela", 6.4, -66.6), + "HT": ("Haiti", 18.9, -72.3), "LB": ("Lebanon", 33.9, 35.9), + "PK": ("Pakistan", 30.4, 69.3), "SO": ("Somalia", 5.2, 46.2), + "LY": ("Libya", 26.3, 17.2), "ET": ("Ethiopia", 9.1, 40.5), +} + + +def _country_risk_events(data: dict) -> list[WorldEvent]: + """Osiris country-risk → locatable events. + + This is a HAND-MAINTAINED geopolitical baseline (osiris marks it + `static: true`), not a live measurement — so salience is capped well below + breaking-news territory and derived from the score alone. It is background + context for the oracle, not a signal that something just happened. + """ + out = [] + for c in (data or {}).get("countries", []) or []: + if not isinstance(c, dict): + continue + code = str(c.get("code") or "").upper() + score = c.get("risk_score") + if code not in _COUNTRY or not isinstance(score, (int, float)): + continue + name, lat, lng = _COUNTRY[code] + tags = ", ".join(t.replace("_", " ") for t in (c.get("tags") or [])) + level = c.get("risk_level", "") + out.append(WorldEvent( + title=f"Country risk: {name} {score}/100 ({level})"[:120], + summary=(f"Standing geopolitical baseline — {tags}." if tags else + "Standing geopolitical baseline.") + " Static reference, not a live event.", + category="instability", source="risk", + lat=lat, lng=lng, + # Ceiling 0.5: a static table must never outrank live signal. A + # 90/100 baseline is context; a missile landing is news. + salience=round(min(0.5, float(score) / 200.0), 2), + )) + return out + + def _space_weather_events(data: dict) -> list[WorldEvent]: """NOAA SWPC — geomagnetic storms + solar flares (satellites, GPS, grids).""" out = [] @@ -756,6 +809,8 @@ async def _fetch_feed(self, c: httpx.AsyncClient, path: str, source: str, catego data = r.json() if source in ("markets", "crypto"): out.extend(_markets_events(data, source)) + elif source == "risk": + out.extend(_country_risk_events(data)) elif source == "futures": out.extend(_futures_events(data)) elif source == "gdacs":