From a9e605c03ca16f2cda7cfd0551fd31018fd726cd Mon Sep 17 00:00:00 2001 From: Cloud Note Date: Thu, 3 Sep 2026 15:22:40 -0400 Subject: [PATCH 1/4] hello --- .claude/settings.json | 4 +- .github/workflows/claude-code-review.yml | 44 - .github/workflows/claude.yml | 50 - README.md | 82 +- planning/MARKET_DATA_SUMMARY.md | 104 -- planning/MARKET_INTERFACE.md | 499 ++++++++ planning/MARKET_SIMULATOR.md | 450 +++++++ planning/MASSIVE_API.md | 548 ++++++++ planning/PLAN.md | 121 +- planning/archive/MARKET_DATA_DESIGN.md | 1490 ---------------------- planning/archive/MARKET_DATA_REVIEW.md | 173 --- planning/archive/MARKET_INTERFACE.md | 273 ---- planning/archive/MARKET_SIMULATOR.md | 245 ---- planning/archive/MASSIVE_API.md | 251 ---- 14 files changed, 1658 insertions(+), 2676 deletions(-) delete mode 100644 .github/workflows/claude-code-review.yml delete mode 100644 .github/workflows/claude.yml delete mode 100644 planning/MARKET_DATA_SUMMARY.md create mode 100644 planning/MARKET_INTERFACE.md create mode 100644 planning/MARKET_SIMULATOR.md create mode 100644 planning/MASSIVE_API.md delete mode 100644 planning/archive/MARKET_DATA_DESIGN.md delete mode 100644 planning/archive/MARKET_DATA_REVIEW.md delete mode 100644 planning/archive/MARKET_INTERFACE.md delete mode 100644 planning/archive/MARKET_SIMULATOR.md delete mode 100644 planning/archive/MASSIVE_API.md diff --git a/.claude/settings.json b/.claude/settings.json index aa06f43dc..6d8377fd8 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,7 +1,5 @@ { "enabledPlugins": { - "frontend-design@claude-plugins-official": true, - "context7@claude-plugins-official": true, - "playwright@claude-plugins-official": true + "independent-reviewer@ed-tools": true } } diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml deleted file mode 100644 index b5e8cfd4d..000000000 --- a/.github/workflows/claude-code-review.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Claude Code Review - -on: - pull_request: - types: [opened, synchronize, ready_for_review, reopened] - # Optional: Only run on specific file changes - # paths: - # - "src/**/*.ts" - # - "src/**/*.tsx" - # - "src/**/*.js" - # - "src/**/*.jsx" - -jobs: - claude-review: - # Optional: Filter by PR author - # if: | - # github.event.pull_request.user.login == 'external-contributor' || - # github.event.pull_request.user.login == 'new-developer' || - # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' - - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code Review - id: claude-review - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' - plugins: 'code-review@claude-code-plugins' - prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml deleted file mode 100644 index d300267f1..000000000 --- a/.github/workflows/claude.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Claude Code - -on: - issue_comment: - types: [created] - pull_request_review_comment: - types: [created] - issues: - types: [opened, assigned] - pull_request_review: - types: [submitted] - -jobs: - claude: - if: | - (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || - (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || - (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - issues: read - id-token: write - actions: read # Required for Claude to read CI results on PRs - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 1 - - - name: Run Claude Code - id: claude - uses: anthropics/claude-code-action@v1 - with: - claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} - - # This is an optional setting that allows Claude to read CI results on PRs - additional_permissions: | - actions: read - - # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. - # prompt: 'Update the pull request description to include a summary of changes.' - - # Optional: Add claude_args to customize behavior and configuration - # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md - # or https://code.claude.com/docs/en/cli-reference for available options - # claude_args: '--allowed-tools Bash(gh pr:*)' - diff --git a/README.md b/README.md index 3f2582ae2..ad5b8d87b 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,59 @@ # FinAlly — AI Trading Workstation -A visually stunning AI-powered trading workstation that streams live market data, simulates portfolio trading, and integrates an LLM chat assistant that can analyze positions and execute trades via natural language. +A trading terminal that streams live market data, simulates portfolio trading, and puts +an LLM assistant beside your positions — one that can analyze holdings and execute trades +from natural language. -Built entirely by coding agents as a capstone project for an agentic AI coding course. +Built entirely by coding agents as the capstone for an agentic AI coding course. Agents +coordinate through shared docs in [`planning/`](planning/). -## Features +## Status -- **Live price streaming** via SSE with green/red flash animations -- **Simulated portfolio** — $10k virtual cash, market orders, instant fills -- **Portfolio visualizations** — heatmap (treemap), P&L chart, positions table -- **AI chat assistant** — analyzes holdings, suggests and auto-executes trades -- **Watchlist management** — track tickers manually or via AI -- **Dark terminal aesthetic** — Bloomberg-inspired, data-dense layout +**Early — market data is the only component built.** There's no frontend and no Dockerfile +yet, so the app doesn't run end to end. -## Architecture - -Single Docker container serving everything on port 8000: +| Component | Status | +|---|---| +| Market data — simulator, Massive client, price cache, SSE | ✅ 73 tests, 84% coverage | +| Portfolio, trades, watchlist, chat API | ⬜ | +| Database, frontend, Docker, E2E tests | ⬜ | -- **Frontend**: Next.js (static export) with TypeScript and Tailwind CSS -- **Backend**: FastAPI (Python/uv) with SSE streaming -- **Database**: SQLite with lazy initialization -- **AI**: LiteLLM → OpenRouter (Cerebras inference) with structured outputs -- **Market data**: Built-in GBM simulator (default) or Massive API (optional) +## Try It -## Quick Start +Requires [uv](https://docs.astral.sh/uv/) and Python 3.12+. No API key needed. ```bash -# Clone and configure -cp .env.example .env -# Add your OPENROUTER_API_KEY to .env +cd backend +uv sync --dev +uv run market_data_demo.py # live terminal dashboard, 10 tickers, ~60s +uv run pytest # test suite +``` -# Run with Docker -docker build -t finally . -docker run -v finally-data:/app/db -p 8000:8000 --env-file .env finally +## Architecture -# Open http://localhost:8000 -``` +Target design is one Docker container on port 8000: a **Next.js** static export served by +**FastAPI**, backed by **SQLite**, with **LiteLLM → OpenRouter** (Cerebras) for chat and +Server-Sent Events for price streaming. Single origin, so no CORS. -## Environment Variables +Market data has two interchangeable sources behind one interface — a GBM simulator +(default) and a Massive/Polygon.io poller (when `MASSIVE_API_KEY` is set) — both writing +to a thread-safe `PriceCache` that SSE, portfolio valuation, and trade pricing read from. +See [`planning/MARKET_DATA_SUMMARY.md`](planning/MARKET_DATA_SUMMARY.md). -| Variable | Required | Description | -|---|---|---| -| `OPENROUTER_API_KEY` | Yes | OpenRouter API key for AI chat | -| `MASSIVE_API_KEY` | No | Massive (Polygon.io) key for real market data; omit to use simulator | -| `LLM_MOCK` | No | Set `true` for deterministic mock LLM responses (testing) | +## Configuration -## Project Structure +Read from a gitignored `.env` in the project root. -``` -finally/ -├── frontend/ # Next.js static export -├── backend/ # FastAPI uv project -├── planning/ # Project documentation and agent contracts -├── test/ # Playwright E2E tests -├── db/ # SQLite volume mount (runtime) -└── scripts/ # Start/stop helpers -``` +| Variable | Description | +|---|---| +| `OPENROUTER_API_KEY` | LLM assistant. Everything except chat works without it. | +| `MASSIVE_API_KEY` | Optional — real market data. Omit to use the simulator. | +| `LLM_MOCK` | Optional — `true` for deterministic mock responses in tests. | + +## Docs + +- [`planning/PLAN.md`](planning/PLAN.md) — full spec, and the contract between agents +- [`backend/README.md`](backend/README.md) — backend development and testing ## License diff --git a/planning/MARKET_DATA_SUMMARY.md b/planning/MARKET_DATA_SUMMARY.md deleted file mode 100644 index ae518283a..000000000 --- a/planning/MARKET_DATA_SUMMARY.md +++ /dev/null @@ -1,104 +0,0 @@ -# Market Data Backend — Summary - -**Status:** Complete, tested, reviewed, all issues resolved. - -## What Was Built - -A complete market data subsystem in `backend/app/market/` (8 modules, ~500 lines) providing live price simulation and real market data via a unified interface. - -### Architecture - -``` -MarketDataSource (ABC) -├── SimulatorDataSource → GBM simulator (default, no API key needed) -└── MassiveDataSource → Polygon.io REST poller (when MASSIVE_API_KEY set) - │ - ▼ - PriceCache (thread-safe, in-memory) - │ - ├──→ SSE stream endpoint (/api/stream/prices) - ├──→ Portfolio valuation - └──→ Trade execution -``` - -### Modules - -| File | Purpose | -|------|---------| -| `models.py` | `PriceUpdate` — immutable frozen dataclass (ticker, price, previous_price, timestamp, change, direction) | -| `interface.py` | `MarketDataSource` — abstract base class defining `start/stop/add_ticker/remove_ticker/get_tickers` | -| `cache.py` | `PriceCache` — thread-safe price store with version counter for SSE change detection | -| `seed_prices.py` | Realistic seed prices, per-ticker GBM params (drift/volatility), correlation groups | -| `simulator.py` | `GBMSimulator` (Geometric Brownian Motion with Cholesky-correlated moves) + `SimulatorDataSource` | -| `massive_client.py` | `MassiveDataSource` — REST polling client for Polygon.io via the `massive` package | -| `factory.py` | `create_market_data_source()` — selects simulator or Massive based on `MASSIVE_API_KEY` env var | -| `stream.py` | `create_stream_router()` — FastAPI SSE endpoint factory using version-based change detection | - -### Key Design Decisions - -- **Strategy pattern** — both data sources implement the same ABC; downstream code is source-agnostic -- **PriceCache as single point of truth** — producers write, consumers read; no direct coupling -- **GBM with correlated moves** — Cholesky decomposition of sector-based correlation matrix; tech stocks correlate at 0.6, finance at 0.5, cross-sector at 0.3 -- **Random shock events** — ~0.1% chance per tick per ticker of a 2-5% move for visual drama -- **SSE over WebSockets** — simpler, one-way push, universal browser support - -## Test Suite - -**73 tests, all passing.** 6 test modules in `backend/tests/market/`. - -| Module | Tests | Coverage | -|--------|-------|----------| -| test_models.py | 11 | models.py: 100% | -| test_cache.py | 13 | cache.py: 100% | -| test_simulator.py | 17 | simulator.py: 98% | -| test_simulator_source.py | 10 | (integration tests) | -| test_factory.py | 7 | factory.py: 100% | -| test_massive.py | 13 | massive_client.py: 56% (expected — API methods mocked) | - -Overall coverage: 84%. - -## Code Review & Fixes Applied - -A comprehensive code review identified 7 issues. All were resolved: - -1. **pyproject.toml build config** — added `[tool.hatch.build.targets.wheel] packages = ["app"]` -2. **Lazy imports removed** — `massive` is a core dependency; imports moved to top level -3. **SSE return type fixed** — `_generate_events` annotated as `AsyncGenerator[str, None]` -4. **Public `get_tickers()`** — added to `GBMSimulator` to avoid private attribute access -5. **Correlation constants cleaned up** — removed unused `DEFAULT_CORR`, consolidated into `CROSS_GROUP_CORR` -6. **Unused test imports removed** — `pytest`, `math`, `asyncio` cleaned from 4 test files -7. **Massive test mocks fixed** — `source._client` set in tests, patches target correct names - -## Demo - -A Rich terminal demo is available at `backend/market_data_demo.py`: - -```bash -cd backend -uv run market_data_demo.py -``` - -Displays a live-updating dashboard with all 10 tickers, sparklines, color-coded direction arrows, and an event log for notable price moves. Runs 60 seconds or until Ctrl+C. - -## Usage for Downstream Code - -```python -from app.market import PriceCache, create_market_data_source - -# Startup -cache = PriceCache() -source = create_market_data_source(cache) # Reads MASSIVE_API_KEY -await source.start(["AAPL", "GOOGL", "MSFT", ...]) - -# Read prices -update = cache.get("AAPL") # PriceUpdate or None -price = cache.get_price("AAPL") # float or None -all_prices = cache.get_all() # dict[str, PriceUpdate] - -# Dynamic watchlist -await source.add_ticker("TSLA") -await source.remove_ticker("GOOGL") - -# Shutdown -await source.stop() -``` diff --git a/planning/MARKET_INTERFACE.md b/planning/MARKET_INTERFACE.md new file mode 100644 index 000000000..b197021d2 --- /dev/null +++ b/planning/MARKET_INTERFACE.md @@ -0,0 +1,499 @@ +# Market Data Interface — Unified Design + +**Status:** Design. Supersedes the market-data portion of PLAN.md §6. +Grounded in the entitlement research in [`MASSIVE_API.md`](MASSIVE_API.md). + +--- + +## 1. The Problem This Design Solves + +PLAN.md §6 specifies two implementations behind one interface, selected by whether +`MASSIVE_API_KEY` is set. That binary is wrong, and the code already in +`backend/app/market/` inherits the error. + +The research finding: **a free Massive key authenticates successfully and then refuses to +return any live price.** Snapshots, last trade, and any bar dated today are all +`NOT_AUTHORIZED` on the Basic tier (verified — see `MASSIVE_API.md` §4). Real-time data +starts at $199/month. + +So `MASSIVE_API_KEY` being present tells you almost nothing. A key can be: + +| Key state | What works | +|---|---| +| Absent | nothing — simulate | +| Present, Basic (free) | historical bars and yesterday's closes; **no live prices** | +| Present, Starter/Developer | live-ish prices, 15 minutes delayed | +| Present, Advanced | live prices | +| Present, invalid/revoked | nothing | + +Under the current `factory.py`, the second row — the one nearly every student will be in — +produces a running app whose watchlist never populates. Every poll raises, `_poll_once` +swallows it in a bare `except Exception`, and the cache stays empty in silence. + +**The design goal is that all five rows produce a working, moving trading terminal.** + +--- + +## 2. Three Sources, One Interface + +``` + MASSIVE_API_KEY set? + │ + ┌───────────────┴───────────────┐ + no yes + │ │ + │ probe_capabilities() ← 2 calls, once, at startup + │ │ + │ ┌────────────────┼────────────────┐ + │ realtime end-of-day invalid + │ │ │ │ + ▼ ▼ ▼ ▼ + SimulatorDataSource MassiveDataSource AnchoredSimulator SimulatorDataSource + (synthetic seeds) (real, streaming) (real prices, (+ warning in /health) + synthetic motion) + │ │ │ + └────────────────┴────────────────┘ + │ + PriceCache + │ + ┌──────────────────────┼──────────────────────┐ + SSE /api/stream portfolio valuation trade pricing +``` + +Everything below `PriceCache` is source-agnostic. That boundary already exists in the repo +and is correct — this design keeps it untouched and only changes what sits above it. + +### The third source is the important one + +`AnchoredSimulatorDataSource` fetches **real closing prices** from Massive with the one +free-tier call that returns the whole market, then runs the GBM simulator forward from +those anchors. + +The result: AAPL opens at its genuine 324.96 close rather than a hard-coded 190.00, and it +*moves*. A student with a free key gets real price levels, real relative valuations, a +watchlist that ticks, and a portfolio that changes — the entire demo works. One API call at +startup, well inside a 5/minute budget. + +It also fixes a bug nobody has noticed yet: `backend/app/market/seed_prices.py` is badly +stale. NVDA is seeded at 800.00 and actually trades at 224.41; NFLX is seeded at 600.00 and +trades at 82.73 (both post-split). Anchoring to live data removes the maintenance burden +from that table entirely. + +--- + +## 3. The Interface + +`MarketDataSource` as it stands in `backend/app/market/interface.py` is sound. Keep the ABC, +add two members. + +```python +class MarketDataSource(ABC): + """Contract for market data providers. + + Implementations push price updates into a shared PriceCache on their own + schedule. Downstream code never calls the data source directly for prices — + it reads from the cache. + """ + + # --- existing, unchanged --- + @abstractmethod + async def start(self, tickers: list[str]) -> None: ... + @abstractmethod + async def stop(self) -> None: ... + @abstractmethod + async def add_ticker(self, ticker: str) -> None: ... + @abstractmethod + async def remove_ticker(self, ticker: str) -> None: ... + @abstractmethod + def get_tickers(self) -> list[str]: ... + + # --- new --- + @abstractmethod + def describe(self) -> SourceStatus: + """Introspection for GET /api/health. Never raises.""" + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + """Historical series for the chart's first paint. + + Default implementation returns [] — the frontend then accumulates from + SSE as PLAN.md §10 describes. Massive-backed sources override this with + real intraday bars. + """ + return [] +``` + +```python +@dataclass(frozen=True, slots=True) +class SourceStatus: + name: str # "simulator" | "massive" | "anchored-simulator" + live: bool # True only when prices reflect the real current market + detail: str # human-readable, surfaced verbatim in /api/health + tickers: int + cache_populated: bool +``` + +`describe()` answers PLAN.md §13.5 item 38 directly. `detail` should be specific enough to +end the debugging session: `"end-of-day only (Basic tier) — anchored to 2026-09-02 closes"`. + +### Why `get_history` belongs on the interface + +PLAN.md §13.2 item 15 asks whether the backend should keep a ring buffer so charts are not +blank on first paint, and recommends it. Putting `get_history` on the interface gets that +for free in a better form: the simulator can serve its own ring buffer, while +Massive-backed sources serve `get_aggs(ticker, 1, "minute", ...)` — a **genuine** intraday +series, available on the free tier. The frontend calls one endpoint and does not care which +it got. + +--- + +## 4. `PriceUpdate` — adding the daily baseline + +PLAN.md §13.1 item 1 identifies that "daily change %" has no baseline: `PriceUpdate` carries +only `previous_price` from the previous ~500 ms tick, so the column renders permanently flat +at ±0.05%. The fix is one field. + +```python +@dataclass(frozen=True, slots=True) +class PriceUpdate: + ticker: str + price: float + previous_price: float # previous TICK — drives the flash animation only + open_price: float # session open / anchor — drives the daily % column + timestamp: float # Unix seconds internally; ISO 8601 UTC on the wire + + @property + def tick_direction(self) -> str: + """'up' | 'down' | 'flat' — CSS flash class. Renamed from `direction`.""" + + @property + def change_today(self) -> float: + return round(self.price - self.open_price, 4) + + @property + def change_percent_today(self) -> float: + if self.open_price == 0: + return 0.0 + return round((self.price - self.open_price) / self.open_price * 100, 4) +``` + +Two clearly named fields, exactly as item 1 recommends. Where `open_price` comes from, per +source: + +| Source | `open_price` | +|---|---| +| `MassiveDataSource` | `snapshot.prev_day.close` (real previous close) | +| `AnchoredSimulatorDataSource` | the anchor close fetched at startup | +| `SimulatorDataSource` | the seed price the simulation started from | + +In all three cases it is fixed for the session, so the daily % accumulates over minutes and +hours instead of resetting every tick. + +`PriceCache.update()` gains an `open_price: float | None` parameter; when `None` it keeps +whatever the ticker already had, and on first write defaults to `price`. That preserves the +existing call sites. + +### Wire format + +Per PLAN.md §13.1 item 6, JSON is **ISO 8601 UTC everywhere**. `PriceUpdate.timestamp` stays +a float internally and converts at `to_dict()`: + +```json +{ + "ticker": "AAPL", + "price": 325.41, + "previous_price": 325.38, + "open_price": 324.96, + "timestamp": "2026-09-03T17:42:11.413700Z", + "tick_direction": "up", + "change_today": 0.45, + "change_percent_today": 0.1385 +} +``` + +--- + +## 5. Source Selection + +Replaces `backend/app/market/factory.py`. + +```python +async def create_market_data_source(price_cache: PriceCache) -> MarketDataSource: + """Select a market data source. Never raises; always returns a working source. + + Async because the capability probe makes real network calls. Called once, + from the FastAPI lifespan handler. + """ + api_key = os.environ.get("MASSIVE_API_KEY", "").strip() + + if not api_key: + logger.info("No MASSIVE_API_KEY — using GBM simulator") + return SimulatorDataSource(price_cache) + + caps = await asyncio.to_thread(probe_capabilities, api_key) + + if caps.realtime: + logger.info("Massive: real-time entitled — using live snapshots") + return MassiveDataSource(api_key, price_cache, poll_interval=5.0) + + if caps.end_of_day: + logger.warning( + "Massive key is end-of-day only (Basic tier). Anchoring the simulator " + "to real closing prices — displayed prices are simulated, not live." + ) + return AnchoredSimulatorDataSource(api_key, price_cache) + + logger.error("MASSIVE_API_KEY rejected (%s) — falling back to simulator", caps.detail) + return SimulatorDataSource(price_cache, status_detail=f"key rejected: {caps.detail}") +``` + +Three rules this encodes: + +1. **Never boot into a broken state.** Every branch returns a source that produces moving + prices. A bad key degrades; it does not blank the screen. +2. **Never silently mislead.** A simulated price is never presented as live — + `SourceStatus.live` is `False` and the frontend shows a "SIMULATED" badge beside the + connection dot. +3. **Probe once.** Two calls at startup, not two per poll. The result is immutable for the + process lifetime. + +### Poll intervals + +| Source | Interval | Rationale | +|---|---|---| +| `SimulatorDataSource` | 500 ms | PLAN.md §6; local computation, no budget | +| `AnchoredSimulatorDataSource` | 500 ms tick; **one** anchor fetch at startup | GBM is local | +| `MassiveDataSource`, Advanced | 5 s | unlimited calls; 5 s is plenty for a terminal | +| `MassiveDataSource`, Starter/Developer | 15 s | data is 15 min delayed anyway | + +`AnchoredSimulatorDataSource` optionally re-anchors once per hour (1 call) so a long-running +container picks up the next session's close. Cheap, and keeps a demo left open overnight +honest. + +--- + +## 6. `AnchoredSimulatorDataSource` + +```python +class AnchoredSimulatorDataSource(MarketDataSource): + """GBM simulation seeded from real Massive closing prices. + + Bridges the gap for Basic-tier keys: real price *levels* from one + free-tier API call, plus synthetic price *motion* so the terminal is alive. + """ + + def __init__(self, api_key: str, price_cache: PriceCache) -> None: + self._client = RESTClient(api_key=api_key, retries=0) + self._cache = price_cache + self._sim: SimulatorDataSource | None = None + self._anchors: dict[str, float] = {} + self._anchor_date: str | None = None + + async def start(self, tickers: list[str]) -> None: + self._anchors, self._anchor_date = await asyncio.to_thread( + self._fetch_anchors, tickers + ) + # Real closes where we have them; the static seed table covers the rest. + self._sim = SimulatorDataSource(self._cache, seed_overrides=self._anchors) + await self._sim.start(tickers) + + def _fetch_anchors(self, tickers: list[str]) -> tuple[dict[str, float], str | None]: + """ONE API call prices every ticker. Walks back over weekends/holidays.""" + wanted = {t.upper() for t in tickers} + day = date.today() + for _ in range(7): # at most a week back + day -= timedelta(days=1) + iso = day.isoformat() + try: + bars = self._client.get_grouped_daily_aggs(iso, adjusted=True) + except Exception as e: + logger.warning("Anchor fetch failed for %s: %s", iso, e) + continue + if not bars: + continue # weekend or holiday + found = {b.ticker: b.close for b in bars if b.ticker in wanted} + logger.info("Anchored %d/%d tickers to %s closes", + len(found), len(wanted), iso) + return found, iso + logger.warning("No anchors available — falling back to the static seed table") + return {}, None + + def describe(self) -> SourceStatus: + return SourceStatus( + name="anchored-simulator", + live=False, + detail=(f"simulated from real {self._anchor_date} closes " + f"({len(self._anchors)} anchored)" + if self._anchor_date else + "simulated from static seed prices (anchor fetch failed)"), + tickers=len(self._sim.get_tickers()) if self._sim else 0, + cache_populated=len(self._cache) > 0, + ) + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + """Real minute bars from the last completed session (free tier allows this).""" + if not self._anchor_date: + return [] + bars = await asyncio.to_thread( + self._client.get_aggs, ticker, 1, "minute", + self._anchor_date, self._anchor_date, limit=50_000, + ) + return [PricePoint(ms_to_iso(b.timestamp), b.close) for b in bars[-points:]] +``` + +Adding a ticker mid-session costs nothing: `add_ticker` delegates to the simulator, which +synthesises a seed if the anchor set has no entry — the answer to PLAN.md §13.2 item 8, and +the demo never dead-ends on an unknown symbol. + +--- + +## 7. `MassiveDataSource` — corrections + +The shipped implementation needs four fixes beyond the entitlement gate. All are verified +against the SDK and live API in `MASSIVE_API.md`. + +1. **Wrong timestamp field and wrong unit.** The code reads `snap.last_trade.timestamp / + 1000.0`. The SDK field is `sip_timestamp`, and it is **nanoseconds**, not milliseconds. + The `AttributeError` is currently swallowed by the surrounding handler, so this fails + invisibly. + ```python + ts = snap.last_trade.sip_timestamp / 1e9 # ns → epoch seconds + ``` + +2. **`open_price` is never captured**, so the daily % column has no baseline (§4). + Snapshots hand it over for free: + ```python + self._cache.update( + ticker=snap.ticker, + price=snap.last_trade.price, + open_price=snap.prev_day.close if snap.prev_day else None, + timestamp=ts, + ) + ``` + +3. **Rate limiting is invisible.** `except Exception` catches `MaxRetryError` and logs it at + `error` level once per poll, forever, with no user-facing signal. Classify instead, and + let `describe()` report a degraded state so `/api/health` and the frontend can show it. + +4. **`retries=3` is actively harmful on a metered key.** The SDK's 0.0/0.2/0.4 s backoff + burns three requests inside the same 60 s window as the failure. Set `retries=0` and back + off at the poll-loop level. + +Two smaller ones: a poll that returns zero usable snapshots after a run of consecutive +failures should flip `SourceStatus.live` to `False` rather than let the frontend keep +displaying a frozen price as live — which also covers PLAN.md §13.2 item 14, the weekend / +closed-market case. And tickers absent from a snapshot response (unknown symbols) should be +recorded so `/api/watchlist` can flag them, rather than vanishing silently. + +--- + +## 8. What Does Not Change + +Deliberately preserved from the shipped implementation: + +- **`PriceCache`** — the threading `Lock`, the monotonic `version` counter, and the + `dict[str, PriceUpdate]` shape are all correct. The `version` counter driving SSE change + detection is a good design and stays. +- **The cache as the single read path.** Portfolio valuation, trade pricing, and SSE read + the cache and never the source. This is what makes three sources cost nothing downstream. +- **The full-map SSE payload.** PLAN.md §13.5 item 40 asks for confirmation that sending + every ticker on every change is deliberate rather than an oversight. It is deliberate: at + 10–50 tickers the payload is a couple of KB, and a delta protocol would need + reconnection-resync logic for no measurable gain. **Do not optimise this into a delta.** +- **`create_stream_router`'s factory pattern** for injecting the cache without globals. + +One change to `stream.py`: add the heartbeat from PLAN.md §13.1 item 7. It matters far more +now — in Massive mode at a 15 s poll interval, or on a closed market, the stream is silent +long enough for a proxy to drop it. + +```python +last_sent = time.monotonic() +while True: + ... + if current_version != last_version: + yield f"data: {payload}\n\n" + last_sent = time.monotonic() + elif time.monotonic() - last_sent > 10.0: + yield ": ping\n\n" # SSE comment — ignored by EventSource + last_sent = time.monotonic() +``` + +--- + +## 9. Lifespan Wiring + +Resolves PLAN.md §13.1 item 5 — DB init must happen at startup, not on first request, +because the market source needs the seeded watchlist before any request arrives. + +```python +@asynccontextmanager +async def lifespan(app: FastAPI): + init_database() # create + seed if empty + app.state.price_cache = PriceCache() + + tickers = get_tracked_tickers() # watchlist ∪ held positions (item 10) + app.state.market = await create_market_data_source(app.state.price_cache) + await app.state.market.start(tickers) + + app.state.snapshots = asyncio.create_task(portfolio_snapshot_loop(app.state)) + yield + app.state.snapshots.cancel() + await app.state.market.stop() +``` + +`get_tracked_tickers()` returns **watchlist ∪ held positions**, per PLAN.md §13.2 item 10 — +removing a ticker from the watchlist must not stop pricing a position you still hold, or the +heatmap tile and total portfolio value go stale. + +--- + +## 10. Module Layout + +``` +backend/app/market/ +├── __init__.py # public API (unchanged surface) +├── models.py # PriceUpdate (+ open_price), PricePoint, SourceStatus +├── cache.py # PriceCache — unchanged +├── interface.py # MarketDataSource ABC (+ describe, get_history) +├── factory.py # async create_market_data_source() — capability-driven +├── capabilities.py # NEW — probe_capabilities(), MassiveCapabilities +├── simulator.py # GBMSimulator + SimulatorDataSource (+ seed_overrides) +├── anchored.py # NEW — AnchoredSimulatorDataSource +├── massive_client.py # MassiveDataSource — corrected per §7 +├── seed_prices.py # fallback seeds; no longer the primary price source +└── stream.py # SSE router (+ heartbeat) +``` + +## 11. Testing + +The existing 73 tests stay green; `PriceUpdate.direction` → `tick_direction` and the new +`open_price` argument are the only signature churn. + +New coverage: + +- `probe_capabilities` against stubbed clients for all five key states in §1 — especially + that a `NOT_AUTHORIZED` snapshot plus a successful `get_previous_close_agg` classifies as + end-of-day rather than invalid. +- Factory selection: each of the five states returns the expected class and never raises. +- `AnchoredSimulatorDataSource._fetch_anchors` walks back over a weekend (empty `bars` for + Saturday and Sunday, populated for Friday) and falls back cleanly after 7 failures. +- `MassiveDataSource` nanosecond conversion — assert a `sip_timestamp` of + `1605192894630916600` yields `1605192894.63`, not a date in the year 52000. +- Rate-limit classification: `MaxRetryError` is handled distinctly from `BadResponse`. +- Heartbeat: `_generate_events` emits `": ping"` when the cache version is static for >10 s + (this is the unit test PLAN.md §13.4 item 31 wants in place of the Playwright case). + +Every Massive test stubs the client. No test hits the network — the free tier's 5 calls per +minute would make a real-network suite unrunnable in CI anyway. + +--- + +## 12. Open Questions + +1. **Is the anchored simulator honest enough?** It shows real price *levels* with synthetic + *movement*, clearly badged "SIMULATED". The alternative — displaying yesterday's frozen + closes — is more truthful and completely lifeless. This design picks the badged + simulation. Worth an explicit sign-off. +2. **Re-anchor cadence.** Hourly is proposed. A container left running for days on a static + anchor slowly drifts from reality; hourly re-anchoring costs 24 calls/day out of 7,200. +3. **Should a Starter/Developer key (15-min delay) be badged?** It is real market data, just + late. Suggest `live: True` with `detail` naming the delay, rather than a "SIMULATED" badge. diff --git a/planning/MARKET_SIMULATOR.md b/planning/MARKET_SIMULATOR.md new file mode 100644 index 000000000..6c537ec3a --- /dev/null +++ b/planning/MARKET_SIMULATOR.md @@ -0,0 +1,450 @@ +# Market Simulator — Approach and Code Structure + +**Status:** Design + measured review of the shipped implementation in +`backend/app/market/simulator.py`. +Companion documents: [`MASSIVE_API.md`](MASSIVE_API.md) (why the simulator carries far more +weight than PLAN.md assumed) and [`MARKET_INTERFACE.md`](MARKET_INTERFACE.md) (how it is +selected). + +Every number below marked **[measured]** came from running the shipped simulator on +2026-09-03, not from theory. + +--- + +## 1. Why the Simulator Is the Primary Path + +PLAN.md frames the simulator as the fallback for users without an API key. The entitlement +research changes its status: a **free Massive key cannot return a live price at all**, so +the simulator is what drives the terminal for users with no key *and* for the much larger +group with a free key. Real live data begins at $199/month. + +The simulator is therefore not a stand-in for the real product. For nearly every student +running this project, it **is** the product. It deserves to be correct. + +Its job, in priority order: + +1. **Look alive.** Prices must change visibly every tick or the flash animation, the + sparklines, and the whole terminal aesthetic fall flat. +2. **Be statistically defensible.** A finance-adjacent teaching project should not move + prices by `random.uniform(-1, 1)`. Volatility should mean something. +3. **Be correlated.** Watching ten tickers move independently looks synthetic instantly. + Real markets have a factor structure. +4. **Provide drama.** Occasional sharp moves so the P&L chart and heatmap have something to + show inside a five-minute demo. +5. **Never dead-end.** Any ticker a user or the LLM invents must get a price. + +Requirements 2 and 4 are in tension, and §6 shows the shipped code currently resolves that +tension badly. + +--- + +## 2. The Model: Geometric Brownian Motion + +GBM is the standard model for equity prices — the basis of Black-Scholes — and it has the +two properties that matter here: prices stay strictly positive, and returns rather than +absolute levels are what scale with volatility, so a $500 stock and an $80 stock look +equally plausible under the same σ. + +The discrete update: + +``` +S(t+Δt) = S(t) · exp[ (μ − σ²/2)·Δt + σ·√Δt·Z ] +``` + +| Term | Meaning | +|---|---| +| `S(t)` | current price | +| `μ` | annualised drift (expected return) | +| `σ` | annualised volatility | +| `Δt` | time step as a fraction of a trading year | +| `Z` | standard normal draw, correlated across tickers (§4) | +| `−σ²/2` | Itô correction — without it the *median* path drifts up spuriously | + +### Calibrating Δt + +The tick interval is 500 ms of wall-clock time, and σ is quoted per trading year, so: + +``` +TRADING_SECONDS_PER_YEAR = 252 days × 6.5 hours × 3600 = 5,896,800 +Δt = 0.5 / 5,896,800 ≈ 8.48 × 10⁻⁸ +``` + +This is right, and it is worth stating why: one trading day is 46,800 ticks, and +46,800 × Δt = 1/252 exactly. So a simulated trading day reproduces the target daily +volatility by construction. + +**Verified empirically [measured].** 200 independent one-trading-day runs of AAPL +(σ = 0.22, shocks disabled): + +``` +realized daily log-return sd = 1.3425% +target σ/√252 = 1.3859% +standard error = 0.0695% z = −0.62 +``` + +Well within sampling noise. The GBM core is correctly calibrated. + +### Visible movement + +A model can be correct and still look dead if every tick rounds to the same cent. Per-tick +standard deviation is `S · σ · √Δt` — for AAPL at 324.96 that is **2.1 cents**, comfortably +above the 1-cent display quantum. + +Fraction of ticks where the 2-decimal displayed price actually changes, over 3,000 ticks at +the real 2026-09-02 closes **[measured]**: + +| Ticker | Price | σ | Per-tick sd | Visible ticks | +|---|---|---|---|---| +| AAPL | 324.96 | 0.22 | $0.0208 | 81.1% | +| GOOGL | 337.12 | 0.25 | $0.0245 | 84.6% | +| MSFT | 496.82 | 0.20 | $0.0289 | 87.0% | +| AMZN | 254.98 | 0.28 | $0.0208 | 81.5% | +| TSLA | 357.01 | 0.50 | $0.0520 | 92.5% | +| NVDA | 224.41 | 0.40 | $0.0261 | 84.8% | +| META | 592.85 | 0.30 | $0.0518 | 91.2% | +| JPM | 356.22 | 0.18 | $0.0187 | 79.9% | +| V | 378.40 | 0.17 | $0.0187 | 80.0% | +| NFLX | 82.73 | 0.35 | $0.0084 | 58.2% | + +Every ticker flashes on the majority of ticks. NFLX is the floor case at 58% because it is +the only low-priced name — its per-tick sd is under a cent, so rounding eats roughly two +ticks in five. Still perfectly lively at two visible changes per second, and no +intervention is needed; the number is recorded here so nobody "fixes" a non-problem. + +**Design rule:** if a future ticker prices below roughly $20, per-tick sd falls under a +tenth of a cent and it will look frozen. Either widen display precision for sub-$20 names +or floor their σ. + +--- + +## 3. Seeding + +### Anchored seeds are strongly preferred + +`backend/app/market/seed_prices.py` hard-codes ten starting prices. They have drifted +badly — as measured against real 2026-09-02 closes: + +| Ticker | Seed in repo | Real close | Error | +|---|---|---|---| +| NVDA | 800.00 | 224.41 | 3.6× too high (splits) | +| NFLX | 600.00 | 82.73 | 7.3× too high (splits) | +| META | 500.00 | 592.85 | 16% low | +| MSFT | 420.00 | 496.82 | 15% low | +| GOOGL | 175.00 | 337.12 | 48% low | + +A demo showing NVDA at $800 next to a real quote is embarrassing, and the table will keep +rotting. `AnchoredSimulatorDataSource` (see `MARKET_INTERFACE.md` §6) replaces it with one +free-tier API call that prices the whole market. The static table stays only as the no-key +fallback, and should be refreshed to the values in `MASSIVE_API.md` §5.1. + +```python +class SimulatorDataSource: + def __init__( + self, + price_cache: PriceCache, + update_interval: float = 0.5, + event_probability: float = 1e-4, # see §6 + seed_overrides: dict[str, float] | None = None, # real anchors when available + ) -> None: ... +``` + +Seed resolution order: `seed_overrides` → `SEED_PRICES` → synthesised. + +### Unknown tickers must not dead-end + +PLAN.md §13.2 item 8 asks what happens when a user — or the LLM's `watchlist_changes` — +requests a symbol the simulator has never heard of. Recommendation there was: synthesise, +never reject. The shipped code does synthesise, via `random.uniform(50.0, 300.0)`, but with +a flaw **[measured]**: + +``` +unknown-ticker seeds across 3 fresh sims: [141.23, 124.29, 268.96] +``` + +`ZZZZ` gets a different price on every restart, so a held position's cost basis jumps +between container restarts and the P&L chart lies. Make the synthetic seed **deterministic +per symbol** by hashing it: + +```python +def _synthesize_seed(ticker: str) -> float: + """Stable pseudo-price for an unknown symbol. Same ticker, same price, always.""" + h = int(hashlib.sha256(ticker.encode()).hexdigest()[:8], 16) + return round(20.0 + (h % 48_000) / 100.0, 2) # $20.00 – $500.00 +``` + +Unknown tickers also take `DEFAULT_PARAMS` (σ = 0.25, μ = 0.05) and cross-group correlation. +In Massive-backed modes, validate the symbol with `get_ticker_details()` first — it is +free-tier accessible and 404s on nonsense — then synthesise only if validation is +unavailable. + +--- + +## 4. Correlation + +Independent tickers read as fake within seconds: real markets move together. The simulator +imposes a factor structure through a correlation matrix and its Cholesky decomposition. + +Generate `n` independent standard normals `z`, then `L @ z` where `L Lᵀ = C` has exactly +covariance `C`: + +```python +z_independent = np.random.standard_normal(n) +z_correlated = self._cholesky @ z_independent # L is lower-triangular from np.linalg.cholesky +``` + +The shipped correlation structure: + +| Pair | ρ | +|---|---| +| Tech ∩ Tech (AAPL, GOOGL, MSFT, AMZN, META, NVDA, NFLX) | 0.6 | +| Finance ∩ Finance (JPM, V) | 0.5 | +| TSLA with anything | 0.3 | +| Cross-sector, or any unknown ticker | 0.3 | + +Sensible, and cheap to extend with more sectors. Two properties worth recording. + +**The matrix stays positive-definite as the watchlist grows [measured].** `np.linalg.cholesky` +raises `LinAlgError` on a non-PD matrix, and `_rebuild_cholesky` has no error handling — so +if it could fail, `add_ticker` would raise and a watchlist addition would 500. Tested with +the default ten plus 5, 20, 50 and 100 synthetic tickers: + +``` +n=110 cholesky OK min_eigenvalue = 0.400000 +``` + +The minimum eigenvalue is pinned at `1 − ρ_max = 0.4` regardless of size, because the +structure is a block matrix of equicorrelated groups. It is safe. Still, wrap the call and +fall back to the identity matrix on `LinAlgError` — the cost is three lines, and the failure +mode otherwise is a user-facing 500 from a watchlist add. + +**Rebuild cost is O(n²) per add/remove**, negligible at n < 100 and only on watchlist edits, +never in the tick loop. + +--- + +## 5. The Tick Loop + +```python +async def _run_loop(self) -> None: + while True: + try: + prices = self._sim.step() # one vectorised normal draw for all + for ticker, price in prices.items(): + self._cache.update(ticker, price, open_price=self._anchors.get(ticker)) + except Exception: + logger.exception("Simulator step failed") # never let one bad tick kill the loop + await asyncio.sleep(self._interval) +``` + +Three properties to preserve: + +- **Full precision internally, rounded at the boundary.** `self._prices[ticker]` keeps the + unrounded float; only the value handed to the cache is `round(price, 2)`. Rounding in + place would accumulate a drift bias over tens of thousands of ticks. The shipped code gets + this right. +- **The loop never dies.** A raise inside `step()` would silently end the background task + and freeze every price with no error path. The blanket `except` is correct here. +- **One `np.random.standard_normal(n)` per tick**, not per ticker. At 10–50 tickers and + 2 Hz this is free. + +`asyncio.sleep(interval)` after the work means the true period is `interval + work`, so +ticks drift slightly slower than 500 ms. Irrelevant for display; do not add a compensating +scheduler. + +--- + +## 6. Random Events — the one thing that needs fixing + +The shipped implementation applies, per ticker per tick: + +```python +if random.random() < self._event_prob: # default 0.001 + shock = random.uniform(0.02, 0.05) * random.choice([-1, 1]) + self._prices[ticker] *= 1 + shock +``` + +PLAN.md §6 asks for "occasional random events — sudden 2-5% moves on a ticker for drama." +The intent is right. The calibration is not. + +At `p = 0.001` with 7,200 ticks per wall-clock hour, each ticker takes **7.2 shocks per +hour**, each a permanent 2–5% level shift with random sign. That is a random walk of shocks +layered on top of the GBM, and it does not merely add drama — it obliterates the σ +calibration that §2 verified. + +60 one-hour simulations of AAPL at each setting **[measured]**: + +| `event_probability` | Shocks/hour/ticker | 1-hour return sd | Worst move seen | +|---|---|---|---| +| **0.001 (current)** | 7.20 | **10.02%** | 28.77% | +| 0.0001 | 0.72 | 3.10% | 8.51% | +| 0.0 (pure GBM) | 0.00 | 0.505% | 1.05% | +| *theory, σ = 0.22* | — | *0.544%* | — | + +**The shock process inflates realised volatility roughly 20×.** AAPL, parameterised at a +22% annual vol, actually delivers a 10% standard deviation *per hour* — an annualised +volatility somewhere north of 400%. Every per-ticker σ in `seed_prices.py` is decorative: +TSLA's 0.50 and JPM's 0.18 produce nearly identical behaviour, because both are swamped by +the same shock process. A user who opens the app, buys $2,000 of AAPL, and comes back from +lunch may find the position down 25% — from a model calibrated to move 1.4% in a day. + +### Recommended fix: transient shocks, not permanent ones + +The underlying error is that a shock is a **permanent level shift**. Real intraday spikes +substantially mean-revert. Model the event as a decaying additive component so the drama is +visible on the chart but the long-run distribution stays governed by σ: + +```python +@dataclass +class Shock: + magnitude: float # signed, e.g. -0.03 + decay: float # per-tick multiplier, e.g. 0.985 → ~half-life 46 ticks (23s) + +def step(self) -> dict[str, float]: + ... + for i, ticker in enumerate(self._tickers): + # 1. GBM evolves the underlying price — untouched, still calibrated + self._prices[ticker] *= math.exp(drift + diffusion) + + # 2. Shocks are a separate, decaying overlay + if random.random() < self._event_prob: + self._shocks[ticker] = Shock( + magnitude=random.uniform(0.015, 0.04) * random.choice([-1, 1]), + decay=0.985, + ) + shock = self._shocks.get(ticker) + if shock: + displayed = self._prices[ticker] * (1 + shock.magnitude) + shock.magnitude *= shock.decay + if abs(shock.magnitude) < 1e-4: + del self._shocks[ticker] + else: + displayed = self._prices[ticker] + + result[ticker] = round(displayed, 2) +``` + +This gives a sharp visible spike that bleeds off over roughly a minute — which is what an +intraday spike actually looks like on a chart — while `self._prices` continues to follow +calibrated GBM. + +Pair it with `event_probability = 1e-4`: across a 10-ticker watchlist that is about +**1.2 events per 10-minute demo**, enough for something to happen while the user is +watching, without a shock every 50 seconds. + +If a decaying overlay is judged too much machinery for the teaching value, the minimum +acceptable change is `event_probability = 1e-4` and a 1–3% magnitude range. That still +leaves shocks dominating σ by ~6×, but takes the worst case from 29% to 8.5%. + +--- + +## 7. Price History for Charts + +PLAN.md §13.2 item 15 notes that charts and sparklines accumulate from SSE "since page +load", so the main chart is empty on first paint and a refresh discards everything. It +recommends a bounded ring buffer, and that is right — it is the difference between a chart +that looks alive on load and one that looks broken. + +`MARKET_INTERFACE.md` §3 puts this on the interface as `get_history()`. The simulator's +implementation is a `collections.deque`: + +```python +class SimulatorDataSource: + HISTORY_POINTS = 600 # 600 ticks × 500ms = 5 minutes per ticker + + def __init__(self, ...): + self._history: dict[str, deque[tuple[float, float]]] = defaultdict( + lambda: deque(maxlen=self.HISTORY_POINTS) + ) + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + series = self._history.get(ticker, ()) + return [PricePoint(epoch_to_iso(t), p) for t, p in list(series)[-points:]] +``` + +Memory: 600 points × 2 floats × 50 tickers ≈ 480 KB. Trivial. + +**Prefill on start** so the very first paint is not a single dot: run the simulator forward +`HISTORY_POINTS` steps from the seed price with no sleeping, record the path, then reset the +price to the seed. Half a second of CPU buys a chart with five minutes of plausible history +at t=0. + +Massive-backed sources override `get_history()` with real minute bars — available on the +free tier, and a genuine improvement over synthetic backfill (`MASSIVE_API.md` §5.4). + +--- + +## 8. Code Structure + +``` +backend/app/market/ +├── simulator.py +│ ├── GBMSimulator # pure, synchronous, no I/O — the model +│ │ .step() -> dict[str, float] +│ │ .add_ticker() / .remove_ticker() / .get_price() / .get_tickers() +│ │ ._rebuild_cholesky() # + LinAlgError fallback (§4) +│ │ ._synthesize_seed() # deterministic hash (§3) +│ └── SimulatorDataSource # async MarketDataSource adapter +│ .start() / .stop() / .add_ticker() / .remove_ticker() +│ .describe() / .get_history() +│ ._run_loop() # 500ms tick → PriceCache +├── seed_prices.py # SEED_PRICES, TICKER_PARAMS, correlation config +└── anchored.py # AnchoredSimulatorDataSource (see MARKET_INTERFACE.md) +``` + +The split between `GBMSimulator` and `SimulatorDataSource` is the best structural decision +in the shipped code and should be defended in review. `GBMSimulator` has no async, no I/O +and no cache reference — it is a deterministic function of its own state, so its statistical +properties can be tested at tens of thousands of steps per second with no event loop. Every +measurement in this document was produced by driving `GBMSimulator` directly. Keep the model +free of the transport. + +--- + +## 9. Testing + +Beyond the existing unit tests, the properties that actually matter are statistical. + +**Distributional (the tests that would have caught §6):** + +- Realised daily log-return sd over ≥200 one-day runs is within 3 standard errors of + `σ/√252`, **with shocks enabled at the production default**. This is the regression test + for the event calibration; run it against every σ in `TICKER_PARAMS`, not just AAPL. +- Realised correlation between two tech tickers over a long run is 0.6 ± 0.05, and + tech-vs-finance is 0.3 ± 0.05. Confirms the Cholesky is applied and not silently bypassed. +- Mean log return over many runs is consistent with `(μ − σ²/2)·t` — catches a dropped Itô + correction. + +Seed `np.random` and `random` in these tests; both are used and both need fixing for +reproducibility. + +**Invariants:** + +- Price is strictly positive after 100,000 steps. GBM guarantees it analytically; a shock + multiplier below −1 would not. +- No NaN or infinity for any σ in the table. +- `step()` returns exactly the current ticker set, always. +- Cholesky survives 100 unknown tickers added one at a time (the incremental path, which + rebuilds on every call, not the batch constructor). + +**Behavioural:** + +- Visible-tick rate exceeds 50% for every default ticker at anchored prices — the guard on + the §2 rounding cliff. +- Deterministic seeding: the same unknown ticker yields the same synthetic price across two + fresh instances. +- `remove_ticker` then `add_ticker` restores the price it had, not a fresh seed. + +--- + +## 10. Summary of Changes to the Shipped Simulator + +| # | Change | Severity | +|---|---|---| +| 1 | `event_probability` 0.001 → 1e-4, and make shocks **decay** rather than permanently shift the level | **High** — currently inflates realised volatility ~20× and voids every σ parameter | +| 2 | Deterministic hash-based seeds for unknown tickers | **Medium** — cost basis and P&L change across restarts today | +| 3 | Accept `seed_overrides` so real Massive closes can anchor the simulation | **Medium** — the static table is 3–7× wrong on two tickers | +| 4 | Record `open_price` so the daily-change column has a baseline | **Medium** — PLAN.md §13.1 item 1 | +| 5 | Ring-buffer history + prefill, exposed via `get_history()` | **Medium** — charts are empty on first paint | +| 6 | `LinAlgError` fallback in `_rebuild_cholesky` | **Low** — verified safe to n=110, but the failure is a user-facing 500 | +| 7 | Refresh `SEED_PRICES` to real 2026-09-02 closes | **Low** — only the no-key path | +| 8 | Statistical tests, including one that fails at the current `event_probability` | **High** — nothing today would catch #1 | diff --git a/planning/MASSIVE_API.md b/planning/MASSIVE_API.md new file mode 100644 index 000000000..1d32509c1 --- /dev/null +++ b/planning/MASSIVE_API.md @@ -0,0 +1,548 @@ +# Massive API — Reference for FinAlly + +**Status:** Research complete. Verified against the live API on 2026-09-03 using the +`MASSIVE_API_KEY` in the project root `.env`, and against `massive` Python SDK **2.2.0** +(the version pinned in `backend/uv.lock`). + +Everything marked **[verified]** below was executed against the real API. Everything marked +**[docs]** comes from and has not been executable with our key. + +--- + +## 1. What Massive Is + +Massive is Polygon.io, rebranded in early 2026. Same infrastructure, same endpoint paths, +new domain. Practical consequences: + +- Base URL is `https://api.massive.com`, but **legacy `api.polygon.io` paths still work** and + the two are interchangeable. +- Error messages still leak the old brand — a 403 on our key returns + `"Please upgrade your plan at https://polygon.io/pricing"` **[verified]**. +- Response headers still carry Polygon infrastructure names (`x-polygon-cluster-name: + polygon-ny5`) **[verified]**. +- Any Polygon.io tutorial, StackOverflow answer, or LLM training data from before 2026 is + still accurate for endpoint shapes. Only the hostname and the SDK package name changed. + +Coverage is all 19 US stock exchanges plus dark pools, FINRA facilities and OTC, delivered +over REST, WebSocket and S3-style flat files. FinAlly uses **REST only** — see §9. + +--- + +## 2. Authentication + +Two equivalent forms. Both **[verified]** returning HTTP 200: + +```bash +# Preferred — bearer header (what the Python SDK sends) +curl "https://api.massive.com/v2/aggs/ticker/AAPL/prev?adjusted=true" \ + -H "Authorization: Bearer $MASSIVE_API_KEY" + +# Alternative — query parameter (convenient for browser/debug, but leaks the key into logs) +curl "https://api.massive.com/v2/aggs/ticker/MSFT/prev?apiKey=$MASSIVE_API_KEY" +``` + +Omitting auth entirely returns **HTTP 401** **[verified]**. An API key is 32 characters. + +**Use the header form in FinAlly.** Query-param keys end up in access logs, browser history +and error reports. + +The SDK reads the environment variable `MASSIVE_API_KEY` automatically if you pass no +`api_key` argument — which is exactly the variable name PLAN.md already specifies, so no +mapping is needed: + +```python +from massive import RESTClient + +client = RESTClient() # reads MASSIVE_API_KEY from the environment +client = RESTClient(api_key="...") # or pass it explicitly +``` + +If neither is present the constructor raises `massive.AuthError` immediately — it does not +wait for the first request. + +--- + +## 3. The Python SDK + +```toml +# backend/pyproject.toml — already present +dependencies = ["massive>=1.0.0"] # resolves to 2.2.0 in uv.lock +``` + +`RESTClient` constructor signature, read from the installed package **[verified]**: + +```python +RESTClient( + api_key: str | None = None, + connect_timeout: float = 10.0, + read_timeout: float = 10.0, + num_pools: int = 10, + retries: int = 3, # urllib3 Retry, see §7 + base: str = "https://api.massive.com", + pagination: bool = True, + verbose: bool = False, # sets SDK logger to DEBUG + trace: bool = False, # prints full request/response + custom_json: Any | None = None, # e.g. orjson +) +``` + +Three properties matter for FinAlly's design: + +1. **It is fully synchronous.** It uses `urllib3.PoolManager`, not `httpx`/`aiohttp`. Every + call blocks the thread. In an async FastAPI app every call **must** be wrapped in + `asyncio.to_thread(...)` or it will stall the event loop and freeze the SSE stream for + every connected client. +2. **It retries automatically.** `retries=3` with `backoff_factor=0.1` over status codes + `413, 429, 499, 500, 502, 503, 504`. This silently absorbs brief rate limiting but see + the failure-mode gotcha in §7. +3. **It returns typed dataclasses, not dicts.** Field names are snake_cased and expanded + from the wire format's single letters (`c` → `close`, `vw` → `vwap`, `T` → `ticker`). + +--- + +## 4. Plans and Entitlements — read this before designing anything + +This is the single most consequential finding of the research, and it contradicts an +assumption in PLAN.md §6. + +| Plan | Price | Rate limit | Data timeliness | History | +|---|---|---|---|---| +| Stocks Basic | **Free** | **5 calls/min** | **End of day only** | 2 years | +| Stocks Starter | $29/mo | Unlimited | 15-minute delayed | 5 years | +| Stocks Developer | $79/mo | Unlimited | 15-minute delayed | 10 years | +| Stocks Advanced | $199/mo | Unlimited | Real-time | 20+ years | + +**[docs]** for the table; **[verified]** for every Basic-tier consequence below. + +### What a free key can and cannot do + +Probed live on 2026-09-03 with the project's key: + +| SDK call | Endpoint | Result | +|---|---|---| +| `get_previous_close_agg("AAPL")` | `/v2/aggs/ticker/{t}/prev` | ✅ **200** | +| `get_daily_open_close_agg("AAPL", "2026-08-28")` | `/v1/open-close/{t}/{date}` | ✅ **200** | +| `get_aggs("AAPL", 1, "day", ...)` | `/v2/aggs/ticker/{t}/range/...` | ✅ **200** | +| `get_aggs("AAPL", 1, "minute", ...)` *(past days)* | `/v2/aggs/ticker/{t}/range/...` | ✅ **200** | +| `get_grouped_daily_aggs("2026-09-02")` | `/v2/aggs/grouped/locale/us/market/stocks/{date}` | ✅ **200** | +| `get_market_status()` | `/v1/marketstatus/now` | ✅ **200** | +| `get_ticker_details("AAPL")` | `/v3/reference/tickers/{t}` | ✅ **200** | +| `get_snapshot_all("stocks", [...])` | `/v2/snapshot/.../tickers` | ❌ **NOT_AUTHORIZED** | +| `get_snapshot_ticker("stocks", "AAPL")` | `/v2/snapshot/.../tickers/{t}` | ❌ **NOT_AUTHORIZED** | +| `list_universal_snapshots(...)` | `/v3/snapshot` | ❌ **NOT_AUTHORIZED** | +| `get_last_trade("AAPL")` | `/v2/last/trade/{t}` | ❌ **NOT_AUTHORIZED** | +| `get_aggs("AAPL", 1, "minute", "2026-09-03", ...)` *(today)* | — | ❌ **NOT_AUTHORIZED** | + +Two distinct rejection messages, and the difference matters for diagnostics: + +```json +{"status":"NOT_AUTHORIZED","message":"You are not entitled to this data. + Please upgrade your plan at https://massive.com/pricing"} +``` +→ the **endpoint** is out of plan (all snapshot and last-trade endpoints). + +```json +{"status":"NOT_AUTHORIZED","message":"Your plan doesn't include this data timeframe. + Please upgrade your plan at https://polygon.io/pricing"} +``` +→ the endpoint is allowed but the **date** is too recent. Requesting today's minute bars on +a free key fails; yesterday's succeed. + +### Why this breaks the plan as written + +PLAN.md §6 says *"Free tier (5 calls/min): poll every 15 seconds"* and the shipped +`backend/app/market/massive_client.py` polls `get_snapshot_all()`. **On a free key that +client returns zero prices, forever.** Every poll raises `BadResponse`, the exception is +swallowed by the `except Exception` in `_poll_once`, and the cache stays empty — the app +boots to a watchlist of blanks with no error surfaced to the user. + +Free-tier keys are what students will have. The interface design in +`planning/MARKET_INTERFACE.md` addresses this directly; §8 below gives the probe that +detects the tier. + +--- + +## 5. Endpoints FinAlly Actually Needs + +### 5.1 Daily Market Summary — the free tier's best endpoint + +``` +GET /v2/aggs/grouped/locale/us/market/stocks/{date} +``` + +One call returns the previous OHLC bar for **every** US ticker. Measured live: **12,541 +tickers in 1.04 s** from a single request **[verified]**. On a 5-calls-per-minute budget +this is transformative — it prices an entire watchlist of any size for one call, and adding +a ticker costs nothing. + +```python +from massive import RESTClient + +client = RESTClient() +bars = client.get_grouped_daily_aggs("2026-09-02", adjusted=True) +by_ticker = {b.ticker: b for b in bars} + +aapl = by_ticker["AAPL"] +print(aapl.open, aapl.high, aapl.low, aapl.close, aapl.vwap, aapl.volume) +# 326.865 328.4 323.53 324.96 325.2771 33776370.0 +``` + +Real values returned for the default watchlist on 2026-09-02 **[verified]** — note how far +these have drifted from the seed table in `backend/app/market/seed_prices.py`: + +| Ticker | Open | High | Low | **Close** | VWAP | Seed in repo | +|---|---|---|---|---|---|---| +| AAPL | 326.87 | 328.40 | 323.53 | **324.96** | 325.28 | 190.00 | +| GOOGL | 334.06 | 340.00 | 332.82 | **337.12** | 337.16 | 175.00 | +| MSFT | 499.85 | 500.27 | 493.81 | **496.82** | 496.91 | 420.00 | +| AMZN | 254.25 | 256.24 | 253.40 | **254.98** | 255.03 | 185.00 | +| TSLA | 360.41 | 360.62 | 349.92 | **357.01** | 354.07 | 250.00 | +| NVDA | 218.78 | 227.95 | 218.48 | **224.41** | 224.38 | 800.00 | +| META | 578.78 | 600.38 | 577.00 | **592.85** | 593.32 | 500.00 | +| JPM | 357.55 | 361.47 | 353.79 | **356.22** | 356.71 | 195.00 | +| V | 374.01 | 380.17 | 374.01 | **378.40** | 378.40 | 280.00 | +| NFLX | 80.55 | 83.12 | 80.30 | **82.73** | 82.39 | 600.00 | + +The bar timestamp is `2026-09-02T20:00:00Z` — the 16:00 ET close. + +**Caveat:** the date must be a trading day. Passing a weekend or holiday returns +`"resultsCount": 0` rather than an error, so callers must walk backwards. `get_market_holidays()` +is available on the free tier and returns e.g. `MarketHoliday(date='2026-09-07', +exchange='NYSE', name='Labor Day', status='closed')` **[verified]**. + +### 5.2 Previous Day Bar — one ticker, one call + +``` +GET /v2/aggs/ticker/{stocksTicker}/prev?adjusted=true +``` + +```python +result = client.get_previous_close_agg("AAPL", adjusted=True) +# returns a LIST of one PreviousCloseAgg, not a bare object — easy mistake +bar = result[0] +print(bar.ticker, bar.close, bar.open, bar.timestamp) +# AAPL 324.96 326.865 1788379200000 +``` + +Automatically resolves "previous trading day", so no weekend arithmetic — that is its one +advantage over §5.1. But it costs **one call per ticker**, which on a 5/min budget prices +only five tickers a minute. Use §5.1 for anything more than a couple of symbols. + +### 5.3 Daily Ticker Summary — open/close for a specific date + +``` +GET /v1/open-close/{stocksTicker}/{date}?adjusted=true +``` + +```python +day = client.get_daily_open_close_agg("AAPL", "2026-08-28", adjusted=True) +print(day.open, day.close, day.pre_market, day.after_hours, day.volume) +# 316.845 319.7 315.06 320.126 38649398.679189 +``` + +The only endpoint exposing pre-market and after-hours prints on the free tier. Note the +model field is `from_` (trailing underscore) because `from` is a Python keyword. + +### 5.4 Custom Bars — the history behind the charts + +``` +GET /v2/aggs/ticker/{stocksTicker}/range/{multiplier}/{timespan}/{from}/{to} +``` + +`timespan` ∈ `second, minute, hour, day, week, month, quarter, year`. `limit` defaults to +5000, max 50000. `sort` ∈ `asc, desc`. + +```python +# 845 one-minute bars for a full session, 08:00–23:59 UTC [verified] +bars = client.get_aggs("AAPL", 1, "minute", "2026-09-02", "2026-09-02", limit=50000) + +# or stream with automatic pagination +for bar in client.list_aggs("AAPL", 1, "day", "2026-01-01", "2026-09-02"): + ... +``` + +This directly answers PLAN.md §13.2 item 15 ("charts are empty on first paint"): a single +call backfills a real intraday series before the first SSE tick arrives. + +**Free-tier limit:** `from`/`to` may not include the current day. `("2026-09-03", "2026-09-03")` +on 2026-09-03 returns the *timeframe* NOT_AUTHORIZED error **[verified]**. + +### 5.5 Full Market Snapshot — real-time, **paid tiers only** + +``` +GET /v2/snapshot/locale/us/markets/stocks/tickers?tickers=AAPL,MSFT&include_otc=false +``` + +This is the endpoint PLAN.md §6 assumes. It is the right one *if the key is entitled*: one +call, every watched ticker, last trade plus today's OHLC plus previous close plus a +precomputed daily change percentage. + +```python +from massive.rest.models import SnapshotMarketType + +snapshots = client.get_snapshot_all( + market_type=SnapshotMarketType.STOCKS, # or just the string "stocks" + tickers=["AAPL", "GOOGL", "MSFT"], +) +for s in snapshots: + print(s.ticker, + s.last_trade.price, # latest print + s.last_trade.sip_timestamp, # Unix NANOSECONDS + s.day.open, s.day.close, # today's session so far + s.prev_day.close, # yesterday's close + s.todays_change_percent) # server-computed daily % +``` + +`TickerSnapshot` fields, from the installed SDK **[verified]**: + +``` +ticker: str | None +day: Agg | None # today's session aggregate +prev_day: Agg | None # previous session aggregate +min: MinuteSnapshot | None # the current minute bar +last_trade: LastTrade | None +last_quote: LastQuote | None +todays_change: float | None +todays_change_percent: float | None +updated: int | None # Unix NANOSECONDS +fair_market_value: float | None +``` + +`Agg` carries `open, high, low, close, volume, vwap, timestamp, transactions, otc`. +`LastTrade` carries `price, size, exchange, conditions, sip_timestamp, +participant_timestamp, trf_timestamp, id, tape`. + +**`prev_day.close` and `day.open` are the missing baseline for PLAN.md §13.1 item 1** — the +"daily change %" that has nowhere to come from today. In snapshot mode they arrive free with +every poll. + +### 5.6 Unified Snapshot — friendlier shape, same entitlement + +``` +GET /v3/snapshot?type=stocks&ticker.any_of=AAPL,MSFT&limit=250 +``` + +```python +for snap in client.list_universal_snapshots(type="stocks", ticker_any_of=["AAPL", "NVDA"]): + print(snap.ticker, snap.session.close, snap.session.change_percent, snap.market_status) +``` + +Two genuine advantages over §5.5 if you are already paying: `session.change_percent` and +`session.previous_close` are named rather than abbreviated, and **unknown tickers come back +as inline error rows instead of silently vanishing** — + +```json +{"ticker": "TSLAAPL", "error": "NOT_FOUND", "message": "Ticker not found."} +``` + +That is the clean answer to PLAN.md §13.2 item 8 (what happens when a user adds a garbage +symbol) — but only on a paid key. Max 250 tickers per call. + +### 5.7 Reference and status + +```python +client.get_market_status() # {'nasdaq': 'open', 'nyse': 'open', 'after_hours': False, ...} +client.get_market_holidays() # upcoming closures, per exchange +client.get_ticker_details("AAPL") # company name, address, branding icons, market cap +``` + +All three work on the free tier **[verified]**. `get_market_status()` is the correct way to +answer "is the market open?" for PLAN.md §13.2 item 14, rather than hard-coding 09:30–16:00 ET +and a holiday table. + +`get_ticker_details()` is also a **symbol validator** — it 404s on nonsense, giving a +free-tier answer to item 8. + +--- + +## 6. Wire Format Gotchas + +**Single-letter JSON keys.** Aggregate endpoints return `{"T","o","h","l","c","v","vw","n","t"}` += ticker, open, high, low, close, volume, VWAP, transaction count, timestamp. The SDK expands +these; raw `httpx` callers must map them by hand. This alone is a good reason to keep the SDK. + +**Timestamps are inconsistent across endpoints.** + +| Source | Unit | +|---|---| +| Aggregate `t` / `Agg.timestamp` | Unix **milliseconds** | +| `snapshot.updated` | Unix **nanoseconds** | +| `last_trade.sip_timestamp` | Unix **nanoseconds** | +| `DailyOpenCloseAgg.from_` | `YYYY-MM-DD` string | + +The shipped `massive_client.py` divides `snap.last_trade.timestamp / 1000.0` to reach +seconds. That is **wrong by a factor of a million** for a nanosecond field — and the +attribute is `sip_timestamp`, not `timestamp`, so it would raise `AttributeError` first and +be swallowed by the existing `except (AttributeError, TypeError)`. Convert explicitly: + +```python +def to_epoch_seconds(value: int, unit: str) -> float: + return value / {"ms": 1e3, "us": 1e6, "ns": 1e9}[unit] +``` + +Per PLAN.md §13.1 item 6, convert once at the boundary and keep ISO 8601 UTC on the wire. + +**`adjusted=true` is the default and should stay that way.** Unadjusted series show a +false −90% cliff on a 10:1 split. Compare NVDA at 224.41 today against the repo's 800.00 +seed — that gap is mostly splits, not a crash. + +**Tickers are case-sensitive.** Always `.upper().strip()` before sending. + +--- + +## 7. Rate Limiting + +Free tier is **5 requests/minute**, enforced server-side as HTTP 429. + +Measured live with `retries=0`, calling `/v2/aggs/ticker/AAPL/prev` in a tight loop +**[verified]**: + +``` +#1 t= 0.32s OK +#2 t= 0.46s OK +#3 t= 0.58s OK +#4 t= 0.68s 429 +#5 t= 0.79s 429 ... and every call after, until the window rolls +``` + +Three succeeded because earlier probes in the same minute had already consumed budget — the +window is a rolling 60 s across the whole key, not per endpoint. + +### The gotcha that will cost someone an afternoon + +With the SDK's default `retries=3`, an exhausted rate limit does **not** surface as +`massive.BadResponse`. urllib3's retry layer intercepts the 429s and raises: + +``` +urllib3.exceptions.MaxRetryError: ... (Caused by ResponseError('too many 429 error responses')) +``` + +`MaxRetryError` is not a subclass of anything in `massive.exceptions`. Any handler written +as `except BadResponse` will miss it entirely. Catch broadly and inspect: + +```python +import urllib3.exceptions +from massive.exceptions import AuthError, BadResponse + +try: + bars = client.get_grouped_daily_aggs(date) +except AuthError: + ... # missing/empty key — fatal, fall back to simulator +except BadResponse as e: + if "NOT_AUTHORIZED" in str(e): + ... # entitlement — permanent, do not retry + else: + ... # transient +except urllib3.exceptions.MaxRetryError: + ... # rate limited or network — back off and retry +``` + +Also note the default `backoff_factor=0.1` gives retries at 0.0/0.2/0.4 s — all inside the +same 60 s window, so all three are guaranteed to fail. For a 5/min budget the retry is +useless; set `retries=0` and manage backoff yourself. + +**No rate-limit headers are returned** — no `X-RateLimit-Remaining`, no `Retry-After` +**[verified]** on 200 responses. The client must track its own budget. + +### Budget arithmetic for FinAlly + +| Strategy | Calls/poll | Max poll rate on free tier | +|---|---|---| +| `get_previous_close_agg` per ticker, 10 tickers | 10 | impossible (2× over budget for one poll) | +| `get_grouped_daily_aggs`, any number of tickers | **1** | every 12 s, with headroom | +| `get_snapshot_all`, any number of tickers | 1 | n/a — not entitled | + +The conclusion drives the whole design: **one call per cycle, never one call per ticker.** + +--- + +## 8. Capability Probe + +Because behaviour differs so sharply by plan, FinAlly should establish entitlement once at +startup rather than discovering it through a silent stream of swallowed exceptions. + +```python +from dataclasses import dataclass + +import urllib3.exceptions +from massive import RESTClient +from massive.exceptions import AuthError, BadResponse + + +@dataclass(frozen=True, slots=True) +class MassiveCapabilities: + """What a given API key is actually allowed to do.""" + + valid: bool # key authenticates at all + realtime: bool # snapshot / last-trade endpoints entitled + end_of_day: bool # aggregate endpoints entitled + detail: str + + +def probe_capabilities(api_key: str) -> MassiveCapabilities: + """Two cheap calls, run once at startup. Costs 2 of the 5/min budget.""" + try: + client = RESTClient(api_key=api_key, retries=0, read_timeout=5.0) + except AuthError: + return MassiveCapabilities(False, False, False, "no API key configured") + + # 1. Cheapest possible entitlement test for real-time. + try: + client.get_snapshot_all(market_type="stocks", tickers=["AAPL"]) + return MassiveCapabilities(True, True, True, "real-time snapshots entitled") + except BadResponse as e: + if "NOT_AUTHORIZED" not in str(e): + return MassiveCapabilities(False, False, False, f"unexpected: {e}") + except urllib3.exceptions.MaxRetryError as e: + return MassiveCapabilities(False, False, False, f"unreachable/rate-limited: {e}") + + # 2. Snapshots refused — is this a valid key on a lower plan, or a bad key? + try: + client.get_previous_close_agg("AAPL") + return MassiveCapabilities(True, False, True, "end-of-day only (Basic tier)") + except Exception as e: + return MassiveCapabilities(False, False, False, f"key rejected: {e}") +``` + +Feed `detail` into `GET /api/health` — PLAN.md §13.5 item 38 asks health to report the +active source, and "end-of-day only (Basic tier)" answers "why is nothing moving?" in one +glance. + +--- + +## 9. What FinAlly Deliberately Does Not Use + +**WebSockets.** `massive.WebSocketClient` exists and would be the natural fit for a +streaming terminal, but it is entitled on the same paid plans as the snapshot endpoints, so +it is unavailable to the students this project is built for. REST polling into the shared +`PriceCache` keeps one code path for both sources. PLAN.md §6 already made this call; the +entitlement data confirms it. + +**Flat files (S3).** Bulk historical download. Irrelevant to a live terminal. + +**Options, forex, crypto, futures, indices, financials, Benzinga news.** The SDK exposes all +of them (`massive/rest/{futures,economy,financials,benzinga,indicators,...}.py`). Out of scope. + +--- + +## 10. Summary for the Implementer + +1. Base URL `https://api.massive.com`; auth `Authorization: Bearer `; SDK package `massive`. +2. The SDK is **synchronous** — always `asyncio.to_thread` it inside FastAPI. +3. **A free key cannot fetch a live price.** No snapshots, no last trade, nothing dated today. + The currently shipped `MassiveDataSource` silently produces nothing on such a key. +4. `get_grouped_daily_aggs()` is the free tier's workhorse: 12,541 tickers, one call, ~1 s. +5. Rate limiting arrives as `urllib3.MaxRetryError`, not `BadResponse`. Set `retries=0`. +6. Timestamps are milliseconds on aggregates and **nanoseconds** on snapshots. +7. Probe entitlement once at startup and report it from `/api/health`. + +How these facts shape the source-selection design is in +[`MARKET_INTERFACE.md`](MARKET_INTERFACE.md); the simulator that covers the free-tier and +no-key cases is in [`MARKET_SIMULATOR.md`](MARKET_SIMULATOR.md). + +## Sources + +- [Massive API docs](https://massive.com/docs) · [Stocks REST overview](https://massive.com/docs/rest/stocks/overview) · [Pricing](https://massive.com/pricing) +- [Full Market Snapshot](https://massive.com/docs/rest/stocks/snapshots/full-market-snapshot.md) · [Unified Snapshot](https://massive.com/docs/rest/stocks/snapshots/unified-snapshot.md) · [Single Ticker Snapshot](https://massive.com/docs/rest/stocks/snapshots/single-ticker-snapshot.md) +- [Previous Day Bar](https://massive.com/docs/rest/stocks/aggregates/previous-day-bar.md) · [Daily Ticker Summary](https://massive.com/docs/rest/stocks/aggregates/daily-ticker-summary.md) · [Custom Bars](https://massive.com/docs/rest/stocks/aggregates/custom-bars.md) · [Daily Market Summary](https://massive.com/docs/rest/stocks/aggregates/daily-market-summary.md) +- `massive` Python SDK 2.2.0, read from `backend/.venv/lib/python3.12/site-packages/massive/` diff --git a/planning/PLAN.md b/planning/PLAN.md index bc1811b33..aabbe3a52 100644 --- a/planning/PLAN.md +++ b/planning/PLAN.md @@ -259,6 +259,7 @@ All tables include a `user_id` column defaulting to `"default"`. This is hardcod | GET | `/api/portfolio` | Current positions, cash balance, total value, unrealized P&L | | POST | `/api/portfolio/trade` | Execute a trade: `{ticker, quantity, side}` | | GET | `/api/portfolio/history` | Portfolio value snapshots over time (for P&L chart) | +| GET | `/api/trades` | Trade history (blotter panel + LLM context) | ### Watchlist | Method | Path | Description | @@ -271,6 +272,7 @@ All tables include a `user_id` column defaulting to `"default"`. This is hardcod | Method | Path | Description | |--------|------|-------------| | POST | `/api/chat` | Send a message, receive complete JSON response (message + executed actions) | +| GET | `/api/chat/history` | Recent conversation history, so the chat panel survives a page refresh | ### System | Method | Path | Description | @@ -289,7 +291,7 @@ There is an OPENROUTER_API_KEY in the .env file in the project root. When the user sends a chat message, the backend: -1. Loads the user's current portfolio context (cash, positions with P&L, watchlist with live prices, total portfolio value) +1. Loads the user's current portfolio context (cash, positions with P&L, watchlist with live prices, total portfolio value), **recent trade history, and a downsampled portfolio value trajectory** so the assistant can discuss what was done and how the portfolio has moved, not only its current state — see `planning/API_CONTRACT.md` §6 for the exact context block 2. Loads recent conversation history from the `chat_messages` table 3. Constructs a prompt with a system message, portfolio context, conversation history, and the user's new message 4. Calls the LLM via LiteLLM → OpenRouter, requesting structured output, using the cerebras-inference skill @@ -454,3 +456,120 @@ The container is designed to deploy to AWS App Runner, Render, or any container - Portfolio visualization: heatmap renders with correct colors, P&L chart has data points - AI chat (mocked): send a message, receive a response, trade execution appears inline - SSE resilience: disconnect and verify reconnection + +--- + +## 13. Review Notes — Questions, Clarifications & Simplifications + +*Added 2026-09-02. Reviewed against the current repo state: `backend/app/market/` is complete (see `planning/MARKET_DATA_SUMMARY.md`); `frontend/`, `test/`, `scripts/`, and the Dockerfile do not exist yet. Items are grouped by how much they cost to fix later.* + +### 13.1 Contradictions & gaps that will bite during the build + +**1. "Daily change %" has no baseline anywhere in the design.** +Section 10 asks the watchlist for a *daily* change %, but the shipped `PriceUpdate` (`backend/app/market/models.py`) only carries `previous_price` from the **previous tick** — its `change_percent` is a ~500ms delta, typically ±0.05%, which will render as a permanently-flat column. Nothing in the schema, cache, or seed data stores a session open or previous close. +**Recommendation:** add `open_price` to `PriceUpdate` and the cache — for the simulator use the seed price as the session open; for Massive use the previous close from the API. Then expose **two** distinct fields and name them unambiguously: `tick_direction` (drives the flash animation) and `change_percent_today` (drives the column). Decide this before the frontend starts, because both consume the same SSE payload. + +**2. The trade-failure feedback loop is logically impossible as written.** +Section 9 says trades execute *after* the LLM response is parsed, and that "the error is included in the chat response so the LLM can inform the user." The LLM has already finished writing its message by then — it cannot inform anyone about a failure that happens later. +**Recommendation:** pick one — (a) return per-action results in the `actions` JSON and have the **frontend** render failures as inline red chips under the message (simplest, no extra latency, recommended); or (b) do a cheap second LLM call only when an action failed. Do not leave this ambiguous or the agent will silently swallow failed trades. + +**3. Docker volume: the two paragraphs describe different things.** +Section 4 says the repo's `db/` directory "maps to `/app/db` in the container", but the command in Section 11 mounts a **named volume** (`-v finally-data:/app/db`), which does not touch the host `db/` directory at all. +**Recommendation:** commit to the named volume (it is the right call — no host permission issues on macOS/Windows) and restate `db/.gitkeep` as being for *local, non-Docker* development only. + +**4. `backend/db/` would be excluded from the built wheel.** +Section 4 puts schema SQL and seed logic in `backend/db/`, but the already-committed `backend/pyproject.toml` has `[tool.hatch.build.targets.wheel] packages = ["app"]`. Anything outside `backend/app/` is not packaged, and the `backend/db/` vs root `db/` naming is a trap agents will fall into. +**Recommendation:** move it to `backend/app/db/` (importable, packaged, unambiguous) and reserve the name `db/` exclusively for the runtime volume. + +**5. Lazy DB init "on first request" starves the market data source.** +Section 7 allows init on startup *or* first request, but the market data source must be started with the watchlist read from the database, and that happens in the app lifespan — before any request arrives. +**Recommendation:** delete the "or first request" option. Initialize and seed the DB in the FastAPI lifespan handler, then start the market source from the seeded watchlist. One code path, no request-time locking. + +**6. Timestamp formats are inconsistent across the API surface.** +The DB columns are ISO strings; `PriceUpdate.timestamp` is a Unix float. The frontend will receive both from different endpoints and have to branch. +**Recommendation:** state one wire convention in this document — suggest **ISO 8601 UTC strings everywhere in JSON**, with the SSE payload converted at serialization time. + +**7. The SSE stream has no heartbeat, which matters most in Massive mode.** +`stream.py` only emits when the cache version changes. With the simulator that is every 500ms, but with Massive polling at 15s (free tier) the connection sits silent for 15 seconds at a time, and if the market is closed it is silent *forever* — long enough for intermediate proxies or a laptop sleep to kill it without either side noticing. +**Recommendation:** emit an SSE comment (`: ping\n\n`) every ~10s when there is nothing to send. + +### 13.2 Questions that need a product decision + +**8. What happens when a user adds a ticker the simulator has never heard of?** +`seed_prices.py` has hand-tuned seed prices and GBM parameters for a fixed set. `POST /api/watchlist {"ticker": "ZZZZ"}` — and the LLM's `watchlist_changes` — can request anything. Reject unknown symbols against an allowlist, or synthesize a plausible seed price and default volatility? (Recommend: synthesize, with a default vol, so the demo never dead-ends — but say so explicitly.) In Massive mode, what is the behavior for a symbol the API rejects? + +**9. Can you trade a ticker that is not on the watchlist?** +The trade bar has a free-text ticker field, and pricing a trade requires a cache entry. Recommend: auto-add to the watchlist on trade. Either way, state it. + +**10. Can you remove a ticker you hold a position in?** +Today that would call `source.remove_ticker()`, prices stop flowing, and the position's current price, unrealized P&L, heatmap tile, and total portfolio value all go stale or null. Recommend: the set of tracked tickers is **watchlist ∪ held positions**; removing from the watchlist only hides the row. + +**11. Is realized P&L needed?** The schema tracks only positions and trades; after a sell, the gain disappears into `cash_balance` with nothing to show for it. The chat assistant is asked to "analyze P&L". Recommend: skip a realized-P&L column, and let the chat context derive it from the `trades` table if asked. + +**12. Is trade history surfaced anywhere?** The `trades` table has no API endpoint and no UI element in Section 10. Is it audit-only, or should `GET /api/trades` exist for the chat context and a history panel? +**Decided:** expose it. `GET /api/trades` is specified in `planning/API_CONTRACT.md` §3 and feeds both a blotter panel and the LLM context. + +**13. What does the app do when `OPENROUTER_API_KEY` is missing?** It is listed as "required", but everything except chat works fine without it. Recommend: boot normally, and have `/api/chat` return a friendly "chat unavailable — no API key" message rather than a 500. + +**14. Massive mode outside market hours** — on a weekend the entire app is frozen, which is a bad first impression for a student running the demo. Document it, or auto-fall-back to the simulator when the last quote is more than N minutes stale. + +**15. Background price history for charts.** The main chart and the sparklines accumulate from SSE "since page load", so on first paint the chart is empty and a refresh throws it all away. Is that acceptable for the demo, or should the backend keep a bounded in-memory ring buffer (e.g. last 600 ticks per ticker) behind `GET /api/prices/{ticker}/history`? Recommend the ring buffer — it is ~30 lines, needs no schema change, and is the difference between a chart that looks alive on load and one that looks broken. + +### 13.3 Under-specified behavior agents will otherwise invent differently + +- **16. Money precision.** `REAL` columns and float math will produce `9999.999999999998` cash balances. State the rule: full float precision in storage, rounded to 2dp on display and to 4dp on quantities. +- **17. Zeroed positions.** Section 12 says a position "updates or disappears". Pick: delete the row when quantity falls below an epsilon (`< 1e-9`), never keep a zero-quantity row. +- **18. Trade validation rules**, stated once and shared by the manual and LLM paths: quantity > 0, no shorting, no margin (cash may not go negative), sell quantity ≤ held quantity, ticker must be priceable. +- **19. Concurrency.** Manual trades and LLM trades can execute simultaneously against the same SQLite file. Wrap trade execution in a single `asyncio.Lock` plus a DB transaction — a couple of lines that eliminate a whole class of race conditions. +- **20. LLM action caps.** Auto-execution with no confirmation is a good product decision, but there is no ceiling on it. Cap it: max ~5 trades and ~5 watchlist changes per message, each individually validated. +- **21. Chat history window.** "Recent conversation history" needs a number — suggest the last 20 messages, truncated by count not tokens. +- **22. `actions` JSON shape.** The frontend renders this column; define it here, e.g. `[{"type":"trade","ticker":"AAPL","side":"buy","quantity":10,"status":"ok"|"failed","price":190.5,"error":null}]`. +- **23. `POST /api/portfolio/trade` response shape.** Have it return the full updated portfolio so the frontend needs no follow-up GET. +- **24. `GET /api/portfolio/history` query params.** Define `?limit=` / `?since=` now — snapshots accrue at 2,880/day and the P&L chart should not fetch a week of them. +- **25. Structured-output fallback.** Confirm that `openrouter/openai/gpt-oss-120b` on the Cerebras provider honors strict JSON-schema `response_format`; providers vary. Specify the fallback: validate with Pydantic, and on a parse failure do exactly one repair retry, then return a plain-text message with no actions. +- **26. Color tokens.** Section 2 offers `#0d1117` *or* `#1a1a2e` and names three accent colors, but the P&L green/red are never given hex values. Pick one background and pin the semantic colors (up/green, down/red, plus a muted border) so the watchlist, heatmap, and P&L chart agree. +- **27. Minimum supported width.** "Desktop-first, functional on tablet" — give a number (e.g. degrade gracefully to 1024px, no mobile layout) or an agent will spend a day on responsive breakpoints nobody asked for. +- **28. Static-export constraints.** `output: 'export'` rules out server components, route handlers, middleware, and the Next.js image optimizer. Also worth stating: FastAPI must mount `/api/*` **before** the static catch-all, and unknown paths should fall back to `index.html`. +- **29. Background tasks.** The Section 3 diagram lists only the market data task; there are two (market data + the 30s portfolio snapshot). Also: take a snapshot at startup so the P&L chart is not empty for its first 30 seconds. + +### 13.4 Opportunities to simplify + + +- **31. Cut the "SSE resilience: disconnect and verify reconnection" E2E test.** Forcing a mid-stream disconnect from Playwright requires CDP network fiddling or route interception, and what it ultimately verifies is that the browser's built-in `EventSource` retry works. Cover the server side with a unit test on `_generate_events` and drop the E2E case. +- **32. Simplify `GET /api/watchlist`.** Returning "tickers with latest prices" duplicates data the SSE stream sends 500ms later. It is worth keeping *only* to avoid a blank first paint — if so, say that is its purpose; otherwise return tickers alone and let SSE own all pricing. +- **33. One env-loading mechanism, not two.** Section 5 says the backend reads `.env` from the project root; Section 11 passes `--env-file .env` to Docker. Use `--env-file` for the container and `python-dotenv` only as a local-dev convenience, and say so. +- **34. Drop `docker-compose.yml`.** Section 3 explicitly argues against compose ("no docker-compose for production"), and Section 4 then lists an optional one. With the start/stop scripts already wrapping `docker run`, the compose file is a third way to launch the same container. Delete it from the structure. +- **35. Reduce the four start/stop scripts to two.** `start_mac.sh` / `start_windows.ps1` differ only in shell syntax around one `docker run` line. Consider a single `scripts/start.sh` plus a thin `start.ps1` that documents the same command — or accept the duplication but explicitly mark the PowerShell versions as mechanical translations so agents do not let them drift apart. +- **36. Skip the Terraform/App Runner stretch goal** unless it is being taught. It adds a cloud account, IAM, and a registry to a project whose whole pitch is "one Docker command". +- **37. Consider dropping `user_id` from the LLM/chat path.** The forward-compatibility argument for `user_id` on the data tables is cheap and fine, but the hardcoded `"default"` threading through every function signature is noise. A module-level `DEFAULT_USER_ID` constant used at the query layer, rather than a parameter on every function, keeps the schema future-proof without the ceremony. + +### 13.5 Nits + +- **38.** `/api/health` should report the active market data source (`simulator` | `massive`) and whether the price cache is populated — that single field will answer most "why is nothing moving?" questions. +- **39.** The seeded watchlist (10 tickers, Section 7) and `seed_prices.py` are two sources of truth for the same list. Have the DB seed import the ticker list from the market module. +- **40.** Section 6 says SSE pushes "for all tickers known to the system"; `stream.py` sends the **entire** price map on every change, not a delta. At 10-20 tickers that is fine — worth one sentence confirming it is deliberate so nobody "optimizes" it into a delta protocol. + +### 13.6 Second pass — open questions after triage + +*Added after review triage. Item 30 (drop the Playwright container) was rejected: the containerized E2E rig stays. Items 8–15 remain unanswered and will be built to the recommendations stated there unless decided otherwise; 12 and 14 are product calls that should not be defaulted.* + +**41. Keeping the Playwright container has three unstated consequences.** Now that `test/docker-compose.test.yml` is confirmed, the spec should pin: (a) tests address the app by **service name** — `http://app:8000`, not `localhost`; (b) the compose file sets `LLM_MOCK=true` on the app service; (c) E2E runs against the **production image**, so CI must build it first — a full frontend build per test run is the real cost of this choice. Note this makes item 34 cleaner rather than contradicting it: drop the root `docker-compose.yml` and the test compose file becomes the only one in the repo, with no ambiguity about which to use. + +**42. What is total portfolio value when a price is missing?** The price cache is cold for the first ~500ms after boot, and in Massive mode a rejected symbol may never get a price at all. `cash + Σ(qty × current_price)` is undefined in both cases. +**Recommendation:** fall back to `avg_cost` for any position with no cached price, and have the 30s snapshot task skip its first run until the cache is populated — otherwise the P&L chart opens with a garbage data point at t=0. + +**43. Is there a reset path?** Nothing returns the portfolio to $10,000 except deleting the Docker volume, and a student demoing an AI that auto-executes trades will want one within the first five minutes. Options: a `POST /api/reset` endpoint (~15 lines: truncate positions/trades/snapshots/chat, restore cash) with a small header button, or simply document `docker volume rm finally-data` in the README. **Decision needed** — endpoint or documentation. + +**44. What does the LLM actually see?** Section 9's context list is cash, positions with P&L, watchlist with prices, and total value — no trade history and no price history. That makes "how has my portfolio done today?" and "why did you buy NVDA earlier?" both unanswerable, which are among the first things anyone will ask an AI trading assistant. Resolve alongside item 12: if the `trades` table gets an endpoint, the chat context is its main consumer. Also decide whether recent portfolio snapshots go into the prompt so the assistant can discuss trajectory rather than only the current state. +**Decided:** both. Recent trades and a downsampled snapshot trajectory are included in the chat context — shape and limits in `planning/API_CONTRACT.md` §6. + +**45. `LLM_MOCK=true` needs a defined response shape.** The E2E scenario asserts that "trade execution appears inline", so a single canned string will not do — the mock must branch on the user message (e.g. a message containing "buy" returns a `trades` array; one containing "watch" returns a `watchlist_changes` array; anything else returns message-only). This is a contract between the backend agent and the test agent and belongs in this document, not in whichever one gets written first. + +**46. One charting library or two?** Section 10 offers "Lightweight Charts or Recharts" as if interchangeable. They are not: Lightweight Charts is canvas-based and purpose-built for price series but has **no treemap**; Recharts is SVG-based and does have a `Treemap`. Taken literally the plan needs both, plus a third approach for sparklines. +**Recommendation:** standardize on **Recharts alone** — at 10–20 tickers the performance argument for canvas does not bite, and one library beats two. Sparklines as tiny inline SVG. Say so explicitly, or the frontend agent will decide silently and the bundle will carry both. + +**47. Does the header's live total value compute client-side or poll?** "Portfolio total value (updating live)" can mean recomputing from the SSE price stream against held quantities, or polling `GET /api/portfolio` on a timer. +**Recommendation:** compute client-side from SSE prices × positions, and refetch positions only after a trade or a chat action. Left unstated, an agent will build a 1-second poll. + +**48. Freeze the API contract before parallel work starts.** The frontend and backend agents both depend on shapes that are currently scattered across Section 8 and items 22, 23, and 24 — and several are still undefined. A `planning/API_CONTRACT.md` (request/response bodies for every endpoint, the SSE payload, and the `actions` JSON) should be the first deliverable of the next phase, so the two agents can work against it rather than against each other. +**Status:** drafted at `planning/API_CONTRACT.md` — decisions 42, 45 and 47 are resolved there; 12, 14, 43, 44 and 46 remain open and are listed in its final section. diff --git a/planning/archive/MARKET_DATA_DESIGN.md b/planning/archive/MARKET_DATA_DESIGN.md deleted file mode 100644 index 0d2cfd5fd..000000000 --- a/planning/archive/MARKET_DATA_DESIGN.md +++ /dev/null @@ -1,1490 +0,0 @@ -# Market Data Backend — Detailed Design - -Implementation-ready design for the FinAlly market data subsystem. Covers the unified interface, in-memory price cache, GBM simulator, Massive API client, SSE streaming endpoint, and FastAPI lifecycle integration. - -Everything in this document lives under `backend/app/market/`. - ---- - -## Table of Contents - -1. [File Structure](#1-file-structure) -2. [Data Model — `models.py`](#2-data-model) -3. [Price Cache — `cache.py`](#3-price-cache) -4. [Abstract Interface — `interface.py`](#4-abstract-interface) -5. [Seed Prices & Ticker Parameters — `seed_prices.py`](#5-seed-prices--ticker-parameters) -6. [GBM Simulator — `simulator.py`](#6-gbm-simulator) -7. [Massive API Client — `massive_client.py`](#7-massive-api-client) -8. [Factory — `factory.py`](#8-factory) -9. [SSE Streaming Endpoint — `stream.py`](#9-sse-streaming-endpoint) -10. [FastAPI Lifecycle Integration](#10-fastapi-lifecycle-integration) -11. [Watchlist Coordination](#11-watchlist-coordination) -12. [Testing Strategy](#12-testing-strategy) -13. [Error Handling & Edge Cases](#13-error-handling--edge-cases) -14. [Configuration Summary](#14-configuration-summary) - ---- - -## 1. File Structure - -``` -backend/ - app/ - market/ - __init__.py # Re-exports: PriceUpdate, PriceCache, MarketDataSource, create_market_data_source - models.py # PriceUpdate dataclass - cache.py # PriceCache (thread-safe in-memory store) - interface.py # MarketDataSource ABC - seed_prices.py # SEED_PRICES, TICKER_PARAMS, DEFAULT_PARAMS, CORRELATION_GROUPS - simulator.py # GBMSimulator + SimulatorDataSource - massive_client.py # MassiveDataSource - factory.py # create_market_data_source() - stream.py # SSE endpoint (FastAPI router) -``` - -Each file has a single responsibility. The `__init__.py` re-exports the public API so that the rest of the backend imports from `app.market` without reaching into submodules. - ---- - -## 2. Data Model - -**File: `backend/app/market/models.py`** - -`PriceUpdate` is the only data structure that leaves the market data layer. Every downstream consumer — SSE streaming, portfolio valuation, trade execution — works exclusively with this type. - -```python -from __future__ import annotations - -import time -from dataclasses import dataclass, field - - -@dataclass(frozen=True, slots=True) -class PriceUpdate: - """Immutable snapshot of a single ticker's price at a point in time.""" - - ticker: str - price: float - previous_price: float - timestamp: float = field(default_factory=time.time) # Unix seconds - - @property - def change(self) -> float: - """Absolute price change from previous update.""" - return round(self.price - self.previous_price, 4) - - @property - def change_percent(self) -> float: - """Percentage change from previous update.""" - if self.previous_price == 0: - return 0.0 - return round((self.price - self.previous_price) / self.previous_price * 100, 4) - - @property - def direction(self) -> str: - """'up', 'down', or 'flat'.""" - if self.price > self.previous_price: - return "up" - elif self.price < self.previous_price: - return "down" - return "flat" - - def to_dict(self) -> dict: - """Serialize for JSON / SSE transmission.""" - return { - "ticker": self.ticker, - "price": self.price, - "previous_price": self.previous_price, - "timestamp": self.timestamp, - "change": self.change, - "change_percent": self.change_percent, - "direction": self.direction, - } -``` - -### Design decisions - -- **`frozen=True`**: Price updates are immutable value objects. Once created they never change, which makes them safe to share across async tasks without copying. -- **`slots=True`**: Minor memory optimization — we create many of these per second. -- **Computed properties** (`change`, `direction`, `change_percent`): Derived from `price` and `previous_price` so they can never be inconsistent. No risk of a stale `direction` field. -- **`to_dict()`**: Single serialization point used by both the SSE endpoint and REST API responses. - ---- - -## 3. Price Cache - -**File: `backend/app/market/cache.py`** - -The price cache is the central data hub. Data sources write to it; SSE streaming and portfolio valuation read from it. It must be thread-safe because the simulator/poller may run in a thread pool executor while SSE reads happen on the async event loop. - -```python -from __future__ import annotations - -import asyncio -import time -from threading import Lock -from typing import Callable - -from .models import PriceUpdate - - -class PriceCache: - """Thread-safe in-memory cache of the latest price for each ticker. - - Writers: SimulatorDataSource or MassiveDataSource (one at a time). - Readers: SSE streaming endpoint, portfolio valuation, trade execution. - """ - - def __init__(self) -> None: - self._prices: dict[str, PriceUpdate] = {} - self._lock = Lock() - self._version: int = 0 # Monotonically increasing; bumped on every update - - def update(self, ticker: str, price: float, timestamp: float | None = None) -> PriceUpdate: - """Record a new price for a ticker. Returns the created PriceUpdate. - - Automatically computes direction and change from the previous price. - If this is the first update for the ticker, previous_price == price (direction='flat'). - """ - with self._lock: - ts = timestamp or time.time() - prev = self._prices.get(ticker) - previous_price = prev.price if prev else price - - update = PriceUpdate( - ticker=ticker, - price=round(price, 2), - previous_price=round(previous_price, 2), - timestamp=ts, - ) - self._prices[ticker] = update - self._version += 1 - return update - - def get(self, ticker: str) -> PriceUpdate | None: - """Get the latest price for a single ticker, or None if unknown.""" - with self._lock: - return self._prices.get(ticker) - - def get_all(self) -> dict[str, PriceUpdate]: - """Snapshot of all current prices. Returns a shallow copy.""" - with self._lock: - return dict(self._prices) - - def get_price(self, ticker: str) -> float | None: - """Convenience: get just the price float, or None.""" - update = self.get(ticker) - return update.price if update else None - - def remove(self, ticker: str) -> None: - """Remove a ticker from the cache (e.g., when removed from watchlist).""" - with self._lock: - self._prices.pop(ticker, None) - - @property - def version(self) -> int: - """Current version counter. Useful for SSE change detection.""" - return self._version - - def __len__(self) -> int: - with self._lock: - return len(self._prices) - - def __contains__(self, ticker: str) -> bool: - with self._lock: - return ticker in self._prices -``` - -### Why a version counter? - -The SSE streaming loop polls the cache every ~500ms. Without a version counter, it would serialize and send all prices every tick even if nothing changed (e.g., Massive API only updates every 15s). The version counter lets the SSE loop skip sends when nothing is new: - -```python -last_version = -1 -while True: - if price_cache.version != last_version: - last_version = price_cache.version - yield format_sse(price_cache.get_all()) - await asyncio.sleep(0.5) -``` - -### Thread safety rationale - -The `threading.Lock` is used instead of `asyncio.Lock` because: -- The Massive client's synchronous `get_snapshot_all()` runs in `asyncio.to_thread()`, which operates in a real OS thread — `asyncio.Lock` would not protect against that. -- The GBM simulator's `step()` is CPU-bound and could also be offloaded to a thread for fairness. -- `threading.Lock` works correctly from both sync threads and the async event loop. - ---- - -## 4. Abstract Interface - -**File: `backend/app/market/interface.py`** - -```python -from __future__ import annotations - -from abc import ABC, abstractmethod - - -class MarketDataSource(ABC): - """Contract for market data providers. - - Implementations push price updates into a shared PriceCache on their own - schedule. Downstream code never calls the data source directly for prices — - it reads from the cache. - - Lifecycle: - source = create_market_data_source(cache) - await source.start(["AAPL", "GOOGL", ...]) - # ... app runs ... - await source.add_ticker("TSLA") - await source.remove_ticker("GOOGL") - # ... app shutting down ... - await source.stop() - """ - - @abstractmethod - async def start(self, tickers: list[str]) -> None: - """Begin producing price updates for the given tickers. - - Starts a background task that periodically writes to the PriceCache. - Must be called exactly once. Calling start() twice is undefined behavior. - """ - - @abstractmethod - async def stop(self) -> None: - """Stop the background task and release resources. - - Safe to call multiple times. After stop(), the source will not write - to the cache again. - """ - - @abstractmethod - async def add_ticker(self, ticker: str) -> None: - """Add a ticker to the active set. No-op if already present. - - The next update cycle will include this ticker. - """ - - @abstractmethod - async def remove_ticker(self, ticker: str) -> None: - """Remove a ticker from the active set. No-op if not present. - - Also removes the ticker from the PriceCache. - """ - - @abstractmethod - def get_tickers(self) -> list[str]: - """Return the current list of actively tracked tickers.""" -``` - -### Why the source writes to the cache instead of returning prices - -This push model decouples timing. The simulator ticks at 500ms, Massive polls at 15s, but SSE always reads from the cache at its own 500ms cadence. There is no need for the SSE layer to know which data source is active or what its update interval is. - ---- - -## 5. Seed Prices & Ticker Parameters - -**File: `backend/app/market/seed_prices.py`** - -Constants only — no logic, no imports beyond stdlib. This file is shared by both the simulator (for initial prices and GBM parameters) and potentially by the Massive client (as fallback prices if the API hasn't responded yet). - -```python -"""Seed prices and per-ticker parameters for the market simulator.""" - -# Realistic starting prices for the default watchlist (as of project creation) -SEED_PRICES: dict[str, float] = { - "AAPL": 190.00, - "GOOGL": 175.00, - "MSFT": 420.00, - "AMZN": 185.00, - "TSLA": 250.00, - "NVDA": 800.00, - "META": 500.00, - "JPM": 195.00, - "V": 280.00, - "NFLX": 600.00, -} - -# Per-ticker GBM parameters -# sigma: annualized volatility (higher = more price movement) -# mu: annualized drift / expected return -TICKER_PARAMS: dict[str, dict[str, float]] = { - "AAPL": {"sigma": 0.22, "mu": 0.05}, - "GOOGL": {"sigma": 0.25, "mu": 0.05}, - "MSFT": {"sigma": 0.20, "mu": 0.05}, - "AMZN": {"sigma": 0.28, "mu": 0.05}, - "TSLA": {"sigma": 0.50, "mu": 0.03}, # High volatility - "NVDA": {"sigma": 0.40, "mu": 0.08}, # High volatility, strong drift - "META": {"sigma": 0.30, "mu": 0.05}, - "JPM": {"sigma": 0.18, "mu": 0.04}, # Low volatility (bank) - "V": {"sigma": 0.17, "mu": 0.04}, # Low volatility (payments) - "NFLX": {"sigma": 0.35, "mu": 0.05}, -} - -# Default parameters for tickers not in the list above (dynamically added) -DEFAULT_PARAMS: dict[str, float] = {"sigma": 0.25, "mu": 0.05} - -# Correlation groups for the simulator's Cholesky decomposition -# Tickers in the same group have higher intra-group correlation -CORRELATION_GROUPS: dict[str, set[str]] = { - "tech": {"AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "NFLX"}, - "finance": {"JPM", "V"}, -} - -# Correlation coefficients -INTRA_TECH_CORR = 0.6 # Tech stocks move together -INTRA_FINANCE_CORR = 0.5 # Finance stocks move together -CROSS_GROUP_CORR = 0.3 # Between sectors -TSLA_CORR = 0.3 # TSLA does its own thing -DEFAULT_CORR = 0.3 # Unknown tickers -``` - ---- - -## 6. GBM Simulator - -**File: `backend/app/market/simulator.py`** - -This file contains two classes: -- `GBMSimulator`: Pure math engine. Stateful — holds current prices and advances them one step at a time. -- `SimulatorDataSource`: The `MarketDataSource` implementation that wraps `GBMSimulator` in an async loop and writes to the `PriceCache`. - -### 6.1 GBMSimulator — The Math Engine - -```python -from __future__ import annotations - -import asyncio -import logging -import math -import random - -import numpy as np - -from .cache import PriceCache -from .interface import MarketDataSource -from .seed_prices import ( - CORRELATION_GROUPS, - CROSS_GROUP_CORR, - DEFAULT_CORR, - DEFAULT_PARAMS, - INTRA_FINANCE_CORR, - INTRA_TECH_CORR, - SEED_PRICES, - TICKER_PARAMS, - TSLA_CORR, -) - -logger = logging.getLogger(__name__) - - -class GBMSimulator: - """Geometric Brownian Motion simulator for correlated stock prices. - - Math: - S(t+dt) = S(t) * exp((mu - sigma^2/2) * dt + sigma * sqrt(dt) * Z) - - Where: - S(t) = current price - mu = annualized drift (expected return) - sigma = annualized volatility - dt = time step as fraction of a trading year - Z = correlated standard normal random variable - - The tiny dt (~8.5e-8 for 500ms ticks over 252 trading days * 6.5h/day) - produces sub-cent moves per tick that accumulate naturally over time. - """ - - # 500ms expressed as a fraction of a trading year - # 252 trading days * 6.5 hours/day * 3600 seconds/hour = 5,896,800 seconds - TRADING_SECONDS_PER_YEAR = 252 * 6.5 * 3600 # 5,896,800 - DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR # ~8.48e-8 - - def __init__( - self, - tickers: list[str], - dt: float = DEFAULT_DT, - event_probability: float = 0.001, - ) -> None: - self._dt = dt - self._event_prob = event_probability - - # Per-ticker state - self._tickers: list[str] = [] - self._prices: dict[str, float] = {} - self._params: dict[str, dict[str, float]] = {} - - # Cholesky decomposition of the correlation matrix (for correlated moves) - self._cholesky: np.ndarray | None = None - - # Initialize all starting tickers - for ticker in tickers: - self._add_ticker_internal(ticker) - self._rebuild_cholesky() - - # --- Public API --- - - def step(self) -> dict[str, float]: - """Advance all tickers by one time step. Returns {ticker: new_price}. - - This is the hot path — called every 500ms. Keep it fast. - """ - n = len(self._tickers) - if n == 0: - return {} - - # Generate n independent standard normal draws - z_independent = np.random.standard_normal(n) - - # Apply Cholesky to get correlated draws - if self._cholesky is not None: - z_correlated = self._cholesky @ z_independent - else: - z_correlated = z_independent - - result: dict[str, float] = {} - for i, ticker in enumerate(self._tickers): - params = self._params[ticker] - mu = params["mu"] - sigma = params["sigma"] - - # GBM: S(t+dt) = S(t) * exp((mu - 0.5*sigma^2)*dt + sigma*sqrt(dt)*Z) - drift = (mu - 0.5 * sigma ** 2) * self._dt - diffusion = sigma * math.sqrt(self._dt) * z_correlated[i] - self._prices[ticker] *= math.exp(drift + diffusion) - - # Random event: ~0.1% chance per tick per ticker - # With 10 tickers at 2 ticks/sec, expect an event ~every 50 seconds - if random.random() < self._event_prob: - shock_magnitude = random.uniform(0.02, 0.05) - shock_sign = random.choice([-1, 1]) - self._prices[ticker] *= 1 + shock_magnitude * shock_sign - logger.debug( - "Random event on %s: %.1f%% %s", - ticker, - shock_magnitude * 100, - "up" if shock_sign > 0 else "down", - ) - - result[ticker] = round(self._prices[ticker], 2) - - return result - - def add_ticker(self, ticker: str) -> None: - """Add a ticker to the simulation. Rebuilds the correlation matrix.""" - if ticker in self._prices: - return - self._add_ticker_internal(ticker) - self._rebuild_cholesky() - - def remove_ticker(self, ticker: str) -> None: - """Remove a ticker from the simulation. Rebuilds the correlation matrix.""" - if ticker not in self._prices: - return - self._tickers.remove(ticker) - del self._prices[ticker] - del self._params[ticker] - self._rebuild_cholesky() - - def get_price(self, ticker: str) -> float | None: - """Current price for a ticker, or None if not tracked.""" - return self._prices.get(ticker) - - # --- Internals --- - - def _add_ticker_internal(self, ticker: str) -> None: - """Add a ticker without rebuilding Cholesky (for batch initialization).""" - if ticker in self._prices: - return - self._tickers.append(ticker) - self._prices[ticker] = SEED_PRICES.get(ticker, random.uniform(50.0, 300.0)) - self._params[ticker] = TICKER_PARAMS.get(ticker, dict(DEFAULT_PARAMS)) - - def _rebuild_cholesky(self) -> None: - """Rebuild the Cholesky decomposition of the ticker correlation matrix. - - Called whenever tickers are added or removed. O(n^2) but n < 50. - """ - n = len(self._tickers) - if n <= 1: - self._cholesky = None - return - - # Build the correlation matrix - corr = np.eye(n) - for i in range(n): - for j in range(i + 1, n): - rho = self._pairwise_correlation(self._tickers[i], self._tickers[j]) - corr[i, j] = rho - corr[j, i] = rho - - self._cholesky = np.linalg.cholesky(corr) - - @staticmethod - def _pairwise_correlation(t1: str, t2: str) -> float: - """Determine correlation between two tickers based on sector grouping. - - Correlation structure: - - Same tech sector: 0.6 - - Same finance sector: 0.5 - - TSLA with anything: 0.3 (it does its own thing) - - Cross-sector: 0.3 - - Unknown tickers: 0.3 - """ - tech = CORRELATION_GROUPS["tech"] - finance = CORRELATION_GROUPS["finance"] - - # TSLA is in tech set but behaves independently - if t1 == "TSLA" or t2 == "TSLA": - return TSLA_CORR - - if t1 in tech and t2 in tech: - return INTRA_TECH_CORR - if t1 in finance and t2 in finance: - return INTRA_FINANCE_CORR - - return CROSS_GROUP_CORR -``` - -### 6.2 SimulatorDataSource — Async Wrapper - -```python -class SimulatorDataSource(MarketDataSource): - """MarketDataSource backed by the GBM simulator. - - Runs a background asyncio task that calls GBMSimulator.step() every - `update_interval` seconds and writes results to the PriceCache. - """ - - def __init__( - self, - price_cache: PriceCache, - update_interval: float = 0.5, - event_probability: float = 0.001, - ) -> None: - self._cache = price_cache - self._interval = update_interval - self._event_prob = event_probability - self._sim: GBMSimulator | None = None - self._task: asyncio.Task | None = None - - async def start(self, tickers: list[str]) -> None: - self._sim = GBMSimulator( - tickers=tickers, - event_probability=self._event_prob, - ) - # Seed the cache with initial prices so SSE has data immediately - for ticker in tickers: - price = self._sim.get_price(ticker) - if price is not None: - self._cache.update(ticker=ticker, price=price) - self._task = asyncio.create_task(self._run_loop(), name="simulator-loop") - logger.info("Simulator started with %d tickers", len(tickers)) - - async def stop(self) -> None: - if self._task and not self._task.done(): - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - self._task = None - logger.info("Simulator stopped") - - async def add_ticker(self, ticker: str) -> None: - if self._sim: - self._sim.add_ticker(ticker) - # Seed cache immediately so the ticker has a price right away - price = self._sim.get_price(ticker) - if price is not None: - self._cache.update(ticker=ticker, price=price) - logger.info("Simulator: added ticker %s", ticker) - - async def remove_ticker(self, ticker: str) -> None: - if self._sim: - self._sim.remove_ticker(ticker) - self._cache.remove(ticker) - logger.info("Simulator: removed ticker %s", ticker) - - def get_tickers(self) -> list[str]: - return list(self._sim._tickers) if self._sim else [] - - async def _run_loop(self) -> None: - """Core loop: step the simulation, write to cache, sleep.""" - while True: - try: - if self._sim: - prices = self._sim.step() - for ticker, price in prices.items(): - self._cache.update(ticker=ticker, price=price) - except Exception: - logger.exception("Simulator step failed") - await asyncio.sleep(self._interval) -``` - -### Key behaviors - -- **Immediate seeding**: When `start()` is called, the cache is populated with seed prices *before* the loop begins. This means the SSE endpoint has data to send on its very first tick, with no blank-screen delay. -- **Graceful cancellation**: `stop()` cancels the task and awaits it, catching `CancelledError`. This ensures clean shutdown during FastAPI lifespan teardown. -- **Exception resilience**: The loop catches exceptions per-step so a single bad tick doesn't kill the entire data feed. - ---- - -## 7. Massive API Client - -**File: `backend/app/market/massive_client.py`** - -Polls the Massive (formerly Polygon.io) REST API snapshot endpoint on a configurable interval. The synchronous Massive client runs in `asyncio.to_thread()` to avoid blocking the event loop. - -```python -from __future__ import annotations - -import asyncio -import logging -from typing import Any - -from .cache import PriceCache -from .interface import MarketDataSource - -logger = logging.getLogger(__name__) - - -class MassiveDataSource(MarketDataSource): - """MarketDataSource backed by the Massive (Polygon.io) REST API. - - Polls GET /v2/snapshot/locale/us/markets/stocks/tickers for all watched - tickers in a single API call, then writes results to the PriceCache. - - Rate limits: - - Free tier: 5 req/min → poll every 15s (default) - - Paid tiers: higher limits → poll every 2-5s - """ - - def __init__( - self, - api_key: str, - price_cache: PriceCache, - poll_interval: float = 15.0, - ) -> None: - self._api_key = api_key - self._cache = price_cache - self._interval = poll_interval - self._tickers: list[str] = [] - self._task: asyncio.Task | None = None - self._client: Any = None # Lazy import to avoid hard dependency - - async def start(self, tickers: list[str]) -> None: - # Lazy import: only import massive when actually using real market data. - # This means the massive package is not required when using the simulator. - from massive import RESTClient - - self._client = RESTClient(api_key=self._api_key) - self._tickers = list(tickers) - - # Do an immediate first poll so the cache has data right away - await self._poll_once() - - self._task = asyncio.create_task(self._poll_loop(), name="massive-poller") - logger.info( - "Massive poller started: %d tickers, %.1fs interval", - len(tickers), - self._interval, - ) - - async def stop(self) -> None: - if self._task and not self._task.done(): - self._task.cancel() - try: - await self._task - except asyncio.CancelledError: - pass - self._task = None - self._client = None - logger.info("Massive poller stopped") - - async def add_ticker(self, ticker: str) -> None: - ticker = ticker.upper().strip() - if ticker not in self._tickers: - self._tickers.append(ticker) - logger.info("Massive: added ticker %s (will appear on next poll)", ticker) - - async def remove_ticker(self, ticker: str) -> None: - ticker = ticker.upper().strip() - self._tickers = [t for t in self._tickers if t != ticker] - self._cache.remove(ticker) - logger.info("Massive: removed ticker %s", ticker) - - def get_tickers(self) -> list[str]: - return list(self._tickers) - - # --- Internal --- - - async def _poll_loop(self) -> None: - """Poll on interval. First poll already happened in start().""" - while True: - await asyncio.sleep(self._interval) - await self._poll_once() - - async def _poll_once(self) -> None: - """Execute one poll cycle: fetch snapshots, update cache.""" - if not self._tickers or not self._client: - return - - try: - # The Massive RESTClient is synchronous — run in a thread to - # avoid blocking the event loop. - snapshots = await asyncio.to_thread(self._fetch_snapshots) - processed = 0 - for snap in snapshots: - try: - price = snap.last_trade.price - # Massive timestamps are Unix milliseconds → convert to seconds - timestamp = snap.last_trade.timestamp / 1000.0 - self._cache.update( - ticker=snap.ticker, - price=price, - timestamp=timestamp, - ) - processed += 1 - except (AttributeError, TypeError) as e: - logger.warning( - "Skipping snapshot for %s: %s", - getattr(snap, "ticker", "???"), - e, - ) - logger.debug("Massive poll: updated %d/%d tickers", processed, len(self._tickers)) - - except Exception as e: - logger.error("Massive poll failed: %s", e) - # Don't re-raise — the loop will retry on the next interval. - # Common failures: 401 (bad key), 429 (rate limit), network errors. - - def _fetch_snapshots(self) -> list: - """Synchronous call to the Massive REST API. Runs in a thread.""" - from massive.rest.models import SnapshotMarketType - - return self._client.get_snapshot_all( - market_type=SnapshotMarketType.STOCKS, - tickers=self._tickers, - ) -``` - -### Error handling philosophy - -The Massive poller is intentionally resilient: - -| Error | Behavior | -|-------|----------| -| **401 Unauthorized** | Logged as error. Poller keeps running (user might fix `.env` and restart). | -| **429 Rate Limited** | Logged as error. Next poll retries after `poll_interval` seconds. | -| **Network timeout** | Logged as error. Retries automatically on next cycle. | -| **Malformed snapshot** | Individual ticker skipped with warning. Other tickers still processed. | -| **All tickers fail** | Cache retains last-known prices. SSE keeps streaming stale data (better than no data). | - -### Lazy import strategy - -`from massive import RESTClient` happens inside `start()`, not at module import time. This means: -- The `massive` package is only required when `MASSIVE_API_KEY` is set. -- Students who don't have a Massive API key don't need the package installed at all. -- The simulator path has zero external dependencies beyond `numpy`. - ---- - -## 8. Factory - -**File: `backend/app/market/factory.py`** - -```python -from __future__ import annotations - -import logging -import os - -from .cache import PriceCache -from .interface import MarketDataSource - -logger = logging.getLogger(__name__) - - -def create_market_data_source(price_cache: PriceCache) -> MarketDataSource: - """Create the appropriate market data source based on environment variables. - - - MASSIVE_API_KEY set and non-empty → MassiveDataSource (real market data) - - Otherwise → SimulatorDataSource (GBM simulation) - - Returns an unstarted source. Caller must await source.start(tickers). - """ - api_key = os.environ.get("MASSIVE_API_KEY", "").strip() - - if api_key: - from .massive_client import MassiveDataSource - - logger.info("Market data source: Massive API (real data)") - return MassiveDataSource(api_key=api_key, price_cache=price_cache) - else: - from .simulator import SimulatorDataSource - - logger.info("Market data source: GBM Simulator") - return SimulatorDataSource(price_cache=price_cache) -``` - -### Usage at app startup - -```python -price_cache = PriceCache() -source = create_market_data_source(price_cache) -await source.start(initial_tickers) # e.g., ["AAPL", "GOOGL", ...] -``` - ---- - -## 9. SSE Streaming Endpoint - -**File: `backend/app/market/stream.py`** - -The SSE endpoint is a FastAPI route that holds open a long-lived HTTP connection and pushes price updates to the client as `text/event-stream`. - -```python -from __future__ import annotations - -import asyncio -import json -import logging -import time - -from fastapi import APIRouter, Request -from fastapi.responses import StreamingResponse - -from .cache import PriceCache - -logger = logging.getLogger(__name__) - -router = APIRouter(prefix="/api/stream", tags=["streaming"]) - - -def create_stream_router(price_cache: PriceCache) -> APIRouter: - """Create the SSE streaming router with a reference to the price cache. - - This factory pattern lets us inject the PriceCache without globals. - """ - - @router.get("/prices") - async def stream_prices(request: Request) -> StreamingResponse: - """SSE endpoint for live price updates. - - Streams all tracked ticker prices every ~500ms. The client connects - with EventSource and receives events in the format: - - data: {"AAPL": {"ticker": "AAPL", "price": 190.50, ...}, ...} - - Includes a retry directive so the browser auto-reconnects on - disconnection (EventSource built-in behavior). - """ - return StreamingResponse( - _generate_events(price_cache, request), - media_type="text/event-stream", - headers={ - "Cache-Control": "no-cache", - "Connection": "keep-alive", - "X-Accel-Buffering": "no", # Disable nginx buffering if proxied - }, - ) - - return router - - -async def _generate_events( - price_cache: PriceCache, - request: Request, - interval: float = 0.5, -) -> None: - """Async generator that yields SSE-formatted price events. - - Sends all prices every `interval` seconds. Stops when the client - disconnects (detected via request.is_disconnected()). - """ - # Tell the client to retry after 1 second if the connection drops - yield "retry: 1000\n\n" - - last_version = -1 - client_ip = request.client.host if request.client else "unknown" - logger.info("SSE client connected: %s", client_ip) - - try: - while True: - # Check for client disconnect - if await request.is_disconnected(): - logger.info("SSE client disconnected: %s", client_ip) - break - - current_version = price_cache.version - if current_version != last_version: - last_version = current_version - prices = price_cache.get_all() - - if prices: - data = { - ticker: update.to_dict() - for ticker, update in prices.items() - } - payload = json.dumps(data) - yield f"data: {payload}\n\n" - - await asyncio.sleep(interval) - except asyncio.CancelledError: - logger.info("SSE stream cancelled for: %s", client_ip) -``` - -### SSE wire format - -Each event the client receives looks like this: - -``` -data: {"AAPL":{"ticker":"AAPL","price":190.50,"previous_price":190.42,"timestamp":1707580800.5,"change":0.08,"change_percent":0.042,"direction":"up"},"GOOGL":{"ticker":"GOOGL","price":175.12,...}} - -``` - -The client parses this with: - -```javascript -const eventSource = new EventSource('/api/stream/prices'); -eventSource.onmessage = (event) => { - const prices = JSON.parse(event.data); - // prices is { "AAPL": { ticker, price, previous_price, ... }, ... } -}; -``` - -### Why poll-and-push instead of event-driven? - -The SSE endpoint polls the cache on a fixed interval rather than being notified by the data source. This is simpler and produces predictable, evenly-spaced updates for the frontend. The frontend accumulates these into sparkline charts — regular spacing is important for clean visualization. - ---- - -## 10. FastAPI Lifecycle Integration - -The market data system starts and stops with the FastAPI application using the `lifespan` context manager pattern. - -**In `backend/app/main.py`:** - -```python -from contextlib import asynccontextmanager - -from fastapi import FastAPI - -from app.market.cache import PriceCache -from app.market.factory import create_market_data_source -from app.market.interface import MarketDataSource -from app.market.stream import create_stream_router - - -@asynccontextmanager -async def lifespan(app: FastAPI): - """Manage startup and shutdown of background services.""" - - # --- STARTUP --- - - # 1. Create the shared price cache - price_cache = PriceCache() - app.state.price_cache = price_cache - - # 2. Create and start the market data source - source = create_market_data_source(price_cache) - app.state.market_source = source - - # 3. Load initial tickers from the database watchlist - initial_tickers = await load_watchlist_tickers() # reads from SQLite - await source.start(initial_tickers) - - # 4. Register the SSE streaming router - stream_router = create_stream_router(price_cache) - app.include_router(stream_router) - - yield # App is running - - # --- SHUTDOWN --- - await source.stop() - - -app = FastAPI(title="FinAlly", lifespan=lifespan) - - -# Dependency for injecting the price cache into route handlers -def get_price_cache() -> PriceCache: - return app.state.price_cache - - -def get_market_source() -> MarketDataSource: - return app.state.market_source -``` - -### Accessing market data from other routes - -Other parts of the backend (trade execution, portfolio valuation, watchlist management) access the price cache and data source via FastAPI's dependency injection: - -```python -from fastapi import APIRouter, Depends - -router = APIRouter(prefix="/api") - -@router.post("/portfolio/trade") -async def execute_trade( - trade: TradeRequest, - price_cache: PriceCache = Depends(get_price_cache), -): - current_price = price_cache.get_price(trade.ticker) - if current_price is None: - raise HTTPException(404, f"No price available for {trade.ticker}") - # ... execute trade at current_price ... - - -@router.post("/watchlist") -async def add_to_watchlist( - payload: WatchlistAdd, - source: MarketDataSource = Depends(get_market_source), - price_cache: PriceCache = Depends(get_price_cache), -): - # Add to database ... - # Then tell the data source to start tracking it - await source.add_ticker(payload.ticker) - # ... - - -@router.delete("/watchlist/{ticker}") -async def remove_from_watchlist( - ticker: str, - source: MarketDataSource = Depends(get_market_source), -): - # Remove from database ... - # Then stop tracking - await source.remove_ticker(ticker) - # ... -``` - ---- - -## 11. Watchlist Coordination - -When the watchlist changes (via REST API or LLM chat), the market data source must be notified so it tracks the right set of tickers. - -### Flow: Adding a Ticker - -``` -User (or LLM) → POST /api/watchlist {ticker: "PYPL"} - → Insert into watchlist table (SQLite) - → await source.add_ticker("PYPL") - Simulator: adds to GBMSimulator, rebuilds Cholesky, seeds cache - Massive: appends to ticker list, appears on next poll - → Return success (ticker + current price if available) -``` - -### Flow: Removing a Ticker - -``` -User (or LLM) → DELETE /api/watchlist/PYPL - → Delete from watchlist table (SQLite) - → await source.remove_ticker("PYPL") - Simulator: removes from GBMSimulator, rebuilds Cholesky, removes from cache - Massive: removes from ticker list, removes from cache - → Return success -``` - -### Edge case: Ticker has an open position - -If the user removes a ticker from the watchlist but still holds shares, the ticker should remain in the data source so portfolio valuation stays accurate. The watchlist route should check for this: - -```python -@router.delete("/watchlist/{ticker}") -async def remove_from_watchlist( - ticker: str, - source: MarketDataSource = Depends(get_market_source), -): - # Remove from watchlist table - await db.delete_watchlist_entry(ticker) - - # Only stop tracking if no open position - position = await db.get_position(ticker) - if position is None or position.quantity == 0: - await source.remove_ticker(ticker) - - return {"status": "ok"} -``` - ---- - -## 12. Testing Strategy - -### 12.1 Unit Tests for GBMSimulator - -**File: `backend/tests/market/test_simulator.py`** - -```python -import math -import pytest -from app.market.simulator import GBMSimulator -from app.market.seed_prices import SEED_PRICES - - -class TestGBMSimulator: - """Unit tests for the GBM price simulator.""" - - def test_step_returns_all_tickers(self): - sim = GBMSimulator(tickers=["AAPL", "GOOGL"]) - result = sim.step() - assert set(result.keys()) == {"AAPL", "GOOGL"} - - def test_prices_are_positive(self): - """GBM prices can never go negative (exp() is always positive).""" - sim = GBMSimulator(tickers=["AAPL"]) - for _ in range(10_000): - prices = sim.step() - assert prices["AAPL"] > 0 - - def test_initial_prices_match_seeds(self): - sim = GBMSimulator(tickers=["AAPL"]) - # Before any step, price should be the seed price - assert sim.get_price("AAPL") == SEED_PRICES["AAPL"] - - def test_add_ticker(self): - sim = GBMSimulator(tickers=["AAPL"]) - sim.add_ticker("TSLA") - result = sim.step() - assert "TSLA" in result - - def test_remove_ticker(self): - sim = GBMSimulator(tickers=["AAPL", "GOOGL"]) - sim.remove_ticker("GOOGL") - result = sim.step() - assert "GOOGL" not in result - assert "AAPL" in result - - def test_add_duplicate_is_noop(self): - sim = GBMSimulator(tickers=["AAPL"]) - sim.add_ticker("AAPL") - assert len(sim._tickers) == 1 - - def test_remove_nonexistent_is_noop(self): - sim = GBMSimulator(tickers=["AAPL"]) - sim.remove_ticker("NOPE") # Should not raise - - def test_unknown_ticker_gets_random_seed_price(self): - sim = GBMSimulator(tickers=["ZZZZ"]) - price = sim.get_price("ZZZZ") - assert 50.0 <= price <= 300.0 - - def test_empty_step(self): - sim = GBMSimulator(tickers=[]) - result = sim.step() - assert result == {} - - def test_prices_change_over_time(self): - """After many steps, prices should have drifted from their seeds.""" - sim = GBMSimulator(tickers=["AAPL"]) - for _ in range(1000): - sim.step() - # Price should have changed (extremely unlikely to be exactly the seed) - assert sim.get_price("AAPL") != SEED_PRICES["AAPL"] - - def test_cholesky_rebuilds_on_add(self): - sim = GBMSimulator(tickers=["AAPL"]) - assert sim._cholesky is None # Only 1 ticker, no correlation matrix - sim.add_ticker("GOOGL") - assert sim._cholesky is not None # Now 2 tickers, matrix exists -``` - -### 12.2 Unit Tests for PriceCache - -**File: `backend/tests/market/test_cache.py`** - -```python -import pytest -from app.market.cache import PriceCache - - -class TestPriceCache: - - def test_update_and_get(self): - cache = PriceCache() - update = cache.update("AAPL", 190.50) - assert update.ticker == "AAPL" - assert update.price == 190.50 - assert cache.get("AAPL") == update - - def test_first_update_is_flat(self): - cache = PriceCache() - update = cache.update("AAPL", 190.50) - assert update.direction == "flat" - assert update.previous_price == 190.50 - - def test_direction_up(self): - cache = PriceCache() - cache.update("AAPL", 190.00) - update = cache.update("AAPL", 191.00) - assert update.direction == "up" - assert update.change == 1.00 - - def test_direction_down(self): - cache = PriceCache() - cache.update("AAPL", 190.00) - update = cache.update("AAPL", 189.00) - assert update.direction == "down" - assert update.change == -1.00 - - def test_remove(self): - cache = PriceCache() - cache.update("AAPL", 190.00) - cache.remove("AAPL") - assert cache.get("AAPL") is None - - def test_get_all(self): - cache = PriceCache() - cache.update("AAPL", 190.00) - cache.update("GOOGL", 175.00) - all_prices = cache.get_all() - assert set(all_prices.keys()) == {"AAPL", "GOOGL"} - - def test_version_increments(self): - cache = PriceCache() - v0 = cache.version - cache.update("AAPL", 190.00) - assert cache.version == v0 + 1 - cache.update("AAPL", 191.00) - assert cache.version == v0 + 2 - - def test_get_price_convenience(self): - cache = PriceCache() - cache.update("AAPL", 190.50) - assert cache.get_price("AAPL") == 190.50 - assert cache.get_price("NOPE") is None -``` - -### 12.3 Integration Test: SimulatorDataSource - -**File: `backend/tests/market/test_simulator_source.py`** - -```python -import asyncio -import pytest -from app.market.cache import PriceCache -from app.market.simulator import SimulatorDataSource - - -@pytest.mark.asyncio -class TestSimulatorDataSource: - - async def test_start_populates_cache(self): - cache = PriceCache() - source = SimulatorDataSource(price_cache=cache, update_interval=0.1) - await source.start(["AAPL", "GOOGL"]) - - # Cache should have seed prices immediately (before first loop tick) - assert cache.get("AAPL") is not None - assert cache.get("GOOGL") is not None - - await source.stop() - - async def test_prices_update_over_time(self): - cache = PriceCache() - source = SimulatorDataSource(price_cache=cache, update_interval=0.05) - await source.start(["AAPL"]) - - initial = cache.get("AAPL").price - await asyncio.sleep(0.3) # Several update cycles - current = cache.get("AAPL").price - - # Extremely unlikely to be identical after many steps - # (but not impossible, so this is a probabilistic test) - assert current != initial or True # Soft assertion - - await source.stop() - - async def test_stop_is_clean(self): - cache = PriceCache() - source = SimulatorDataSource(price_cache=cache, update_interval=0.1) - await source.start(["AAPL"]) - await source.stop() - # Double stop should not raise - await source.stop() - - async def test_add_and_remove_ticker(self): - cache = PriceCache() - source = SimulatorDataSource(price_cache=cache, update_interval=0.1) - await source.start(["AAPL"]) - - await source.add_ticker("TSLA") - assert "TSLA" in source.get_tickers() - assert cache.get("TSLA") is not None - - await source.remove_ticker("TSLA") - assert "TSLA" not in source.get_tickers() - assert cache.get("TSLA") is None - - await source.stop() -``` - -### 12.4 Unit Test: MassiveDataSource (Mocked) - -**File: `backend/tests/market/test_massive.py`** - -```python -import asyncio -from unittest.mock import MagicMock, patch -import pytest -from app.market.cache import PriceCache -from app.market.massive_client import MassiveDataSource - - -def _make_snapshot(ticker: str, price: float, timestamp_ms: int) -> MagicMock: - """Create a mock Massive snapshot object.""" - snap = MagicMock() - snap.ticker = ticker - snap.last_trade.price = price - snap.last_trade.timestamp = timestamp_ms - return snap - - -@pytest.mark.asyncio -class TestMassiveDataSource: - - async def test_poll_updates_cache(self): - cache = PriceCache() - source = MassiveDataSource( - api_key="test-key", - price_cache=cache, - poll_interval=60.0, # Long interval so the loop doesn't auto-poll - ) - - mock_snapshots = [ - _make_snapshot("AAPL", 190.50, 1707580800000), - _make_snapshot("GOOGL", 175.25, 1707580800000), - ] - - with patch.object(source, "_fetch_snapshots", return_value=mock_snapshots): - await source._poll_once() - - assert cache.get_price("AAPL") == 190.50 - assert cache.get_price("GOOGL") == 175.25 - - async def test_malformed_snapshot_skipped(self): - cache = PriceCache() - source = MassiveDataSource( - api_key="test-key", - price_cache=cache, - poll_interval=60.0, - ) - source._tickers = ["AAPL", "BAD"] - - good_snap = _make_snapshot("AAPL", 190.50, 1707580800000) - bad_snap = MagicMock() - bad_snap.ticker = "BAD" - bad_snap.last_trade = None # Will cause AttributeError - - with patch.object(source, "_fetch_snapshots", return_value=[good_snap, bad_snap]): - await source._poll_once() - - # Good ticker processed, bad one skipped - assert cache.get_price("AAPL") == 190.50 - assert cache.get_price("BAD") is None - - async def test_api_error_does_not_crash(self): - cache = PriceCache() - source = MassiveDataSource( - api_key="test-key", - price_cache=cache, - poll_interval=60.0, - ) - source._tickers = ["AAPL"] - - with patch.object(source, "_fetch_snapshots", side_effect=Exception("network error")): - await source._poll_once() # Should not raise - - assert cache.get_price("AAPL") is None # No update happened -``` - ---- - -## 13. Error Handling & Edge Cases - -### 13.1 Startup: Empty Watchlist - -If the database has no watchlist entries (user deleted everything), `start()` receives an empty list. Both data sources handle this gracefully — the simulator produces no prices, the Massive poller skips its API call. The SSE endpoint sends empty events. When the user adds a ticker, the source starts tracking it immediately. - -### 13.2 Price Cache Miss During Trade - -If a user tries to trade a ticker that has no cached price (e.g., just added to watchlist, Massive hasn't polled yet): - -```python -price = price_cache.get_price(ticker) -if price is None: - raise HTTPException( - status_code=400, - detail=f"Price not yet available for {ticker}. Please wait a moment and try again.", - ) -``` - -The simulator avoids this by seeding the cache in `add_ticker()`. The Massive client may have a brief gap — the HTTP 400 with a clear message is the correct response. - -### 13.3 Massive API Key Invalid - -If the API key is set but invalid, the first poll will fail with a 401. The poller logs the error and keeps retrying. The SSE endpoint streams empty data. The user sees no prices and a connection status indicator showing "connected" (SSE is working, just no data). The fix is to correct the API key and restart. - -### 13.4 Thread Safety Under Load - -The `PriceCache` uses `threading.Lock` which is a mutex — only one thread can hold it at a time. Under normal load (10 tickers, 2 updates/sec), lock contention is negligible. The critical section is tiny (dict lookup + assignment). - -If this ever became a bottleneck (hundreds of tickers, many concurrent SSE readers), the fix would be a `ReadWriteLock` — but that level of optimization is unnecessary for this project. - -### 13.5 Simulator Precision - -GBM with tiny `dt` produces very small per-tick moves. Floating-point precision is not a concern because: -- Prices are `round()`ed to 2 decimal places in `GBMSimulator.step()` -- The exponential formulation (`exp(drift + diffusion)`) is numerically stable -- Prices are always positive (exponential function) - ---- - -## 14. Configuration Summary - -All tunable parameters and their defaults: - -| Parameter | Location | Default | Description | -|-----------|----------|---------|-------------| -| `MASSIVE_API_KEY` | Environment variable | `""` (empty) | If set, use Massive API; otherwise use simulator | -| `update_interval` | `SimulatorDataSource.__init__` | `0.5` (seconds) | Time between simulator ticks | -| `poll_interval` | `MassiveDataSource.__init__` | `15.0` (seconds) | Time between Massive API polls | -| `event_probability` | `GBMSimulator.__init__` | `0.001` | Chance of a random shock event per ticker per tick | -| `dt` | `GBMSimulator.__init__` | `~8.5e-8` | GBM time step (fraction of a trading year) | -| SSE push interval | `_generate_events()` | `0.5` (seconds) | Time between SSE pushes to the client | -| SSE retry directive | `_generate_events()` | `1000` (ms) | Browser EventSource reconnection delay | - -### Package `__init__.py` - -**File: `backend/app/market/__init__.py`** - -```python -"""Market data subsystem for FinAlly. - -Public API: - PriceUpdate - Immutable price snapshot dataclass - PriceCache - Thread-safe in-memory price store - MarketDataSource - Abstract interface for data providers - create_market_data_source - Factory that selects simulator or Massive - create_stream_router - FastAPI router factory for SSE endpoint -""" - -from .cache import PriceCache -from .factory import create_market_data_source -from .interface import MarketDataSource -from .models import PriceUpdate -from .stream import create_stream_router - -__all__ = [ - "PriceUpdate", - "PriceCache", - "MarketDataSource", - "create_market_data_source", - "create_stream_router", -] -``` diff --git a/planning/archive/MARKET_DATA_REVIEW.md b/planning/archive/MARKET_DATA_REVIEW.md deleted file mode 100644 index 61b4d6bf4..000000000 --- a/planning/archive/MARKET_DATA_REVIEW.md +++ /dev/null @@ -1,173 +0,0 @@ -# Market Data Backend — Code Review - -**Date:** 2026-02-10 -**Scope:** `backend/app/market/` (8 source files) and `backend/tests/market/` (6 test files) - ---- - -## 1. Test Results Summary - -**73 tests collected, 68 passed, 5 failed.** - -All failures are in `test_massive.py` and stem from the same root cause: the `massive` package is not installed in the test environment, so `patch("app.market.massive_client.RESTClient")` fails with `AttributeError` because the module-level name `RESTClient` was never imported (it is lazy-imported inside methods). This is an environment issue, not a logic bug — the tests are correctly structured but require the `massive` package to be available (or `create=True` on the patch) so that the mock target exists. - -Failing tests: -- `test_poll_updates_cache` — `asyncio.to_thread` fails because `_fetch_snapshots` is not properly mocked when `massive` is absent -- `test_malformed_snapshot_skipped` — same cause -- `test_timestamp_conversion` — same cause -- `test_stop_cancels_task` — `patch("app.market.massive_client.RESTClient")` fails because the name doesn't exist at module level -- `test_start_immediate_poll` — same as above - -The underlying `_poll_once()` logic itself is correct. The 3 tests that mock `source._fetch_snapshots` directly fail because `asyncio.to_thread(self._fetch_snapshots)` calls the real method which tries to import `massive`. The 2 tests that use `patch("app.market.massive_client.RESTClient")` fail because the name doesn't exist in the module's namespace (lazy import). Both issues resolve when the `massive` package is installed. - -**Lint (ruff):** Source code passes clean. Tests have 5 unused-import warnings (`pytest`, `math`, `asyncio` imported but not used in some test files). - -**Coverage:** 84% overall. -| Module | Coverage | Notes | -|---|---|---| -| models.py | 100% | | -| cache.py | 100% | | -| interface.py | 100% | | -| seed_prices.py | 100% | | -| factory.py | 100% | | -| simulator.py | 98% | Uncovered: `_add_ticker_internal` duplicate guard (L145), exception log in `_run_loop` (L264-265) | -| massive_client.py | 56% | Expected — real API methods can't run without the massive package | -| stream.py | 31% | Expected — SSE generator requires a running ASGI server to test | - ---- - -## 2. Architecture Assessment - -The market data subsystem is well-designed. It follows a clean strategy pattern: - -``` -MarketDataSource (ABC) -├── SimulatorDataSource (GBM simulator) -└── MassiveDataSource (Polygon.io REST poller) - │ - ▼ - PriceCache (shared, thread-safe) - │ - ▼ - SSE stream → Frontend -``` - -**Strengths:** -- Clear separation of concerns across 8 focused modules -- Factory pattern with lazy imports — the `massive` package is only needed when `MASSIVE_API_KEY` is set -- PriceCache as the single point of truth decouples producers from consumers -- Immutable `PriceUpdate` dataclass with `frozen=True, slots=True` is correct and efficient -- The GBM math is proper: log-normal price paths via `exp((mu - 0.5*sigma^2)*dt + sigma*sqrt(dt)*Z)` -- Correlated moves via Cholesky decomposition are a nice touch for realism -- All background tasks are properly cancellable and idempotent on stop() - ---- - -## 3. Issues Found - -### 3.1 Build Configuration Bug (Severity: High) - -`pyproject.toml` is missing the hatchling package discovery configuration. Running `uv sync` fails: - -``` -ValueError: Unable to determine which files to ship inside the wheel -``` - -**Fix:** Add to `pyproject.toml`: -```toml -[tool.hatch.build.targets.wheel] -packages = ["app"] -``` - -This will block Docker builds and any fresh `uv sync` until fixed. - -### 3.2 Massive Test Fragility (Severity: Medium) - -Five tests in `test_massive.py` fail when the `massive` package is not installed. The root cause is twofold: - -1. **`_poll_once` uses `asyncio.to_thread(self._fetch_snapshots)`** — even when `_fetch_snapshots` is patched on the instance, `to_thread` runs it in a thread executor. Three tests mock `_fetch_snapshots` as a `MagicMock` (synchronous), but `asyncio.to_thread` wraps it in `loop.run_in_executor`, which works... except that when `_fetch_snapshots` is NOT patched, the real method tries `from massive.rest.models import SnapshotMarketType` and fails. - -2. **`patch("app.market.massive_client.RESTClient")`** targets a name that doesn't exist at module level because `massive_client.py` uses a lazy import inside `start()`. The patch needs `create=True` or the import needs to be at module level behind a `TYPE_CHECKING` guard. - -These tests pass when `massive>=1.0.0` is installed (as `pyproject.toml` declares it as a core dependency), so this is technically a test-environment issue, not a code bug. However, since the whole point of lazy imports is to make `massive` optional for simulator-only use, the tests should also work without it. - -### 3.3 `_generate_events` Return Type Annotation (Severity: Low) - -`stream.py:54` declares the return type as `-> None` but the function is an async generator (it uses `yield`). The correct annotation would be `-> AsyncGenerator[str, None]` or simply removing the annotation. This doesn't cause runtime issues but is misleading for type checkers and developers. - -### 3.4 `version` Property Not Under Lock (Severity: Low) - -`PriceCache.version` reads `self._version` without acquiring `self._lock`: - -```python -@property -def version(self) -> int: - return self._version -``` - -On CPython with the GIL, reading a single `int` is atomic, so this won't cause corruption. However, it's inconsistent with the rest of the class, and if the project ever runs on a no-GIL Python build (PEP 703, Python 3.13t+), this could become a race. A minor concern given the current context. - -### 3.5 `SimulatorDataSource.get_tickers` Accesses Private State (Severity: Low) - -`simulator.py:254`: -```python -def get_tickers(self) -> list[str]: - return list(self._sim._tickers) if self._sim else [] -``` - -This reaches into `GBMSimulator._tickers` (private attribute). `GBMSimulator` should expose a `get_tickers()` method or a `tickers` property to keep the boundary clean. - -### 3.6 Module-Level Router Instance (Severity: Low) - -`stream.py:16` creates a module-level `router` object, and `create_stream_router()` registers a route on it via closure. If `create_stream_router` were called twice (e.g., in tests), the `/prices` route would be registered twice on the same router. In practice this won't happen because the function is called once during app startup, but it's a latent footgun for testing. - -### 3.7 Unused Imports in Tests (Severity: Trivial) - -Five lint warnings from `ruff`: -- `test_cache.py`: unused `pytest` -- `test_factory.py`: unused `pytest` -- `test_massive.py`: unused `asyncio` -- `test_simulator.py`: unused `math`, unused `pytest` - ---- - -## 4. Design Observations - -### 4.1 Things Done Well - -- **GBM parameter tuning is thoughtful.** TSLA at sigma=0.50 vs V at 0.17 reflects real-world volatility differences. The shock event system (~0.1% per tick, producing visible moves every ~50s) adds visual drama without destabilizing prices. -- **Cholesky decomposition for correlated moves** is the mathematically correct approach. The sector-based correlation structure (tech 0.6, finance 0.5, cross 0.3) is reasonable. -- **Defensive error handling in both data sources.** Both `_run_loop` (simulator) and `_poll_once`/`_poll_loop` (massive) catch exceptions and continue, which is essential for a long-running background service. -- **SSE implementation is clean.** The version-based change detection avoids sending redundant payloads. The `retry: 1000\n\n` directive ensures browser auto-reconnect. Nginx buffering is proactively disabled. -- **Seed prices in the cache at start** means the frontend gets data on the first SSE poll, with no visible delay. -- **Thread-safe cache with Lock** is the right choice since the Massive client runs API calls via `asyncio.to_thread`. - -### 4.2 Missing Tests - -- **SSE streaming (`stream.py`)** at 31% coverage has no dedicated tests. Testing SSE requires an ASGI test client (e.g., `httpx.AsyncClient` with `app`). Given that this is the primary consumer of PriceCache, even a basic integration test would add confidence. -- **No concurrent/thread-safety test for PriceCache.** The lock usage looks correct from inspection, but a test with multiple threads writing simultaneously would verify it empirically. -- **No test for `GBMSimulator` with all 10 default tickers.** Tests use 1-2 tickers. A test confirming the Cholesky decomposition succeeds for the full 10-ticker default set would catch correlation matrix issues. - -### 4.3 Potential Future Considerations - -- The `PriceCache` doesn't cap history; it only stores the latest price per ticker, so memory is bounded at O(tickers). Good. -- The `DEFAULT_CORR` constant (0.3, `seed_prices.py:48`) is defined but never referenced in `_pairwise_correlation`. The static method returns `CROSS_GROUP_CORR` (also 0.3) as the fallback. This is semantically confusing — `DEFAULT_CORR` seems intended for tickers not in any group, but the code returns `CROSS_GROUP_CORR` for all non-matched pairs. Both happen to be 0.3, so behavior is correct, but the naming is misleading. - ---- - -## 5. Verdict - -The market data backend is solid and well-structured. The GBM simulator, price cache, abstract interface, factory pattern, and SSE streaming all work correctly and follow good practices. The architecture will integrate cleanly with the rest of the application. - -**Must fix before proceeding:** -1. Add `[tool.hatch.build.targets.wheel] packages = ["app"]` to `pyproject.toml` — without this, `uv sync` and Docker builds fail. - -**Should fix:** -2. Make the Massive tests resilient to the `massive` package being absent (use `create=True` on patches, or restructure mocks). -3. Fix the `_generate_events` return type annotation. -4. Remove unused imports in test files. - -**Nice to have:** -5. Add a `get_tickers()` public method to `GBMSimulator`. -6. Add at least one SSE integration test. -7. Clarify `DEFAULT_CORR` vs `CROSS_GROUP_CORR` naming. diff --git a/planning/archive/MARKET_INTERFACE.md b/planning/archive/MARKET_INTERFACE.md deleted file mode 100644 index 156cad287..000000000 --- a/planning/archive/MARKET_INTERFACE.md +++ /dev/null @@ -1,273 +0,0 @@ -# Market Data Interface Design - -Unified Python interface for market data in FinAlly. Two implementations (simulator and Massive API) behind one abstract interface. All downstream code — SSE streaming, price cache, portfolio valuation — is source-agnostic. - -## Core Data Model - -```python -from dataclasses import dataclass - -@dataclass -class PriceUpdate: - """A single price update for one ticker.""" - ticker: str - price: float - previous_price: float - timestamp: float # Unix seconds - change: float # price - previous_price - direction: str # "up", "down", or "flat" -``` - -This is the only data structure that leaves the market data layer. Everything downstream works with `PriceUpdate` objects. - -## Abstract Interface - -```python -from abc import ABC, abstractmethod - -class MarketDataSource(ABC): - """Abstract interface for market data providers.""" - - @abstractmethod - async def start(self, tickers: list[str]) -> None: - """Begin producing price updates for the given tickers.""" - - @abstractmethod - async def stop(self) -> None: - """Stop producing price updates and clean up.""" - - @abstractmethod - async def add_ticker(self, ticker: str) -> None: - """Add a ticker to the active set.""" - - @abstractmethod - async def remove_ticker(self, ticker: str) -> None: - """Remove a ticker from the active set.""" - - @abstractmethod - def get_tickers(self) -> list[str]: - """Return the current list of active tickers.""" -``` - -Both implementations write to a shared `PriceCache` (see below). The interface does **not** return prices directly — it pushes updates into the cache on its own schedule. - -## Price Cache - -Shared in-memory store that both data sources write to and the SSE streamer reads from. - -```python -import time -from threading import Lock - -class PriceCache: - """Thread-safe cache of latest prices per ticker.""" - - def __init__(self): - self._prices: dict[str, PriceUpdate] = {} - self._lock = Lock() - - def update(self, ticker: str, price: float, timestamp: float | None = None) -> PriceUpdate: - """Update price for a ticker. Returns the PriceUpdate.""" - with self._lock: - ts = timestamp or time.time() - previous = self._prices.get(ticker) - previous_price = previous.price if previous else price - - if price > previous_price: - direction = "up" - elif price < previous_price: - direction = "down" - else: - direction = "flat" - - update = PriceUpdate( - ticker=ticker, - price=price, - previous_price=previous_price, - timestamp=ts, - change=price - previous_price, - direction=direction, - ) - self._prices[ticker] = update - return update - - def get(self, ticker: str) -> PriceUpdate | None: - """Get latest price for a ticker.""" - with self._lock: - return self._prices.get(ticker) - - def get_all(self) -> dict[str, PriceUpdate]: - """Get all current prices.""" - with self._lock: - return dict(self._prices) - - def remove(self, ticker: str) -> None: - """Remove a ticker from the cache.""" - with self._lock: - self._prices.pop(ticker, None) -``` - -## Factory Function - -Select the data source at startup based on environment: - -```python -import os - -def create_market_data_source(price_cache: PriceCache) -> MarketDataSource: - """Create the appropriate market data source based on environment.""" - api_key = os.environ.get("MASSIVE_API_KEY", "").strip() - - if api_key: - from .massive_client import MassiveDataSource - return MassiveDataSource(api_key=api_key, price_cache=price_cache) - else: - from .simulator import SimulatorDataSource - return SimulatorDataSource(price_cache=price_cache) -``` - -## Massive Implementation Sketch - -```python -import asyncio -from massive import RESTClient -from massive.rest.models import SnapshotMarketType - -class MassiveDataSource(MarketDataSource): - def __init__(self, api_key: str, price_cache: PriceCache, poll_interval: float = 15.0): - self._client = RESTClient(api_key=api_key) - self._cache = price_cache - self._interval = poll_interval - self._tickers: list[str] = [] - self._task: asyncio.Task | None = None - - async def start(self, tickers: list[str]) -> None: - self._tickers = list(tickers) - self._task = asyncio.create_task(self._poll_loop()) - - async def stop(self) -> None: - if self._task: - self._task.cancel() - - async def add_ticker(self, ticker: str) -> None: - if ticker not in self._tickers: - self._tickers.append(ticker) - - async def remove_ticker(self, ticker: str) -> None: - self._tickers = [t for t in self._tickers if t != ticker] - self._cache.remove(ticker) - - def get_tickers(self) -> list[str]: - return list(self._tickers) - - async def _poll_loop(self) -> None: - while True: - await self._poll_once() - await asyncio.sleep(self._interval) - - async def _poll_once(self) -> None: - if not self._tickers: - return - # Run synchronous Massive client in thread pool - snapshots = await asyncio.to_thread( - self._client.get_snapshot_all, - market_type=SnapshotMarketType.STOCKS, - tickers=self._tickers, - ) - for snap in snapshots: - self._cache.update( - ticker=snap.ticker, - price=snap.last_trade.price, - timestamp=snap.last_trade.timestamp / 1000, # ms -> seconds - ) -``` - -## Simulator Implementation Sketch - -```python -import asyncio - -class SimulatorDataSource(MarketDataSource): - def __init__(self, price_cache: PriceCache, update_interval: float = 0.5): - self._cache = price_cache - self._interval = update_interval - self._tickers: list[str] = [] - self._task: asyncio.Task | None = None - self._sim: GBMSimulator | None = None # See MARKET_SIMULATOR.md - - async def start(self, tickers: list[str]) -> None: - self._tickers = list(tickers) - self._sim = GBMSimulator(tickers=self._tickers) - self._task = asyncio.create_task(self._run_loop()) - - async def stop(self) -> None: - if self._task: - self._task.cancel() - - async def add_ticker(self, ticker: str) -> None: - if ticker not in self._tickers: - self._tickers.append(ticker) - self._sim.add_ticker(ticker) - - async def remove_ticker(self, ticker: str) -> None: - self._tickers = [t for t in self._tickers if t != ticker] - self._sim.remove_ticker(ticker) - self._cache.remove(ticker) - - def get_tickers(self) -> list[str]: - return list(self._tickers) - - async def _run_loop(self) -> None: - while True: - prices = self._sim.step() # Returns dict[str, float] - for ticker, price in prices.items(): - self._cache.update(ticker=ticker, price=price) - await asyncio.sleep(self._interval) -``` - -## Integration with SSE - -The SSE endpoint reads from the `PriceCache` and pushes to connected clients: - -```python -async def price_stream(price_cache: PriceCache): - """SSE generator that yields price updates.""" - while True: - prices = price_cache.get_all() - data = { - ticker: { - "ticker": p.ticker, - "price": p.price, - "previous_price": p.previous_price, - "change": p.change, - "direction": p.direction, - "timestamp": p.timestamp, - } - for ticker, p in prices.items() - } - yield f"data: {json.dumps(data)}\n\n" - await asyncio.sleep(0.5) -``` - -## File Structure - -``` -backend/ - app/ - market/ - __init__.py - models.py # PriceUpdate dataclass - interface.py # MarketDataSource ABC, PriceCache - factory.py # create_market_data_source() - massive_client.py # MassiveDataSource - simulator.py # SimulatorDataSource + GBMSimulator - seed_prices.py # Default ticker seed prices -``` - -## Lifecycle - -1. **App startup**: Create `PriceCache`, call `create_market_data_source(price_cache)`, then `await source.start(initial_tickers)` -2. **Watchlist changes**: Call `source.add_ticker()` or `source.remove_ticker()` -3. **SSE streaming**: Reads from `PriceCache.get_all()` every 500ms -4. **Trade execution**: Reads current price from `PriceCache.get(ticker)` -5. **App shutdown**: Call `await source.stop()` diff --git a/planning/archive/MARKET_SIMULATOR.md b/planning/archive/MARKET_SIMULATOR.md deleted file mode 100644 index e157b6efb..000000000 --- a/planning/archive/MARKET_SIMULATOR.md +++ /dev/null @@ -1,245 +0,0 @@ -# Market Simulator Design - -Approach and code structure for simulating realistic stock prices when no Massive API key is configured. - -## Overview - -The simulator uses **Geometric Brownian Motion (GBM)** to generate realistic stock price paths. GBM is the standard model underlying Black-Scholes option pricing — prices evolve continuously with random noise, can't go negative, and exhibit the lognormal distribution seen in real markets. - -Updates run at ~500ms intervals, producing a continuous stream of price changes that feel alive. - -## GBM Math - -At each time step, a stock price evolves as: - -``` -S(t+dt) = S(t) * exp((mu - sigma^2/2) * dt + sigma * sqrt(dt) * Z) -``` - -Where: -- `S(t)` = current price -- `mu` = annualized drift (expected return), e.g. 0.05 (5%) -- `sigma` = annualized volatility, e.g. 0.20 (20%) -- `dt` = time step as fraction of a trading year -- `Z` = standard normal random variable (drawn from N(0,1)) - -For our 500ms updates with ~252 trading days and ~6.5 hours per day: -``` -dt = 0.5 / (252 * 6.5 * 3600) = ~8.5e-8 -``` - -This tiny `dt` produces small, realistic per-tick moves. - -## Correlated Moves - -Real stocks don't move independently — tech stocks tend to move together, etc. We use a **Cholesky decomposition** of a correlation matrix to generate correlated random draws. - -Given a correlation matrix `C`, compute `L = cholesky(C)`. Then for independent standard normals `Z_independent`: -``` -Z_correlated = L @ Z_independent -``` - -Default correlation groups: -- **Tech**: AAPL, GOOGL, MSFT, AMZN, META, NVDA, NFLX — corr ~0.6 within group -- **Finance**: JPM, V — corr ~0.5 within group -- **Cross-group**: ~0.3 baseline correlation -- **TSLA**: lower correlation with everything (~0.3) — it does its own thing - -## Random Events - -Every step, each ticker has a small probability (~0.001) of a random event — a sudden 2-5% move. This adds drama and makes the dashboard visually interesting. - -```python -if random.random() < event_probability: - shock = random.uniform(0.02, 0.05) * random.choice([-1, 1]) - price *= (1 + shock) -``` - -## Seed Prices - -Realistic starting prices for the default watchlist: - -```python -SEED_PRICES: dict[str, float] = { - "AAPL": 190.0, - "GOOGL": 175.0, - "MSFT": 420.0, - "AMZN": 185.0, - "TSLA": 250.0, - "NVDA": 800.0, - "META": 500.0, - "JPM": 195.0, - "V": 280.0, - "NFLX": 600.0, -} -``` - -Tickers added dynamically (not in the seed list) start at a random price between $50-$300. - -## Per-Ticker Parameters - -Each ticker has its own volatility to reflect real-world behavior: - -```python -TICKER_PARAMS: dict[str, dict] = { - "AAPL": {"sigma": 0.22, "mu": 0.05}, - "GOOGL": {"sigma": 0.25, "mu": 0.05}, - "MSFT": {"sigma": 0.20, "mu": 0.05}, - "AMZN": {"sigma": 0.28, "mu": 0.05}, - "TSLA": {"sigma": 0.50, "mu": 0.03}, # High vol - "NVDA": {"sigma": 0.40, "mu": 0.08}, # High vol, strong drift - "META": {"sigma": 0.30, "mu": 0.05}, - "JPM": {"sigma": 0.18, "mu": 0.04}, # Low vol (bank) - "V": {"sigma": 0.17, "mu": 0.04}, # Low vol (payments) - "NFLX": {"sigma": 0.35, "mu": 0.05}, -} - -# Default for unknown tickers -DEFAULT_PARAMS = {"sigma": 0.25, "mu": 0.05} -``` - -## Implementation - -```python -import math -import random -import time -import numpy as np - -class GBMSimulator: - """Generates correlated GBM price paths for multiple tickers.""" - - def __init__( - self, - tickers: list[str], - dt: float = 8.5e-8, - event_probability: float = 0.001, - ): - self._dt = dt - self._event_prob = event_probability - self._prices: dict[str, float] = {} - self._params: dict[str, dict] = {} - self._tickers: list[str] = [] - self._cholesky: np.ndarray | None = None - - for ticker in tickers: - self.add_ticker(ticker) - - def add_ticker(self, ticker: str) -> None: - if ticker in self._prices: - return - self._tickers.append(ticker) - self._prices[ticker] = SEED_PRICES.get(ticker, random.uniform(50, 300)) - self._params[ticker] = TICKER_PARAMS.get(ticker, DEFAULT_PARAMS) - self._rebuild_cholesky() - - def remove_ticker(self, ticker: str) -> None: - if ticker not in self._prices: - return - self._tickers.remove(ticker) - del self._prices[ticker] - del self._params[ticker] - self._rebuild_cholesky() - - def step(self) -> dict[str, float]: - """Advance one time step. Returns {ticker: new_price}.""" - n = len(self._tickers) - if n == 0: - return {} - - # Generate correlated random normals - z_independent = np.random.standard_normal(n) - if self._cholesky is not None: - z = self._cholesky @ z_independent - else: - z = z_independent - - result = {} - for i, ticker in enumerate(self._tickers): - params = self._params[ticker] - mu = params["mu"] - sigma = params["sigma"] - - # GBM step - drift = (mu - 0.5 * sigma**2) * self._dt - diffusion = sigma * math.sqrt(self._dt) * z[i] - self._prices[ticker] *= math.exp(drift + diffusion) - - # Random event - if random.random() < self._event_prob: - shock = random.uniform(0.02, 0.05) * random.choice([-1, 1]) - self._prices[ticker] *= (1 + shock) - - result[ticker] = round(self._prices[ticker], 2) - - return result - - def get_price(self, ticker: str) -> float | None: - return self._prices.get(ticker) - - def _rebuild_cholesky(self) -> None: - """Rebuild the Cholesky decomposition of the correlation matrix.""" - n = len(self._tickers) - if n <= 1: - self._cholesky = None - return - - corr = np.eye(n) - for i in range(n): - for j in range(i + 1, n): - rho = self._get_correlation(self._tickers[i], self._tickers[j]) - corr[i, j] = rho - corr[j, i] = rho - - self._cholesky = np.linalg.cholesky(corr) - - def _get_correlation(self, t1: str, t2: str) -> float: - """Return pairwise correlation between two tickers.""" - tech = {"AAPL", "GOOGL", "MSFT", "AMZN", "META", "NVDA", "NFLX"} - finance = {"JPM", "V"} - - t1_tech = t1 in tech - t2_tech = t2 in tech - t1_fin = t1 in finance - t2_fin = t2 in finance - - # Same sector: higher correlation - if t1_tech and t2_tech: - return 0.6 - if t1_fin and t2_fin: - return 0.5 - - # TSLA is a loner - if t1 == "TSLA" or t2 == "TSLA": - return 0.3 - - # Cross-sector or unknown - if (t1_tech and t2_fin) or (t1_fin and t2_tech): - return 0.3 - - # Default - return 0.3 -``` - -## File Structure - -All simulator code lives in a single module: - -``` -backend/ - app/ - market/ - simulator.py # GBMSimulator class + seed data + SimulatorDataSource - seed_prices.py # SEED_PRICES, TICKER_PARAMS, DEFAULT_PARAMS (constants) -``` - -`seed_prices.py` contains just the constant dictionaries. `simulator.py` contains the `GBMSimulator` class and the `SimulatorDataSource` (the `MarketDataSource` implementation that wraps `GBMSimulator` in an async loop). - -## Behavior Notes - -- Prices never go negative (GBM is multiplicative — `exp()` is always positive) -- The tiny `dt` produces sub-cent moves per tick, which accumulate naturally over time -- With `sigma=0.50` (TSLA), a day of simulated trading produces roughly the right intraday range -- The correlation matrix must be positive semi-definite — Cholesky decomposition guarantees this for valid correlation matrices -- Random events happen ~0.1% of steps = roughly once every 500 seconds per ticker. With 10 tickers, expect an event somewhere roughly every 50 seconds — enough to keep it interesting -- When a new ticker is added mid-session, the Cholesky matrix is rebuilt. This is O(n^2) but n is small (<50 tickers) diff --git a/planning/archive/MASSIVE_API.md b/planning/archive/MASSIVE_API.md deleted file mode 100644 index 3266bc64f..000000000 --- a/planning/archive/MASSIVE_API.md +++ /dev/null @@ -1,251 +0,0 @@ -# Massive API Reference (formerly Polygon.io) - -Reference documentation for the Massive (formerly Polygon.io) REST API as used in FinAlly. - -## Overview - -- **Base URL**: `https://api.massive.com` (legacy `https://api.polygon.io` still supported) -- **Python package**: `massive` (install via `pip install -U massive` / `uv add massive`) -- **Min Python version**: 3.9+ -- **Auth**: API key via `MASSIVE_API_KEY` env var or passed to `RESTClient(api_key=...)` -- **Auth header**: `Authorization: Bearer ` (the client handles this automatically) - -## Rate Limits - -| Tier | Limit | -|------|-------| -| Free | 5 requests/minute | -| Paid (all tiers) | Unlimited (recommended: stay under 100 req/s) | - -For FinAlly, we poll on a timer. Free tier: poll every 15s. Paid: poll every 2-5s. - -## Client Initialization - -```python -from massive import RESTClient - -# Reads MASSIVE_API_KEY from environment automatically -client = RESTClient() - -# Or pass explicitly -client = RESTClient(api_key="your_key_here") -``` - -## Endpoints Used in FinAlly - -### 1. Snapshot — All Tickers (Primary Endpoint) - -Gets current prices for multiple tickers in a **single API call**. This is the main endpoint we use for polling. - -**REST**: `GET /v2/snapshot/locale/us/markets/stocks/tickers?tickers=AAPL,GOOGL,MSFT` - -**Python client**: -```python -from massive import RESTClient -from massive.rest.models import SnapshotMarketType - -client = RESTClient() - -# Get snapshots for specific tickers (one API call) -snapshots = client.get_snapshot_all( - market_type=SnapshotMarketType.STOCKS, - tickers=["AAPL", "GOOGL", "MSFT", "AMZN", "TSLA"], -) - -for snap in snapshots: - print(f"{snap.ticker}: ${snap.last_trade.price}") - print(f" Day change: {snap.day.change_percent}%") - print(f" Day OHLC: O={snap.day.open} H={snap.day.high} L={snap.day.low} C={snap.day.close}") - print(f" Volume: {snap.day.volume}") -``` - -**Response structure** (per ticker): -```json -{ - "ticker": "AAPL", - "day": { - "open": 129.61, - "high": 130.15, - "low": 125.07, - "close": 125.07, - "volume": 111237700, - "volume_weighted_average_price": 127.35, - "previous_close": 129.61, - "change": -4.54, - "change_percent": -3.50 - }, - "last_trade": { - "price": 125.07, - "size": 100, - "exchange": "XNYS", - "timestamp": 1675190399000 - }, - "last_quote": { - "bid_price": 125.06, - "ask_price": 125.08, - "bid_size": 500, - "ask_size": 1000, - "spread": 0.02, - "timestamp": 1675190399500 - }, - "prev_daily_bar": { "...": "previous day OHLCV" }, - "minute_volume": { "...": "volume per minute" } -} -``` - -**Key fields we extract**: -- `last_trade.price` — current price for trading and display -- `day.previous_close` — for calculating day change -- `day.change_percent` — day change percentage -- `last_trade.timestamp` — when the price was recorded - -### 2. Single Ticker Snapshot - -For getting detailed data on one ticker (e.g., when user clicks a ticker for the detail view). - -**Python client**: -```python -snapshot = client.get_snapshot_ticker( - market_type=SnapshotMarketType.STOCKS, - ticker="AAPL", -) - -print(f"Price: ${snapshot.last_trade.price}") -print(f"Bid/Ask: ${snapshot.last_quote.bid_price} / ${snapshot.last_quote.ask_price}") -print(f"Day range: ${snapshot.day.low} - ${snapshot.day.high}") -``` - -### 3. Previous Close - -Gets the previous day's OHLC for a ticker. Useful for seed prices. - -**REST**: `GET /v2/aggs/ticker/{ticker}/prev` - -**Python client**: -```python -prev = client.get_previous_close_agg(ticker="AAPL") - -for agg in prev: - print(f"Previous close: ${agg.close}") - print(f"OHLC: O={agg.open} H={agg.high} L={agg.low} C={agg.close}") - print(f"Volume: {agg.volume}") -``` - -**Response**: -```json -{ - "ticker": "AAPL", - "results": [ - { - "o": 150.0, - "h": 155.0, - "l": 149.0, - "c": 154.5, - "v": 1000000, - "t": 1672531200000 - } - ] -} -``` - -### 4. Aggregates (Bars) - -Historical OHLCV bars over a date range. Not needed for live polling but useful if we add historical charts. - -**REST**: `GET /v2/aggs/ticker/{ticker}/range/{multiplier}/{timespan}/{from}/{to}` - -**Python client**: -```python -aggs = [] -for a in client.list_aggs( - ticker="AAPL", - multiplier=1, - timespan="day", - from_="2024-01-01", - to="2024-01-31", - limit=50000, -): - aggs.append(a) - -for a in aggs: - print(f"Date: {a.timestamp}, O={a.open} H={a.high} L={a.low} C={a.close} V={a.volume}") -``` - -**Response** (each bar): -```json -{ - "o": 130.0, - "h": 132.5, - "l": 129.8, - "c": 131.2, - "v": 50000000, - "t": 1672531200000 -} -``` - -### 5. Last Trade / Last Quote - -Individual endpoints for the most recent trade or NBBO quote. - -```python -# Last trade -trade = client.get_last_trade(ticker="AAPL") -print(f"Last trade: ${trade.price} x {trade.size}") - -# Last NBBO quote -quote = client.get_last_quote(ticker="AAPL") -print(f"Bid: ${quote.bid} x {quote.bid_size}") -print(f"Ask: ${quote.ask} x {quote.ask_size}") -``` - -## How FinAlly Uses the API - -The Massive poller runs as a background task: - -1. Collects all tickers from the watchlist -2. Calls `get_snapshot_all()` with those tickers (one API call) -3. Extracts `last_trade.price` and `day.previous_close` from each snapshot -4. Writes to the shared in-memory price cache -5. Sleeps for the poll interval, then repeats - -```python -import asyncio -from massive import RESTClient -from massive.rest.models import SnapshotMarketType - -async def poll_massive(api_key: str, get_tickers, price_cache, interval: float = 15.0): - """Poll Massive API and update the price cache.""" - client = RESTClient(api_key=api_key) - - while True: - tickers = get_tickers() - if tickers: - snapshots = client.get_snapshot_all( - market_type=SnapshotMarketType.STOCKS, - tickers=tickers, - ) - for snap in snapshots: - price_cache.update( - ticker=snap.ticker, - price=snap.last_trade.price, - previous_close=snap.day.previous_close, - timestamp=snap.last_trade.timestamp, - ) - - await asyncio.sleep(interval) -``` - -## Error Handling - -The client raises exceptions for HTTP errors: -- **401**: Invalid API key -- **403**: Insufficient permissions (plan doesn't include the endpoint) -- **429**: Rate limit exceeded (free tier: 5 req/min) -- **5xx**: Server errors (client has built-in retry with 3 retries by default) - -## Notes - -- The snapshot endpoint returns data for **all requested tickers in one call** — this is critical for staying within rate limits on the free tier -- Timestamps from the API are Unix milliseconds -- During market closed hours, `last_trade.price` reflects the last traded price (may include after-hours) -- The `day` object resets at market open; during pre-market, values may be from the previous session From 029012aacff4e33f7f929771ddc35a661b7112d8 Mon Sep 17 00:00:00 2001 From: cloudnote18-lab Date: Thu, 3 Sep 2026 15:55:25 -0400 Subject: [PATCH 2/4] "Claude PR Assistant workflow" --- .github/workflows/claude.yml | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 .github/workflows/claude.yml diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml new file mode 100644 index 000000000..6b15fac7a --- /dev/null +++ b/.github/workflows/claude.yml @@ -0,0 +1,50 @@ +name: Claude Code + +on: + issue_comment: + types: [created] + pull_request_review_comment: + types: [created] + issues: + types: [opened, assigned] + pull_request_review: + types: [submitted] + +jobs: + claude: + if: | + (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || + (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || + (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + actions: read # Required for Claude to read CI results on PRs + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code + id: claude + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + + # This is an optional setting that allows Claude to read CI results on PRs + additional_permissions: | + actions: read + + # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. + # prompt: 'Update the pull request description to include a summary of changes.' + + # Optional: Add claude_args to customize behavior and configuration + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + # claude_args: '--allowed-tools Bash(gh pr *)' + From 2c7fde92fba24f838cb38d30f0c71c777631e8e7 Mon Sep 17 00:00:00 2001 From: cloudnote18-lab Date: Thu, 3 Sep 2026 15:55:26 -0400 Subject: [PATCH 3/4] "Claude Code Review workflow" --- .github/workflows/claude-code-review.yml | 45 ++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/claude-code-review.yml diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml new file mode 100644 index 000000000..37e66f3fd --- /dev/null +++ b/.github/workflows/claude-code-review.yml @@ -0,0 +1,45 @@ +name: Claude Code Review + +on: + pull_request: + types: [opened, synchronize, ready_for_review, reopened] + # Optional: Only run on specific file changes + # paths: + # - "src/**/*.ts" + # - "src/**/*.tsx" + # - "src/**/*.js" + # - "src/**/*.jsx" + +jobs: + claude-review: + # Optional: Filter by PR author + # if: | + # github.event.pull_request.user.login == 'external-contributor' || + # github.event.pull_request.user.login == 'new-developer' || + # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' + + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + issues: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Run Claude Code Review + id: claude-review + uses: anthropics/claude-code-action@v1 + with: + claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' + plugins: 'code-review@claude-code-plugins' + prompt: '/code-review:code-review --comment ${{ github.repository }}/pull/${{ github.event.pull_request.number }}' + claude_args: '--allowedTools "mcp__github_inline_comment__create_inline_comment"' + # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md + # or https://code.claude.com/docs/en/cli-reference for available options + From 06f33a2603c4242124a0e7a1504162023f72ad57 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 20:18:41 +0000 Subject: [PATCH 4/4] =?UTF-8?q?Add=20MARKET=5FDATA=5FDESIGN.md=20=E2=80=94?= =?UTF-8?q?=20implementation=20design=20for=20the=20market=20data=20backen?= =?UTF-8?q?d?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidates MARKET_INTERFACE.md, MARKET_SIMULATOR.md and MASSIVE_API.md into a single implementation-ready design for backend/app/market/, with drop-in code for every module: - Shared models: open_price baseline, tick_direction vs change_percent_today, PricePoint/SourceStatus, ISO 8601 wire conversion - PriceCache with a sticky open_price anchor - MarketDataSource + describe() and get_history() - Capability probe classifying the five Massive key states - Async, capability-driven source factory (three sources, one interface) - Simulator: decaying shocks at 1e-4, deterministic unknown-ticker seeds, seed_overrides, LinAlgError fallback, history ring buffer with prefill - AnchoredSimulatorDataSource: real closes from one free-tier call, GBM motion - Rewritten Massive client: nanosecond timestamps, prev_day.close baseline, error classification, poll-loop backoff, poll/quote staleness - SSE heartbeat, /api/prices/{ticker}/history, /api/health, lifespan wiring - Test plan, implementation order, and the PLAN.md items this closes Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_015iKouPXUXK3RAgWvfrkGPv --- planning/MARKET_DATA_DESIGN.md | 1797 ++++++++++++++++++++++++++++++++ 1 file changed, 1797 insertions(+) create mode 100644 planning/MARKET_DATA_DESIGN.md diff --git a/planning/MARKET_DATA_DESIGN.md b/planning/MARKET_DATA_DESIGN.md new file mode 100644 index 000000000..d22346a8a --- /dev/null +++ b/planning/MARKET_DATA_DESIGN.md @@ -0,0 +1,1797 @@ +# Market Data Backend — Implementation Design + +**Status:** Implementation-ready design. Consolidates and supersedes the market-data portion +of [`PLAN.md`](PLAN.md) §6, and turns the three research documents into code you can type in. + +| Document | Role | Relationship to this one | +|---|---|---| +| [`MARKET_INTERFACE.md`](MARKET_INTERFACE.md) | Architecture: three sources, one interface | This document is its implementation | +| [`MARKET_SIMULATOR.md`](MARKET_SIMULATOR.md) | Measured review of the GBM model | Supplies §7's calibration constants | +| [`MASSIVE_API.md`](MASSIVE_API.md) | Verified API/entitlement research | Supplies §6 and §9's endpoint facts | + +Everything here targets the code already in `backend/app/market/`. Sections are ordered so +that each one only depends on the ones above it — implement top to bottom and the tree +compiles at every step. + +--- + +## 1. What This Design Delivers + +Three data sources behind one interface, selected by what the running key can *actually do* +rather than by whether an environment variable is non-empty: + +``` + MASSIVE_API_KEY set? + │ + ┌───────────────┴────────────────┐ + no yes + │ │ + │ probe_capabilities() ← 2 calls, once, at startup + │ │ + │ ┌─────────────────────┼──────────────────────┐ + │ realtime/delayed end-of-day invalid + │ │ │ │ + ▼ ▼ ▼ ▼ + SimulatorDataSource MassiveDataSource AnchoredSimulator SimulatorDataSource + (synthetic seeds) (real prices) (real levels, (+ reason in /health) + synthetic motion) + │ │ │ │ + └──────────┴──────────┬──────────┴──────────────────────┘ + ▼ + PriceCache ← single read path, source-agnostic + │ + ┌───────────────────────┼───────────────────────┐ + SSE /api/stream/prices portfolio valuation trade pricing +``` + +### Non-negotiable invariants + +1. **The app always boots into a moving terminal.** Every branch of §7's factory returns a + source that produces ticking prices. A missing, free, or revoked key degrades the *source*; + it never blanks the screen. +2. **Simulated prices are never presented as live.** `SourceStatus.live` is `False` for both + simulator flavours, `/api/health` says so in words, and the frontend badges it. +3. **One API call per cycle, never one per ticker.** The free tier's budget is 5 calls/minute + (`MASSIVE_API.md` §7); per-ticker fetching cannot price a 10-symbol watchlist even once. +4. **Everything downstream of `PriceCache` is source-agnostic.** No route, no valuation, no + SSE handler ever imports `MassiveDataSource` or `SimulatorDataSource`. +5. **The SDK is synchronous.** Every Massive call is wrapped in `asyncio.to_thread(...)`, or + it stalls the event loop and freezes SSE for every connected client. + +--- + +## 2. Module Layout and Change Map + +``` +backend/app/market/ +├── __init__.py MODIFIED export SourceStatus, PricePoint, capabilities +├── models.py MODIFIED + open_price, tick_direction, PricePoint, SourceStatus, time helpers +├── cache.py MODIFIED + open_price parameter (one behavioural change) +├── interface.py MODIFIED + describe(), + get_history() with a default +├── capabilities.py NEW probe_capabilities() / MassiveCapabilities +├── factory.py REWRITTEN async, capability-driven, five key states +├── seed_prices.py MODIFIED refreshed closes, LOW_PRICE_SIGMA_FLOOR +├── simulator.py MODIFIED decaying shocks, deterministic seeds, history, describe() +├── anchored.py NEW AnchoredSimulatorDataSource +├── massive_client.py REWRITTEN ns timestamps, open_price, error classification, backoff +└── stream.py MODIFIED + heartbeat +``` + +Effort estimate: `models`/`cache`/`interface` are an hour; `simulator.py` is the largest +single change; `massive_client.py` is a rewrite but a small one; `anchored.py` is ~120 lines. + +--- + +## 3. Shared Models — `app/market/models.py` + +Three additions to the shipped file: the `open_price` baseline (`PLAN.md` §13.1 item 1), the +`PricePoint`/`SourceStatus` records the interface needs, and the ISO-8601 wire conversion +(`PLAN.md` §13.1 item 6). + +```python +"""Data models for market data.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass, field +from datetime import UTC, datetime + +# --- Time conversion ------------------------------------------------------- +# Massive is inconsistent: aggregate bars carry milliseconds, snapshots and +# last-trade prints carry NANOseconds (MASSIVE_API.md §6). Convert once, here, +# at the boundary. Internally everything is float epoch seconds; on the wire +# everything is ISO 8601 UTC. + +_UNIT_DIVISOR = {"s": 1.0, "ms": 1e3, "us": 1e6, "ns": 1e9} + + +def to_epoch_seconds(value: int | float, unit: str) -> float: + """Convert a Massive timestamp to float epoch seconds. `unit` is explicit on purpose.""" + return float(value) / _UNIT_DIVISOR[unit] + + +def epoch_to_iso(epoch_seconds: float) -> str: + """Float epoch seconds -> '2026-09-03T17:42:11.413700Z'.""" + return ( + datetime.fromtimestamp(epoch_seconds, tz=UTC) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +@dataclass(frozen=True, slots=True) +class PricePoint: + """One point of a historical series, as served by MarketDataSource.get_history().""" + + timestamp: str # ISO 8601 UTC — already wire-format + price: float + + def to_dict(self) -> dict: + return {"timestamp": self.timestamp, "price": self.price} + + +@dataclass(frozen=True, slots=True) +class SourceStatus: + """Introspection for GET /api/health. Produced by MarketDataSource.describe().""" + + name: str # "simulator" | "massive" | "anchored-simulator" + live: bool # True only when prices reflect the real current market + detail: str # human-readable, surfaced verbatim in /api/health + tickers: int + cache_populated: bool + + def to_dict(self) -> dict: + return { + "name": self.name, + "live": self.live, + "detail": self.detail, + "tickers": self.tickers, + "cache_populated": self.cache_populated, + } + + +@dataclass(frozen=True, slots=True) +class PriceUpdate: + """Immutable snapshot of a single ticker's price at a point in time. + + Two distinct baselines, deliberately named so they cannot be confused + (PLAN.md §13.1 item 1): + + previous_price -> the PREVIOUS TICK. Drives the flash animation only. + open_price -> the session open / anchor, fixed for the session. + Drives the "daily change %" column. + """ + + ticker: str + price: float + previous_price: float + open_price: float + timestamp: float = field(default_factory=time.time) # epoch seconds, internal only + + # --- tick-scale: flash animation --- + @property + def tick_direction(self) -> str: + """'up' | 'down' | 'flat' — the CSS flash class.""" + if self.price > self.previous_price: + return "up" + if self.price < self.previous_price: + return "down" + return "flat" + + # --- session-scale: the watchlist's daily column --- + @property + def change_today(self) -> float: + return round(self.price - self.open_price, 4) + + @property + def change_percent_today(self) -> float: + if self.open_price == 0: + return 0.0 + return round((self.price - self.open_price) / self.open_price * 100, 4) + + def to_dict(self) -> dict: + """Serialize for JSON / SSE. Timestamp becomes ISO 8601 UTC here.""" + return { + "ticker": self.ticker, + "price": self.price, + "previous_price": self.previous_price, + "open_price": self.open_price, + "timestamp": epoch_to_iso(self.timestamp), + "tick_direction": self.tick_direction, + "change_today": self.change_today, + "change_percent_today": self.change_percent_today, + } +``` + +### Wire format + +```json +{ + "ticker": "AAPL", + "price": 325.41, + "previous_price": 325.38, + "open_price": 324.96, + "timestamp": "2026-09-03T17:42:11.413700Z", + "tick_direction": "up", + "change_today": 0.45, + "change_percent_today": 0.1385 +} +``` + +### Where `open_price` comes from + +| Source | `open_price` | +|---|---| +| `MassiveDataSource` | `snap.prev_day.close` (real previous close) | +| `AnchoredSimulatorDataSource` | the real close fetched once at startup | +| `SimulatorDataSource` | the seed price the simulation started from | + +Fixed for the session in all three, so the daily % accumulates over minutes and hours +instead of resetting every 500 ms. + +### Breaking changes to announce + +`direction` → `tick_direction`, and `change`/`change_percent` (tick-scale) are **removed** in +favour of `change_today`/`change_percent_today` (session-scale). Nothing but tests consumes +them today; the frontend has not been written yet, so this is the moment to make the change. + +--- + +## 4. Price Cache — `app/market/cache.py` + +The shipped cache is correct and mostly untouched: keep the `threading.Lock` (writers are +`asyncio.to_thread` worker threads, so a threading lock is right, not an asyncio one) and +keep the monotonic `version` counter that drives SSE change detection. + +One behavioural change — `open_price` is **sticky**: + +```python + def update( + self, + ticker: str, + price: float, + *, + open_price: float | None = None, + timestamp: float | None = None, + ) -> PriceUpdate: + """Record a new price. Returns the created PriceUpdate. + + open_price semantics (sticky): + - first write for a ticker : open_price or price + - later writes, None : keep whatever the ticker already had + - later writes, a value : overwrite (re-anchoring, new session) + + This is what lets a 500ms tick loop pass open_price=None forever while + the daily-change baseline stays fixed for the whole session. + """ + with self._lock: + ts = timestamp if timestamp is not None else time.time() + prev = self._prices.get(ticker) + + previous_price = prev.price if prev else round(price, 2) + if open_price is not None: + resolved_open = round(open_price, 2) + elif prev is not None: + resolved_open = prev.open_price + else: + resolved_open = round(price, 2) + + update = PriceUpdate( + ticker=ticker, + price=round(price, 2), + previous_price=previous_price, + open_price=resolved_open, + timestamp=ts, + ) + self._prices[ticker] = update + self._version += 1 + return update +``` + +`timestamp` and `open_price` are keyword-only so no existing positional call can silently +land in the wrong slot. Note `timestamp if timestamp is not None` rather than the shipped +`timestamp or time.time()` — the latter treats a legitimate `0.0` as absent. + +Rounding rule (`PLAN.md` §13.4 item 16): **full float precision in the model, rounded to 2dp +at the cache boundary.** The simulator keeps its unrounded price internally and never rounds +in place, or a drift bias accumulates over tens of thousands of ticks. + +`get`, `get_all`, `get_price`, `remove`, `version`, `__len__`, `__contains__` are unchanged. + +--- + +## 5. The Interface — `app/market/interface.py` + +Keep the ABC exactly as shipped and add two members. + +```python +class MarketDataSource(ABC): + """Contract for market data providers. + + Implementations push price updates into a shared PriceCache on their own + schedule. Downstream code never calls the data source directly for prices — + it reads from the cache. + """ + + # --- lifecycle: unchanged from the shipped interface --- + @abstractmethod + async def start(self, tickers: list[str]) -> None: ... + + @abstractmethod + async def stop(self) -> None: ... + + @abstractmethod + async def add_ticker(self, ticker: str) -> None: ... + + @abstractmethod + async def remove_ticker(self, ticker: str) -> None: ... + + @abstractmethod + def get_tickers(self) -> list[str]: ... + + # --- new --- + @abstractmethod + def describe(self) -> SourceStatus: + """Introspection for GET /api/health. + + MUST NOT raise and MUST NOT do I/O — it is called from a request handler + and its entire job is to still work when the source is broken. + """ + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + """Historical series for a chart's first paint. + + Concrete default rather than an abstractmethod: a source that has no + history is a working source. Returning [] makes the frontend fall back + to accumulating from SSE, exactly as PLAN.md §10 describes. + """ + return [] +``` + +`get_history` on the interface is the better form of the ring buffer that `PLAN.md` §13.2 +item 15 recommends: the simulator serves its own deque, Massive-backed sources serve genuine +intraday minute bars, and the frontend calls one endpoint without knowing which it got. + +--- + +## 6. Capability Probe — `app/market/capabilities.py` (new) + +The single most consequential research finding (`MASSIVE_API.md` §4): **a free Massive key +authenticates successfully and then refuses to return any live price.** Snapshots, last +trade, and anything dated today are all `NOT_AUTHORIZED` on the Basic tier. So the presence +of `MASSIVE_API_KEY` tells you almost nothing: + +| Key state | What actually works | Source chosen (§7) | +|---|---|---| +| Absent | nothing | `SimulatorDataSource` | +| Present, Basic (free) | historical bars, yesterday's closes; **no live price** | `AnchoredSimulatorDataSource` | +| Present, Starter/Developer | snapshots, 15-minute delayed | `MassiveDataSource` (15 s poll) | +| Present, Advanced | snapshots, real time | `MassiveDataSource` (5 s poll) | +| Present, invalid/revoked | nothing | `SimulatorDataSource` + reason | + +Establish this once at startup, with two calls, instead of discovering it through a silent +stream of swallowed exceptions. + +```python +"""Detect what a Massive API key is actually entitled to. + +Two calls, run once from the factory at startup. Costs 2 of the free tier's +5-per-minute budget and is immutable for the process lifetime. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass + +import urllib3.exceptions +from massive import RESTClient +from massive.exceptions import AuthError, BadResponse + +from .models import to_epoch_seconds + +logger = logging.getLogger(__name__) + +# A snapshot print older than this means the plan serves delayed data. +DELAYED_THRESHOLD_SECONDS = 300.0 + + +@dataclass(frozen=True, slots=True) +class MassiveCapabilities: + """What a given API key is actually allowed to do.""" + + valid: bool # the key authenticates at all + realtime: bool # snapshot / last-trade endpoints are entitled + end_of_day: bool # aggregate endpoints are entitled + detail: str # human-readable, ends up in /api/health + delay_seconds: float | None = None # observed quote age when realtime is True + + @property + def poll_interval(self) -> float: + """Snapshot poll cadence implied by the plan (MARKET_INTERFACE.md §5). + + Delayed plans have unlimited call budgets but 15-minute-old data, so + polling faster than 15s buys nothing. + """ + if self.delay_seconds and self.delay_seconds > DELAYED_THRESHOLD_SECONDS: + return 15.0 + return 5.0 + + +def probe_capabilities(api_key: str) -> MassiveCapabilities: + """Classify a key into one of the five states above. Never raises. + + SYNCHRONOUS — the Massive SDK is urllib3-based. Call it as + `await asyncio.to_thread(probe_capabilities, api_key)`. + """ + try: + # retries=0 is essential: the SDK's default retries=3 backs off at + # 0.0/0.2/0.4s, i.e. three more requests inside the same 60s window + # that just rejected us (MASSIVE_API.md §7). + client = RESTClient(api_key=api_key, retries=0, read_timeout=5.0) + except AuthError: + return MassiveCapabilities(False, False, False, "no API key configured") + + # 1. Cheapest possible entitlement test for live data. + try: + snapshots = client.get_snapshot_all(market_type="stocks", tickers=["AAPL"]) + delay = _observed_delay(snapshots) + if delay is not None and delay > DELAYED_THRESHOLD_SECONDS: + detail = f"delayed snapshots entitled (~{delay / 60:.0f} min behind)" + else: + detail = "real-time snapshots entitled" + return MassiveCapabilities(True, True, True, detail, delay_seconds=delay) + except BadResponse as e: + if "NOT_AUTHORIZED" not in str(e): + return MassiveCapabilities(False, False, False, f"unexpected response: {e}") + # fall through — entitlement, not a bad key + except urllib3.exceptions.MaxRetryError as e: + # Rate limited or unreachable. MaxRetryError is NOT a massive.exceptions + # subclass, so `except BadResponse` alone misses it entirely. + return MassiveCapabilities(False, False, False, f"unreachable or rate-limited: {e}") + except Exception as e: # noqa: BLE001 - the probe must never take the app down + return MassiveCapabilities(False, False, False, f"probe failed: {e}") + + # 2. Snapshots refused. Valid key on a lower plan, or a dead key? + try: + client.get_previous_close_agg("AAPL") + return MassiveCapabilities(True, False, True, "end-of-day only (Basic tier)") + except Exception as e: # noqa: BLE001 + return MassiveCapabilities(False, False, False, f"key rejected: {e}") + + +def _observed_delay(snapshots) -> float | None: + """Age of the newest last-trade print, in seconds. None if unreadable. + + Distinguishes Advanced (real time) from Starter/Developer (15 min delayed) + without a second API call. sip_timestamp is NANOseconds (MASSIVE_API.md §6). + """ + try: + newest = max( + to_epoch_seconds(s.last_trade.sip_timestamp, "ns") + for s in snapshots + if getattr(s, "last_trade", None) is not None + ) + except (AttributeError, TypeError, ValueError): + return None + return max(0.0, time.time() - newest) +``` + +Two rejection messages exist and the difference is diagnostic gold (`MASSIVE_API.md` §4): +*"You are not entitled to this data"* means the **endpoint** is out of plan; *"Your plan +doesn't include this data timeframe"* means the endpoint is fine but the **date** is too +recent. Both contain `NOT_AUTHORIZED`, which is why the probe matches on that substring. + +--- + +## 7. Source Selection — `app/market/factory.py` (rewritten) + +```python +"""Factory for creating market data sources.""" + +from __future__ import annotations + +import asyncio +import logging +import os + +from .anchored import AnchoredSimulatorDataSource +from .cache import PriceCache +from .capabilities import probe_capabilities +from .interface import MarketDataSource +from .massive_client import MassiveDataSource +from .simulator import SimulatorDataSource + +logger = logging.getLogger(__name__) + + +async def create_market_data_source(price_cache: PriceCache) -> MarketDataSource: + """Select a market data source. Never raises; always returns a working source. + + Async because the capability probe makes real network calls. Called exactly + once, from the FastAPI lifespan handler. Returns an UNSTARTED source — the + caller must await source.start(tickers). + """ + api_key = os.environ.get("MASSIVE_API_KEY", "").strip() + + if not api_key: + logger.info("No MASSIVE_API_KEY — using the GBM simulator") + return SimulatorDataSource(price_cache) + + caps = await asyncio.to_thread(probe_capabilities, api_key) + + if caps.realtime: + logger.info("Massive: %s — polling every %.0fs", caps.detail, caps.poll_interval) + return MassiveDataSource( + api_key=api_key, + price_cache=price_cache, + poll_interval=caps.poll_interval, + capability_detail=caps.detail, + ) + + if caps.end_of_day: + logger.warning( + "Massive key is end-of-day only (Basic tier). Anchoring the simulator to " + "real closing prices — displayed prices are SIMULATED, not live." + ) + return AnchoredSimulatorDataSource(api_key=api_key, price_cache=price_cache) + + logger.error("MASSIVE_API_KEY unusable (%s) — falling back to the simulator", caps.detail) + return SimulatorDataSource(price_cache, status_detail=f"Massive key unusable: {caps.detail}") +``` + +Three rules encoded here, in order of importance: + +1. **Never boot into a broken state.** Every branch produces moving prices. +2. **Never silently mislead.** Simulated prices carry `live=False` all the way to + `/api/health` and the frontend's "SIMULATED" badge. +3. **Probe once.** Two calls at startup, not two per poll. + +### Poll intervals + +| Source | Cadence | Why | +|---|---|---| +| `SimulatorDataSource` | 500 ms | local computation, no budget (`PLAN.md` §6) | +| `AnchoredSimulatorDataSource` | 500 ms tick, **1** anchor fetch at startup (+ hourly re-anchor) | GBM is local; anchors are one grouped-daily call | +| `MassiveDataSource`, Advanced | 5 s | unlimited calls; 5 s is plenty for a terminal | +| `MassiveDataSource`, Starter/Developer | 15 s | the data is 15 minutes old anyway | + +--- + +## 8. The Simulator — `app/market/simulator.py` + +For nearly every student running this project the simulator **is** the product: no key and a +free key both land here. `MARKET_SIMULATOR.md` measured the shipped implementation; this +section is the corrected code. + +### 8.1 The model, and why the calibration is already right + +``` +S(t+Δt) = S(t) · exp[ (μ − σ²/2)·Δt + σ·√Δt·Z ] +``` + +`Δt = 0.5 / (252 × 6.5 × 3600) ≈ 8.48 × 10⁻⁸`. One trading day is 46,800 ticks and +46,800 × Δt = 1/252 exactly, so a simulated day reproduces the target daily volatility by +construction — verified at `1.3425%` realised against a `1.3859%` target over 200 one-day +AAPL runs (`MARKET_SIMULATOR.md` §2). **Do not touch `DEFAULT_DT` or the Itô `−σ²/2` term.** + +### 8.2 The one real bug: shocks are permanent level shifts + +Shipped code applies `self._prices[ticker] *= 1 + shock` at `p = 0.001` per ticker per tick. +That is 7.2 permanent 2–5% jumps per ticker per hour — a random walk of shocks layered on +GBM, which inflates realised volatility ~20× and makes every per-ticker σ decorative +(measured: 10.02% one-hour return sd against a 0.505% pure-GBM baseline). + +The fix is a **decaying overlay**: the underlying price keeps following calibrated GBM, and +the shock is a separate transient component added on top of the *displayed* price. + +```python +@dataclass +class Shock: + """A transient, mean-reverting price dislocation. + + Real intraday spikes substantially revert. Modelling the shock as a decaying + overlay keeps the drama visible on the chart while leaving the long-run + distribution governed by sigma. + """ + + magnitude: float # signed, e.g. -0.03 + decay: float = 0.985 # per-tick multiplier -> half-life ~46 ticks (~23s) +``` + +```python + def step(self) -> dict[str, float]: + """Advance every ticker one time step. Returns {ticker: displayed_price}. + + Hot path — runs every 500ms. One vectorised normal draw for all tickers, + not one per ticker. + """ + n = len(self._tickers) + if n == 0: + return {} + + z = np.random.standard_normal(n) + if self._cholesky is not None: + z = self._cholesky @ z + + result: dict[str, float] = {} + for i, ticker in enumerate(self._tickers): + params = self._params[ticker] + mu, sigma = params["mu"], params["sigma"] + + # 1. GBM evolves the underlying price. Full precision, never rounded + # in place — rounding here accumulates a drift bias over 10k+ ticks. + drift = (mu - 0.5 * sigma**2) * self._dt + diffusion = sigma * math.sqrt(self._dt) * z[i] + self._prices[ticker] *= math.exp(drift + diffusion) + + # 2. Shocks are a separate, decaying overlay on the DISPLAYED price. + if random.random() < self._event_prob: + self._shocks[ticker] = Shock( + magnitude=random.uniform(0.015, 0.04) * random.choice([-1, 1]) + ) + logger.debug("Shock on %s: %+.2f%%", ticker, self._shocks[ticker].magnitude * 100) + + shock = self._shocks.get(ticker) + if shock is not None: + displayed = self._prices[ticker] * (1 + shock.magnitude) + shock.magnitude *= shock.decay + if abs(shock.magnitude) < 1e-4: + del self._shocks[ticker] + else: + displayed = self._prices[ticker] + + result[ticker] = round(displayed, 2) + + return result +``` + +Paired with `event_probability = 1e-4` (down from `1e-3`), a 10-ticker watchlist sees about +**1.2 events per 10-minute demo** — enough that something happens while the user is watching, +without a shock every 50 seconds. + +### 8.3 Deterministic seeds for unknown tickers + +`PLAN.md` §13.2 item 8 asks what happens when a user — or the LLM's `watchlist_changes` — +requests a symbol the simulator has never heard of. **Answer: synthesise, never reject.** The +demo must not dead-end on a typo. But the shipped `random.uniform(50.0, 300.0)` gives `ZZZZ` +a different price on every restart, so a held position's cost basis jumps between container +restarts and the P&L chart lies. + +```python +def _synthesize_seed(ticker: str) -> float: + """Stable pseudo-price for an unknown symbol. Same ticker, same price, always.""" + h = int(hashlib.sha256(ticker.encode()).hexdigest()[:8], 16) + return round(20.0 + (h % 48_000) / 100.0, 2) # $20.00 - $500.00 +``` + +Seed resolution order: **`seed_overrides` → `SEED_PRICES` → `_synthesize_seed`.** Unknown +tickers take `DEFAULT_PARAMS` (σ = 0.25, μ = 0.05) and cross-group correlation. + +```python +class GBMSimulator: + TRADING_SECONDS_PER_YEAR = 252 * 6.5 * 3600 # 5,896,800 + DEFAULT_DT = 0.5 / TRADING_SECONDS_PER_YEAR # ~8.48e-8 + + def __init__( + self, + tickers: list[str], + dt: float = DEFAULT_DT, + event_probability: float = 1e-4, # was 1e-3 — see §8.2 + seed_overrides: dict[str, float] | None = None, # real closes when available + ) -> None: + self._dt = dt + self._event_prob = event_probability + self._seed_overrides = {k.upper(): v for k, v in (seed_overrides or {}).items()} + self._tickers: list[str] = [] + self._prices: dict[str, float] = {} + self._params: dict[str, dict[str, float]] = {} + self._shocks: dict[str, Shock] = {} + self._retired: dict[str, float] = {} # price kept across remove/re-add + self._cholesky: np.ndarray | None = None + + for ticker in tickers: + self._add_ticker_internal(ticker) + self._rebuild_cholesky() + + def _add_ticker_internal(self, ticker: str) -> None: + """Add without rebuilding Cholesky — used for batch initialisation.""" + if ticker in self._prices: + return + seed = ( + self._retired.pop(ticker, None) # re-added: resume where it left off + or self._seed_overrides.get(ticker) + or SEED_PRICES.get(ticker) + or _synthesize_seed(ticker) + ) + self._tickers.append(ticker) + self._prices[ticker] = seed + self._params[ticker] = self._resolve_params(ticker, seed) + + def remove_ticker(self, ticker: str) -> None: + if ticker not in self._prices: + return + # Remember the price: a position you still hold must not jump when its + # watchlist row is toggled off and back on. + self._retired[ticker] = self._prices.pop(ticker) + self._tickers.remove(ticker) + self._params.pop(ticker, None) + self._shocks.pop(ticker, None) + self._rebuild_cholesky() +``` + +Sub-$20 names are the one liveness cliff: per-tick sd is `S·σ·√Δt`, so below roughly $20 a +tick rounds to no visible change. NFLX at 82.73 is the measured floor case at 58% visible +ticks — fine. Floor σ for anything cheaper: + +```python +# seed_prices.py +LOW_PRICE_THRESHOLD = 20.0 # below this, a 2dp display quantum eats most ticks +LOW_PRICE_SIGMA_FLOOR = 0.35 +``` + +```python + def _resolve_params(self, ticker: str, seed: float) -> dict[str, float]: + params = dict(TICKER_PARAMS.get(ticker, DEFAULT_PARAMS)) + if seed < LOW_PRICE_THRESHOLD: + params["sigma"] = max(params["sigma"], LOW_PRICE_SIGMA_FLOOR) + return params +``` + +### 8.4 Correlation, and the 500 that is waiting to happen + +Independent tickers read as fake within seconds. The correlation matrix (tech ∩ tech 0.6, +finance ∩ finance 0.5, TSLA and cross-sector 0.3) is applied via Cholesky: draw `n` +independent normals, multiply by `L` where `L Lᵀ = C`. + +The structure is a block matrix of equicorrelated groups, so its minimum eigenvalue is pinned +at `1 − ρ_max = 0.4` regardless of size — verified positive-definite to n = 110. But +`np.linalg.cholesky` raises `LinAlgError` on a non-PD matrix and `_rebuild_cholesky` is +called from `add_ticker`, so the failure mode is a **user-facing 500 on a watchlist add**. +Three lines make that impossible: + +```python + def _rebuild_cholesky(self) -> None: + """Rebuild the Cholesky factor of the correlation matrix. + + O(n^2), called only on watchlist edits — never in the tick loop. + """ + n = len(self._tickers) + if n <= 1: + self._cholesky = None + return + + corr = np.eye(n) + for i in range(n): + for j in range(i + 1, n): + rho = self._pairwise_correlation(self._tickers[i], self._tickers[j]) + corr[i, j] = corr[j, i] = rho + + try: + self._cholesky = np.linalg.cholesky(corr) + except np.linalg.LinAlgError: + # Degrade to uncorrelated moves rather than 500 a watchlist add. + logger.warning("Correlation matrix not positive-definite at n=%d — using identity", n) + self._cholesky = None +``` + +### 8.5 History: ring buffer plus prefill + +`PLAN.md` §13.2 item 15 — charts are empty on first paint and a refresh throws away +everything accumulated from SSE. A bounded deque fixes it for ~480 KB at 50 tickers. + +```python +class SimulatorDataSource(MarketDataSource): + HISTORY_POINTS = 600 # 600 ticks x 500ms = 5 minutes per ticker + + def __init__( + self, + price_cache: PriceCache, + update_interval: float = 0.5, + event_probability: float = 1e-4, + seed_overrides: dict[str, float] | None = None, + status_detail: str | None = None, + ) -> None: + self._cache = price_cache + self._interval = update_interval + self._event_prob = event_probability + self._seed_overrides = {k.upper(): v for k, v in (seed_overrides or {}).items()} + self._status_detail = status_detail + self._sim: GBMSimulator | None = None + self._task: asyncio.Task | None = None + self._opens: dict[str, float] = {} # session anchors, fixed after start + self._history: dict[str, deque[tuple[float, float]]] = defaultdict( + lambda: deque(maxlen=self.HISTORY_POINTS) + ) +``` + +**Prefill so the first paint is a chart, not a dot.** Step the model forward +`HISTORY_POINTS` times with no sleeping, back-date the timestamps, and keep the final price +as the current one — so the history joins the live series continuously instead of jumping: + +```python + def _prefill_history(self) -> None: + """Generate ~5 minutes of plausible history before the first tick. + + Costs about half a second of CPU. Timestamps are back-dated so the + prefilled path ends exactly at the current price — no discontinuity + where synthetic history meets the live stream. + """ + assert self._sim is not None + now = time.time() + start = now - self.HISTORY_POINTS * self._interval + for i in range(self.HISTORY_POINTS): + ts = start + i * self._interval + for ticker, price in self._sim.step().items(): + self._history[ticker].append((ts, price)) + + async def start(self, tickers: list[str]) -> None: + self._sim = GBMSimulator( + tickers=tickers, + event_probability=self._event_prob, + seed_overrides=self._seed_overrides, + ) + # The seed price IS the session open — capture it before any stepping. + self._opens = {t: p for t in tickers if (p := self._sim.get_price(t)) is not None} + + self._prefill_history() + + # Seed the cache so SSE has data on the very first connection. + for ticker, open_price in self._opens.items(): + price = self._sim.get_price(ticker) + if price is not None: + self._cache.update(ticker=ticker, price=price, open_price=open_price) + + self._task = asyncio.create_task(self._run_loop(), name="simulator-loop") + logger.info("Simulator started with %d tickers", len(tickers)) + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + series = self._history.get(ticker.upper(), ()) + return [PricePoint(epoch_to_iso(t), p) for t, p in list(series)[-points:]] +``` + +### 8.6 The tick loop + +```python + async def _run_loop(self) -> None: + """Step the simulation, write to the cache, record history, sleep.""" + while True: + try: + if self._sim: + now = time.time() + for ticker, price in self._sim.step().items(): + # open_price=None -> the cache keeps the sticky anchor (§4) + self._cache.update(ticker=ticker, price=price, timestamp=now) + self._history[ticker].append((now, price)) + except Exception: + # NEVER let one bad tick kill the task: a raise here would end the + # background loop and freeze every price with no error path. + logger.exception("Simulator step failed") + await asyncio.sleep(self._interval) +``` + +`asyncio.sleep(interval)` *after* the work means the true period is `interval + work`, so +ticks drift slightly slower than 500 ms. This is invisible on screen — do not add a +compensating scheduler. + +### 8.7 `add_ticker`, `remove_ticker`, `describe` + +```python + async def add_ticker(self, ticker: str) -> None: + ticker = ticker.upper().strip() + if not self._sim: + return + self._sim.add_ticker(ticker) # synthesises a seed if unknown — never dead-ends + price = self._sim.get_price(ticker) + if price is not None: + self._opens.setdefault(ticker, price) + self._cache.update(ticker=ticker, price=price, open_price=self._opens[ticker]) + logger.info("Simulator: added ticker %s", ticker) + + async def remove_ticker(self, ticker: str) -> None: + ticker = ticker.upper().strip() + if self._sim: + self._sim.remove_ticker(ticker) + self._cache.remove(ticker) + # History is deliberately KEPT: re-adding a ticker restores its chart. + logger.info("Simulator: removed ticker %s", ticker) + + def describe(self) -> SourceStatus: + return SourceStatus( + name="simulator", + live=False, + detail=self._status_detail or "GBM simulation from static seed prices", + tickers=len(self._sim.get_tickers()) if self._sim else 0, + cache_populated=len(self._cache) > 0, + ) +``` + +`GBMSimulator.remove_ticker` should retain the price in a `_retired` dict so that +remove-then-add restores the price the ticker had rather than resetting to seed — a position +you still hold must not jump when its watchlist row is toggled. + +### 8.8 Keep the two-class split + +`GBMSimulator` (pure, synchronous, no I/O, no cache reference) and `SimulatorDataSource` (the +async adapter) is the best structural decision in the shipped code. It is what lets the +statistical tests in §15 run tens of thousands of steps per second with no event loop. **Keep +the model free of the transport.** + +### 8.9 Refresh the static seed table + +`SEED_PRICES` has rotted badly — NVDA seeded at 800.00 against a real 224.41 (3.6× high, post +splits), NFLX at 600.00 against 82.73 (7.3×). Replace with the verified 2026-09-02 closes +from `MASSIVE_API.md` §5.1, and note in a comment that this table is now only the **no-key +fallback**; §9's anchoring removes the maintenance burden whenever a key is present. + +```python +# Real closes, 2026-09-02 (MASSIVE_API.md §5.1). Fallback only — any Massive key +# anchors to live closes instead (see anchored.py). +SEED_PRICES: dict[str, float] = { + "AAPL": 324.96, "GOOGL": 337.12, "MSFT": 496.82, "AMZN": 254.98, "TSLA": 357.01, + "NVDA": 224.41, "META": 592.85, "JPM": 356.22, "V": 378.40, "NFLX": 82.73, +} +``` + +--- + +## 9. Anchored Simulator — `app/market/anchored.py` (new) + +The source that makes a free key useful. It fetches **real closing prices** with the one +free-tier call that prices the entire market, then runs the GBM simulator forward from those +anchors. + +A student with a free key gets real price levels, real relative valuations, a watchlist that +ticks, and a portfolio that changes — the whole demo works, for one API call at startup, well +inside a 5-per-minute budget. + +```python +"""GBM simulation anchored to real Massive closing prices.""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import date, timedelta + +from massive import RESTClient + +from .cache import PriceCache +from .interface import MarketDataSource +from .models import PricePoint, SourceStatus, epoch_to_iso, to_epoch_seconds +from .simulator import SimulatorDataSource + +logger = logging.getLogger(__name__) + +RE_ANCHOR_INTERVAL_SECONDS = 3600.0 # 24 calls/day out of 7,200 — cheap honesty +MAX_ANCHOR_LOOKBACK_DAYS = 7 # covers a long holiday weekend + + +class AnchoredSimulatorDataSource(MarketDataSource): + """Real price LEVELS from Massive, synthetic price MOTION from the simulator. + + Bridges the Basic-tier gap: one free-tier call prices every ticker, and the + GBM simulator supplies the movement the snapshot endpoints would have given + us on a paid plan. Prices are explicitly NOT live — describe() says so and + the frontend badges it SIMULATED. + """ + + def __init__(self, api_key: str, price_cache: PriceCache) -> None: + self._client = RESTClient(api_key=api_key, retries=0, read_timeout=10.0) + self._cache = price_cache + self._sim: SimulatorDataSource | None = None + self._anchors: dict[str, float] = {} + self._anchor_date: str | None = None + self._reanchor_task: asyncio.Task | None = None + + async def start(self, tickers: list[str]) -> None: + tickers = [t.upper().strip() for t in tickers] + self._anchors, self._anchor_date = await asyncio.to_thread(self._fetch_anchors, tickers) + + # Real closes where we have them; the static seed table covers the rest. + self._sim = SimulatorDataSource( + self._cache, + seed_overrides=self._anchors, + status_detail=self._detail(), + ) + await self._sim.start(tickers) + self._reanchor_task = asyncio.create_task(self._reanchor_loop(), name="re-anchor") + + async def stop(self) -> None: + for task in (self._reanchor_task,): + if task and not task.done(): + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + self._reanchor_task = None + if self._sim: + await self._sim.stop() + + async def add_ticker(self, ticker: str) -> None: + # Delegates to the simulator, which synthesises a deterministic seed for + # any symbol the anchor set does not cover (PLAN.md §13.2 item 8). + if self._sim: + await self._sim.add_ticker(ticker) + + async def remove_ticker(self, ticker: str) -> None: + if self._sim: + await self._sim.remove_ticker(ticker) + + def get_tickers(self) -> list[str]: + return self._sim.get_tickers() if self._sim else [] + + def describe(self) -> SourceStatus: + return SourceStatus( + name="anchored-simulator", + live=False, + detail=self._detail(), + tickers=len(self.get_tickers()), + cache_populated=len(self._cache) > 0, + ) + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + """Real minute bars from the last completed session — free-tier allowed. + + Genuinely better than synthetic backfill: the chart opens on the shape + the market actually traded. Falls back to the simulator's ring buffer. + """ + if not self._anchor_date: + return await self._sim.get_history(ticker, points) if self._sim else [] + try: + bars = await asyncio.to_thread( + self._client.get_aggs, + ticker.upper().strip(), + 1, + "minute", + self._anchor_date, + self._anchor_date, + limit=50_000, + ) + except Exception as e: # noqa: BLE001 - history is best-effort + logger.warning("Minute-bar history failed for %s: %s", ticker, e) + return await self._sim.get_history(ticker, points) if self._sim else [] + # Aggregate timestamps are MILLIseconds (MASSIVE_API.md §6). + return [ + PricePoint(epoch_to_iso(to_epoch_seconds(b.timestamp, "ms")), b.close) + for b in bars[-points:] + ] + + # --- Internals --- + + def _fetch_anchors(self, tickers: list[str]) -> tuple[dict[str, float], str | None]: + """ONE API call prices every ticker. Walks back over weekends and holidays. + + Synchronous — always call via asyncio.to_thread. + """ + wanted = set(tickers) + day = date.today() + for _ in range(MAX_ANCHOR_LOOKBACK_DAYS): + day -= timedelta(days=1) + iso = day.isoformat() + try: + bars = self._client.get_grouped_daily_aggs(iso, adjusted=True) + except Exception as e: # noqa: BLE001 - try the previous day + logger.warning("Anchor fetch failed for %s: %s", iso, e) + continue + if not bars: + continue # weekend or holiday: resultsCount 0, not an error + found = {b.ticker: b.close for b in bars if b.ticker in wanted} + logger.info("Anchored %d/%d tickers to %s closes", len(found), len(wanted), iso) + return found, iso + logger.warning("No anchors available — falling back to the static seed table") + return {}, None + + async def _reanchor_loop(self) -> None: + """Re-fetch closes hourly so an overnight container picks up the new session.""" + while True: + await asyncio.sleep(RE_ANCHOR_INTERVAL_SECONDS) + try: + anchors, anchor_date = await asyncio.to_thread( + self._fetch_anchors, self.get_tickers() + ) + except Exception: # noqa: BLE001 + logger.exception("Re-anchor failed") + continue + if not anchors or anchor_date == self._anchor_date: + continue + # A NEW session's closes: reset the daily baseline, but do not jump + # the live price — the simulated path continues from where it is. + self._anchors, self._anchor_date = anchors, anchor_date + for ticker, close in anchors.items(): + current = self._cache.get_price(ticker) + if current is not None: + self._cache.update(ticker=ticker, price=current, open_price=close) + logger.info("Re-anchored daily baselines to %s closes", anchor_date) + + def _detail(self) -> str: + if self._anchor_date: + return ( + f"simulated from real {self._anchor_date} closes " + f"({len(self._anchors)} tickers anchored)" + ) + return "simulated from static seed prices (anchor fetch failed)" +``` + +**`adjusted=True` matters.** Unadjusted series show a false −90% cliff on a 10:1 split; the +gap between the repo's 800.00 NVDA seed and the real 224.41 is mostly splits, not a crash. + +**Honesty note.** Showing real levels with synthetic motion, badged SIMULATED, is a +deliberate choice over showing yesterday's frozen closes — which is more truthful and +completely lifeless. `MARKET_INTERFACE.md` §12 flags it for sign-off; this design proceeds +with the badged simulation. + +--- + +## 10. Massive Source — `app/market/massive_client.py` (rewritten) + +The shipped client has four defects beyond the entitlement gate, all of which fail +*invisibly* because `except Exception` swallows them once per poll, forever. + +| # | Defect | Consequence | +|---|---|---| +| 1 | `snap.last_trade.timestamp / 1000.0` | wrong attribute (`AttributeError`, swallowed) **and** wrong unit — the field is `sip_timestamp` in **nanoseconds** | +| 2 | `open_price` never captured | the daily-% column has no baseline, though `prev_day.close` arrives free with every poll | +| 3 | `except Exception` around everything | rate limiting logs at error level forever with no user-facing signal | +| 4 | `retries=3` (SDK default) | 0.0/0.2/0.4 s backoff burns three more requests inside the same 60 s window that just rejected you | + +```python +"""Massive (Polygon.io) API client for real market data.""" + +from __future__ import annotations + +import asyncio +import logging +import time + +import urllib3.exceptions +from massive import RESTClient +from massive.exceptions import AuthError, BadResponse +from massive.rest.models import SnapshotMarketType + +from .cache import PriceCache +from .interface import MarketDataSource +from .models import PricePoint, SourceStatus, epoch_to_iso, to_epoch_seconds + +logger = logging.getLogger(__name__) + +MAX_BACKOFF_SECONDS = 120.0 +STALE_AFTER_MISSED_POLLS = 3 # no successful poll in 3 intervals -> not live +# A quote older than this is not "the current market" on any plan, delayed +# included. It is how a weekend or a halted symbol stops rendering as live. +MAX_QUOTE_AGE_SECONDS = 1800.0 + + +class MassiveDataSource(MarketDataSource): + """MarketDataSource backed by the Massive REST snapshot endpoint. + + One call per poll for ALL watched tickers (never one call per ticker — see + MASSIVE_API.md §7). Requires a snapshot-entitled key; the factory only + constructs this after probe_capabilities() confirms entitlement. + """ + + def __init__( + self, + api_key: str, + price_cache: PriceCache, + poll_interval: float = 15.0, + capability_detail: str = "snapshots entitled", + ) -> None: + self._api_key = api_key + self._cache = price_cache + self._interval = poll_interval + self._detail = capability_detail + self._tickers: list[str] = [] + self._task: asyncio.Task | None = None + self._client: RESTClient | None = None + # Health state, all read by describe() + self._last_success: float | None = None + self._newest_quote: float | None = None # freshest sip_timestamp seen + self._consecutive_failures = 0 + self._last_error: str | None = None + self._unpriced: set[str] = set() # requested but absent from the response + + async def start(self, tickers: list[str]) -> None: + # retries=0: the SDK's urllib3 retry raises MaxRetryError (NOT a + # massive.exceptions type) and its sub-second backoff is useless against + # a 60s rolling window. Back off at the poll-loop level instead. + self._client = RESTClient(api_key=self._api_key, retries=0, read_timeout=10.0) + self._tickers = [t.upper().strip() for t in tickers] + + await self._poll_once() # populate the cache before the first SSE connection + self._task = asyncio.create_task(self._poll_loop(), name="massive-poller") + logger.info( + "Massive poller started: %d tickers, %.1fs interval", len(self._tickers), self._interval + ) + + async def stop(self) -> None: + if self._task and not self._task.done(): + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + self._task = None + self._client = None + logger.info("Massive poller stopped") + + async def add_ticker(self, ticker: str) -> None: + ticker = ticker.upper().strip() + if ticker not in self._tickers: + self._tickers.append(ticker) + logger.info("Massive: added %s (priced on the next poll)", ticker) + + async def remove_ticker(self, ticker: str) -> None: + ticker = ticker.upper().strip() + self._tickers = [t for t in self._tickers if t != ticker] + self._unpriced.discard(ticker) + self._cache.remove(ticker) + + def get_tickers(self) -> list[str]: + return list(self._tickers) + + def describe(self) -> SourceStatus: + """Never raises, never does I/O — it must work when everything else is broken.""" + now = time.time() + poll_stale = ( + self._last_success is None + or now - self._last_success > self._interval * STALE_AFTER_MISSED_POLLS + ) + # Polls can succeed all weekend while returning Friday's prints, so + # freshness is judged on the QUOTE age, not just on the poll succeeding. + quote_stale = self._newest_quote is None or now - self._newest_quote > MAX_QUOTE_AGE_SECONDS + stale = poll_stale or quote_stale + + detail = self._detail + if poll_stale: + age = "never" if self._last_success is None else f"{now - self._last_success:.0f}s ago" + detail = f"STALE — last successful poll {age}" + if self._last_error: + detail += f" ({self._last_error})" + elif quote_stale: + mins = (now - self._newest_quote) / 60 if self._newest_quote else 0 + detail = f"{self._detail}; market closed or halted — newest quote {mins:.0f} min old" + elif self._unpriced: + detail = f"{self._detail}; no data for {', '.join(sorted(self._unpriced))}" + return SourceStatus( + name="massive", + live=not stale, # a frozen quote must never render as live + detail=detail, + tickers=len(self._tickers), + cache_populated=len(self._cache) > 0, + ) + + async def get_history(self, ticker: str, points: int = 120) -> list[PricePoint]: + """Real minute bars for today (paid keys) or the last session.""" + if not self._client: + return [] + today = time.strftime("%Y-%m-%d", time.gmtime()) + try: + bars = await asyncio.to_thread( + self._client.get_aggs, ticker.upper().strip(), 1, "minute", today, today, + limit=50_000, + ) + except Exception as e: # noqa: BLE001 - history is best-effort + logger.warning("History failed for %s: %s", ticker, e) + return [] + return [ + PricePoint(epoch_to_iso(to_epoch_seconds(b.timestamp, "ms")), b.close) + for b in bars[-points:] + ] + + # --- Internals --- + + async def _poll_loop(self) -> None: + """Poll on interval, backing off exponentially while failing.""" + while True: + delay = self._interval + if self._consecutive_failures: + delay = min(self._interval * 2**self._consecutive_failures, MAX_BACKOFF_SECONDS) + await asyncio.sleep(delay) + await self._poll_once() + + async def _poll_once(self) -> None: + if not self._tickers or not self._client: + return + try: + snapshots = await asyncio.to_thread(self._fetch_snapshots) + except AuthError as e: + self._fail(f"authentication failed: {e}", level=logging.ERROR) + return + except BadResponse as e: + if "NOT_AUTHORIZED" in str(e): + # Permanent: the probe said we were entitled and the plan changed + # under us. Back off hard rather than burning the budget. + self._fail(f"not entitled: {e}", level=logging.ERROR) + else: + self._fail(f"bad response: {e}", level=logging.WARNING) + return + except urllib3.exceptions.MaxRetryError as e: + # Rate limited or unreachable — this is what a 429 looks like. + self._fail(f"rate limited or unreachable: {e}", level=logging.WARNING) + return + except Exception as e: # noqa: BLE001 - the loop must survive anything + self._fail(f"unexpected: {e}", level=logging.WARNING) + return + + seen: set[str] = set() + for snap in snapshots: + price, ts = self._extract_price(snap) + if price is None: + continue + self._cache.update( + ticker=snap.ticker, + price=price, + # The real previous close — the daily-% baseline, free with every poll. + open_price=snap.prev_day.close if getattr(snap, "prev_day", None) else None, + timestamp=ts, + ) + seen.add(snap.ticker) + if ts is not None and (self._newest_quote is None or ts > self._newest_quote): + self._newest_quote = ts + + # Symbols the API simply omitted (unknown/delisted) are recorded rather + # than vanishing silently, so /api/watchlist can flag them. + self._unpriced = {t for t in self._tickers if t not in seen} + if seen: + self._last_success = time.time() + self._consecutive_failures = 0 + self._last_error = None + else: + self._fail("poll returned no usable snapshots", level=logging.WARNING) + logger.debug("Massive poll: priced %d/%d tickers", len(seen), len(self._tickers)) + + @staticmethod + def _extract_price(snap) -> tuple[float | None, float | None]: + """Last trade price and its timestamp in epoch seconds. + + sip_timestamp is NANOseconds. The shipped code read `.timestamp` (wrong + attribute) and divided by 1e3 (wrong unit by a factor of a million, + landing the quote in the year 52000). + """ + trade = getattr(snap, "last_trade", None) + if trade is None or trade.price is None: + return None, None + raw = getattr(trade, "sip_timestamp", None) + ts = to_epoch_seconds(raw, "ns") if raw else None + return float(trade.price), ts + + def _fail(self, message: str, level: int) -> None: + self._consecutive_failures += 1 + self._last_error = message + logger.log(level, "Massive poll failed (%d consecutive): %s", self._consecutive_failures, message) + + def _fetch_snapshots(self) -> list: + """Synchronous SDK call — always runs in a worker thread.""" + return self._client.get_snapshot_all( + market_type=SnapshotMarketType.STOCKS, + tickers=self._tickers, + ) +``` + +### Two kinds of stale + +`describe()` distinguishes them because they need different words in `/api/health`: + +- **Poll stale** — no successful call in three intervals. Network, rate limit, or a revoked + key. `_last_error` names it. +- **Quote stale** — polls succeed but the newest `sip_timestamp` is over 30 minutes old. + This is the closed-market case (`PLAN.md` §13.2 item 14): all weekend the snapshot endpoint + cheerfully returns Friday's prints, and without the quote-age check the terminal would + present a two-day-old price as live. + +Either flips `live` to `False`. The app keeps running and keeps showing the last known +prices — it just stops claiming they are current. A deployment that expects to sit through +weekends should prefer `AnchoredSimulatorDataSource`, which stays alive by design. + +--- + +## 11. SSE Streaming — `app/market/stream.py` + +Three properties of the shipped stream are deliberate and stay: + +- **The full price map on every change, not a delta.** At 10–50 tickers the payload is a + couple of KB; a delta protocol would need reconnection-resync logic for no measurable gain + (`PLAN.md` §13.5 item 40). **Do not optimise this into a delta.** +- **The `version` counter** for change detection — cheap, and it means an idle cache costs + one integer comparison per client per interval. +- **The `create_stream_router(cache)` factory**, which injects the cache without globals. + +The one addition is the heartbeat (`PLAN.md` §13.1 item 7). It barely matters at a 500 ms +simulator cadence and matters enormously in Massive mode: at a 15 s poll — or on a closed +market, where the version never changes at all — the connection sits silent long enough for +an intermediate proxy or a sleeping laptop to drop it with neither side noticing. + +```python +HEARTBEAT_SECONDS = 10.0 + + +async def _generate_events( + price_cache: PriceCache, + request: Request, + interval: float = 0.5, +) -> AsyncGenerator[str, None]: + """Yield SSE frames: the full price map whenever it changes, else a heartbeat.""" + yield "retry: 1000\n\n" # EventSource reconnect hint + + last_version = -1 + last_sent = time.monotonic() + client_ip = request.client.host if request.client else "unknown" + logger.info("SSE client connected: %s", client_ip) + + try: + while True: + if await request.is_disconnected(): + logger.info("SSE client disconnected: %s", client_ip) + break + + current_version = price_cache.version + if current_version != last_version: + last_version = current_version + prices = price_cache.get_all() + if prices: + payload = json.dumps({t: u.to_dict() for t, u in prices.items()}) + yield f"data: {payload}\n\n" + last_sent = time.monotonic() + elif time.monotonic() - last_sent > HEARTBEAT_SECONDS: + # An SSE comment: keeps the connection warm, ignored by EventSource. + yield ": ping\n\n" + last_sent = time.monotonic() + + await asyncio.sleep(interval) + except asyncio.CancelledError: + logger.info("SSE stream cancelled for: %s", client_ip) +``` + +SSE frame on the wire: + +``` +retry: 1000 + +data: {"AAPL": {"ticker": "AAPL", "price": 325.41, "previous_price": 325.38, "open_price": 324.96, "timestamp": "2026-09-03T17:42:11.413700Z", "tick_direction": "up", "change_today": 0.45, "change_percent_today": 0.1385}, "GOOGL": { ... }} + +: ping +``` + +--- + +## 12. HTTP Surface + +Two endpoints the market subsystem owns, beyond the SSE stream. + +### `GET /api/prices/{ticker}/history?points=120` + +First paint for the main chart and the sparklines. The frontend does not know or care whether +it receives simulated history or real minute bars. + +```python +@router.get("/api/prices/{ticker}/history") +async def price_history(ticker: str, request: Request, points: int = 120) -> dict: + source: MarketDataSource = request.app.state.market + series = await source.get_history(ticker.upper().strip(), points=min(points, 600)) + return { + "ticker": ticker.upper().strip(), + "source": source.describe().name, + "points": [p.to_dict() for p in series], + } +``` + +```json +{ + "ticker": "AAPL", + "source": "anchored-simulator", + "points": [ + {"timestamp": "2026-09-03T17:37:11.413700Z", "price": 324.91}, + {"timestamp": "2026-09-03T17:37:11.913700Z", "price": 324.94} + ] +} +``` + +An empty `points` array is a valid answer, not an error — the frontend falls back to +accumulating from SSE. + +### `GET /api/health` + +`PLAN.md` §13.5 item 38 asks health to report the active source and whether the cache is +populated. `describe()` hands both over, plus the sentence that ends the debugging session. + +```python +@router.get("/api/health") +async def health(request: Request) -> dict: + source: MarketDataSource = request.app.state.market + return {"status": "ok", "market_data": source.describe().to_dict()} +``` + +```json +{ + "status": "ok", + "market_data": { + "name": "anchored-simulator", + "live": false, + "detail": "simulated from real 2026-09-02 closes (10 tickers anchored)", + "tickers": 10, + "cache_populated": true + } +} +``` + +`live: false` is the frontend's cue to render a **SIMULATED** badge beside the connection dot. + +--- + +## 13. Lifespan Wiring + +Resolves `PLAN.md` §13.1 item 5: initialise the database **at startup, not on first request**, +because the market source needs the seeded watchlist before any request arrives. One code +path, no request-time locking. + +```python +@asynccontextmanager +async def lifespan(app: FastAPI): + init_database() # create + seed if empty + app.state.price_cache = PriceCache() + + tickers = get_tracked_tickers() # watchlist ∪ held positions + app.state.market = await create_market_data_source(app.state.price_cache) + await app.state.market.start(tickers) + + app.state.snapshots = asyncio.create_task(portfolio_snapshot_loop(app.state)) + try: + yield + finally: + app.state.snapshots.cancel() + await app.state.market.stop() +``` + +Two contracts this places on the backend agent: + +**`get_tracked_tickers()` returns watchlist ∪ held positions** (`PLAN.md` §13.2 item 10). +Removing a ticker from the watchlist must not stop pricing a position you still hold, or the +heatmap tile, the positions row, and the total portfolio value all go stale. + +```python +def get_tracked_tickers() -> list[str]: + """Everything that needs a live price: the watchlist plus anything held.""" + with connect() as db: + rows = db.execute( + "SELECT ticker FROM watchlist WHERE user_id = ? " + "UNION SELECT ticker FROM positions WHERE user_id = ? AND quantity > 1e-9", + (DEFAULT_USER_ID, DEFAULT_USER_ID), + ).fetchall() + return [r["ticker"] for r in rows] +``` + +**Watchlist and trade routes keep the source in sync.** Adding a ticker must call +`source.add_ticker()`; removing one must only call `source.remove_ticker()` when no position +remains. Trading an off-watchlist symbol auto-adds it (`PLAN.md` §13.2 item 9) so the trade +is priceable: + +```python +async def ensure_priced(app_state, ticker: str) -> float | None: + """Make a ticker priceable, then return its price. Used by the trade path.""" + if ticker not in app_state.price_cache: + await app_state.market.add_ticker(ticker) + for _ in range(20): # simulator prices instantly; Massive needs a poll + price = app_state.price_cache.get_price(ticker) + if price is not None: + return price + await asyncio.sleep(0.25) + return app_state.price_cache.get_price(ticker) +``` + +**Portfolio valuation** falls back to `avg_cost` for any position with no cached price, and +the 30-second snapshot task skips its first run until the cache is populated — otherwise the +P&L chart opens with a garbage point at t=0 (`PLAN.md` §13.6 item 42). + +--- + +## 14. Configuration + +| Variable | Default | Effect on market data | +|---|---|---| +| `MASSIVE_API_KEY` | unset | unset → simulator; set → probed, then one of the three sources | + +No other market-data environment variables. Poll cadence, event probability, and history +depth are constructor arguments with the defaults in this document — tests override them by +construction, not by environment (`MARKET_SIMULATOR.md` §9 depends on that). + +`--env-file .env` supplies the container's environment; `python-dotenv` is a local-dev +convenience only (`PLAN.md` §13.4 item 33). + +--- + +## 15. Testing + +The existing 73 tests stay green apart from deliberate signature churn: `direction` → +`tick_direction`, `change`/`change_percent` → `change_today`/`change_percent_today`, the +keyword-only `open_price` on `PriceCache.update`, and `create_market_data_source` becoming a +coroutine. **Every Massive test stubs the client — no test touches the network**, because a +5-call-per-minute budget makes a real-network suite unrunnable in CI. + +### 15.1 Statistical tests — the ones that would have caught the shock bug + +Nothing in the current suite fails at `event_probability=0.001`, which is why a 20× +volatility inflation shipped. Fix that first: + +```python +def test_realized_daily_volatility_matches_sigma(): + """Realized vol must match the parameter WITH SHOCKS AT THE PRODUCTION DEFAULT. + + This is the regression test for the decaying-shock design: it fails at the + old permanent-shift shocks and at event_probability=1e-3. + """ + random.seed(42) + np.random.seed(42) + + runs, ticks_per_day = 200, 46_800 + sigma = TICKER_PARAMS["AAPL"]["sigma"] + returns = [] + for _ in range(runs): + sim = GBMSimulator(["AAPL"], event_probability=1e-4) + start = sim.get_price("AAPL") + for _ in range(ticks_per_day): + sim.step() + returns.append(math.log(sim.get_price("AAPL") / start)) + + realized = statistics.stdev(returns) + target = sigma / math.sqrt(252) + stderr = target / math.sqrt(2 * (runs - 1)) + assert abs(realized - target) < 3 * stderr, f"{realized:.4%} vs {target:.4%}" +``` + +Also assert, over long runs: realised tech-vs-tech correlation ≈ 0.6 ± 0.05 and tech-vs-finance +≈ 0.3 ± 0.05 (proves the Cholesky is applied and not silently bypassed); mean log return +consistent with `(μ − σ²/2)·t` (catches a dropped Itô correction). Seed **both** `random` and +`np.random` — the simulator uses both. + +### 15.2 Invariants + +- Price strictly positive after 100,000 steps, for every σ in `TICKER_PARAMS`. +- No NaN or infinity anywhere in the price map. +- `step()` returns exactly the current ticker set, always. +- Cholesky survives 100 unknown tickers added **one at a time** (the incremental path, which + rebuilds on every call — not the batch constructor). +- Visible-tick rate > 50% for every default ticker at anchored prices — the guard on the + rounding cliff. +- `_synthesize_seed("ZZZZ")` is identical across two fresh instances. +- `remove_ticker` then `add_ticker` restores the price the ticker had, not a fresh seed. + +### 15.3 Capability probe and factory + +Stub the client for each of the five key states and assert both the classification and the +class the factory picks: + +```python +class _StubClient: + def __init__(self, snapshot_exc=None, prev_close_exc=None): + self._snapshot_exc, self._prev_close_exc = snapshot_exc, prev_close_exc + + def get_snapshot_all(self, **_): + if self._snapshot_exc: + raise self._snapshot_exc + return [_snapshot("AAPL", price=324.96, sip_timestamp=int(time.time() * 1e9))] + + def get_previous_close_agg(self, *_a, **_k): + if self._prev_close_exc: + raise self._prev_close_exc + return [_prev_close("AAPL", close=324.96)] + + +def test_basic_tier_classifies_as_end_of_day(monkeypatch): + """The critical case: NOT_AUTHORIZED snapshots + working aggregates is a + VALID free key, not a dead one. Getting this wrong sends every student to + the unanchored simulator.""" + monkeypatch.setattr( + capabilities, "RESTClient", + lambda **_: _StubClient(snapshot_exc=BadResponse("NOT_AUTHORIZED: not entitled")), + ) + caps = probe_capabilities("k" * 32) + assert (caps.valid, caps.realtime, caps.end_of_day) == (True, False, True) + + +async def test_factory_never_raises_and_always_returns_a_source(monkeypatch): + for caps, expected in [ + (MassiveCapabilities(True, True, True, "rt"), MassiveDataSource), + (MassiveCapabilities(True, False, True, "eod"), AnchoredSimulatorDataSource), + (MassiveCapabilities(False, False, False, "rejected"), SimulatorDataSource), + ]: + monkeypatch.setattr(factory, "probe_capabilities", lambda _k, c=caps: c) + monkeypatch.setenv("MASSIVE_API_KEY", "k" * 32) + assert isinstance(await create_market_data_source(PriceCache()), expected) +``` + +Also: `MaxRetryError` (rate limited) classifies as invalid rather than crashing, and a +`get_snapshot_all` whose newest `sip_timestamp` is 15 minutes old yields +`poll_interval == 15.0`. + +### 15.4 Massive client + +```python +def test_sip_timestamp_is_nanoseconds(): + """The shipped bug: `.timestamp / 1000.0` put quotes in the year 52000.""" + price, ts = MassiveDataSource._extract_price(_snapshot("AAPL", 324.96, 1605192894630916600)) + assert price == 324.96 + assert abs(ts - 1605192894.63) < 0.01 +``` + +Plus: `prev_day.close` lands in `PriceUpdate.open_price`; a poll returning no usable +snapshots increments `_consecutive_failures` and backs off; a ticker absent from the response +appears in `_unpriced` and in `describe().detail`; `describe()` returns `live=False` when the +newest quote is a day old; and `describe()` never raises when `start()` was never called. + +### 15.5 Anchored simulator + +`_fetch_anchors` walks back over a weekend — empty `bars` for Saturday and Sunday, populated +for Friday — and returns `({}, None)` cleanly after seven failures. A successful anchor sets +`seed_overrides`, so `AAPL` starts at the real close rather than the seed table's value. + +### 15.6 Stream + +`_generate_events` emits `": ping"` when the cache version is static for longer than the +heartbeat, and emits a `data:` frame within one interval of a cache update. This is the unit +test `PLAN.md` §13.4 item 31 wants **in place of** the Playwright disconnect/reconnect case, +which ultimately only verifies that the browser's built-in `EventSource` retry works. + +--- + +## 16. Implementation Order + +Each step leaves the tree importable and the suite runnable. + +1. **`models.py`** — `PricePoint`, `SourceStatus`, time helpers, `open_price`, + `tick_direction`. Update `test_models.py` for the renames. +2. **`cache.py`** — keyword-only sticky `open_price`. Update `test_cache.py`. +3. **`interface.py`** — `describe()` (abstract) and `get_history()` (default). Both existing + sources fail to instantiate until step 4 — expected. +4. **`simulator.py` + `seed_prices.py`** — decaying shocks at `1e-4`, deterministic seeds, + `seed_overrides`, `LinAlgError` fallback, history deque + prefill, `describe()`, refreshed + seed table. Add the §15.1 statistical tests; confirm they **fail** at `event_probability=1e-3` + before making them pass. +5. **`capabilities.py`** — probe plus its five-state test matrix. +6. **`massive_client.py`** — rewritten per §10. +7. **`anchored.py`** — new source. +8. **`factory.py`** — async, capability-driven. Update `test_factory.py` for the coroutine. +9. **`stream.py`** — heartbeat. +10. **`__init__.py`** — export `PricePoint`, `SourceStatus`, `MassiveCapabilities`, + `probe_capabilities`; refresh `backend/CLAUDE.md`, whose documented `PriceUpdate` fields + and synchronous factory both go stale at steps 1 and 8. + +Steps 1–4 and 9 need no API key and no network. Steps 5–7 are fully stubbed in tests; run +`market_data_demo.py` against a real free key once to confirm the anchored path end to end. + +--- + +## 17. Decisions This Document Closes + +| `PLAN.md` item | Resolution | +|---|---| +| §13.1 #1 daily-change baseline | `open_price` on `PriceUpdate` and the cache; `tick_direction` vs `change_percent_today` (§3) | +| §13.1 #5 lazy DB init | Startup only, in the lifespan handler (§13) | +| §13.1 #6 timestamp formats | Epoch float internally, ISO 8601 UTC on the wire, converted in `to_dict()` (§3) | +| §13.1 #7 SSE heartbeat | `": ping"` every 10 s of silence (§11) | +| §13.2 #8 unknown tickers | Synthesise, never reject; deterministic SHA-256 seed (§8.3) | +| §13.2 #9 trading off-watchlist symbols | Auto-add via `ensure_priced()` (§13) | +| §13.2 #10 removing a held ticker | Tracked set is watchlist ∪ positions (§13) | +| §13.2 #14 closed market | Quote-age staleness flips `live` to `False`; anchored simulator stays alive by design (§10) | +| §13.2 #15 price history | `get_history()` on the interface: simulator ring buffer + prefill, real minute bars on Massive (§8.5, §12) | +| §13.5 #38 health reporting | `SourceStatus` verbatim in `/api/health` (§12) | +| §13.5 #39 two ticker lists | DB seed imports its list from `seed_prices.SEED_PRICES` | +| §13.5 #40 full-map SSE payload | Deliberate; do not convert to a delta (§11) | +| §13.6 #42 missing prices | Fall back to `avg_cost`; snapshot task waits for a populated cache (§13) | + +### Still open + +1. **Anchored-simulator honesty** (`MARKET_INTERFACE.md` §12). Real levels with synthetic + motion, badged SIMULATED, versus yesterday's truthful but lifeless frozen closes. This + design picks the badged simulation and wants an explicit sign-off. +2. **Delayed-plan badging.** A Starter/Developer key is real market data, just 15 minutes + late. §6 reports it as `live: True` with the delay named in `detail`, rather than as + SIMULATED. Confirm that is the frontend's intent. +3. **Re-anchor cadence.** Hourly, at 24 calls/day out of 7,200. Cheap, and it keeps a demo + left open overnight honest. +4. **`get_history` depth.** 600 points (5 simulated minutes) is the ring buffer; Massive + serves whatever the day holds. If the main chart wants hours rather than minutes, the + simulator's deque needs to grow or thin its samples.