diff --git a/.env.example b/.env.example index 20f0d54..bc45045 100644 --- a/.env.example +++ b/.env.example @@ -57,9 +57,14 @@ QDRANT_PORT=6333 # ── Redis (hot cache) ──────────────────────────────────────────────────────── # Docker Compose overrides REDIS_HOST to `redis` for api/pipeline-worker. +# Cloud (Upstash): set REDIS_URL instead — takes precedence over HOST/PORT. REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD=cortex_local +# REDIS_URL=rediss://default:token@host.upstash.io:6379 + +# Production API: seed demo graph on container start (true = first deploy only). +CORTEX_SEED_DEMO=false # ── Kafka ──────────────────────────────────────────────────────────────────── # External listener on host :9092. Compose sets KAFKA_BOOTSTRAP_SERVERS=kafka:29092 for workers. diff --git a/.github/workflows/production-smoke.yml b/.github/workflows/production-smoke.yml new file mode 100644 index 0000000..588cd18 --- /dev/null +++ b/.github/workflows/production-smoke.yml @@ -0,0 +1,46 @@ +# Optional production uptime smoke — hits live Railway + Vercel proxy. +name: Production smoke + +on: + schedule: + # Every 6 hours — keeps Railway warm-ish and catches regressions. + - cron: "0 */6 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + production-api: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Smoke Railway API + env: + CORTEX_PRODUCTION_URL: ${{ secrets.CORTEX_PRODUCTION_URL }} + run: | + URL="${CORTEX_PRODUCTION_URL:-https://cortex-api-production-fbd5.up.railway.app}" + python scripts/staging_smoke.py --url "$URL" --query "Why CockroachDB for payments?" + + production-dashboard: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Smoke Vercel dashboard proxy + env: + CORTEX_DASHBOARD_URL: ${{ secrets.CORTEX_DASHBOARD_URL }} + run: | + URL="${CORTEX_DASHBOARD_URL:-https://frontend-ten-rouge-99.vercel.app}" + python scripts/staging_smoke.py --url "$URL" --query "Why CockroachDB for payments?" diff --git a/.gitignore b/.gitignore index 768ea40..4a1b295 100644 --- a/.gitignore +++ b/.gitignore @@ -22,8 +22,8 @@ htmlcov/ coverage.xml *.cover -# Environment -.env +# Vercel local project link (never commit credentials) +.vercel/ .env.* !.env.example diff --git a/.vercelignore b/.vercelignore new file mode 100644 index 0000000..ad4aacb --- /dev/null +++ b/.vercelignore @@ -0,0 +1,27 @@ +# Cortex monorepo — Vercel hosts the Vite dashboard only. +# Whitelist: only frontend + root deploy config are uploaded (keeps bundle < 500 MB). +# Use leading paths where noted so frontend/scripts/ is never excluded by mistake. + +* +!frontend +!frontend/** +!vercel.json +!middleware.ts +!.vercelignore + +# Belt-and-suspenders: never upload Python backend even if patterns change +/api/ +/pipeline/ +/connectors/ +/extraction/ +/graph/ +/intelligence/ +/memory/ +/scoring/ +/shared/ +/mcp/ +/tests/ +pyproject.toml +uv.lock +.python-version +*.py diff --git a/Makefile b/Makefile index c061417..13cec92 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ -.PHONY: demo demo-dry-run test ci stack init-kafka pipeline-restart pipeline-local verify-pipeline verify-github verify-jira verify-linear verify-connectors +.PHONY: demo demo-dry-run test ci stack init-kafka pipeline-restart pipeline-local verify-pipeline verify-github verify-jira verify-linear verify-connectors portfolio-demo seed-dev import-oss import-oss-graph seed-cloud verify-dual verify-dual-production verify-production + +# Production API URL for smoke tests (override on CLI). +CORTEX_PRODUCTION_URL ?= https://cortex-api-production-fbd5.up.railway.app # Local portfolio demo: Docker infra + migrations + seed + API + worker + frontend. demo: @@ -45,3 +48,38 @@ verify-linear: python scripts/verify_slack_pipeline.py --source linear --timeout 120 verify-connectors: verify-pipeline verify-github verify-jira verify-linear + +# Dual-workspace: synthetic dev seed vs OSS import smoke. +seed-dev: + uv run python scripts/seed_demo.py --workspace local-dev + +import-oss: + uv run python scripts/import_github_org.py --org tiangolo --repo fastapi --dry-run + +# Direct Neo4j import (no Kafka) — use for cloud / portfolio backends. +import-oss-graph: + uv run python scripts/import_github_graph.py --org tiangolo --repo fastapi --workspace oss-tiangolo-fastapi --limit 30 + +# Synthetic OSS workspace seed when GitHub import is unavailable. +seed-oss-fastapi: + uv run python scripts/seed_oss_fastapi_demo.py --workspace oss-tiangolo-fastapi + +# Seed + OSS import against remote Neo4j (set NEO4J_URI / credentials in env). +seed-cloud: seed-dev seed-oss-fastapi + +verify-dual: + uv run python scripts/dual_workspace_smoke.py --workspaces local-dev,oss-tiangolo-fastapi + +verify-production: + uv run python scripts/staging_smoke.py --url $(CORTEX_PRODUCTION_URL) --query "Why CockroachDB for payments?" + +verify-dual-production: + uv run python scripts/dual_workspace_smoke.py --url $(CORTEX_PRODUCTION_URL) --workspaces local-dev,oss-tiangolo-fastapi + +# Vercel frontend build smoke (injects API rewrites when CORTEX_API_ORIGIN is set). +verify-vercel-build: + cd frontend && npm run build + +# Portfolio demo: Docker API must be up; starts cloudflared and prints Vercel env instructions. +portfolio-demo: + @./scripts/start_portfolio_demo.sh diff --git a/README.md b/README.md index 12482eb..06aad31 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,11 @@ | In 3 minutes | Command | |---|---| -| **Run the demo** | `make demo` → [localhost:3000](http://localhost:3000) | -| **Ask a question** | Workspace `local-dev` → *Why CockroachDB for payments?* | -| **Wire an agent** | Add the MCP block below — `cortex_query`, `cortex_inject`, `cortex_remember` | +| **Live demo** | **[frontend-ten-rouge-99.vercel.app](https://frontend-ten-rouge-99.vercel.app)** — workspace `local-dev`, ask *Why CockroachDB for payments?* | +| **Run locally** | `make demo` → [localhost:3000](http://localhost:3000) | +| **Wire an agent** | [MCP setup in 60s](docs/MCP_SETUP.md) — `cortex_query`, `cortex_inject`, `cortex_remember` | + +> **Portfolio / LinkedIn:** Share the [live demo link](https://frontend-ten-rouge-99.vercel.app). For a 24/7 backend (no laptop required), deploy the API to Render/Railway — [docs/PORTFOLIO_DEMO.md](docs/PORTFOLIO_DEMO.md). --- @@ -217,6 +219,25 @@ This brings up Kafka, Neo4j, Redis, Postgres, applies graph migrations, writes t **Recording a video or GIF for the README:** see [docs/DEMO_RECORDING.md](docs/DEMO_RECORDING.md). **Validating real webhooks (Slack / GitHub / Jira):** [docs/CONNECTOR_VALIDATION.md](docs/CONNECTOR_VALIDATION.md). +### Deploy preview (Vercel dashboard) + +Host the **dashboard only** on Vercel (`frontend/` as root directory). The API and pipeline worker run on Docker, Railway, or similar — do not deploy the repo root as FastAPI on Vercel (full `uv.lock` exceeds Lambda size limits). + +| Step | Action | +|------|--------| +| 1 | Deploy `frontend/` with `CORTEX_API_ORIGIN=https://your-api.example.com` | +| 2 | Edge `middleware.ts` proxies same-origin `/query` to the API at runtime | +| 3 | Avoid `VITE_API_URL` pointing at tunnel URLs (511 errors) | +| 4 | API on Railway/Render via `railway.toml` or `render.yaml` — see [docs/DEPLOY.md](docs/DEPLOY.md) | + +Full guide: [docs/DEPLOY.md](docs/DEPLOY.md). **Dual workspaces** (synthetic `local-dev` vs real OSS imports): [docs/DATA_SOURCES.md](docs/DATA_SOURCES.md). + +```bash +make seed-dev # local-dev synthetic decisions +make import-oss # dry-run GitHub PR import preview +make verify-dual # compare queries across workspaces +``` + ### Context API (REST) | Method | Path | Purpose | diff --git a/SESSIONS.md b/SESSIONS.md index bee5e17..c031871 100644 --- a/SESSIONS.md +++ b/SESSIONS.md @@ -528,3 +528,48 @@ 1. Merge PR #15 after CI green 2. Phase 6: dashboard polish + demo video 3. Staging smoke with `CORTEX_CMVK_BACKEND=openai` + +--- + +## Session — 2026-06-11 — Vercel v1 deployment + dual-workspace restore +**Duration:** ~45m +**Phase:** Phase 6 — deploy preview + dual-workspace tooling + +### Built +- Restored deleted dual-workspace files from HEAD (`import_github_org.py`, `DEPLOY.md`, `vercel.json`, tests) +- **`frontend/middleware.ts`** — runtime Edge proxy via `CORTEX_API_ORIGIN` (fixes build-time rewrite limitation on Vercel) +- **`railway.toml`**, **`render.yaml`** — API v1 deploy blueprints +- **`api/memory.py`** — `REDIS_URL` support for Upstash cloud Redis +- **`docs/DEPLOY.md`**, **`README.md`** — Railway/Render + middleware deploy guide +- Vercel production redeploy: **https://frontend-ten-rouge-99.vercel.app** — `/health` and `/query` verified end-to-end + +### State at end +- Branch **`feature/dual-workspace-import`** — uncommitted changes (middleware, railway.toml, render.yaml, REDIS_URL) +- **370 passed** (full suite, `--no-cov`) +- `CORTEX_API_ORIGIN` on Vercel points at ephemeral localtunnel — replace with Railway URL for stable demo + +### Next session starts with +1. `railway login` + deploy API; seed Neo4j Aura; update `CORTEX_API_ORIGIN` +2. Commit + PR for dual-workspace + Vercel middleware +3. Optional: `import_github_org.py` live run for `oss-tiangolo-fastapi` + +--- + +## Session — 2026-06-11 — Portfolio demo link live +**Duration:** ~20m +**Phase:** Phase 6 — public demo for portfolio / LinkedIn + +### Built +- **`docs/PORTFOLIO_DEMO.md`** — shareable demo URL, recruiter walkthrough, 24/7 Render/Railway path +- **`scripts/start_portfolio_demo.sh`** + **`make portfolio-demo`** — cloudflared tunnel helper +- **README** — Live demo row points to Vercel production URL +- Vercel `CORTEX_API_ORIGIN` updated; production verified (`/health` 200, `/query` 3 hits) + +### State at end +- **Portfolio URL:** https://frontend-ten-rouge-99.vercel.app +- Demo works while Docker API + `cloudflared` run on Mac; Render/Railway needed for 24/7 LinkedIn traffic +- Uncommitted: middleware, railway.toml, render.yaml, portfolio docs + +### Next session starts with +1. Deploy API to Render/Railway + Aura for always-on demo +2. Commit + push deployment slice diff --git a/api/Dockerfile b/api/Dockerfile index ad86aa8..1708bd5 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -22,4 +22,4 @@ RUN pip install --no-cache-dir --upgrade pip \ EXPOSE 8000 -CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["sh", "-c", "uvicorn api.main:app --host 0.0.0.0 --port ${PORT:-8000}"] diff --git a/api/Dockerfile.query b/api/Dockerfile.query new file mode 100644 index 0000000..9ded5e2 --- /dev/null +++ b/api/Dockerfile.query @@ -0,0 +1,30 @@ +# Query-only Cortex API for cloud deploy (Render/Railway portfolio demo). +# Semantic search off (CORTEX_SEMANTIC_ENABLED=false) — no torch/spacy/mlflow. +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV CORTEX_SEMANTIC_ENABLED=false + +COPY api/requirements-query.txt ./requirements-query.txt +RUN pip install --no-cache-dir --upgrade pip \ + && pip install --no-cache-dir -r requirements-query.txt + +COPY api ./api +COPY connectors ./connectors +COPY extraction ./extraction +COPY graph ./graph +COPY scoring ./scoring +COPY shared ./shared +COPY pipeline ./pipeline +COPY memory ./memory +COPY intelligence ./intelligence +COPY scripts ./scripts + +EXPOSE 8000 + +RUN chmod +x scripts/start_api_production.sh + +CMD ["scripts/start_api_production.sh"] diff --git a/api/main.py b/api/main.py index 59d1461..fa5d088 100644 --- a/api/main.py +++ b/api/main.py @@ -245,15 +245,26 @@ async def query( workspace_id=request.workspace_id, ) + try: + coverage_score = await memory().workspace_coverage(request.workspace_id) + except Exception: + coverage_score = 0.0 + return QueryResponse( query=request.query, workspace_id=request.workspace_id, results=results, total=len(results), latency_ms=latency_ms, + coverage_score=coverage_score, ) +def _inject_result_limit(max_tokens: int) -> int: + """Map agent token budget to decision count (~400 tokens each), always ≥1.""" + return max(1, min(max_tokens // 400, 10)) + + @app.post( "/inject", response_model=InjectResponse, @@ -266,7 +277,7 @@ async def inject( ) -> InjectResponse: """Inject relevant organizational memory into an AI agent's context window. - Called by the MCP server (mcp/server.ts) on every agent tool invocation. + Called by the MCP server (mcp/server.js) on every agent tool invocation. Returns the most relevant decisions ranked by importance × trust × recency. Ranks injectable decisions by importance × trust (see scoring.trust_scorer). @@ -285,7 +296,7 @@ async def inject( context=request.context, workspace_id=request.workspace_id, caller_roles=roles, - limit=min(request.max_tokens // 400, 10), + limit=_inject_result_limit(request.max_tokens), ) except Exception as exc: log.error("inject.failed", error=str(exc), agent_id=request.agent_id) diff --git a/api/memory.py b/api/memory.py index d0cf5e8..a058bb3 100644 --- a/api/memory.py +++ b/api/memory.py @@ -35,14 +35,22 @@ def _build_redis_client() -> Any | None: except ImportError: return None + redis_url = os.environ.get("REDIS_URL", "").strip() try: - client = redis.Redis( - host=os.environ.get("REDIS_HOST", "localhost"), - port=int(os.environ.get("REDIS_PORT", "6379")), - password=os.environ.get("REDIS_PASSWORD"), - socket_connect_timeout=1, - decode_responses=True, - ) + if redis_url: + client = redis.from_url( + redis_url, + socket_connect_timeout=1, + decode_responses=True, + ) + else: + client = redis.Redis( + host=os.environ.get("REDIS_HOST", "localhost"), + port=int(os.environ.get("REDIS_PORT", "6379")), + password=os.environ.get("REDIS_PASSWORD"), + socket_connect_timeout=1, + decode_responses=True, + ) client.ping() return client except Exception: @@ -136,6 +144,10 @@ async def query_decisions( return results + async def workspace_coverage(self, workspace_id: str) -> float: + """Return heuristic completeness score for a workspace.""" + return await self._graph.workspace_coverage_score(workspace_id) + async def inject_decisions( self, *, diff --git a/api/requirements-query.txt b/api/requirements-query.txt new file mode 100644 index 0000000..5e90b5f --- /dev/null +++ b/api/requirements-query.txt @@ -0,0 +1,22 @@ +# Minimal deps for query-only Cortex API (portfolio demo on Render/Railway). +# Skips torch, spacy, mlflow, sentence-transformers — semantic search disabled. +fastapi>=0.111.0 +uvicorn[standard]>=0.29.0 +pydantic>=2.7.0 +pydantic-settings>=2.3.0 +python-dotenv>=1.0.1 +neo4j>=5.20.0 +redis>=5.0.4 +structlog>=24.2.0 +httpx>=0.27.0 +tenacity>=8.3.0 +python-multipart>=0.0.9 +prometheus-client>=0.20.0 +python-jose[cryptography]>=3.3.0 +passlib[bcrypt]>=1.7.4 +opentelemetry-api>=1.42.0 +opentelemetry-sdk>=1.42.0 +opentelemetry-exporter-otlp-proto-http>=1.42.0 +opentelemetry-instrumentation-fastapi>=0.51b0 +confluent-kafka>=2.4.0 +openai>=1.30.0 diff --git a/api/schemas.py b/api/schemas.py index f4f531f..bf8be00 100644 --- a/api/schemas.py +++ b/api/schemas.py @@ -44,6 +44,12 @@ class QueryResponse(BaseModel): results: list[DecisionResult] total: int latency_ms: float + coverage_score: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Heuristic workspace memory completeness (0–1)", + ) class InjectRequest(BaseModel): diff --git a/docs/AURA_MIGRATION.md b/docs/AURA_MIGRATION.md new file mode 100644 index 0000000..2b0eef1 --- /dev/null +++ b/docs/AURA_MIGRATION.md @@ -0,0 +1,76 @@ +# Migrate production Neo4j from Railway to Aura + +Railway’s community Neo4j image is fine for first deploy but lacks managed backups and stable memory limits. **Neo4j Aura Free** is the recommended production graph for the portfolio demo. + +## Why migrate + +| Railway Neo4j | Neo4j Aura Free | +|---------------|-----------------| +| 256MB heap, manual ops | Managed, auto-patches | +| No backups | Daily backups (paid tiers) / export on free | +| Crash loops under load | Stable for demo scale | +| Internal DNS only | `neo4j+s://` public bolt | + +Decision **P-003** in [DECISIONS.md](../DECISIONS.md): Aura for production graph; Railway hosts API + Redis only. + +## Steps + +### 1. Create Aura instance + +1. Sign in at [console.neo4j.io](https://console.neo4j.io). +2. **New instance** → **Free** → region close to Railway API (e.g. `us-east-1`). +3. Save connection URI, user (`neo4j`), and password. + +### 2. Export from Railway (if data exists) + +From a machine that can reach Railway Neo4j: + +```bash +export NEO4J_URI=bolt://neo4j-db.railway.internal:7687 # or public proxy if configured +export NEO4J_USER=neo4j +export NEO4J_PASSWORD= + +uv run python graph/migrate.py +# Optional: dump via neo4j-admin if you have shell access on the container +``` + +Simplest path for demo data: **re-seed on Aura** instead of dump/restore: + +```bash +export NEO4J_URI=neo4j+s://xxxx.databases.neo4j.io +export NEO4J_USER=neo4j +export NEO4J_PASSWORD= + +uv run python graph/migrate.py +CORTEX_SEED_DEMO=true uv run python scripts/seed_demo.py --workspace local-dev --scale small +uv run python scripts/import_github_graph.py --org tiangolo --repo fastapi --workspace oss-tiangolo-fastapi --limit 30 +``` + +### 3. Point Railway API at Aura + +Railway → **cortex-api** → Variables: + +```env +NEO4J_URI=neo4j+s://xxxx.databases.neo4j.io +NEO4J_USER=neo4j +NEO4J_PASSWORD= +CORTEX_SEED_DEMO=false +``` + +Redeploy API. Verify: + +```bash +curl -s https://cortex-api-production-fbd5.up.railway.app/health | jq . +``` + +### 4. Rotate credentials + +1. Change Aura password in console after migration. +2. Update Railway env vars (never commit passwords). +3. If the old Railway Neo4j password appeared in chat/logs, rotate it and decommission the Railway Neo4j service. + +### 5. Uptime + +Keep Railway Neo4j stopped or deleted after cutover to avoid split-brain confusion. + +See [DEPLOY.md](./DEPLOY.md) and [PORTFOLIO_DEMO.md](./PORTFOLIO_DEMO.md). diff --git a/docs/CUSTOM_DOMAIN.md b/docs/CUSTOM_DOMAIN.md new file mode 100644 index 0000000..abb6952 --- /dev/null +++ b/docs/CUSTOM_DOMAIN.md @@ -0,0 +1,49 @@ +# Custom domain for the Cortex dashboard (Vercel) + +Replace `frontend-ten-rouge-99.vercel.app` with a branded URL (e.g. `cortex.askmy-stack.dev`). + +## 1. Add domain in Vercel + +1. Vercel → Project **frontend** → **Settings** → **Domains**. +2. Add your domain (apex or subdomain). +3. Follow Vercel’s DNS instructions (CNAME to `cname.vercel-dns.com` or A records for apex). + +## 2. Keep API proxy working + +No change to `CORTEX_API_ORIGIN` — middleware still proxies `/query` and `/health` to Railway. + +Ensure **Environment Variables** include: + +```env +CORTEX_API_ORIGIN=https://cortex-api-production-fbd5.up.railway.app +``` + +Apply to **Production** (and Preview if you use preview URLs). + +## 3. Optional: API subdomain + +If you want `api.cortex.example.com` instead of middleware-only: + +1. Deploy API with public URL (Railway custom domain or Render). +2. Set `CORTEX_API_ORIGIN` to that URL **or** set `VITE_API_URL` (requires rebuild — prefer middleware). + +Recommended for portfolio: **dashboard on custom domain, API stays on Railway**, proxy via middleware (no CORS). + +## 4. Update links + +After DNS propagates, update: + +- [README.md](../README.md) live demo badge +- [docs/PORTFOLIO_DEMO.md](./PORTFOLIO_DEMO.md) +- LinkedIn / portfolio site + +## 5. Verify + +```bash +curl -s https://your-domain.example/health | jq . +curl -s -X POST https://your-domain.example/query \ + -H 'content-type: application/json' \ + -d '{"query":"Why CockroachDB for payments?","workspace_id":"local-dev","limit":3}' +``` + +Both should return JSON from the Railway API. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md new file mode 100644 index 0000000..afc9fba --- /dev/null +++ b/docs/DATA_SOURCES.md @@ -0,0 +1,70 @@ +# Cortex data workspaces + +Cortex isolates organizational memory by **`workspace_id`**. Use separate workspaces for synthetic demo data vs real open-source imports. + +## Workspace naming + +| Workspace | Source | How to populate | +|-----------|--------|-----------------| +| `local-dev` | Synthetic demo catalog | `make demo` or `python scripts/seed_demo.py --workspace local-dev` | +| `oss--` | Real GitHub merged PRs | `python scripts/import_github_org.py --org tiangolo --repo fastapi` | +| `oss-adr` / `oss-adr-` | ADR markdown files | `python scripts/import_adr_markdown.py --path docs/adr` | + +## Path A — GitHub (full pipeline) + +Exercises Kafka → extractor → importance/trust → Neo4j. + +```bash +# Preview without Kafka +python scripts/import_github_org.py --org tiangolo --repo fastapi --dry-run + +# Publish (requires Kafka + pipeline-worker) +python scripts/import_github_org.py \ + --org tiangolo \ + --repo fastapi \ + --workspace oss-tiangolo-fastapi \ + --limit 30 + +# Optional: GITHUB_TOKEN in .env for higher rate limits +``` + +After import, wait for `pipeline-worker` to process, then: + +```bash +python scripts/dual_workspace_smoke.py --workspaces local-dev,oss-tiangolo-fastapi +``` + +## Path B — ADR markdown (direct graph write) + +High-quality decisions without LLM extraction variance. Uses the same scoring loop as `seed_demo.py`. + +```bash +python scripts/import_adr_markdown.py --path /path/to/docs/adr --workspace oss-adr-myproject --dry-run +python scripts/import_adr_markdown.py --path /path/to/docs/adr --workspace oss-adr-myproject +``` + +Supports common MADR sections: Context, Decision, Consequences, Status. + +## Compare workspaces + +```bash +make seed-dev # local-dev only +make verify-dual # query local-dev vs oss-tiangolo-fastapi +``` + +Dashboard presets: **local-dev**, **oss-tiangolo-fastapi**, **oss-adr** (Connection bar). + +## Open data sources (future) + +| Source | Maps to | Notes | +|--------|---------|-------| +| GitHub REST API | `RawEvent` via `normalise_github_event` | Implemented | +| ADR markdown repos | `DecisionEvent` direct | Implemented | +| Kaggle GitHub issues CSV | Custom CSV → `RawEvent` | Not yet shipped | +| GH Archive | Batch JSON → inject script | Not yet shipped | + +## Licenses and ethics + +- Only import **public** repositories and datasets. +- Do not import private Slack exports or employee PII without GDPR review. +- Document dataset provenance in PR descriptions when used for portfolio demos. diff --git a/docs/DEMO_RECORDING.md b/docs/DEMO_RECORDING.md index 1176429..f63b522 100644 --- a/docs/DEMO_RECORDING.md +++ b/docs/DEMO_RECORDING.md @@ -11,6 +11,22 @@ Use this when capturing the **~3 minute** portfolio demo and any **README GIF**. - [http://localhost:3000](http://localhost:3000) — dashboard loads, **API health** is green. - [http://localhost:8000/docs](http://localhost:8000/docs) — OpenAPI loads. +## Cloud demo script (Vercel + Railway) + +Use this when recording for LinkedIn without running Docker locally. + +| Time | Action | +|------|--------| +| 0:00 | Open https://frontend-ten-rouge-99.vercel.app — mention 24/7 Vercel + Railway stack. | +| 0:15 | **Ask** tab → confirm workspace **`local-dev`**. | +| 0:30 | Query *Why CockroachDB for payments?* — show results, trust scores, **coverage %**. | +| 1:00 | Switch workspace to **`oss-tiangolo-fastapi`** (if imported) — compare real OSS decisions. | +| 1:30 | **Explore** memory map; **Review** contradictions if seeded. | +| 2:00 | Show [MCP_SETUP.md](./MCP_SETUP.md) — wire Cursor in one JSON block. | +| 2:30 | GitHub README live demo link; close with organizational memory thesis. | + +Custom domain steps: [CUSTOM_DOMAIN.md](./CUSTOM_DOMAIN.md). + ## Suggested script (voiceover optional) | Time | Action | diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md new file mode 100644 index 0000000..360e8c1 --- /dev/null +++ b/docs/DEPLOY.md @@ -0,0 +1,182 @@ +# Deploying Cortex + +Cortex is a **multi-service stack** (Kafka, Neo4j, Redis, API, pipeline worker, dashboard). Vercel hosts the **dashboard only**; the API and worker run elsewhere. + +## Recommended split + +| Component | Host | Notes | +|-----------|------|-------| +| Dashboard | **Vercel** (`frontend/`) | Vite static + API rewrites | +| API | Railway / Render / Fly | `api/Dockerfile` | +| pipeline-worker | Same as API | `pipeline/Dockerfile` | +| Neo4j | Neo4j Aura | Bolt URI in env | +| Redis | Upstash | Query cache | +| Kafka | Upstash Kafka / Confluent Cloud | Event bus | + +## Vercel (frontend) + +**Do not deploy the Python API on Vercel.** If the build log shows `Using Python 3.12 from pyproject.toml` or installs `uv.lock`, Vercel is treating the repo as FastAPI — that bundle (~5 GB) exceeds Lambda limits. + +**Fix (pick one):** + +| Approach | Settings | +|----------|----------| +| **Recommended** | Project Settings → General → **Root Directory** → `frontend` → Save → Redeploy | +| **Repo root** | Keep Root Directory `.` — root `vercel.json` + `.vercelignore` upload only `frontend/` (excludes Python/`api/` so the bundle stays under Vercel limits) | + +After changing Root Directory, clear the Framework Preset override if it still says **FastAPI** — it should be **Vite** or **Other**. + +**Project settings (Root Directory = `frontend`)** + +- Root Directory: `frontend` +- Framework: Vite +- Build: `npm run build` +- API proxy: `middleware.ts` reads `CORTEX_API_ORIGIN` at **runtime** (Vercel parses `vercel.json` before build — build-time rewrites cannot inject env vars) + +**Environment variables** + +| Variable | Purpose | +|----------|---------| +| `CORTEX_API_ORIGIN` | Public API URL — Edge Middleware proxies `/query`, `/health`, etc. server-side | +| ~~`VITE_API_URL`~~ | **Do not** point at `loca.lt` — causes 511 tunnel interstitial errors | + +**Do not** import the repo root with Framework Preset **FastAPI**. That installs the full `uv.lock` (~5 GB) and exceeds Lambda limits. + +```bash +cd frontend +npx vercel deploy --prod +``` + +Set `CORTEX_API_ORIGIN=https://your-api.example.com` in the Vercel project before deploy. + +## Local full stack + +```bash +make demo +open http://localhost:3000 # dashboard (nginx → API) +open http://localhost:8000/docs +``` + +## Dev preview (laptop + tunnel) + +For short-lived public demos while the API runs locally: + +```bash +cloudflared tunnel --url http://localhost:8000 +# Set CORTEX_API_ORIGIN to the trycloudflare.com URL on Vercel, redeploy frontend +``` + +Prefer **cloudflared** over localtunnel — localtunnel returns **511** for browser `fetch()` calls. + +## Dual-workspace testing + +See [DATA_SOURCES.md](./DATA_SOURCES.md) for `local-dev` vs `oss-*` workspaces. + +```bash +python scripts/seed_demo.py --workspace local-dev +python scripts/import_github_org.py --org tiangolo --repo fastapi --dry-run +make verify-dual +``` + +## Railway / Render (API v1) + +Deploy **only the API** — not Kafka or the pipeline worker. Pre-seed Neo4j with `scripts/seed_demo.py` so `/query` works without live ingestion. + +### Railway + +```bash +# Install CLI: https://docs.railway.com/guides/cli +railway login +railway init # link this repo +railway up # uses railway.toml → api/Dockerfile +``` + +**Required environment variables** (Railway → Variables): + +| Variable | Example | +|----------|---------| +| `NEO4J_URI` | `neo4j+s://xxxx.databases.neo4j.io` | +| `NEO4J_USER` | `neo4j` | +| `NEO4J_PASSWORD` | Aura password | +| `REDIS_URL` | `rediss://default:token@host.upstash.io:6379` | +| `CORTEX_API_KEYS` | `demo-readonly:authenticated` (optional abuse control) | +| `CORTEX_SEMANTIC_ENABLED` | `false` | +| `CORTEX_SEED_DEMO` | `true` **first deploy only** — then set `false` so restarts skip re-seed | + +Copy the public Railway URL (e.g. `https://cortex-api-production.up.railway.app`), set `CORTEX_API_ORIGIN` on Vercel, and redeploy the frontend. + +**Production startup:** `scripts/start_api_production.sh` runs migrations on every boot. Demo seed runs only when `CORTEX_SEED_DEMO=true`. + +### Render + +Use [render.yaml](../render.yaml) as a Blueprint, or create a **Web Service** with Docker runtime and `api/Dockerfile` as the Dockerfile path. Same env vars as Railway. + +### Seed production graph (one-time) + +**Option A — first Railway deploy:** set `CORTEX_SEED_DEMO=true` on the API service, deploy once, then set `CORTEX_SEED_DEMO=false`. + +**Option B — from laptop** with Neo4j credentials in `.env`: + +```bash +export NEO4J_URI=neo4j+s://... +export NEO4J_USER=neo4j +export NEO4J_PASSWORD=... +uv run python graph/migrate.py +uv run python scripts/seed_demo.py --workspace local-dev --scale small +uv run python scripts/import_github_graph.py --org tiangolo --repo fastapi --workspace oss-tiangolo-fastapi --limit 30 +make verify-dual-production +``` + +Direct graph import (no Kafka): [scripts/import_github_graph.py](../scripts/import_github_graph.py). + +### Neo4j Aura migration + +See [AURA_MIGRATION.md](./AURA_MIGRATION.md). Rotate passwords after any credential exposure — update Railway env vars only, never commit secrets. + +### Optional demo API key (abuse control) + +For public demos, use a read-only key so anonymous traffic cannot hammer write endpoints: + +```bash +CORTEX_API_KEYS=demo-readonly:authenticated +``` + +Dashboard users paste the key in **Connection** settings. Open `/query` and `/health` remain usable without a key when `CORTEX_API_KEYS` is unset. + +### Wire Vercel → API + +```bash +# Vercel project → Settings → Environment Variables +CORTEX_API_ORIGIN=https://your-api.railway.app + +cd frontend && npx vercel deploy --prod +``` + +Verify: + +```bash +curl -s https://your-vercel-app.vercel.app/health +curl -s -X POST https://your-vercel-app.vercel.app/query \ + -H "Content-Type: application/json" \ + -d '{"query":"Why CockroachDB?","workspace_id":"local-dev","limit":5}' +``` + +## Auth on preview + +```bash +CORTEX_API_KEYS=preview-key:admin;authenticated +CORTEX_DEMO_API_KEY=preview-key +``` + +`make demo` and `scripts/demo.sh` send `Authorization` when these are set. + +## Optional cloud webhook path (v2) + +Full ingestion uses Kafka + pipeline-worker locally. For cloud v1 without Kafka, use direct graph import: + +```bash +make import-oss-graph # real GitHub PRs → Neo4j +make seed-oss-fastapi # synthetic OSS demo fallback +``` + +Future v2: deploy pipeline-worker as a second Railway service + Upstash Kafka, or add `POST /webhooks/github` → synchronous extract/write for demo scale. See [LAUNCH.md](./LAUNCH.md). diff --git a/docs/LAUNCH.md b/docs/LAUNCH.md new file mode 100644 index 0000000..66ebd01 --- /dev/null +++ b/docs/LAUNCH.md @@ -0,0 +1,60 @@ +# Phase 7 — Open-source launch checklist + +Use this when posting to HN, LinkedIn, and GitHub. + +## Assets + +- [ ] Live demo: https://frontend-ten-rouge-99.vercel.app (or custom domain — [CUSTOM_DOMAIN.md](./CUSTOM_DOMAIN.md)) +- [ ] 3-min video: [DEMO_RECORDING.md](./DEMO_RECORDING.md) (cloud + local scripts) +- [ ] README **Live demo** badge at top +- [ ] MCP wiring: [MCP_SETUP.md](./MCP_SETUP.md) + +## Smoke before posting + +```bash +make verify-production # health + query against Railway +make verify-dual-production # local-dev + oss-tiangolo-fastapi +``` + +## Hacker News (Show HN draft) + +**Title:** Show HN: Cortex – organizational memory OS for AI agents (Neo4j + MCP) + +**Body:** + +Cortex captures **decisions** (not documents) from Slack, GitHub, Jira, etc., stores them in a knowledge graph, and injects relevant context into agents via MCP at inference time. + +Live demo (no install): https://frontend-ten-rouge-99.vercel.app +Try workspace `local-dev`, query: *Why CockroachDB for payments?* + +Stack: Kafka ingestion pipeline, Neo4j graph, importance/trust scoring, contradiction detection, FastAPI query layer, React dashboard, TypeScript MCP server. + +GitHub: https://github.com/askmy-stack/Cortex + +Feedback welcome — especially on the “active injection vs passive RAG” thesis. + +## LinkedIn post (draft) + +Shipped a live demo of **Cortex** — an organizational memory OS for AI-native teams. + +Instead of dumping docs into a vector DB, Cortex: +- Extracts **decisions** from Slack, GitHub, Jira +- Scores importance + trust at ingestion +- Stores relationships in **Neo4j** +- Injects context into any MCP agent at inference time + +Try it: https://frontend-ten-rouge-99.vercel.app +Query: *Why CockroachDB for payments?* (workspace: local-dev) + +Open source: github.com/askmy-stack/Cortex +#AI #MLOps #OpenSource #KnowledgeGraph + +## Optional v2 (post-launch) + +- Cloud webhook → API → graph write (bypass Kafka for demo) — see [DEPLOY.md](./DEPLOY.md) +- Railway worker service for live GitHub ingestion +- Contradiction pair in Review tab for demo narrative + +## CI + +Scheduled production smoke: `.github/workflows/production-smoke.yml` (optional `CORTEX_PRODUCTION_URL` secret). diff --git a/docs/MCP_SETUP.md b/docs/MCP_SETUP.md new file mode 100644 index 0000000..ad28d23 --- /dev/null +++ b/docs/MCP_SETUP.md @@ -0,0 +1,67 @@ +# Wire Cortex MCP in 60 seconds + +Cortex ships a **stdio MCP server** that calls your live API (`POST /query`, `POST /inject`). Agents get organizational memory without custom integration code. + +## Prerequisites + +- Node.js 18+ +- A running Cortex API (local `make demo`, or production Railway URL) + +## Cursor / Claude Desktop config + +Add to your MCP settings (Cursor: **Settings → MCP**, or `~/.cursor/mcp.json`): + +```json +{ + "mcpServers": { + "cortex": { + "command": "node", + "args": ["/absolute/path/to/Cortex/mcp/server.js"], + "env": { + "CORTEX_API_URL": "https://cortex-api-production-fbd5.up.railway.app", + "CORTEX_API_KEY": "" + } + } + } +} +``` + +For **local** development: + +```json +"CORTEX_API_URL": "http://localhost:8000" +``` + +When the cloud API uses optional auth, set `CORTEX_API_KEY` to your read-only demo key (see [DEPLOY.md](./DEPLOY.md)). + +## Available tools + +| Tool | Purpose | +|------|---------| +| `cortex_query` | Natural-language search over organizational decisions | +| `cortex_remember` | Submit explicit memory into the ingestion pipeline | +| `cortex_inject` | Active context injection for an agent prompt | + +## Verify + +1. Restart Cursor after saving MCP config. +2. In chat, ask the agent to call `cortex_query` with workspace `local-dev` and query *Why CockroachDB for payments?* +3. You should see structured decision JSON — not raw Slack threads. + +## Architecture note (interviews) + +```text +Your IDE agent → MCP (stdio) → Cortex API → Neo4j graph (+ Redis cache) +``` + +The MCP server is **agent-neutral** — any MCP-compatible client works. Cloud v1 deploys the API only; MCP runs beside your agent locally (zero extra Railway cost). + +## Troubleshooting + +| Symptom | Fix | +|---------|-----| +| `401 Unauthorized` | Set `CORTEX_API_KEY` or remove keys on API for open demo | +| `503` / connection refused | Check `CORTEX_API_URL`; Railway free tier may cold-start ~30s | +| Empty results | Confirm workspace id (`local-dev` on cloud seed) | + +See also: [README.md](../README.md), [PORTFOLIO_DEMO.md](./PORTFOLIO_DEMO.md). diff --git a/docs/PORTFOLIO_DEMO.md b/docs/PORTFOLIO_DEMO.md new file mode 100644 index 0000000..8ea1cf3 --- /dev/null +++ b/docs/PORTFOLIO_DEMO.md @@ -0,0 +1,108 @@ +# Cortex — Portfolio & LinkedIn Demo + +Use this page when linking Cortex from your portfolio site, GitHub README, or LinkedIn posts. + +## Live demo (share this URL) + +**https://frontend-ten-rouge-99.vercel.app** + +### 30-second walkthrough (for recruiters) + +1. Open the link — Cortex dashboard loads (Vercel CDN). +2. Confirm workspace is **`local-dev`** (preset chip). +3. Go to **Ask** and query: **`Why CockroachDB for payments?`** +4. Point out: decisions from Slack/GitHub/Jira, trust scores, graph relationships — not raw chat logs. + +### What to say in one line + +> *Cortex is an organizational memory OS — it captures decisions from every tool, stores them in a knowledge graph, and injects relevant context into AI agents at inference time via MCP.* + +--- + +## How the public demo is wired + +```text +Browser → Vercel (React dashboard) + ↓ Edge Middleware (CORTEX_API_ORIGIN) + API (FastAPI + Neo4j + Redis) +``` + +- **Dashboard:** Vercel project `frontend` (static Vite build). +- **API proxy:** `frontend/middleware.ts` forwards `/query`, `/health`, etc. server-side. +- **Do not** set `VITE_API_URL` on Vercel — same-origin `/query` avoids CORS and tunnel 511 errors. + +--- + +## Keeping the demo online + +| Mode | Uptime | Setup | +|------|--------|--------| +| **Production** (live) | 24/7 | Railway API + Neo4j + Redis on `cortex-api-demo` project | +| **Dev tunnel** (fallback) | While your Mac + Docker + `cloudflared` run | `make portfolio-demo` | + +### Quick dev tunnel (laptop demo) + +```bash +make portfolio-demo +``` + +Starts `cloudflared` to `localhost:8000`, prints the tunnel URL, and reminds you to set `CORTEX_API_ORIGIN` on Vercel if the URL changed. + +### Production backend (deployed) + +| Service | URL | +|---------|-----| +| API | https://cortex-api-production-fbd5.up.railway.app | +| Dashboard | https://frontend-ten-rouge-99.vercel.app | +| Railway project | `cortex-api-demo` | + +Vercel `CORTEX_API_ORIGIN` → Railway API URL (no laptop required). + +### Re-deploy from scratch + +1. `railway login` → `railway init` (or use existing `cortex-api-demo` project). +2. Add **Redis** (`railway add --database redis`) and **Neo4j** (`neo4j:5.20-community` image). +3. Deploy API: `railway up --service cortex-api` (uses `api/Dockerfile.query`). +4. Set env on the API service: + + ```env + NEO4J_URI=neo4j+s://... + NEO4J_USER=neo4j + NEO4J_PASSWORD=... + REDIS_URL=rediss://... + CORTEX_SEMANTIC_ENABLED=false + CORTEX_API_KEYS=preview-key:admin;authenticated + ``` + +5. Seed the graph once from your machine: + + ```bash + export NEO4J_URI=neo4j+s://... + uv run python graph/migrate.py + uv run python scripts/seed_demo.py --workspace local-dev --scale small + ``` + + Or set `CORTEX_SEED_DEMO=true` on first API deploy, then **`CORTEX_SEED_DEMO=false`** after boot. + +6. Vercel → Project **frontend** → Environment Variables → `CORTEX_API_ORIGIN` = your public API URL (e.g. `https://cortex-api.onrender.com`). No frontend redeploy needed — middleware reads it at runtime. + +**Custom domain:** [CUSTOM_DOMAIN.md](./CUSTOM_DOMAIN.md) +**Neo4j Aura:** [AURA_MIGRATION.md](./AURA_MIGRATION.md) +**Launch checklist:** [LAUNCH.md](./LAUNCH.md) + +--- + +## Vercel project settings + +| Setting | Value | +|---------|--------| +| Root Directory | `frontend` | +| Framework | Vite | +| Production URL | https://frontend-ten-rouge-99.vercel.app | +| Env var | `CORTEX_API_ORIGIN` = public API base URL (no trailing slash) | + +Redeploy frontend after code changes: + +```bash +cd frontend && npx vercel deploy --prod +``` diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..3cffee0 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,4 @@ +.vercel +dist +node_modules +.env*.local diff --git a/frontend/middleware.ts b/frontend/middleware.ts new file mode 100644 index 0000000..c665959 --- /dev/null +++ b/frontend/middleware.ts @@ -0,0 +1,72 @@ +/** + * Vercel Edge Middleware — proxy API routes to CORTEX_API_ORIGIN at runtime. + * + * Build-time vercel.json rewrites cannot read env vars on Vercel (config is + * parsed before the build step). Middleware reads CORTEX_API_ORIGIN on each + * request so production deploys pick up dashboard → API wiring without rebuilds. + */ + +export const config = { + matcher: [ + "/health", + "/metrics", + "/query", + "/inject", + "/remember", + "/docs", + "/openapi.json", + "/decisions/:path*", + "/contradictions/:path*", + "/gdpr/:path*", + "/webhooks/:path*", + ], +}; + +const API_PREFIXES = config.matcher.map((m) => m.replace("/:path*", "")); + +function isApiRoute(pathname: string): boolean { + return API_PREFIXES.some( + (prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`), + ); +} + +export default async function middleware(request: Request): Promise { + const origin = (process.env.CORTEX_API_ORIGIN ?? "").trim().replace(/\/$/, ""); + if (!origin) { + return new Response( + JSON.stringify({ + detail: + "CORTEX_API_ORIGIN is not set on Vercel. Add your Railway/Render API URL in Project Settings.", + }), + { status: 503, headers: { "content-type": "application/json" } }, + ); + } + + const url = new URL(request.url); + if (!isApiRoute(url.pathname)) { + return new Response("Not found", { status: 404 }); + } + + const target = `${origin}${url.pathname}${url.search}`; + const headers = new Headers(request.headers); + headers.delete("host"); + + const init: RequestInit = { + method: request.method, + headers, + redirect: "manual", + }; + if (request.method !== "GET" && request.method !== "HEAD") { + init.body = request.body; + } + + try { + return await fetch(target, init); + } catch (error) { + const message = error instanceof Error ? error.message : "upstream fetch failed"; + return new Response( + JSON.stringify({ detail: `API unreachable at ${origin}: ${message}` }), + { status: 502, headers: { "content-type": "application/json" } }, + ); + } +} diff --git a/frontend/package.json b/frontend/package.json index 19f6c5a..f45a16c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "vite", "typecheck": "tsc --noEmit", - "build": "tsc --noEmit && vite build", + "build": "node scripts/vercel-api-rewrites.mjs && tsc --noEmit && vite build", "preview": "vite preview", "test": "vitest run", "test:watch": "vitest", diff --git a/frontend/scripts/vercel-api-rewrites.mjs b/frontend/scripts/vercel-api-rewrites.mjs new file mode 100644 index 0000000..8b390db --- /dev/null +++ b/frontend/scripts/vercel-api-rewrites.mjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +/** + * Inject Vercel rewrites so the dashboard calls /query on the same origin. + * Vercel proxies to CORTEX_API_ORIGIN server-side — avoids localtunnel 511 + * interstitials and CORS when the API runs on your laptop via cloudflared. + * + * Updates frontend/vercel.json (Root Directory = frontend) and repo-root + * vercel.json (Root Directory = .) when the latter exists. + */ +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const frontendRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = join(frontendRoot, ".."); + +const origin = String(process.env.CORTEX_API_ORIGIN ?? "") + .trim() + .replace(/\/$/, ""); + +const apiRewrites = origin + ? [ + { source: "/health", destination: `${origin}/health` }, + { source: "/metrics", destination: `${origin}/metrics` }, + { source: "/query", destination: `${origin}/query` }, + { source: "/inject", destination: `${origin}/inject` }, + { source: "/remember", destination: `${origin}/remember` }, + { source: "/docs", destination: `${origin}/docs` }, + { source: "/openapi.json", destination: `${origin}/openapi.json` }, + { source: "/decisions/:path*", destination: `${origin}/decisions/:path*` }, + { source: "/contradictions/:path*", destination: `${origin}/contradictions/:path*` }, + { source: "/gdpr/:path*", destination: `${origin}/gdpr/:path*` }, + { source: "/webhooks/:path*", destination: `${origin}/webhooks/:path*` }, + ] + : []; + +const spaFallback = { source: "/((?!assets/).*)", destination: "/index.html" }; + +function applyRewrites(configPath) { + const base = JSON.parse(readFileSync(configPath, "utf8")); + base.rewrites = [...apiRewrites, spaFallback]; + writeFileSync(configPath, `${JSON.stringify(base, null, 2)}\n`); +} + +const targets = [join(frontendRoot, "vercel.json")]; +const rootConfig = join(repoRoot, "vercel.json"); +if (existsSync(rootConfig)) { + targets.push(rootConfig); +} + +for (const configPath of targets) { + applyRewrites(configPath); +} + +if (origin) { + console.log(`vercel.json: proxying API routes → ${origin} (${targets.length} file(s))`); +} else { + console.log( + `vercel.json: no CORTEX_API_ORIGIN — SPA only (${targets.length} file(s); use nginx/Vite proxy locally)`, + ); +} diff --git a/frontend/src/components/layout/WorkspaceBar.tsx b/frontend/src/components/layout/WorkspaceBar.tsx index edb11fa..5480598 100644 --- a/frontend/src/components/layout/WorkspaceBar.tsx +++ b/frontend/src/components/layout/WorkspaceBar.tsx @@ -2,7 +2,7 @@ import { useId, useState } from "react"; import { useApp } from "../../context/AppContext"; import { hasApiKeyConfigured } from "../../api/client"; -const PRESETS = ["local-dev", "acme-demo"] as const; +const PRESETS = ["local-dev", "oss-tiangolo-fastapi", "oss-adr"] as const; export function WorkspaceBar() { const { workspaceId, setWorkspaceId, apiKey, setApiKey, saveApiKey } = useApp(); diff --git a/frontend/src/index.css b/frontend/src/index.css index 8fd06b5..5f46067 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -547,6 +547,38 @@ a { background: var(--accent-glow); } +.demo-hint { + margin-bottom: 1rem; + font-size: 0.9rem; + color: var(--text-muted); +} + +.link-button { + padding: 0; + border: none; + background: none; + color: var(--accent); + font: inherit; + font-weight: 600; + cursor: pointer; + text-decoration: underline; + text-underline-offset: 2px; +} + +.link-button:hover { + opacity: 0.85; +} + +.coverage-badge { + display: inline-block; + padding: 0.1rem 0.45rem; + border-radius: 999px; + background: var(--accent-glow); + color: var(--accent); + font-size: 0.85em; + font-weight: 600; +} + .btn { padding: 0.55rem 1.1rem; border-radius: var(--radius); diff --git a/frontend/src/lib/errors.ts b/frontend/src/lib/errors.ts index c6cc789..3ab34ff 100644 --- a/frontend/src/lib/errors.ts +++ b/frontend/src/lib/errors.ts @@ -45,6 +45,13 @@ export async function parseHttpError(response: Response): Promise { const base = "Service unavailable (503). Neo4j or another dependency may be down."; return detail ? `${base} ${detail.slice(0, 120)}` : base; } + if (response.status === 511 || detail.includes("Tunnel website ahead")) { + return ( + "API tunnel blocked the request (511). The dashboard must use same-origin /query via " + + "Vercel rewrites (CORTEX_API_ORIGIN), not a localtunnel URL in VITE_API_URL. " + + "See README deploy notes." + ); + } if (!detail) return `Request failed (${status})`; return `Request failed (${status}): ${detail.slice(0, 200)}`; } diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index e98e7b1..cf89dd8 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -29,6 +29,7 @@ export type QueryResponse = { results: DecisionResult[]; total: number; latency_ms: number; + coverage_score?: number; }; export type InjectResponse = { diff --git a/frontend/src/views/AskView.tsx b/frontend/src/views/AskView.tsx index cf951dd..a5a59b3 100644 --- a/frontend/src/views/AskView.tsx +++ b/frontend/src/views/AskView.tsx @@ -113,6 +113,20 @@ export function AskView() { subtitle="Natural language search over captured decisions — who decided, what systems are affected, and why." /> +

+ Live demo: workspace local-dev · try{" "} + +

+
@@ -212,6 +226,15 @@ export function AskView() {

{lastQuery.total} result{lastQuery.total === 1 ? "" : "s"} ·{" "} {lastQuery.latency_ms}ms + {typeof lastQuery.coverage_score === "number" ? ( + <> + {" "} + · coverage{" "} + + {Math.round(lastQuery.coverage_score * 100)}% + + + ) : null}

@@ -160,7 +165,7 @@ export function ReviewView() { diff --git a/frontend/vercel.json b/frontend/vercel.json new file mode 100644 index 0000000..cc7b312 --- /dev/null +++ b/frontend/vercel.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "framework": "vite", + "buildCommand": "npm run build", + "outputDirectory": "dist", + "rewrites": [ + { + "source": "/((?!assets/).*)", + "destination": "/index.html" + } + ] +} diff --git a/graph/query.py b/graph/query.py index 8db2862..78bbfcd 100644 --- a/graph/query.py +++ b/graph/query.py @@ -387,6 +387,34 @@ async def health(self) -> bool: except Exception: return False + async def workspace_coverage_score(self, workspace_id: str) -> float: + """Estimate memory completeness for a workspace (0–1 heuristic). + + Uses decision and system counts vs portfolio demo targets. Full Phase 8 + coverage scoring will replace this with domain-aware completeness. + """ + driver = await self._driver_instance() + async with driver.session() as session: + result = await session.run( + """ + MATCH (d:Decision {workspace_id: $workspace_id}) + WHERE d.status <> 'archived' + WITH count(d) AS decisions + OPTIONAL MATCH (s:System {workspace_id: $workspace_id}) + WITH decisions, count(s) AS systems + RETURN decisions, systems + """, + workspace_id=workspace_id, + ) + record = await result.single() + if record is None: + return 0.0 + decisions = int(record.get("decisions") or 0) + systems = int(record.get("systems") or 0) + decision_ratio = min(1.0, decisions / 50.0) + system_ratio = min(1.0, systems / 15.0) + return round(min(1.0, 0.7 * decision_ratio + 0.3 * system_ratio), 3) + async def find_conflict_candidates( self, *, diff --git a/middleware.ts b/middleware.ts new file mode 100644 index 0000000..0b65504 --- /dev/null +++ b/middleware.ts @@ -0,0 +1,76 @@ +/** + * Vercel Edge Middleware — proxy API routes to CORTEX_API_ORIGIN at runtime. + * + * Used when the Vercel project Root Directory is the repo root (`cortex` project). + * When Root Directory = `frontend`, see frontend/middleware.ts. + */ + +export const config = { + matcher: [ + "/health", + "/metrics", + "/query", + "/inject", + "/remember", + "/docs", + "/openapi.json", + "/decisions/:path*", + "/contradictions/:path*", + "/gdpr/:path*", + "/webhooks/:path*", + ], +}; + +const API_PREFIXES = config.matcher.map((m) => m.replace("/:path*", "")); + +function isApiRoute(pathname: string): boolean { + return API_PREFIXES.some( + (prefix) => pathname === prefix || pathname.startsWith(`${prefix}/`), + ); +} + +export default async function middleware(request: Request): Promise { + const origin = String( + (globalThis as typeof globalThis & { process?: { env?: Record } }) + .process?.env?.CORTEX_API_ORIGIN ?? "", + ) + .trim() + .replace(/\/$/, ""); + if (!origin) { + return new Response( + JSON.stringify({ + detail: + "CORTEX_API_ORIGIN is not set on Vercel. Add your Railway/Render API URL in Project Settings.", + }), + { status: 503, headers: { "content-type": "application/json" } }, + ); + } + + const url = new URL(request.url); + if (!isApiRoute(url.pathname)) { + return new Response("Not found", { status: 404 }); + } + + const target = `${origin}${url.pathname}${url.search}`; + const headers = new Headers(request.headers); + headers.delete("host"); + + const init: RequestInit = { + method: request.method, + headers, + redirect: "manual", + }; + if (request.method !== "GET" && request.method !== "HEAD") { + init.body = request.body; + } + + try { + return await fetch(target, init); + } catch (error) { + const message = error instanceof Error ? error.message : "upstream fetch failed"; + return new Response( + JSON.stringify({ detail: `API unreachable at ${origin}: ${message}` }), + { status: 502, headers: { "content-type": "application/json" } }, + ); + } +} diff --git a/railway.toml b/railway.toml new file mode 100644 index 0000000..0955945 --- /dev/null +++ b/railway.toml @@ -0,0 +1,18 @@ +# Cortex API — Railway deployment (v1 preview) +# https://docs.railway.com/guides/dockerfiles +# +# Deploy: +# railway login && railway init +# railway up --dockerfile api/Dockerfile +# +# Set env vars in Railway dashboard (see docs/DEPLOY.md). + +[build] +builder = "DOCKERFILE" +dockerfilePath = "api/Dockerfile.query" + +[deploy] +healthcheckPath = "/health" +healthcheckTimeout = 300 +restartPolicyType = "ON_FAILURE" +restartPolicyMaxRetries = 3 diff --git a/render.yaml b/render.yaml new file mode 100644 index 0000000..7ecc9f1 --- /dev/null +++ b/render.yaml @@ -0,0 +1,33 @@ +# Cortex API — Render Blueprint (v1 preview alternative to Railway) +# https://render.com/docs/blueprint-spec +# +# Connect repo → New Blueprint → point at this file. +# Fill secret env vars in the Render dashboard after first deploy. + +services: + - type: web + name: cortex-api + runtime: docker + dockerfilePath: ./api/Dockerfile + dockerContext: . + plan: free + healthCheckPath: /health + envVars: + - key: ENVIRONMENT + value: production + - key: NEO4J_URI + sync: false + - key: NEO4J_USER + value: neo4j + - key: NEO4J_PASSWORD + sync: false + - key: REDIS_URL + sync: false + - key: CORTEX_API_KEYS + sync: false + - key: CORTEX_SEMANTIC_ENABLED + value: "false" + - key: EXTRACTION_BACKEND + value: openai + - key: OPENAI_API_KEY + sync: false diff --git a/scripts/dual_workspace_smoke.py b/scripts/dual_workspace_smoke.py new file mode 100644 index 0000000..4dd679e --- /dev/null +++ b/scripts/dual_workspace_smoke.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Compare query results across multiple Cortex workspaces. + +Useful for validating synthetic seed data (local-dev) vs real OSS imports (oss-*). + +Usage: + python scripts/dual_workspace_smoke.py --url http://localhost:8000 + python scripts/dual_workspace_smoke.py --workspaces local-dev,oss-tiangolo-fastapi +""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.error +import urllib.request + +DEFAULT_QUERIES = [ + "Why CockroachDB for payments?", + "cache session strategy", + "architecture migration decision", +] + +DEFAULT_WORKSPACES = ["local-dev", "oss-tiangolo-fastapi"] + + +def post_query(base_url: str, *, query: str, workspace_id: str, limit: int) -> dict: + """POST /query and return parsed JSON.""" + body = json.dumps( + {"query": query, "workspace_id": workspace_id, "limit": limit}, + ).encode("utf-8") + req = urllib.request.Request( + f"{base_url.rstrip('/')}/query", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Dual-workspace query smoke test") + parser.add_argument("--url", default="http://localhost:8000", help="Cortex API base URL") + parser.add_argument( + "--workspaces", + default=",".join(DEFAULT_WORKSPACES), + help="Comma-separated workspace ids", + ) + parser.add_argument( + "--queries", + default="", + help="Comma-separated queries (default: built-in set)", + ) + parser.add_argument("--limit", type=int, default=3, help="Results per query") + args = parser.parse_args() + + workspaces = [w.strip() for w in args.workspaces.split(",") if w.strip()] + queries = ( + [q.strip() for q in args.queries.split(",") if q.strip()] + if args.queries + else DEFAULT_QUERIES + ) + + print(f"API: {args.url}") + print(f"Workspaces: {', '.join(workspaces)}") + print() + + failed = 0 + for query in queries: + print(f"Query: {query!r}") + row: list[str] = [] + for ws in workspaces: + try: + payload = post_query(args.url, query=query, workspace_id=ws, limit=args.limit) + results = payload.get("results") or [] + if results: + top = results[0].get("content", "")[:60] + row.append(f"{ws}: {len(results)} hit(s) — {top!r}…") + else: + row.append(f"{ws}: 0 results") + except urllib.error.HTTPError as exc: + failed += 1 + row.append(f"{ws}: HTTP {exc.code}") + except urllib.error.URLError as exc: + failed += 1 + row.append(f"{ws}: unreachable ({exc.reason})") + for line in row: + print(f" {line}") + print() + + if failed: + print(f"FAIL: {failed} request(s) failed", file=sys.stderr) + return 1 + print("OK: all workspace queries completed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/import_adr_markdown.py b/scripts/import_adr_markdown.py new file mode 100644 index 0000000..0eba283 --- /dev/null +++ b/scripts/import_adr_markdown.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +"""Import Architecture Decision Records from markdown files into Neo4j. + +Parses common MADR-style ADR markdown and writes Decision nodes via the same +scoring pipeline as ``seed_demo.py`` (no Kafka / LLM extraction). + +Usage: + python scripts/import_adr_markdown.py --path ./vendor/adr --workspace oss-adr-fastapi --dry-run + python scripts/import_adr_markdown.py --path docs/adr --glob '*.md' +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +import uuid +from datetime import UTC, datetime +from pathlib import Path + +_REPO = Path(__file__).resolve().parents[1] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from dotenv import load_dotenv + +from shared.models import DecisionEvent, Provenance + +SECTION_HEADING = re.compile(r"^#{1,3}\s+(.+)$", re.MULTILINE) + + +def adr_uuid(path_key: str) -> str: + """Deterministic id for idempotent MERGE on re-import.""" + return str(uuid.uuid5(uuid.NAMESPACE_URL, f"cortex.adr:{path_key}")) + + +def parse_adr_markdown(text: str, *, path_key: str) -> dict[str, str | list[str]] | None: + """Extract title, status, context, decision, consequences from MADR-like markdown.""" + lines = text.strip().splitlines() + if not lines: + return None + + title = lines[0].lstrip("#").strip() + if not title: + return None + + sections: dict[str, str] = {} + current: str | None = None + buf: list[str] = [] + + def flush() -> None: + nonlocal buf, current + if current and buf: + sections[current] = "\n".join(buf).strip() + buf = [] + + for line in lines[1:]: + m = SECTION_HEADING.match(line) + if m: + flush() + current = m.group(1).strip().lower() + elif current: + buf.append(line) + flush() + + decision_text = ( + sections.get("decision") + or sections.get("decision outcome") + or sections.get("decision outcome / rationale") + or "" + ) + context = sections.get("context") or sections.get("issue") or "" + consequences = sections.get("consequences") or sections.get("consequences / implications") or "" + + content_parts = [title] + if decision_text: + content_parts.append(decision_text) + elif context: + content_parts.append(context[:500]) + content = ". ".join(p for p in content_parts if p) + + rationale: list[str] = [] + if context: + rationale.append(context[:400]) + if consequences: + rationale.append(consequences[:400]) + + status_raw = (sections.get("status") or "accepted").lower() + event_type = "update" if "supersed" in status_raw or "deprecated" in status_raw else "decision" + + return { + "title": title, + "content": content[:4000], + "rationale": rationale, + "status": status_raw, + "event_type": event_type, + "path_key": path_key, + } + + +def build_decision_from_adr( + parsed: dict[str, str | list[str]], + workspace_id: str, + *, + channel: str, +) -> DecisionEvent: + """Materialize parsed ADR fields into a DecisionEvent.""" + now = datetime.now(UTC) + path_key = str(parsed["path_key"]) + event_id = adr_uuid(path_key) + raw_id = adr_uuid(f"raw-{path_key}") + + importance = 0.82 if parsed["event_type"] == "decision" else 0.72 + trust = 0.85 + + return DecisionEvent( + event_id=event_id, + source_raw_event_id=raw_id, + workspace_id=workspace_id, + event_type=str(parsed["event_type"]), # type: ignore[arg-type] + content=str(parsed["content"]), + made_by=["adr-import"], + affects=[channel.split("/")[0] if "/" in channel else "architecture"], + rationale=list(parsed["rationale"]), # type: ignore[arg-type] + extraction_confidence=0.95, + importance_score=importance, + trust_score=trust, + provenance=Provenance( + source="manual", + channel=channel, + original_timestamp=now, + extractor_version="adr-import-1.0", + extractor_model="markdown-parser", + verified_by=[], + raw_event_id=raw_id, + ), + extracted_at=now, + status="active" if "accept" in str(parsed["status"]) else "under_review", + ) + + +def collect_adr_files(root: Path, glob_pattern: str) -> list[Path]: + """Return markdown files under root matching glob.""" + if root.is_file(): + return [root] if root.suffix.lower() in {".md", ".markdown"} else [] + return sorted(root.rglob(glob_pattern)) + + +def write_decisions(decisions: list[DecisionEvent]) -> tuple[int, int]: + """Score and persist decisions; return (written, skipped).""" + from graph.writer import GraphWriter + from memory.quarantine import persist_quarantine + from scoring.write_pipeline import DecisionScoringPipeline, write_reject_reason + + scoring = DecisionScoringPipeline() + writer = GraphWriter() + written = 0 + skipped = 0 + try: + for decision in decisions: + scoring.score(decision) + reject = write_reject_reason(decision) + if reject is not None: + persist_quarantine(decision, reject) + skipped += 1 + continue + writer.write(decision) + written += 1 + finally: + writer.close() + return written, skipped + + +def main() -> int: + parser = argparse.ArgumentParser(description="Import ADR markdown into Neo4j") + parser.add_argument("--path", type=Path, required=True, help="ADR file or directory") + parser.add_argument("--glob", default="*.md", help="Glob under directory (default: *.md)") + parser.add_argument("--workspace", default=None, help="Cortex workspace id") + parser.add_argument( + "--channel", + default=None, + help="Provenance channel label (default: adr/)", + ) + parser.add_argument("--dry-run", action="store_true", help="Parse only; do not write") + args = parser.parse_args() + + load_dotenv(_REPO / ".env") + workspace = args.workspace or os.environ.get("CORTEX_WORKSPACE_ID", "oss-adr") + + files = collect_adr_files(args.path, args.glob) + if not files: + print(f"No ADR files found under {args.path}", file=sys.stderr) + return 1 + + decisions: list[DecisionEvent] = [] + for file_path in files: + text = file_path.read_text(encoding="utf-8") + rel = file_path.relative_to(args.path) if args.path.is_dir() else file_path.name + path_key = str(rel) + parsed = parse_adr_markdown(text, path_key=path_key) + if parsed is None: + continue + channel = args.channel or f"adr/{path_key}" + decisions.append(build_decision_from_adr(parsed, workspace, channel=channel)) + + print(f"Parsed {len(decisions)} ADR(s) from {len(files)} file(s) → workspace={workspace!r}") + + if args.dry_run: + for d in decisions[:3]: + print(f" - {d.event_id}: {d.content[:80]}…") + if len(decisions) > 3: + print(f" … and {len(decisions) - 3} more") + return 0 + + written, skipped = write_decisions(decisions) + print(f"ADR import complete: wrote {written}, skipped {skipped} (quarantine/importance)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/import_github_graph.py b/scripts/import_github_graph.py new file mode 100644 index 0000000..8dbb6f9 --- /dev/null +++ b/scripts/import_github_graph.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Import merged GitHub PRs directly into Neo4j (no Kafka). + +Use for cloud demos where pipeline-worker is not deployed. Fetches PRs from +the GitHub REST API, normalises to RawEvent, extracts decisions, scores, and +writes via GraphWriter — same path as the extraction worker minus Kafka. + +Usage: + python scripts/import_github_graph.py --org tiangolo --repo fastapi --dry-run + python scripts/import_github_graph.py --org tiangolo --repo fastapi \\ + --workspace oss-tiangolo-fastapi --limit 30 +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +_REPO = Path(__file__).resolve().parents[1] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from dotenv import load_dotenv + +from connectors.github.producer import normalise_github_event +from extraction.decision_extractor import DecisionExtractor +from graph.writer import GraphWriter +from memory.quarantine import persist_quarantine +from scoring.write_pipeline import DecisionScoringPipeline, write_reject_reason +from scripts.import_github_org import ( + fetch_merged_prs, + is_decision_like, + pr_to_payload, +) +from shared.models import RawEvent + + +def _process_event( + raw: RawEvent, + *, + extractor: DecisionExtractor, + scoring: DecisionScoringPipeline, + writer: GraphWriter, +) -> str | None: + """Extract, score, and write one raw event; return decision id or None.""" + decision = extractor.extract(raw) + if decision is None: + return None + scoring.score(decision) + reject = write_reject_reason(decision) + if reject is not None: + persist_quarantine(decision, reject) + return None + try: + return writer.write(decision) + except ValueError: + return None + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Import merged GitHub PRs directly into Neo4j", + ) + parser.add_argument("--org", required=True, help="GitHub org or user") + parser.add_argument("--repo", required=True, help="Repository name") + parser.add_argument( + "--workspace", + default=None, + help="Cortex workspace id (default: oss--)", + ) + parser.add_argument("--limit", type=int, default=30, help="Max merged PRs to fetch") + parser.add_argument( + "--all-merged", + action="store_true", + help="Import all merged PRs (default: decision-like PRs only)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print planned RawEvents without writing to Neo4j", + ) + args = parser.parse_args() + + load_dotenv(_REPO / ".env") + workspace = args.workspace or f"oss-{args.org}-{args.repo}" + repo_full = f"{args.org}/{args.repo}" + decision_only = not args.all_merged + + print(f"Fetching merged PRs from {repo_full} (limit={args.limit})…") + prs = fetch_merged_prs(args.org, args.repo, limit=args.limit) + if decision_only: + prs = [pr for pr in prs if is_decision_like(pr)] + + print(f"Selected {len(prs)} pull request(s) for workspace={workspace!r}") + + raw_events: list[RawEvent] = [] + for pr in prs: + payload = pr_to_payload(pr, repo_full) + raw = normalise_github_event(payload, "pull_request", workspace) + if raw is not None: + raw_events.append(raw) + + if args.dry_run: + for raw in raw_events[:5]: + print(json.dumps(json.loads(raw.model_dump_json()), indent=2)[:500], "…") + if len(raw_events) > 5: + print(f"… and {len(raw_events) - 5} more") + print(f"\nDry run — would process {len(raw_events)} event(s) into Neo4j") + return 0 + + if not raw_events: + print("No events to import.", file=sys.stderr) + return 1 + + extractor = DecisionExtractor() + scoring = DecisionScoringPipeline() + writer = GraphWriter() + written: list[str] = [] + skipped = 0 + try: + for raw in raw_events: + event_id = _process_event( + raw, + extractor=extractor, + scoring=scoring, + writer=writer, + ) + if event_id: + written.append(event_id) + else: + skipped += 1 + finally: + writer.close() + + print(f"Import complete for workspace={workspace!r}") + print(f" Wrote {len(written)} decisions") + if skipped: + print(f" Skipped {skipped} events (no extract / below threshold / quarantine)") + return 0 if written else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/import_github_org.py b/scripts/import_github_org.py new file mode 100644 index 0000000..596f75a --- /dev/null +++ b/scripts/import_github_org.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Import merged GitHub PRs from a public repo into the Cortex Kafka pipeline. + +Fetches closed merged pull requests via the GitHub REST API, filters for +decision-like titles/bodies, normalises to RawEvent, and publishes to +``cortex.raw.github.events`` for the extraction worker. + +Usage: + python scripts/import_github_org.py --org tiangolo --repo fastapi --dry-run + python scripts/import_github_org.py --org tiangolo --repo fastapi --workspace oss-fastapi --limit 30 +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +_REPO = Path(__file__).resolve().parents[1] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from dotenv import load_dotenv + +from connectors.github.producer import GitHubKafkaProducer, normalise_github_event +from scripts.inject_github_event import build_merged_pr_payload + +DECISION_PATTERN = re.compile( + r"\b(decision|rfc|adr|migrate|migration|deprecat|architect|proposal|" + r"we decided|chosen|replace|switch to)\b", + re.IGNORECASE, +) + +GITHUB_API = "https://api.github.com" + + +def _github_headers() -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "cortex-import-github-org", + } + token = os.environ.get("GITHUB_TOKEN", "").strip() + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def fetch_merged_prs( + owner: str, + repo: str, + *, + limit: int, + page_size: int = 100, +) -> list[dict[str, Any]]: + """Return up to ``limit`` merged pull requests (newest first).""" + collected: list[dict[str, Any]] = [] + page = 1 + while len(collected) < limit: + url = ( + f"{GITHUB_API}/repos/{owner}/{repo}/pulls" + f"?state=closed&sort=updated&direction=desc&per_page={page_size}&page={page}" + ) + req = urllib.request.Request(url, headers=_github_headers()) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + batch: list[dict[str, Any]] = json.loads(resp.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + raise RuntimeError(f"GitHub API error {exc.code}: {exc.reason}") from exc + + if not batch: + break + + for pr in batch: + if pr.get("merged_at"): + collected.append(pr) + if len(collected) >= limit: + break + page += 1 + if len(batch) < page_size: + break + time.sleep(0.5) + + return collected[:limit] + + +def is_decision_like(pr: dict[str, Any]) -> bool: + """Heuristic filter for PRs likely to contain organizational decisions.""" + text = f"{pr.get('title', '')}\n{pr.get('body') or ''}" + return bool(DECISION_PATTERN.search(text)) + + +def pr_to_payload(pr: dict[str, Any], repo_full_name: str) -> dict[str, Any]: + """Map GitHub pull object to webhook-shaped payload.""" + user = pr.get("user") or {} + head = pr.get("head") or {} + return build_merged_pr_payload( + title=str(pr.get("title", "")), + body=str(pr.get("body") or ""), + repo=repo_full_name, + number=int(pr.get("number", 0)), + user_login=str(user.get("login", "unknown")), + updated_at=str(pr.get("updated_at") or pr.get("merged_at") or ""), + created_at=str(pr.get("created_at") or ""), + head_ref=str(head.get("ref", "feature/import")), + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Import merged GitHub PRs into cortex.raw.github.events", + ) + parser.add_argument("--org", required=True, help="GitHub org or user") + parser.add_argument("--repo", required=True, help="Repository name") + parser.add_argument( + "--workspace", + default=None, + help="Cortex workspace id (default: oss--)", + ) + parser.add_argument("--limit", type=int, default=50, help="Max merged PRs to fetch") + parser.add_argument( + "--all-merged", + action="store_true", + help="Import all merged PRs (default: filter decision-like PRs only)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print planned RawEvents without publishing to Kafka", + ) + args = parser.parse_args() + + load_dotenv(_REPO / ".env") + workspace = args.workspace or f"oss-{args.org}-{args.repo}" + repo_full = f"{args.org}/{args.repo}" + decision_only = not args.all_merged + + print(f"Fetching merged PRs from {repo_full} (limit={args.limit})…") + prs = fetch_merged_prs(args.org, args.repo, limit=args.limit) + if decision_only: + prs = [pr for pr in prs if is_decision_like(pr)] + + print(f"Selected {len(prs)} pull request(s) for workspace={workspace!r}") + + events: list[tuple[str, dict[str, Any]]] = [] + for pr in prs: + payload = pr_to_payload(pr, repo_full) + raw = normalise_github_event(payload, "pull_request", workspace) + if raw is None: + continue + events.append((raw.event_id, json.loads(raw.model_dump_json()))) + + if args.dry_run: + for _eid, body in events[:5]: + print(json.dumps(body, indent=2)[:500], "…") + if len(events) > 5: + print(f"… and {len(events) - 5} more") + print(f"\nDry run — would publish {len(events)} event(s) to cortex.raw.github.events") + return 0 + + if not events: + print("No events to publish.", file=sys.stderr) + return 1 + + producer = GitHubKafkaProducer() + try: + for _eid, body in events: + from shared.models import RawEvent + + raw_event = RawEvent.model_validate(body) + producer.publish(raw_event) + producer.flush(timeout=30.0) + finally: + producer.close() + + print(f"Published {len(events)} event(s) to cortex.raw.github.events") + print("Run pipeline-worker and query with workspace_id=", repr(workspace)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/inject_github_event.py b/scripts/inject_github_event.py index 46a80fd..df8eba4 100644 --- a/scripts/inject_github_event.py +++ b/scripts/inject_github_event.py @@ -31,20 +31,31 @@ DEFAULT_TITLE = "Decision: migrate payments to CockroachDB" -def build_merged_pr_payload(*, title: str, body: str, repo: str) -> dict: +def build_merged_pr_payload( + *, + title: str, + body: str, + repo: str, + number: int = 42, + user_login: str = "priya", + updated_at: str = "2026-06-10T18:00:00Z", + created_at: str = "2026-06-10T17:00:00Z", + head_ref: str = "feature/decision", +) -> dict: """Build a GitHub ``pull_request`` closed+merged webhook payload.""" return { "action": "closed", "pull_request": { - "number": 42, + "number": number, "title": title, - "body": body, + "body": body or "", "merged": True, - "user": {"login": "priya"}, + "merged_at": updated_at, + "user": {"login": user_login}, "base": {"ref": "main"}, - "head": {"ref": "feature/cockroachdb"}, - "updated_at": "2026-06-10T18:00:00Z", - "created_at": "2026-06-10T17:00:00Z", + "head": {"ref": head_ref}, + "updated_at": updated_at, + "created_at": created_at, "labels": [], "requested_reviewers": [], }, diff --git a/scripts/seed_oss_fastapi_demo.py b/scripts/seed_oss_fastapi_demo.py new file mode 100644 index 0000000..75959fd --- /dev/null +++ b/scripts/seed_oss_fastapi_demo.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Seed oss-tiangolo-fastapi workspace with FastAPI-themed demo decisions. + +Use when GitHub import (import_github_graph.py) is unavailable — e.g. cloud +Neo4j without Kafka or without GitHub token. Idempotent via deterministic UUIDs. + +Usage: + python scripts/seed_oss_fastapi_demo.py + python scripts/seed_oss_fastapi_demo.py --dry-run +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parents[1] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from dotenv import load_dotenv + +from scripts.demo_catalog import _spec, build_decision_from_spec + +WORKSPACE = "oss-tiangolo-fastapi" + +_OSS_SPECS: list[dict[str, object]] = [ + _spec( + "oss-async-deps", + source="github", + channel="pull_request", + content="Adopt async dependency injection pattern for database sessions in FastAPI routers.", + made_by=["person:tiangolo"], + affects=["system:fastapi-core"], + rationale=["Aligns with ASGI concurrency model; reduces thread pool pressure."], + importance=0.72, + trust=0.8, + ), + _spec( + "oss-openapi-3-1", + source="github", + channel="pull_request", + content="Standardize on OpenAPI 3.1 schema generation for all public FastAPI examples.", + made_by=["person:tiangolo"], + affects=["system:fastapi-docs", "system:openapi-generator"], + rationale=["Improves SDK compatibility for downstream API clients."], + importance=0.74, + trust=0.82, + ), + _spec( + "oss-pydantic-v2", + source="github", + channel="issue_comment", + content="Migrate validation layer to Pydantic v2 with model_config instead of Config class.", + made_by=["person:tiangolo"], + affects=["system:fastapi-core"], + rationale=["Performance gains and parity with ecosystem tooling."], + importance=0.76, + trust=0.83, + ), + _spec( + "oss-starlette-pin", + source="github", + channel="pull_request", + content="Pin Starlette minor version in release branches to avoid breaking middleware hooks.", + made_by=["person:tiangolo"], + affects=["system:fastapi-core", "system:starlette"], + rationale=["Prevents surprise regressions in long-lived LTS installs."], + importance=0.7, + trust=0.79, + ), + _spec( + "oss-security-defaults", + source="github", + channel="pull_request", + content="Enable secure cookie defaults and document HTTPS requirements in deployment guides.", + made_by=["person:tiangolo"], + affects=["system:fastapi-docs", "system:fastapi-security"], + rationale=["Reduces misconfiguration in production deployments."], + importance=0.78, + trust=0.85, + ), +] + + +def main() -> int: + parser = argparse.ArgumentParser(description="Seed oss-tiangolo-fastapi demo workspace") + parser.add_argument("--workspace", default=WORKSPACE) + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + load_dotenv(_REPO / ".env") + decisions = [build_decision_from_spec(spec, args.workspace) for spec in _OSS_SPECS] + + if args.dry_run: + print(f"Dry run — would write {len(decisions)} decisions to {args.workspace!r}") + for d in decisions: + print(f" - {d.event_id}: {d.content[:64]}...") + return 0 + + from graph.writer import GraphWriter + from memory.quarantine import persist_quarantine + from scoring.write_pipeline import DecisionScoringPipeline, write_reject_reason + + scoring = DecisionScoringPipeline() + writer = GraphWriter() + written: list[str] = [] + try: + for decision in decisions: + scoring.score(decision) + reject = write_reject_reason(decision) + if reject is not None: + persist_quarantine(decision, reject) + continue + written.append(writer.write(decision)) + finally: + writer.close() + + print(f"OSS seed complete for workspace={args.workspace!r}: {len(written)} decisions") + return 0 if written else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/start_api_production.sh b/scripts/start_api_production.sh new file mode 100755 index 0000000..02f5748 --- /dev/null +++ b/scripts/start_api_production.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Production API entrypoint — migrate graph schema, optional one-time seed, then serve. +# +# CORTEX_SEED_DEMO defaults to false so redeploys do not re-seed. Set to "true" only +# on first boot or when intentionally resetting demo data. +set -eu + +echo "Running Neo4j migrations..." +python graph/migrate.py + +if [ "${CORTEX_SEED_DEMO:-false}" = "true" ]; then + echo "Seeding local-dev demo workspace (CORTEX_SEED_DEMO=true)..." + python scripts/seed_demo.py --workspace local-dev --scale small +else + echo "Skipping demo seed (CORTEX_SEED_DEMO is not true)." +fi + +echo "Starting uvicorn on port ${PORT:-8000}..." +exec uvicorn api.main:app --host 0.0.0.0 --port "${PORT:-8000}" diff --git a/scripts/start_portfolio_demo.sh b/scripts/start_portfolio_demo.sh new file mode 100755 index 0000000..1393554 --- /dev/null +++ b/scripts/start_portfolio_demo.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Start cloudflared tunnel to local Cortex API for Vercel portfolio demo. +# Requires: docker compose api on :8000, cloudflared installed (brew install cloudflared). +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +API_URL="${CORTEX_API_URL:-http://localhost:8000}" +VERCEL_DEMO_URL="${VERCEL_DEMO_URL:-https://frontend-ten-rouge-99.vercel.app}" + +if ! curl -sf "${API_URL}/health" >/dev/null 2>&1; then + echo "ERROR: Cortex API not reachable at ${API_URL}" >&2 + echo "Start the stack: make demo (or: docker compose --profile api up -d)" >&2 + exit 1 +fi + +if ! command -v cloudflared >/dev/null 2>&1; then + echo "ERROR: cloudflared not found. Install: brew install cloudflared" >&2 + exit 1 +fi + +echo "Starting cloudflared tunnel → ${API_URL}" +echo "When the trycloudflare.com URL appears below:" +echo " 1. Vercel → frontend project → Settings → Environment Variables" +echo " 2. Set CORTEX_API_ORIGIN to that URL (Production)" +echo " 3. Share demo: ${VERCEL_DEMO_URL}" +echo "" +echo "For 24/7 demo without this laptop, see docs/PORTFOLIO_DEMO.md" +echo "---" + +exec cloudflared tunnel --url "${API_URL}" diff --git a/tests/api/test_edge_cases.py b/tests/api/test_edge_cases.py index e924216..4e39184 100644 --- a/tests/api/test_edge_cases.py +++ b/tests/api/test_edge_cases.py @@ -93,6 +93,26 @@ def test_inject_rejects_short_context() -> None: assert response.status_code == 422 +def test_inject_uses_at_least_one_result_slot_for_small_token_budget() -> None: + """max_tokens=100 is valid; inject must not pass limit=0 to memory.""" + client = TestClient(app) + mock_memory = AsyncMock() + mock_memory.inject_decisions = AsyncMock(return_value=[]) + with patch("api.main.memory", return_value=mock_memory): + response = client.post( + "/inject", + json={ + "context": "payments service database migration context", + "workspace_id": "ws-1", + "agent_id": "agent-1", + "max_tokens": 100, + }, + ) + assert response.status_code == 200 + mock_memory.inject_decisions.assert_awaited_once() + assert mock_memory.inject_decisions.await_args.kwargs["limit"] == 1 + + def test_query_limit_bounds() -> None: client = TestClient(app) response = client.post( diff --git a/tests/api/test_memory_resilience.py b/tests/api/test_memory_resilience.py index db1c697..6a875cc 100644 --- a/tests/api/test_memory_resilience.py +++ b/tests/api/test_memory_resilience.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -92,6 +92,21 @@ async def test_query_propagates_graph_failure() -> None: ) +def test_build_redis_client_uses_redis_url(monkeypatch: pytest.MonkeyPatch) -> None: + """REDIS_URL (Upstash) takes precedence over REDIS_HOST for cloud deploys.""" + monkeypatch.setenv("REDIS_URL", "redis://localhost:6379/0") + monkeypatch.delenv("REDIS_HOST", raising=False) + + with patch("redis.from_url") as from_url: + client = MagicMock() + client.ping.return_value = True + from_url.return_value = client + built = MemoryService._build_redis_client() + + from_url.assert_called_once() + assert built is client + + def test_redis_health_unreachable_when_client_none() -> None: svc = MemoryService() svc._redis = None diff --git a/tests/scripts/test_dual_workspace_smoke.py b/tests/scripts/test_dual_workspace_smoke.py new file mode 100644 index 0000000..7f8837f --- /dev/null +++ b/tests/scripts/test_dual_workspace_smoke.py @@ -0,0 +1,46 @@ +"""Unit tests for scripts/dual_workspace_smoke.py.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from unittest.mock import patch + +_REPO = Path(__file__).resolve().parents[2] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from scripts import dual_workspace_smoke as dws + + +def test_post_query_parses_json() -> None: + payload = {"results": [{"content": "CockroachDB migration"}], "total": 1} + body = json.dumps(payload).encode("utf-8") + + class FakeResp: + def read(self) -> bytes: + return body + + def __enter__(self) -> FakeResp: + return self + + def __exit__(self, *args: object) -> None: + return None + + with patch("urllib.request.urlopen", return_value=FakeResp()): + out = dws.post_query("http://localhost:8000", query="test", workspace_id="local-dev", limit=3) + assert out["total"] == 1 + + +def test_main_ok() -> None: + fake = {"results": [{"content": "hit"}], "total": 1} + with ( + patch.object(dws, "post_query", return_value=fake), + patch.object( + sys, + "argv", + ["dual_workspace_smoke.py", "--workspaces", "local-dev", "--queries", "test"], + ), + ): + assert dws.main() == 0 diff --git a/tests/scripts/test_import_adr_markdown.py b/tests/scripts/test_import_adr_markdown.py new file mode 100644 index 0000000..e0efb27 --- /dev/null +++ b/tests/scripts/test_import_adr_markdown.py @@ -0,0 +1,71 @@ +"""Unit tests for scripts/import_adr_markdown.py (no Neo4j).""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parents[2] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from scripts import import_adr_markdown as iam + +SAMPLE_ADR = """# Use Redis for session cache + +## Status + +Accepted + +## Context + +In-memory sessions do not survive horizontal scaling. + +## Decision + +We will store sessions in Redis with a 24h TTL. + +## Consequences + +Requires Redis cluster in staging and production. +""" + + +def test_parse_adr_markdown_extracts_sections() -> None: + parsed = iam.parse_adr_markdown(SAMPLE_ADR, path_key="001-redis.md") + assert parsed is not None + assert parsed["title"] == "Use Redis for session cache" + assert "Redis" in str(parsed["content"]) + assert parsed["event_type"] == "decision" + assert len(parsed["rationale"]) >= 1 + + +def test_adr_uuid_stable() -> None: + assert iam.adr_uuid("001-redis.md") == iam.adr_uuid("001-redis.md") + assert iam.adr_uuid("a") != iam.adr_uuid("b") + + +def test_build_decision_from_adr_workspace() -> None: + parsed = iam.parse_adr_markdown(SAMPLE_ADR, path_key="001-redis.md") + assert parsed is not None + decision = iam.build_decision_from_adr(parsed, "oss-adr-test", channel="adr/001-redis.md") + assert decision.workspace_id == "oss-adr-test" + assert decision.importance_score >= 0.7 + assert decision.provenance.channel == "adr/001-redis.md" + + +def test_collect_adr_files_single_file(tmp_path: Path) -> None: + adr = tmp_path / "0001-test.md" + adr.write_text(SAMPLE_ADR, encoding="utf-8") + files = iam.collect_adr_files(adr, "*.md") + assert files == [adr] + + +def test_collect_adr_files_directory(tmp_path: Path) -> None: + sub = tmp_path / "adr" + sub.mkdir() + (sub / "a.md").write_text(SAMPLE_ADR, encoding="utf-8") + (sub / "skip.txt").write_text("nope", encoding="utf-8") + files = iam.collect_adr_files(sub, "*.md") + assert len(files) == 1 + assert files[0].name == "a.md" diff --git a/tests/scripts/test_import_github_org.py b/tests/scripts/test_import_github_org.py new file mode 100644 index 0000000..f9d1fa4 --- /dev/null +++ b/tests/scripts/test_import_github_org.py @@ -0,0 +1,87 @@ +"""Unit tests for scripts/import_github_org.py (no GitHub / Kafka).""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +_REPO = Path(__file__).resolve().parents[2] +if str(_REPO) not in sys.path: + sys.path.insert(0, str(_REPO)) + +from scripts import import_github_org as igo + + +SAMPLE_PR = { + "number": 123, + "title": "RFC: migrate session cache to Redis", + "body": "We decided to replace in-memory sessions.", + "merged_at": "2026-01-15T12:00:00Z", + "updated_at": "2026-01-15T12:00:00Z", + "created_at": "2026-01-10T09:00:00Z", + "user": {"login": "alice"}, + "head": {"ref": "feature/redis-cache"}, +} + + +def test_is_decision_like_matches_keywords() -> None: + assert igo.is_decision_like(SAMPLE_PR) is True + assert igo.is_decision_like({"title": "Fix typo", "body": "n/a"}) is False + + +def test_pr_to_payload_includes_repo_and_number() -> None: + payload = igo.pr_to_payload(SAMPLE_PR, "tiangolo/fastapi") + assert payload["pull_request"]["number"] == 123 + assert payload["repository"]["full_name"] == "tiangolo/fastapi" + assert payload["pull_request"]["user"]["login"] == "alice" + + +def test_main_dry_run_filters_decision_prs() -> None: + prs = [ + SAMPLE_PR, + {"number": 1, "title": "chore: bump deps", "body": "", "merged_at": "2026-01-01T00:00:00Z"}, + ] + with ( + patch.object(igo, "fetch_merged_prs", return_value=prs), + patch.object(sys, "argv", ["import_github_org.py", "--org", "o", "--repo", "r", "--dry-run"]), + ): + assert igo.main() == 0 + + +def test_main_dry_run_all_merged() -> None: + prs = [ + SAMPLE_PR, + { + "number": 2, + "title": "chore: bump deps", + "body": "", + "merged_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + "created_at": "2026-01-01T00:00:00Z", + "user": {"login": "bot"}, + "head": {"ref": "chore"}, + }, + ] + with ( + patch.object(igo, "fetch_merged_prs", return_value=prs), + patch.object( + sys, + "argv", + ["import_github_org.py", "--org", "o", "--repo", "r", "--all-merged", "--dry-run"], + ), + ): + assert igo.main() == 0 + + +def test_main_publish_calls_producer() -> None: + producer = MagicMock() + with ( + patch.object(igo, "fetch_merged_prs", return_value=[SAMPLE_PR]), + patch.object(igo, "GitHubKafkaProducer", return_value=producer), + patch.object(sys, "argv", ["import_github_org.py", "--org", "o", "--repo", "r"]), + ): + assert igo.main() == 0 + assert producer.publish.call_count == 1 + producer.flush.assert_called_once() + producer.close.assert_called_once() diff --git a/tests/scripts/test_post_deploy_scripts.py b/tests/scripts/test_post_deploy_scripts.py new file mode 100644 index 0000000..ab215bd --- /dev/null +++ b/tests/scripts/test_post_deploy_scripts.py @@ -0,0 +1,57 @@ +"""Tests for post-deploy seed/import scripts.""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +_REPO = Path(__file__).resolve().parents[2] + + +def test_seed_oss_fastapi_dry_run() -> None: + proc = subprocess.run( + [ + sys.executable, + str(_REPO / "scripts" / "seed_oss_fastapi_demo.py"), + "--dry-run", + ], + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0, proc.stderr + assert "oss-tiangolo-fastapi" in proc.stdout + + +def test_import_github_graph_dry_run() -> None: + fake_prs = [ + { + "number": 1, + "title": "RFC: migrate architecture to async", + "body": "We decided to switch dependency injection pattern.", + "merged_at": "2024-01-01T00:00:00Z", + "updated_at": "2024-01-01T00:00:00Z", + "created_at": "2024-01-01T00:00:00Z", + "user": {"login": "tiangolo"}, + "head": {"ref": "feature/async"}, + } + ] + import scripts.import_github_graph as import_mod + + argv = [ + "import_github_graph.py", + "--org", + "tiangolo", + "--repo", + "fastapi", + "--workspace", + "oss-tiangolo-fastapi", + "--limit", + "3", + "--dry-run", + ] + with patch.object(sys, "argv", argv): + with patch.object(import_mod, "fetch_merged_prs", return_value=fake_prs): + assert import_mod.main() == 0 diff --git a/vercel.json b/vercel.json new file mode 100644 index 0000000..7f4447c --- /dev/null +++ b/vercel.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://openapi.vercel.sh/vercel.json", + "installCommand": "cd frontend && npm install", + "buildCommand": "cd frontend && npm run build", + "outputDirectory": "frontend/dist", + "framework": "vite", + "rewrites": [ + { + "source": "/((?!assets/).*)", + "destination": "/index.html" + } + ] +}