diff --git a/.gitignore b/.gitignore index 45b44f1..adf496d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,5 +13,3 @@ build/ .vercel data/raw/ data/processed/ -artifacts/models/ -artifacts/reports/ diff --git a/README.md b/README.md index 42c09d3..af9e7d4 100644 --- a/README.md +++ b/README.md @@ -19,23 +19,24 @@ FlightOps AI is a tested vertical slice of an operations platform: it accepts re | Backend engineering | Typed FastAPI routes, validation, layered service/repository design, OpenAPI docs | | Data engineering | Normalized UTC events, indexed SQLite persistence, latest-event queries, aggregate summaries | | Reliability | Idempotent writes, bounded inputs, typed 404s, health endpoint, retry-safe behavior | -| Explainable AI | Versioned baseline risk score with per-factor contributions—clearly identified as rules, not a trained model | +| Applied ML | Evaluated T-24h delay model trained on 90,000 official BTS records, portable JSON inference, calibration, model card, and error slices | | Observability | Prometheus-compatible request and ingestion counters | | Cloud delivery | Non-root Docker image, Compose setup, Vercel entrypoint, automated GitHub Actions tests | ### At a glance -- **5 operational endpoints** for ingestion, risk, summaries, health, and metrics +- **7 operational endpoints** for ingestion, risk, trained prediction, model evidence, summaries, health, and metrics - **4 scored risk signals** plus an explicit cancellation adjustment - **Idempotency by event ID**, so producer retries do not duplicate records -- **6 automated tests** covering API behavior, validation, scoring, summaries, and metrics +- **8 automated tests** covering API behavior, validation, rules, trained inference, summaries, and metrics +- **90,000 official BTS records** in a checksum-tracked, reproducible chronological evaluation - **One-command local start** with Docker Compose ## Two-minute recruiter tour -1. Read the [risk engine](app/risk.py) to see transparent feature contributions and model versioning. -2. Read the [API test](tests/test_api.py) to see idempotency, validation, risk, summaries, and metrics working together. -3. Read the [architecture notes](docs/architecture.md) for current tradeoffs and the production evolution path. +1. Read the [model card](docs/ml/model-card.md) for the chronological evaluation, calibration, operating threshold, error slices, and limitations. +2. Read the [portable inference code](app/ml_model.py) to see how inspectable JSON coefficients serve predictions without unsafe pickle loading. +3. Read the [API tests](tests/test_api.py) to see idempotency, validation, rules, trained inference, summaries, and metrics working together. 4. Open the [live API explorer](https://flightops-ai-mu.vercel.app/docs) to exercise every endpoint without local setup. ## Architecture @@ -46,9 +47,11 @@ flowchart LR A --> S["Operations service"] S --> R["SQLite event repository"] S --> E["Explainable risk engine"] + A --> P["T-24h schedule model"] A --> M["Service metrics"] R --> Q["Operational summary"] E --> O["Risk score + factor explanations"] + P --> O2["Delay probability + model version"] ``` The repository boundary keeps persistence replaceable. SQLite makes the demo reproducible with zero infrastructure; PostgreSQL is the planned durable production store. See [docs/architecture.md](docs/architecture.md) and [ADR 0001](docs/adr/0001-start-with-a-tested-vertical-slice.md). @@ -110,6 +113,15 @@ curl http://127.0.0.1:8000/v1/operations/summary curl http://127.0.0.1:8000/metrics ``` +Request a schedule-only trained-model prediction: + +```bash +curl -X POST http://127.0.0.1:8000/v1/predictions/delay \ + -H 'content-type: application/json' \ + -d '{"flight_date":"2026-08-10","reporting_airline":"DL","origin":"IAD","destination":"ATL","crs_departure_time":815,"crs_elapsed_time":115,"distance":534,"distance_group":3}' +curl http://127.0.0.1:8000/v1/models/delay/metadata +``` + Posting the same `event_id` again returns `200` with `"created": false`; the database keeps one event. That makes upstream retries safe. ## API surface @@ -120,8 +132,25 @@ Posting the same `event_id` again returns `200` with `"created": false`; the dat | `POST` | `/v1/events` | Validate and idempotently store a flight event | | `GET` | `/v1/flights/{flight_id}/risk` | Score the latest event and explain each contribution | | `GET` | `/v1/operations/summary` | Return fleet-level event and high-risk counts | +| `POST` | `/v1/predictions/delay` | Predict T-24h arrival-delay probability from schedule-only fields | +| `GET` | `/v1/models/delay/metadata` | Expose model version, threshold, evaluation metrics, lineage, and limitations | | `GET` | `/metrics` | Export Prometheus-compatible counters | +## Trained-model evidence + +The deployed `bts-schedule-logistic-v1` model was trained on a deterministic 90,000-row sample from official BTS monthly files. Training uses Jan-Dec 2024, threshold selection uses Jan-Mar 2025, and the untouched test period is Apr-Jun 2025. + +| Test metric | Result | +| --- | ---: | +| ROC-AUC | 0.640 | +| PR-AUC | 0.335 | +| Precision | 0.304 | +| Recall | 0.748 | +| F1 | 0.432 | +| Brier score | 0.176 | + +These are retrospective public-data results, not production accuracy claims. Reproduce the download and training with the [ML runbook](ml/README.md), then inspect the [model card](docs/ml/model-card.md), [data manifest](artifacts/data/bts-sample-manifest.json), and [error slices](artifacts/reports/error-slices.csv). + ## Quality checks ```bash @@ -132,7 +161,7 @@ GitHub Actions installs the project in a clean Python 3.12 runner and executes t ## Decisions and honest limitations -- The current score is a deterministic, versioned baseline—not a trained ML model. This makes the behavior testable and gives a future model a measurable benchmark. +- The event-risk endpoint remains a deterministic rules fallback; the separate schedule-prediction endpoint uses the evaluated trained model and never mixes post-departure fields into T-24h inference. - SQLite is appropriate for a local demonstration. On Vercel it uses ephemeral `/tmp` storage, so the public demo is not a durable system of record. - Metrics are process-local. A production deployment would export them to managed observability infrastructure. - Authentication, rate limiting, a durable event queue, and infrastructure-as-code belong in a later production milestone. @@ -147,13 +176,13 @@ GitHub Actions installs the project in a clean Python 3.12 runner and executes t - [x] Docker packaging and CI - [ ] PostgreSQL migrations and query-performance evidence - [ ] React + TypeScript operations dashboard -- [ ] Versioned delay-prediction model with an evaluation report +- [x] Versioned delay-prediction model with an evaluation report - [ ] Queue, retries, caching, and failure-injection tests - [ ] Evaluated incident/runbook assistant with citations - [ ] AWS deployment with Terraform and a cost estimate - [ ] Load-test report, SLO, and incident write-up -The trained-model milestone is specified in [Issue #2](https://github.com/mitulpatel123/flightops-ai/issues/2) and begins with a leakage-resistant [data contract](docs/ml/data-contract.md). +The trained-model milestone is specified in [Issue #2](https://github.com/mitulpatel123/flightops-ai/issues/2) and governed by a leakage-resistant [data contract](docs/ml/data-contract.md). ## Project integrity diff --git a/app/main.py b/app/main.py index ef8842e..ecc6e83 100644 --- a/app/main.py +++ b/app/main.py @@ -7,7 +7,15 @@ from fastapi.responses import RedirectResponse from app.db import EventRepository -from app.models import DelayRisk, FlightEvent, IngestResponse, OperationsSummary +from app.ml_model import get_delay_model +from app.models import ( + DelayRisk, + FlightEvent, + IngestResponse, + ModelPrediction, + OperationsSummary, + ScheduledFlight, +) from app.service import FlightNotFoundError, OperationsService @@ -41,8 +49,8 @@ async def lifespan(_: FastAPI): api = FastAPI( title="FlightOps AI", - version="0.1.0", - description="Airline operations event ingestion and explainable delay-risk API.", + version="0.2.0", + description="Airline operations ingestion, explainable risk, and evaluated T-24h delay prediction API.", lifespan=lifespan, ) @@ -52,7 +60,7 @@ def index() -> RedirectResponse: @api.get("/health") def health() -> dict[str, str]: - return {"status": "ok", "version": "0.1.0"} + return {"status": "ok", "version": "0.2.0"} @api.post("/v1/events", response_model=IngestResponse, status_code=status.HTTP_201_CREATED) def ingest_event(event: FlightEvent, response: Response) -> IngestResponse: @@ -77,6 +85,15 @@ def get_summary() -> OperationsSummary: metrics.increment("summary_requests") return service.summary() + @api.post("/v1/predictions/delay", response_model=ModelPrediction) + def predict_delay(flight: ScheduledFlight) -> ModelPrediction: + metrics.increment("delay_predictions") + return get_delay_model().predict(flight) + + @api.get("/v1/models/delay/metadata") + def delay_model_metadata() -> dict: + return get_delay_model().metadata() + @api.get("/metrics", response_class=Response) def get_metrics() -> Response: return Response(metrics.render(), media_type="text/plain; version=0.0.4") diff --git a/app/ml_model.py b/app/ml_model.py new file mode 100644 index 0000000..ffc709f --- /dev/null +++ b/app/ml_model.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import json +import math +from functools import lru_cache +from pathlib import Path +from typing import Any + +from app.models import ModelPrediction, ScheduledFlight + + +MODEL_PATH = Path(__file__).resolve().parent.parent / "artifacts" / "models" / "delay-logistic-v1.json" + + +class PortableLogisticModel: + """Small, inspectable JSON logistic model with no pickle execution risk.""" + + def __init__(self, payload: dict[str, Any]) -> None: + self.payload = payload + self.threshold = float(payload["threshold"]) + self.version = str(payload["model_version"]) + + @classmethod + def from_path(cls, path: Path = MODEL_PATH) -> "PortableLogisticModel": + return cls(json.loads(path.read_text(encoding="utf-8"))) + + def predict(self, flight: ScheduledFlight) -> ModelPrediction: + values = _model_values(flight) + linear = float(self.payload["intercept"]) + + numeric = self.payload["numeric"] + for name, value in values.items(): + if name not in numeric: + continue + spec = numeric[name] + scale = float(spec["scale"]) or 1.0 + standardized = (float(value) - float(spec["mean"])) / scale + linear += standardized * float(spec["coefficient"]) + + categorical = self.payload["categorical"] + for name in ("Reporting_Airline", "Origin", "Dest"): + category = str(values[name]) + linear += float(categorical[name].get(category, 0.0)) + + probability = 1.0 / (1.0 + math.exp(-max(-35.0, min(35.0, linear)))) + probability = round(probability, 4) + return ModelPrediction( + probability=probability, + predicted_delayed=probability >= self.threshold, + threshold=self.threshold, + model_version=self.version, + caveat="Retrospective BTS schedule-data estimate; not a live operational guarantee.", + ) + + def metadata(self) -> dict[str, Any]: + return { + "model_version": self.version, + "prediction_time": self.payload["prediction_time"], + "threshold": self.threshold, + "trained_at": self.payload["trained_at"], + "data_manifest_sha256": self.payload["data_manifest_sha256"], + "test_metrics": self.payload["test_metrics"], + "limitations": self.payload["limitations"], + } + + +def _model_values(flight: ScheduledFlight) -> dict[str, float | str]: + hour = flight.crs_departure_time // 100 + angle = 2.0 * math.pi * hour / 24.0 + return { + "Month": float(flight.flight_date.month), + "DayOfWeek": float(flight.flight_date.isoweekday()), + "DepHourSin": math.sin(angle), + "DepHourCos": math.cos(angle), + "CRSElapsedTime": float(flight.crs_elapsed_time), + "Distance": float(flight.distance), + "DistanceGroup": float(flight.distance_group), + "Reporting_Airline": flight.reporting_airline, + "Origin": flight.origin, + "Dest": flight.destination, + } + + +@lru_cache(maxsize=1) +def get_delay_model() -> PortableLogisticModel: + return PortableLogisticModel.from_path() diff --git a/app/models.py b/app/models.py index a4a8157..7a98eb7 100644 --- a/app/models.py +++ b/app/models.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import date, datetime, timezone from typing import Literal from pydantic import BaseModel, ConfigDict, Field, field_validator @@ -61,3 +61,39 @@ class OperationsSummary(BaseModel): delayed_events: int cancelled_events: int high_risk_flights: int + + +class ScheduledFlight(BaseModel): + """Schedule-only fields available at least 24 hours before departure.""" + + model_config = ConfigDict(str_strip_whitespace=True) + + flight_date: date + reporting_airline: str = Field(min_length=2, max_length=8) + origin: str = Field(min_length=3, max_length=4) + destination: str = Field(min_length=3, max_length=4) + crs_departure_time: int = Field(ge=0, le=2359) + crs_elapsed_time: float = Field(gt=0, le=1440) + distance: float = Field(gt=0, le=12000) + distance_group: int = Field(ge=1, le=25) + + @field_validator("reporting_airline", "origin", "destination") + @classmethod + def normalize_code(cls, value: str) -> str: + return value.upper() + + @field_validator("crs_departure_time") + @classmethod + def validate_clock_time(cls, value: int) -> int: + if value % 100 >= 60: + raise ValueError("crs_departure_time must be HHMM") + return value + + +class ModelPrediction(BaseModel): + probability: float = Field(ge=0.0, le=1.0) + predicted_delayed: bool + threshold: float = Field(ge=0.0, le=1.0) + model_version: str + prediction_time: str = "T-24h" + caveat: str diff --git a/artifacts/data/bts-sample-manifest.json b/artifacts/data/bts-sample-manifest.json new file mode 100644 index 0000000..ffc7279 --- /dev/null +++ b/artifacts/data/bts-sample-manifest.json @@ -0,0 +1,235 @@ +{ + "schema_version": 1, + "source": "BTS Reporting Carrier On-Time Performance", + "source_table": "Reporting Carrier On-Time Performance (1987-present)", + "period": { + "start": "2024-01", + "end": "2025-06" + }, + "sampling": { + "method": "deterministic uniform sample after cohort filtering", + "rows_per_month": 5000, + "random_seed": 1062 + }, + "required_columns": [ + "Year", + "Month", + "DayofMonth", + "DayOfWeek", + "FlightDate", + "Reporting_Airline", + "Origin", + "Dest", + "CRSDepTime", + "CRSArrTime", + "CRSElapsedTime", + "Distance", + "DistanceGroup", + "ArrDel15", + "Cancelled", + "Diverted" + ], + "processed_file": "bts-schedule-sample.csv.gz", + "processed_rows": 90000, + "processed_sha256": "298bc459eae629fa393961e2844be617019d800a05e63c29209548d5261c6b12", + "sources": [ + { + "month": "2024-01", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_1.zip", + "zip_bytes": 27573265, + "zip_sha256": "fe089b45523f9d4ac0ccd0e176a543d274e914ee5ae5bd384dbaafc7dc06ebfd", + "rows_total": 547271, + "rows_cancelled": 20389, + "rows_diverted": 1512, + "rows_eligible": 525370, + "rows_sampled": 5000 + }, + { + "month": "2024-02", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_2.zip", + "zip_bytes": 25086854, + "zip_sha256": "8463683d100504a97847497d2f2281008fd9add9639bf7436a9747032df15f9a", + "rows_total": 519221, + "rows_cancelled": 3002, + "rows_diverted": 950, + "rows_eligible": 515269, + "rows_sampled": 5000 + }, + { + "month": "2024-03", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_3.zip", + "zip_bytes": 29313663, + "zip_sha256": "e2c5021406f480e813a52a2ed795c251eb1169613da6a416daf88236db8c9013", + "rows_total": 591767, + "rows_cancelled": 5120, + "rows_diverted": 1234, + "rows_eligible": 585413, + "rows_sampled": 5000 + }, + { + "month": "2024-04", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_4.zip", + "zip_bytes": 28540589, + "zip_sha256": "d8b1e88626a7375d351962917fe5ec132660c1636dc8b91f778627c769e0e15a", + "rows_total": 582185, + "rows_cancelled": 4035, + "rows_diverted": 1235, + "rows_eligible": 576915, + "rows_sampled": 5000 + }, + { + "month": "2024-05", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_5.zip", + "zip_bytes": 31139902, + "zip_sha256": "03e4ae26c2d1247de02d0b9a4d87160f8021718abf475a1421de9c45b7d68b04", + "rows_total": 609743, + "rows_cancelled": 8244, + "rows_diverted": 2273, + "rows_eligible": 599226, + "rows_sampled": 5000 + }, + { + "month": "2024-06", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_6.zip", + "zip_bytes": 31226761, + "zip_sha256": "4f329439fd6892aaf89b32908191b2a007e70f775e2551181a22e704ef28a0ab", + "rows_total": 611132, + "rows_cancelled": 7876, + "rows_diverted": 1984, + "rows_eligible": 601272, + "rows_sampled": 5000 + }, + { + "month": "2024-07", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_7.zip", + "zip_bytes": 34427174, + "zip_sha256": "4d992fee134137b3fa43094d7e8cf4a8e6feaa6a422cffb4063f1ed6c7f3f731", + "rows_total": 634613, + "rows_cancelled": 18372, + "rows_diverted": 2303, + "rows_eligible": 613938, + "rows_sampled": 5000 + }, + { + "month": "2024-08", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_8.zip", + "zip_bytes": 30854511, + "zip_sha256": "58d651379af90cc44635657e95fe098567ff954046aec21d6d2cd18424a317f9", + "rows_total": 619025, + "rows_cancelled": 12714, + "rows_diverted": 2008, + "rows_eligible": 604303, + "rows_sampled": 5000 + }, + { + "month": "2024-09", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_9.zip", + "zip_bytes": 28433978, + "zip_sha256": "c0df7503b0050ca46099dab1edfffa33464a0a1031ad712c542af6a4ecece24c", + "rows_total": 582622, + "rows_cancelled": 3499, + "rows_diverted": 861, + "rows_eligible": 578262, + "rows_sampled": 5000 + }, + { + "month": "2024-10", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_10.zip", + "zip_bytes": 29418453, + "zip_sha256": "a417136d6c1663736437dfeeacefa5b93b82e9db322c863c2f57b19bdd3f04fb", + "rows_total": 615497, + "rows_cancelled": 6284, + "rows_diverted": 699, + "rows_eligible": 608514, + "rows_sampled": 5000 + }, + { + "month": "2024-11", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_11.zip", + "zip_bytes": 28149354, + "zip_sha256": "26b6e89e7c1f66163a412380c743c648eb470a9017074de4efbd209f1e09860a", + "rows_total": 575404, + "rows_cancelled": 2629, + "rows_diverted": 842, + "rows_eligible": 571933, + "rows_sampled": 5000 + }, + { + "month": "2024-12", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2024_12.zip", + "zip_bytes": 30150370, + "zip_sha256": "a324577aa76d1f0aa36239f8010b6b5270fea0ff5ac846177753b76a42c5d01c", + "rows_total": 590581, + "rows_cancelled": 4151, + "rows_diverted": 1598, + "rows_eligible": 584832, + "rows_sampled": 5000 + }, + { + "month": "2025-01", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2025_1.zip", + "zip_bytes": 27108664, + "zip_sha256": "868387dcedaef1b8d8392e608e642219fde50d594c3cf2094aec858167629ccf", + "rows_total": 539747, + "rows_cancelled": 16312, + "rows_diverted": 1166, + "rows_eligible": 522269, + "rows_sampled": 5000 + }, + { + "month": "2025-02", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2025_2.zip", + "zip_bytes": 25302583, + "zip_sha256": "12dc8dbdb3c8b3c20d4c00a8b188196ae202f781a8760871ae502a9c733d0ecd", + "rows_total": 504884, + "rows_cancelled": 7405, + "rows_diverted": 1003, + "rows_eligible": 496476, + "rows_sampled": 5000 + }, + { + "month": "2025-03", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2025_3.zip", + "zip_bytes": 30544825, + "zip_sha256": "9c80fbc2112cdbf3f0613ec2001ba019b3ad75511cd69657080736e7eda9cef4", + "rows_total": 600872, + "rows_cancelled": 6923, + "rows_diverted": 1648, + "rows_eligible": 592301, + "rows_sampled": 5000 + }, + { + "month": "2025-04", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2025_4.zip", + "zip_bytes": 29369186, + "zip_sha256": "f3718431e988e2afd53a1714dfca37f67355b6fc06b02eb506bffa39300d608c", + "rows_total": 583950, + "rows_cancelled": 4914, + "rows_diverted": 1306, + "rows_eligible": 577730, + "rows_sampled": 5000 + }, + { + "month": "2025-05", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2025_5.zip", + "zip_bytes": 30830593, + "zip_sha256": "17c3419d7169b313b2bc8ddd3fd57e1e2d4d959c35f2da17f6a5610d4bac7d63", + "rows_total": 605648, + "rows_cancelled": 6344, + "rows_diverted": 1730, + "rows_eligible": 597574, + "rows_sampled": 5000 + }, + { + "month": "2025-06", + "url": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_2025_6.zip", + "zip_bytes": 31131411, + "zip_sha256": "087b75e4e059516ca1181e58955d062f341933adb2706e5b2404ed10c9ecbdc6", + "rows_total": 611575, + "rows_cancelled": 9736, + "rows_diverted": 2367, + "rows_eligible": 599472, + "rows_sampled": 5000 + } + ] +} diff --git a/artifacts/models/delay-logistic-v1.json b/artifacts/models/delay-logistic-v1.json new file mode 100644 index 0000000..14a24c4 --- /dev/null +++ b/artifacts/models/delay-logistic-v1.json @@ -0,0 +1,769 @@ +{ + "schema_version": 1, + "model_version": "bts-schedule-logistic-v1", + "prediction_time": "T-24h", + "trained_at": "2026-08-08T23:15:27+00:00", + "intercept": -1.5603805620340776, + "threshold": 0.175, + "numeric": { + "Month": { + "mean": 6.5, + "scale": 3.452052529534663, + "coefficient": -0.06904248137175817 + }, + "DayOfWeek": { + "mean": 3.9826, + "scale": 2.0121540464553567, + "coefficient": 0.06004599083514959 + }, + "DepHourSin": { + "mean": -0.0694287515884979, + "scale": 0.7562569854214362, + "coefficient": -0.4623763541236258 + }, + "DepHourCos": { + "mean": -0.36427913162566267, + "scale": 0.5390322204805547, + "coefficient": -0.011436796063000602 + }, + "CRSElapsedTime": { + "mean": 146.14968333333334, + "scale": 71.96332453571789, + "coefficient": -0.5330682193025378 + }, + "Distance": { + "mean": 829.1637666666667, + "scale": 592.7595787898713, + "coefficient": 0.6371182855584719 + }, + "DistanceGroup": { + "mean": 3.7896, + "scale": 2.3214216563706533, + "coefficient": -0.10058673842141007 + } + }, + "categorical": { + "Reporting_Airline": { + "9E": -0.48392428211644983, + "AA": 0.09565773666494619, + "AS": 0.056089307561541055, + "B6": 0.04023445011841561, + "DL": -0.28552503738632967, + "F9": 0.3321463048123611, + "G4": -0.08832641154436584, + "HA": 0.30998902424180447, + "MQ": -0.28779640777580373, + "NK": -0.03870953146606562, + "OH": 0.0060531657473753015, + "OO": -0.17100862086840812, + "UA": -0.3552795180949323, + "WN": -0.04148518096625318, + "YX": -0.6484955609586689 + }, + "Origin": { + "ABE": -0.4746999806072368, + "ABI": -0.2022852686747384, + "ABQ": -0.012122518266237019, + "ABR": 0.5630812393618314, + "ABY": 0.8350877196312992, + "ACK": -0.26006793274089396, + "ACT": -0.3889745979743958, + "ACV": -0.6105457161758201, + "ACY": -0.3920003249272332, + "ADQ": -0.03718233051781902, + "AEX": -0.2519657442565574, + "AGS": 0.18034136034718412, + "AKN": -0.568888446607395, + "ALB": -0.2642898598293121, + "ALW": -0.44696802303362165, + "AMA": -0.09487013634653757, + "ANC": -0.7978063062416624, + "APN": 0.6336416349887166, + "ASE": 0.1328495456913728, + "ATL": 0.1466482087088538, + "ATW": 0.7965838263102872, + "AUS": 0.1557281481365779, + "AVL": 0.1281080667690173, + "AVP": 0.15570873722618514, + "AZA": -0.5651700862156971, + "AZO": 0.10072158288412104, + "BDL": 0.06274677092532652, + "BET": 0.09890578999783453, + "BFF": -0.4143158745992813, + "BFL": -0.5704334034153512, + "BGM": 0.730732174058305, + "BGR": 0.2293464098902489, + "BHM": 0.3297403688531353, + "BIH": 0.10646576264019444, + "BIL": 0.6394953659692092, + "BIS": -0.45216406389229735, + "BJI": -0.12405985071754931, + "BLI": -0.47686792755386187, + "BLV": -0.2386194637960928, + "BMI": -0.39037264490667256, + "BNA": -0.017476060683459478, + "BOI": -0.3605831044357738, + "BOS": 0.28254727681521624, + "BPT": -0.5861890950886454, + "BQK": 0.6930413099103774, + "BQN": -0.01677200782345667, + "BRD": -0.01572085733335766, + "BRO": 0.0693021482096051, + "BRW": 0.8109466722785788, + "BTM": -0.8583711126190998, + "BTR": 0.21856862047158934, + "BTV": 0.19831192621921376, + "BUF": -0.018540679844474283, + "BUR": -0.43767857706426205, + "BWI": 0.26670475348620587, + "BZN": 0.1462497030814667, + "CAE": -0.1691213561223503, + "CAK": 0.04565143880545856, + "CDC": -0.7698917180145679, + "CDV": -0.11935660527039196, + "CHA": -0.14283472877698927, + "CHO": 0.44325294482197863, + "CHS": -0.08775627789350252, + "CID": -0.39545175549876127, + "CIU": 0.2031954048874683, + "CKB": 0.642549757076299, + "CLE": -0.1749039377163145, + "CLL": 1.003949673061079, + "CLT": 0.2601443395317738, + "CMH": -0.026890379480674204, + "CMI": -0.6043137463722003, + "CMX": 0.5892536054963158, + "COD": 0.5213715918523097, + "COS": 0.275982036407636, + "COU": 0.814470536436657, + "CPR": -0.7257234185373772, + "CRP": -0.4947772771801187, + "CRW": 0.3361938385892243, + "CSG": 0.13636882060540134, + "CVG": -0.031762809470608735, + "CWA": -0.47841404569250445, + "CYS": 0.010509258467572263, + "DAB": -0.03156737317384574, + "DAL": -0.0400318874938516, + "DAY": -0.11131770720543997, + "DCA": 0.14539668649626072, + "DDC": 0.8707491323844864, + "DEC": -0.05752343781626272, + "DEN": 0.2028162088967015, + "DFW": 0.2653739738201609, + "DHN": 0.6775439749470163, + "DIK": -0.5700439215771561, + "DLG": -0.1715005694573443, + "DLH": 0.4428665288547115, + "DRO": 0.24597865405999475, + "DSM": 0.16566130028646903, + "DTW": 0.15923967299796887, + "DVL": 0.029538753704059378, + "EAR": -0.20771846447573802, + "ECP": -0.5921258279371214, + "EGE": 0.1530337991885375, + "EKO": -0.22484650700272038, + "ELM": -0.4228923098739207, + "ELP": -0.4182165358413368, + "ESC": -0.08855152659369216, + "EUG": -0.3075770780821744, + "EVV": -0.12264367113304596, + "EWR": 0.33526522258464936, + "EYW": 0.5375584749919439, + "FAI": -0.12710716421088877, + "FAR": -0.09459337514814642, + "FAT": -0.1402264659540186, + "FAY": 0.44830074975987, + "FCA": 0.3129187049015375, + "FLG": -0.07360602098811155, + "FLL": 0.48538471185628285, + "FNT": -0.26696763495074455, + "FOD": -0.38526803179808333, + "FSD": 0.12861321196496378, + "FSM": -0.020410772876480444, + "FWA": 0.6820177264764921, + "GCC": -0.6669863177438207, + "GCK": 0.7000915468427866, + "GEG": -0.31818335518184765, + "GFK": 0.48166483013016725, + "GGG": -0.5670730626818555, + "GJT": 0.22155894597634748, + "GNV": -0.4279680846947521, + "GPT": 0.2738208370287874, + "GRB": -0.3409881195225502, + "GRI": 0.03145967201650538, + "GRK": 0.4185253271809064, + "GRR": 0.42015207886744316, + "GSO": 0.1803166963091754, + "GSP": -0.08337918013302856, + "GST": -0.23816579432486845, + "GTF": 0.13985719117092005, + "GTR": -0.525681137152373, + "GUC": 0.4152744190427218, + "GUM": 0.10699110585189267, + "HDN": -0.9301918423163272, + "HGR": 0.05573170508120374, + "HHH": 0.6618924256062791, + "HIB": 0.653493356740141, + "HLN": -0.24899817255062176, + "HNL": -0.9196449795222561, + "HOB": -0.37301515163453636, + "HOU": -0.16028666702306707, + "HPN": 0.09037608546939499, + "HRL": -0.9867007784184189, + "HSV": -0.34065500997031467, + "HTS": -0.30356517530418486, + "HYA": 0.7499653852421314, + "HYS": 0.5156166807429544, + "IAD": 0.3629774236464757, + "IAG": -0.153076439057169, + "IAH": 0.32420707896440404, + "ICT": 0.35848452366402545, + "IDA": 0.24183337021228826, + "ILM": 0.2431575557002933, + "IMT": 0.021805938436700874, + "IND": 0.30693507614675986, + "INL": -0.3034780086458087, + "ISP": -0.345060151934244, + "ITH": -0.7056815260365517, + "ITO": -1.0508845755996037, + "JAC": -0.08670470451891737, + "JAN": -0.74723313494043, + "JAX": -0.2799510381807764, + "JFK": 0.3393328930313235, + "JLN": 0.3383379917776585, + "JMS": -0.3672382087549472, + "JNU": -0.8269809716259264, + "JST": 0.006292412104842642, + "KOA": -1.036358597286775, + "KTN": -0.558329965688365, + "LAN": 0.9221941532938968, + "LAR": -0.7003426002451174, + "LAS": 0.06479896200438237, + "LAW": -0.33173909953557623, + "LAX": -0.08019007435890112, + "LBB": 0.10054771738731658, + "LBE": 0.13105986773160438, + "LBF": -0.32877181937656125, + "LBL": -0.3734795899852874, + "LCH": 0.22555570106602296, + "LCK": 1.1386329970077413, + "LEX": 0.04126688717300533, + "LFT": 0.32324163868589856, + "LGA": 0.18609899087385467, + "LGB": -0.8334063330942456, + "LIH": -1.1722897462721489, + "LIT": -0.059496018914245226, + "LNK": -0.4235385885629282, + "LRD": 0.17051137959429585, + "LSE": 0.6585211252358306, + "LWS": 0.4947988576339847, + "MAF": -0.19009167584917336, + "MBS": 0.5808881561501184, + "MCI": -0.16960211562752733, + "MCO": 0.33121831142847313, + "MDT": -0.5330800194408661, + "MDW": 0.055642218678715695, + "MEI": 0.06544463619081602, + "MEM": 0.34774022873496124, + "MFE": 0.3052915086675692, + "MFR": 0.154863715931111, + "MGM": 0.24904202140895754, + "MHK": -0.7055211993140831, + "MHT": 0.03554373228354781, + "MIA": 0.4732721190945591, + "MKE": 0.12790388332366187, + "MLB": -0.11010733223290987, + "MLI": 0.09012592261701283, + "MLU": 0.7625195201152942, + "MOB": -0.36995121590867613, + "MOT": 0.6927678398701083, + "MQT": 0.12413111011514019, + "MRY": -0.09048929560907781, + "MSN": -0.059991640710461674, + "MSO": 0.3698079318887037, + "MSP": 0.15859192658633992, + "MSY": 0.011539015553367497, + "MTJ": -0.9063860948715227, + "MVY": -0.2152118973528027, + "MYR": -0.2043109042945402, + "OAJ": -0.3756919570916334, + "OAK": -0.17651103411093116, + "OGG": -0.9487339470453106, + "OKC": -0.18869945544514535, + "OMA": 0.22344543401005448, + "OME": -0.11751200753849306, + "ONT": -0.2238929186306232, + "ORD": 0.34926404634909003, + "ORF": 0.3827600492747014, + "ORH": 0.4805641258824872, + "OTH": 0.2811741410859042, + "OTZ": -0.43970784307712085, + "PAE": -0.1665126754683194, + "PBG": 0.048250974397067804, + "PBI": 0.0854443489005524, + "PDX": -0.26625754023868203, + "PGD": 0.10651375277414808, + "PHL": 0.12085006880358182, + "PHX": -0.21413625002264708, + "PIA": 0.09952703139885011, + "PIB": -0.4791257174170812, + "PIE": -0.2538416767862975, + "PIH": -0.5351069277947776, + "PIT": -0.09112622013473384, + "PLN": 0.17914133216824668, + "PNS": -0.0041988852306839445, + "PQI": -0.11337154356935489, + "PRC": 0.4718458259265579, + "PSC": -0.8764541242161273, + "PSE": 0.33282105010982915, + "PSG": 0.09289688474228401, + "PSM": -0.2834795839189017, + "PSP": -0.48732133039882797, + "PVD": -0.2978845472101114, + "PVU": 0.4493595721859399, + "PWM": 0.6176088314196442, + "RAP": 0.5498570743135336, + "RDD": -1.3983637212602, + "RDM": 0.4081726253304548, + "RDU": 0.30442487740979424, + "RFD": 0.38726552899073724, + "RHI": -0.0789595728780362, + "RIC": 0.4689693648756582, + "RIW": -0.3001204798539488, + "RKS": -0.36611899701826567, + "RNO": 0.1923165609473959, + "ROA": 0.32412126431473326, + "ROC": -0.22506616016720846, + "ROW": 1.0534216810264005, + "RST": -0.364343043823178, + "RSW": 0.12011968788865064, + "SAF": 0.8440711083745628, + "SAN": 0.14871320033623608, + "SAT": -0.3292540450958017, + "SAV": 0.16188390293331303, + "SBA": -0.7672812118024037, + "SBN": 0.14540640122082887, + "SBP": -0.7523516091225724, + "SCC": -0.5687407864823163, + "SCE": -0.23235248508432615, + "SCK": 0.1886694871553393, + "SDF": -0.0716614443740309, + "SEA": -0.018761277847787597, + "SFB": 0.5811667348749802, + "SFO": 0.12280984010891505, + "SGF": 0.4429109501173236, + "SGU": -0.21981188839192078, + "SHR": 0.7568119296406635, + "SHV": 0.5680748232893988, + "SIT": -0.8279356878131575, + "SJC": -0.4642364280860071, + "SJT": -0.03363915121117147, + "SJU": 0.016620064423609037, + "SLC": -0.08765347296359145, + "SLN": -0.18623397987914309, + "SMF": -0.14166463433484652, + "SMX": 0.5468888188751666, + "SNA": -0.1697138297534782, + "SPN": -0.1868993819560774, + "SPS": 0.22532606479464076, + "SRQ": 0.08727829668620185, + "STL": -0.01046009874774625, + "STS": 0.5575187634965354, + "STT": -0.41113650798487594, + "STX": -0.1282188284246486, + "SUN": -0.18150058872151834, + "SUX": 0.20035515594572473, + "SWF": -0.5980129859157275, + "SWO": 0.4883007801110665, + "SYR": 0.31935613482856506, + "TLH": 0.47493040317793367, + "TOL": 0.7433217273189814, + "TPA": 0.11785146110080624, + "TRI": 0.07322748737000809, + "TTN": -0.29294733699645337, + "TUL": -0.060076490946652135, + "TUS": -0.12170352824911736, + "TVC": 0.09099975483861492, + "TWF": -0.18237307372221923, + "TXK": 0.26877714760115456, + "TYR": 0.6367051274262906, + "TYS": 0.1768715139837389, + "USA": 0.49839654559066593, + "VCT": -0.013767684631071295, + "VEL": -0.12116438017073136, + "VLD": 0.29406026537189567, + "VPS": 0.2961766123134771, + "WRG": -0.13178397401078712, + "WYS": -0.2887733037167503, + "XNA": -0.07377918982428697, + "XWA": 0.048639320757691604, + "YAK": -0.36670272478796245, + "YUM": -0.04358308779729816 + }, + "Dest": { + "ABE": -0.011693401135624892, + "ABI": 0.19559610221854287, + "ABQ": 0.2533165562883105, + "ABR": 0.4421303827646665, + "ABY": -0.03810561077779959, + "ACK": -0.02893623511131086, + "ACT": -0.584257577615387, + "ACV": 0.6263667304058519, + "ACY": 0.0876699960856562, + "ADK": -0.077390510400247, + "ADQ": 0.12950551150357134, + "AEX": -0.7635083460411455, + "AGS": 0.10760365117535473, + "AKN": -0.07574618145489687, + "ALB": -0.07329767635567525, + "ALW": 0.3311422099685977, + "AMA": -0.05197401468741255, + "ANC": 0.25967082248514284, + "APN": -0.335776829039522, + "ASE": -0.5205660407510904, + "ATL": 0.0597762698923959, + "ATW": 0.014696830679234097, + "AUS": 0.4191052619013035, + "AVL": 0.3063805874250809, + "AVP": 0.17685812693689332, + "AZA": -0.06283026262091425, + "AZO": -0.93436542227347, + "BDL": -0.18432235248600184, + "BET": 0.7140804003074267, + "BFF": -0.24766997689263484, + "BFL": 0.11322328856260772, + "BGM": 0.9353007039183119, + "BGR": 0.12519170235105134, + "BHM": 0.017363909762588073, + "BIH": 1.052681357551488, + "BIL": 0.09173225087146357, + "BIS": -0.37938993966933504, + "BJI": -0.4800101491820273, + "BLI": -0.4462610542390032, + "BLV": 0.3888730252155243, + "BMI": -0.2867605151871236, + "BNA": -0.03095282217594734, + "BOI": 0.16381472603469793, + "BOS": 0.2542230122452732, + "BPT": -0.8496308326829253, + "BQK": -0.2328553423137658, + "BQN": 0.0011996849195144584, + "BRD": -0.04199531757075415, + "BRO": -0.8556944745896943, + "BRW": -0.3178355104966698, + "BTM": 0.3987107528022035, + "BTR": 0.193249922488843, + "BTV": 0.1312664358628959, + "BUF": 0.24599375178956634, + "BUR": 0.19806698227685093, + "BWI": -0.2119306946797077, + "BZN": 0.21574996959414394, + "CAE": -0.26461321185454, + "CAK": 0.31096041888863457, + "CDC": -0.32033723307440637, + "CDV": -0.5603534604653492, + "CHA": 0.30697583984769433, + "CHO": -0.25780890691972747, + "CHS": -0.03514081114499421, + "CID": 0.3980718103457356, + "CIU": 0.1624575812560013, + "CKB": -0.195184031955406, + "CLE": 0.12112529720886225, + "CLL": -0.3207910623976856, + "CLT": 0.13043334375289595, + "CMH": -0.13863334684162196, + "CMI": -0.6586906202849568, + "CMX": 0.2774249424679501, + "COD": 0.4106383459018597, + "COS": 0.004455718950102871, + "COU": -0.5542413767508356, + "CPR": -0.5588131168074892, + "CRP": -0.35698079266680643, + "CRW": 0.30897915935444403, + "CSG": -0.2649058876411105, + "CVG": 0.16139941629569352, + "CWA": 0.1242025370496024, + "CYS": -0.41403435078612516, + "DAB": -0.4755996585663601, + "DAL": 0.15357968497038785, + "DAY": -0.31109879129082624, + "DCA": 0.07112842512586756, + "DDC": -0.35242588744822545, + "DEC": -0.6046850300568439, + "DEN": 0.1551189155579979, + "DFW": 0.41875021180939065, + "DHN": -0.1660977497553023, + "DIK": -0.5693332444709474, + "DLH": -1.1643651074337404, + "DRO": -0.24433410832973484, + "DSM": 0.2685260937754408, + "DTW": 0.1075401809371529, + "DVL": -0.07449435867512834, + "ECP": 0.056775291252632865, + "EGE": -0.8192664926640753, + "EKO": -0.6668227509779423, + "ELM": -0.47025777111960426, + "ELP": 0.20377009889613037, + "ESC": -0.40347030393273675, + "EUG": 0.30049545990738735, + "EVV": -0.5450339924763036, + "EWR": 0.5209110906686792, + "EYW": 0.6967032847973484, + "FAI": 0.28777617768992747, + "FAR": 0.28106756778153774, + "FAT": 0.05772846126748088, + "FAY": -0.38703606710448524, + "FCA": 0.20014023364408695, + "FLG": -0.03171308593328711, + "FLL": 0.29907752219960804, + "FNT": 0.18709431094316187, + "FOD": 1.0924921314678138, + "FSD": 0.019826912508413414, + "FSM": 0.1383481550853744, + "FWA": -0.16090365313333094, + "GCC": 0.8668506056135966, + "GCK": -0.54564004274656, + "GEG": -0.08127306033896949, + "GFK": -0.4645394241169935, + "GGG": 0.45818504623167955, + "GJT": 0.09344621398016176, + "GNV": 0.11277082913995158, + "GPT": 0.41379740404504617, + "GRB": 0.33675939223701973, + "GRI": 0.1590263297645069, + "GRK": -0.3184160788728656, + "GRR": -0.3121920657044299, + "GSO": 0.0020153306224383236, + "GSP": -0.27747573671781345, + "GTF": -0.22951599444118437, + "GTR": 0.24481518779535125, + "GUC": -0.3446241335343941, + "GUM": 0.3307777730681405, + "HDN": -1.1039426647675623, + "HHH": -0.006569642745998316, + "HIB": -0.360941896115812, + "HLN": -0.5737474757492325, + "HNL": 0.2523878963521082, + "HOB": 0.09428540211273323, + "HOU": -0.09347603739458529, + "HPN": 0.12890742563134705, + "HRL": -0.07872177275475875, + "HSV": 0.4557446881610164, + "HTS": -0.013150003326571133, + "HYA": -0.2720851875189131, + "HYS": -0.11951886817920516, + "IAD": -0.03264529852600438, + "IAG": 0.7083414494339046, + "IAH": 0.3692261926011391, + "ICT": 0.3757781066096225, + "IDA": 0.42775446281255186, + "ILM": -0.07391678405489296, + "IMT": 0.9697978591195323, + "IND": 0.42354983311550626, + "INL": -0.33036317013227945, + "ISP": 0.022853458498682342, + "ITH": -0.44688389237728915, + "ITO": -0.2990578700769602, + "JAC": -0.19755242918775873, + "JAN": -0.7432326620260845, + "JAX": -0.0731992858059952, + "JFK": 0.3145206611628288, + "JLN": -0.26868306503604117, + "JMS": -0.06644449709008193, + "JNU": -0.12128589120517172, + "JST": 0.6176117368340579, + "KOA": -0.4577254385998994, + "KTN": -0.14271683165366264, + "LAN": -0.003320760343473804, + "LAR": 0.203062294264496, + "LAS": 0.31365702559881475, + "LAW": -0.8933798689196738, + "LAX": 0.22359921594154772, + "LBB": -0.29370572288700575, + "LBE": 0.2868304570516104, + "LBF": -0.8342308599371, + "LBL": -0.15600681365192903, + "LCH": -0.5145495786512061, + "LCK": 0.25376304390948506, + "LEX": 0.2483560579405228, + "LFT": -0.14924144161991793, + "LGA": 0.17034588777085377, + "LGB": -0.01133065741820342, + "LIH": -0.23887190760016333, + "LIT": -0.08333598195481018, + "LNK": 0.3774191277123835, + "LRD": -0.21670190279152254, + "LSE": 0.45293883132395074, + "LWS": 0.19372943230371037, + "MAF": 0.43831899430665167, + "MBS": 0.4672856657473662, + "MCI": 0.07607377797351376, + "MCO": 0.1705924861898515, + "MCW": 0.12148750425403497, + "MDT": 0.1809934011559542, + "MDW": 0.1004556365596684, + "MEI": -0.22709709578868426, + "MEM": -0.2746776655176146, + "MFE": 0.22004368402701477, + "MFR": -0.16420131734261015, + "MGM": -0.44450173995703823, + "MHK": 0.11207110476760719, + "MHT": 0.14881988360433923, + "MIA": 0.20989861570013488, + "MKE": -0.008609596093663895, + "MLB": -0.40575751832033297, + "MLI": -0.09742832638417377, + "MLU": 0.31327478855873525, + "MOB": -0.05933566535344387, + "MOT": 0.9638042861004871, + "MQT": 0.4773003037226291, + "MRY": 0.12364949302345725, + "MSN": -0.10269567628859133, + "MSO": 0.5243509301811438, + "MSP": 0.20649173770299445, + "MSY": 0.05957577146250439, + "MTJ": -0.41764445535635425, + "MVY": -0.4300783565539455, + "MYR": 0.22767058101835033, + "OAJ": -0.5926282342366345, + "OAK": 0.019175681924999004, + "OGG": 0.3325897522770823, + "OKC": 0.1309974932524023, + "OMA": 0.2749012296068003, + "OME": -0.1363476945201411, + "ONT": 0.03874465906811033, + "ORD": 0.3506779765925713, + "ORF": 0.10273078721719686, + "ORH": -0.09665433490866608, + "OTH": -0.11052332081642795, + "OTZ": -0.5239082761025239, + "PAE": 0.21874082871293582, + "PBG": -0.5369054902307445, + "PBI": 0.3026504906884024, + "PDX": 0.31654852978093684, + "PGD": -0.17559325842069048, + "PHL": 0.21839846242075045, + "PHX": 0.033896921948841, + "PIA": 0.0676667846585828, + "PIB": -0.7053976893703049, + "PIE": 0.013909823610551937, + "PIH": 0.22134840113545104, + "PIT": -0.05751458924577192, + "PLN": 0.13744604100934357, + "PNS": -0.18225981331007599, + "PPG": -0.1466272835877919, + "PRC": 0.292735143465446, + "PSC": 0.17576983169710256, + "PSE": -0.2369071849929535, + "PSG": -0.30128437397915, + "PSM": -0.2781866069765762, + "PSP": 0.04009899914023455, + "PVD": -0.09242279751857817, + "PVU": -0.372391437215464, + "PWM": -0.23633676186236868, + "RAP": -0.045709954655266026, + "RDD": -0.44681932816800546, + "RDM": -0.13769730967821944, + "RDU": 0.08818807782746373, + "RFD": 0.06836197812585126, + "RHI": 0.3392541991131297, + "RIC": 0.16972721845474625, + "RIW": -0.5953472303055709, + "RKS": -0.1451076373392507, + "RNO": 0.063701251950546, + "ROA": -0.371993028823612, + "ROC": 0.2709190791155854, + "ROW": -0.12704475920274746, + "RST": 0.4353607996626555, + "RSW": 0.20047793837591077, + "SAF": -0.10383139385870843, + "SAN": 0.4324574926727702, + "SAT": 0.04265918356192729, + "SAV": 0.3301958597695698, + "SBA": 0.3693119910158246, + "SBN": -0.4242306910075, + "SBP": 0.6037255388119888, + "SCC": -0.25225198062021553, + "SCE": 0.02016267530544188, + "SCK": -0.530635945981123, + "SDF": 0.04370575534073813, + "SEA": 0.3394788292975139, + "SFB": 0.2365152998773465, + "SFO": 0.7380156630793888, + "SGF": 0.34971158416138814, + "SGU": 0.06944042706636497, + "SHR": -0.6193568362607194, + "SHV": -0.06797516195365631, + "SIT": 0.7790456435998684, + "SJC": 0.058206628831789606, + "SJT": -0.28424828665751, + "SJU": 0.2563134416560802, + "SLC": 0.03564443564471042, + "SLN": 0.7001050669437743, + "SMF": 0.21750092194392617, + "SMX": 0.7043587924222892, + "SNA": -0.07038434501275564, + "SPI": -0.09086438981201397, + "SPN": 0.6630126409649486, + "SPS": 0.013893031974417314, + "SRQ": -0.2352768092009231, + "STC": -0.061384081051898666, + "STL": 0.046496577055210715, + "STS": 0.2923515425272969, + "STT": 0.4641766864507468, + "STX": 0.10168343095400037, + "SUN": -0.6265225372379997, + "SUX": -0.028174169085139894, + "SWF": -0.33242951711645036, + "SWO": -0.06347814950747901, + "SYR": 0.05888318549833369, + "TLH": -0.4990243541036958, + "TOL": -0.3324038234438433, + "TPA": 0.3074768952031531, + "TRI": -0.03392741389501613, + "TTN": 0.38535969086783295, + "TUL": -0.06800075959648678, + "TUS": 0.06119170654586386, + "TVC": 0.18621889988499432, + "TWF": -0.5504629417016731, + "TXK": 0.3680547938396049, + "TYR": -0.48243840296998264, + "TYS": 0.08973098349570159, + "USA": 0.05563783579325712, + "VCT": -0.620373042293391, + "VLD": -0.6043512887017412, + "VPS": 0.06230625030276924, + "WRG": 0.019992340611574833, + "WYS": -0.24397910639684964, + "XNA": 0.06763730001185614, + "XWA": 0.3297032262312306, + "YAK": -0.41449491250113163, + "YUM": -0.07576783642799192 + } + }, + "data_manifest_sha256": "b2e487ae5f3bc2737c45ba915994c124231df08e5b094cca6130a7ad90144091", + "test_metrics": { + "rows": 15000, + "positive_prevalence": 0.240733, + "roc_auc": 0.640094, + "pr_auc": 0.335026, + "precision": 0.304108, + "recall": 0.748269, + "f1": 0.432458, + "brier": 0.175904, + "threshold": 0.175, + "confusion_matrix": { + "tn": 5206, + "fp": 6183, + "fn": 909, + "tp": 2702 + }, + "expected_cost": 10728.0 + }, + "limitations": [ + "Retrospective public BTS data is not a live airline feed.", + "Schedule-only features omit weather, maintenance, crew, and network disruptions.", + "Performance can drift outside the evaluated April-June 2025 period." + ] +} diff --git a/artifacts/reports/delay-model-metrics.json b/artifacts/reports/delay-model-metrics.json new file mode 100644 index 0000000..537798b --- /dev/null +++ b/artifacts/reports/delay-model-metrics.json @@ -0,0 +1,224 @@ +{ + "schema_version": 1, + "model_version": "bts-schedule-logistic-v1", + "data_manifest_sha256": "b2e487ae5f3bc2737c45ba915994c124231df08e5b094cca6130a7ad90144091", + "split_summary": { + "train": { + "rows": 60000, + "positive_prevalence": 0.207383 + }, + "validation": { + "rows": 15000, + "positive_prevalence": 0.192333 + }, + "test": { + "rows": 15000, + "positive_prevalence": 0.240733 + } + }, + "source_exclusions": { + "train": { + "source_rows": 7079061, + "cancelled": 96315, + "diverted": 17499 + }, + "validation": { + "source_rows": 1645503, + "cancelled": 30640, + "diverted": 3817 + }, + "test": { + "source_rows": 1801173, + "cancelled": 20994, + "diverted": 5403 + } + }, + "missingness_before_imputation": { + "train": { + "Month": 0.0, + "DayOfWeek": 0.0, + "DepHourSin": 0.0, + "DepHourCos": 0.0, + "CRSElapsedTime": 0.0, + "Distance": 0.0, + "DistanceGroup": 0.0, + "Reporting_Airline": 0.0, + "Origin": 0.0, + "Dest": 0.0 + }, + "validation": { + "Month": 0.0, + "DayOfWeek": 0.0, + "DepHourSin": 0.0, + "DepHourCos": 0.0, + "CRSElapsedTime": 0.0, + "Distance": 0.0, + "DistanceGroup": 0.0, + "Reporting_Airline": 0.0, + "Origin": 0.0, + "Dest": 0.0 + }, + "test": { + "Month": 0.0, + "DayOfWeek": 0.0, + "DepHourSin": 0.0, + "DepHourCos": 0.0, + "CRSElapsedTime": 0.0, + "Distance": 0.0, + "DistanceGroup": 0.0, + "Reporting_Airline": 0.0, + "Origin": 0.0, + "Dest": 0.0 + } + }, + "model_configuration": { + "logistic_regression": { + "solver": "liblinear", + "max_iter": 400, + "search_budget": "single documented baseline" + }, + "hist_gradient_boosting": { + "max_iter": 80, + "max_leaf_nodes": 31, + "learning_rate": 0.08, + "l2_regularization": 1.0, + "search_budget": "one fixed configuration; no test-set tuning" + } + }, + "cost_assumption": { + "false_negative": 5.0, + "false_positive": 1.0 + }, + "majority_class": { + "rows": 15000, + "positive_prevalence": 0.240733, + "roc_auc": 0.5, + "pr_auc": 0.240733, + "precision": 0.0, + "recall": 0.0, + "f1": 0.0, + "brier": 0.182781, + "threshold": 1.0, + "confusion_matrix": { + "tn": 11389, + "fp": 0, + "fn": 3611, + "tp": 0 + } + }, + "logistic_regression": { + "validation": { + "rows": 15000, + "positive_prevalence": 0.192333, + "roc_auc": 0.60521, + "pr_auc": 0.254973, + "precision": 0.232683, + "recall": 0.735875, + "f1": 0.353568, + "brier": 0.154299, + "threshold": 0.175, + "confusion_matrix": { + "tn": 5114, + "fp": 7001, + "fn": 762, + "tp": 2123 + }, + "expected_cost": 10811.0 + }, + "test": { + "rows": 15000, + "positive_prevalence": 0.240733, + "roc_auc": 0.640094, + "pr_auc": 0.335026, + "precision": 0.304108, + "recall": 0.748269, + "f1": 0.432458, + "brier": 0.175904, + "threshold": 0.175, + "confusion_matrix": { + "tn": 5206, + "fp": 6183, + "fn": 909, + "tp": 2702 + }, + "expected_cost": 10728.0 + } + }, + "hist_gradient_boosting": { + "validation": { + "rows": 15000, + "positive_prevalence": 0.192333, + "roc_auc": 0.593811, + "pr_auc": 0.249842, + "precision": 0.22511, + "recall": 0.729636, + "f1": 0.344067, + "brier": 0.153222, + "threshold": 0.17, + "confusion_matrix": { + "tn": 4869, + "fp": 7246, + "fn": 780, + "tp": 2105 + }, + "expected_cost": 11146.0 + }, + "test": { + "rows": 15000, + "positive_prevalence": 0.240733, + "roc_auc": 0.661223, + "pr_auc": 0.370512, + "precision": 0.291989, + "recall": 0.804486, + "f1": 0.428466, + "brier": 0.171872, + "threshold": 0.17, + "confusion_matrix": { + "tn": 4345, + "fp": 7044, + "fn": 706, + "tp": 2905 + }, + "expected_cost": 10574.0 + } + }, + "calibration": [ + { + "bin": "(-0.001, 0.1]", + "rows": 1397, + "mean_probability": 0.081552, + "observed_rate": 0.108089 + }, + { + "bin": "(0.1, 0.2]", + "rows": 5970, + "mean_probability": 0.14784, + "observed_rate": 0.174372 + }, + { + "bin": "(0.2, 0.3]", + "rows": 4710, + "mean_probability": 0.248463, + "observed_rate": 0.287686 + }, + { + "bin": "(0.3, 0.4]", + "rows": 2457, + "mean_probability": 0.341435, + "observed_rate": 0.359788 + }, + { + "bin": "(0.4, 0.5]", + "rows": 443, + "mean_probability": 0.432205, + "observed_rate": 0.38149 + }, + { + "bin": "(0.5, 0.6]", + "rows": 23, + "mean_probability": 0.52987, + "observed_rate": 0.478261 + } + ], + "error_slice_rows": 215 +} diff --git a/artifacts/reports/error-slices.csv b/artifacts/reports/error-slices.csv new file mode 100644 index 0000000..5d96899 --- /dev/null +++ b/artifacts/reports/error-slices.csv @@ -0,0 +1,216 @@ +dimension,value,rows,positive_prevalence,roc_auc,pr_auc,precision,recall,f1,brier,threshold,confusion_matrix +carrier,AA,2074,0.271938,0.631229,0.353384,0.310563,0.870567,0.457809,0.190215,0.175,"{'tn': 420, 'fp': 1090, 'fn': 73, 'tp': 491}" +carrier,AS,524,0.290076,0.596562,0.382469,0.334347,0.723684,0.45738,0.205099,0.175,"{'tn': 153, 'fp': 219, 'fn': 42, 'tp': 110}" +carrier,B6,479,0.25261,0.654612,0.33403,0.293651,0.917355,0.44489,0.179675,0.175,"{'tn': 91, 'fp': 267, 'fn': 10, 'tp': 111}" +carrier,DL,2238,0.21269,0.623468,0.294302,0.272242,0.642857,0.3825,0.163163,0.175,"{'tn': 944, 'fp': 818, 'fn': 170, 'tp': 306}" +carrier,F9,433,0.351039,0.61512,0.45294,0.378947,0.947368,0.541353,0.221315,0.175,"{'tn': 45, 'fp': 236, 'fn': 8, 'tp': 144}" +carrier,G4,326,0.273006,0.682027,0.408652,0.357843,0.820225,0.498294,0.187096,0.175,"{'tn': 106, 'fp': 131, 'fn': 16, 'tp': 73}" +carrier,HA,147,0.197279,0.667008,0.278334,0.28,0.482759,0.35443,0.156362,0.175,"{'tn': 82, 'fp': 36, 'fn': 15, 'tp': 14}" +carrier,MQ,621,0.238325,0.578967,0.312419,0.269542,0.675676,0.385356,0.180884,0.175,"{'tn': 202, 'fp': 271, 'fn': 48, 'tp': 100}" +carrier,NK,402,0.226368,0.653475,0.357298,0.271429,0.835165,0.409704,0.166419,0.175,"{'tn': 107, 'fp': 204, 'fn': 15, 'tp': 76}" +carrier,OH,527,0.309298,0.613876,0.393818,0.37464,0.797546,0.509804,0.213041,0.175,"{'tn': 147, 'fp': 217, 'fn': 33, 'tp': 130}" +carrier,OO,1786,0.205487,0.594477,0.260817,0.245614,0.648501,0.356287,0.161852,0.175,"{'tn': 688, 'fp': 731, 'fn': 129, 'tp': 238}" +carrier,UA,1653,0.22565,0.626426,0.311696,0.291762,0.683646,0.408982,0.169764,0.175,"{'tn': 661, 'fp': 619, 'fn': 118, 'tp': 255}" +carrier,WN,3081,0.236287,0.679908,0.347107,0.324219,0.798077,0.461111,0.169731,0.175,"{'tn': 1142, 'fp': 1211, 'fn': 147, 'tp': 581}" +carrier,YX,709,0.222849,0.679329,0.326982,0.354369,0.462025,0.401099,0.171257,0.175,"{'tn': 418, 'fp': 133, 'fn': 85, 'tp': 73}" +origin,ABQ,58,0.172414,0.577083,0.305653,0.206897,0.6,0.307692,0.142352,0.175,"{'tn': 25, 'fp': 23, 'fn': 4, 'tp': 6}" +origin,ANC,38,0.157895,0.520833,0.320234,0.25,0.166667,0.2,0.135223,0.175,"{'tn': 29, 'fp': 3, 'fn': 5, 'tp': 1}" +origin,ATL,634,0.293375,0.63118,0.402054,0.354115,0.763441,0.483816,0.204716,0.175,"{'tn': 189, 'fp': 259, 'fn': 44, 'tp': 142}" +origin,AUS,205,0.204878,0.667105,0.308862,0.267176,0.833333,0.404624,0.154773,0.175,"{'tn': 67, 'fp': 96, 'fn': 7, 'tp': 35}" +origin,AVL,29,0.275862,0.845238,0.712607,0.411765,0.875,0.56,0.171834,0.175,"{'tn': 11, 'fp': 10, 'fn': 1, 'tp': 7}" +origin,BDL,65,0.2,0.668639,0.354824,0.323529,0.846154,0.468085,0.152033,0.175,"{'tn': 29, 'fp': 23, 'fn': 2, 'tp': 11}" +origin,BHM,40,0.15,0.764706,0.422685,0.26087,1.0,0.413793,0.117478,0.175,"{'tn': 17, 'fp': 17, 'fn': 0, 'tp': 6}" +origin,BNA,224,0.232143,0.600067,0.303883,0.27027,0.576923,0.368098,0.176629,0.175,"{'tn': 91, 'fp': 81, 'fn': 22, 'tp': 30}" +origin,BOI,62,0.177419,0.750446,0.516145,0.35,0.636364,0.451613,0.132307,0.175,"{'tn': 38, 'fp': 13, 'fn': 4, 'tp': 7}" +origin,BOS,298,0.238255,0.662778,0.331467,0.341463,0.788732,0.476596,0.173412,0.175,"{'tn': 119, 'fp': 108, 'fn': 15, 'tp': 56}" +origin,BUF,44,0.181818,0.725694,0.334613,0.315789,0.75,0.444444,0.139115,0.175,"{'tn': 23, 'fp': 13, 'fn': 2, 'tp': 6}" +origin,BUR,68,0.161765,0.674641,0.342294,0.258065,0.727273,0.380952,0.128037,0.175,"{'tn': 34, 'fp': 23, 'fn': 3, 'tp': 8}" +origin,BWI,204,0.27451,0.716458,0.459155,0.351724,0.910714,0.507463,0.179786,0.175,"{'tn': 54, 'fp': 94, 'fn': 5, 'tp': 51}" +origin,CHS,53,0.169811,0.75,0.363214,0.275862,0.888889,0.421053,0.127498,0.175,"{'tn': 23, 'fp': 21, 'fn': 1, 'tp': 8}" +origin,CLE,80,0.2375,0.684642,0.377361,0.342105,0.684211,0.45614,0.174103,0.175,"{'tn': 36, 'fp': 25, 'fn': 6, 'tp': 13}" +origin,CLT,415,0.310843,0.575229,0.36337,0.327586,0.883721,0.477987,0.213438,0.175,"{'tn': 52, 'fp': 234, 'fn': 15, 'tp': 114}" +origin,CMH,95,0.2,0.76108,0.442378,0.4,0.736842,0.518519,0.145306,0.175,"{'tn': 55, 'fp': 21, 'fn': 5, 'tp': 14}" +origin,COS,32,0.46875,0.494118,0.51776,0.461538,0.8,0.585366,0.302014,0.175,"{'tn': 3, 'fp': 14, 'fn': 3, 'tp': 12}" +origin,CVG,69,0.26087,0.632898,0.456151,0.354839,0.611111,0.44898,0.190936,0.175,"{'tn': 31, 'fp': 20, 'fn': 7, 'tp': 11}" +origin,DAL,166,0.253012,0.693164,0.396973,0.357895,0.809524,0.49635,0.178437,0.175,"{'tn': 63, 'fp': 61, 'fn': 8, 'tp': 34}" +origin,DCA,286,0.265734,0.634586,0.373575,0.343195,0.763158,0.473469,0.190497,0.175,"{'tn': 99, 'fp': 111, 'fn': 18, 'tp': 58}" +origin,DEN,692,0.284682,0.664775,0.40609,0.352201,0.852792,0.498516,0.193652,0.175,"{'tn': 186, 'fp': 309, 'fn': 29, 'tp': 168}" +origin,DFW,635,0.341732,0.625752,0.437899,0.382845,0.843318,0.526619,0.22251,0.175,"{'tn': 123, 'fp': 295, 'fn': 34, 'tp': 183}" +origin,DSM,32,0.125,0.3125,0.112981,0.117647,0.5,0.190476,0.131482,0.175,"{'tn': 13, 'fp': 15, 'fn': 2, 'tp': 2}" +origin,DTW,238,0.222689,0.636206,0.299545,0.273438,0.660377,0.38674,0.168353,0.175,"{'tn': 92, 'fp': 93, 'fn': 18, 'tp': 35}" +origin,ELP,41,0.170732,0.726891,0.317747,0.2,0.285714,0.235294,0.135993,0.175,"{'tn': 26, 'fp': 8, 'fn': 5, 'tp': 2}" +origin,EWR,217,0.281106,0.636034,0.433891,0.365217,0.688525,0.477273,0.198371,0.175,"{'tn': 83, 'fp': 73, 'fn': 19, 'tp': 42}" +origin,EYW,25,0.16,0.404762,0.158968,0.173913,1.0,0.296296,0.170547,0.175,"{'tn': 2, 'fp': 19, 'fn': 0, 'tp': 4}" +origin,FAT,25,0.12,0.772727,0.277778,0.230769,1.0,0.375,0.101257,0.175,"{'tn': 12, 'fp': 10, 'fn': 0, 'tp': 3}" +origin,FLL,165,0.30303,0.624435,0.433037,0.324324,0.96,0.484848,0.202322,0.175,"{'tn': 15, 'fp': 100, 'fn': 2, 'tp': 48}" +origin,GEG,48,0.25,0.728009,0.485837,0.421053,0.666667,0.516129,0.182471,0.175,"{'tn': 25, 'fp': 11, 'fn': 4, 'tp': 8}" +origin,GRR,47,0.234043,0.67803,0.319051,0.289474,1.0,0.44898,0.172453,0.175,"{'tn': 9, 'fp': 27, 'fn': 0, 'tp': 11}" +origin,HNL,96,0.145833,0.746516,0.302907,0.375,0.428571,0.4,0.116418,0.175,"{'tn': 72, 'fp': 10, 'fn': 8, 'tp': 6}" +origin,HOU,120,0.2,0.621528,0.291435,0.228571,0.666667,0.340426,0.155952,0.175,"{'tn': 42, 'fp': 54, 'fn': 8, 'tp': 16}" +origin,HPN,27,0.111111,0.458333,0.24537,0.076923,0.333333,0.125,0.113821,0.175,"{'tn': 12, 'fp': 12, 'fn': 2, 'tp': 1}" +origin,IAD,149,0.228188,0.706266,0.476301,0.28,0.823529,0.41791,0.160081,0.175,"{'tn': 43, 'fp': 72, 'fn': 6, 'tp': 28}" +origin,IAH,248,0.258065,0.549083,0.293868,0.283333,0.796875,0.418033,0.193797,0.175,"{'tn': 55, 'fp': 129, 'fn': 13, 'tp': 51}" +origin,ICT,33,0.151515,0.778571,0.403896,0.2,1.0,0.333333,0.122833,0.175,"{'tn': 8, 'fp': 20, 'fn': 0, 'tp': 5}" +origin,IND,107,0.242991,0.632479,0.386336,0.328358,0.846154,0.473118,0.176101,0.175,"{'tn': 36, 'fp': 45, 'fn': 4, 'tp': 22}" +origin,JAX,59,0.271186,0.635174,0.351085,0.322581,0.625,0.425532,0.199175,0.175,"{'tn': 22, 'fp': 21, 'fn': 6, 'tp': 10}" +origin,JFK,235,0.238298,0.596568,0.294244,0.273292,0.785714,0.40553,0.181198,0.175,"{'tn': 62, 'fp': 117, 'fn': 12, 'tp': 44}" +origin,KOA,38,0.157895,0.661458,0.255909,0.166667,0.166667,0.166667,0.130678,0.175,"{'tn': 27, 'fp': 5, 'fn': 5, 'tp': 1}" +origin,LAS,415,0.250602,0.612246,0.329005,0.29927,0.788462,0.433862,0.183324,0.175,"{'tn': 119, 'fp': 192, 'fn': 22, 'tp': 82}" +origin,LAX,410,0.165854,0.637147,0.245122,0.221719,0.720588,0.3391,0.135918,0.175,"{'tn': 170, 'fp': 172, 'fn': 19, 'tp': 49}" +origin,LGA,288,0.267361,0.630455,0.400641,0.36,0.584416,0.445545,0.195761,0.175,"{'tn': 131, 'fp': 80, 'fn': 32, 'tp': 45}" +origin,LGB,39,0.051282,0.695946,0.54,0.5,0.5,0.5,0.048829,0.175,"{'tn': 36, 'fp': 1, 'fn': 1, 'tp': 1}" +origin,LIH,30,0.1,0.691358,0.281046,0.166667,0.333333,0.222222,0.08754,0.175,"{'tn': 22, 'fp': 5, 'fn': 2, 'tp': 1}" +origin,MCI,94,0.276596,0.593326,0.345409,0.341463,0.538462,0.41791,0.209794,0.175,"{'tn': 41, 'fp': 27, 'fn': 12, 'tp': 14}" +origin,MCO,335,0.256716,0.714509,0.425783,0.317269,0.918605,0.471642,0.173173,0.175,"{'tn': 79, 'fp': 170, 'fn': 7, 'tp': 79}" +origin,MDW,157,0.286624,0.68244,0.446941,0.388889,0.777778,0.518519,0.195229,0.175,"{'tn': 57, 'fp': 55, 'fn': 10, 'tp': 35}" +origin,MEM,56,0.25,0.642007,0.391965,0.315789,0.857143,0.461538,0.178846,0.175,"{'tn': 16, 'fp': 26, 'fn': 2, 'tp': 12}" +origin,MIA,196,0.255102,0.640137,0.374315,0.273224,1.0,0.429185,0.184791,0.175,"{'tn': 13, 'fp': 133, 'fn': 0, 'tp': 50}" +origin,MKE,56,0.25,0.511905,0.293146,0.266667,0.571429,0.363636,0.200258,0.175,"{'tn': 20, 'fp': 22, 'fn': 6, 'tp': 8}" +origin,MSN,33,0.181818,0.722222,0.452473,0.357143,0.833333,0.5,0.137674,0.175,"{'tn': 18, 'fp': 9, 'fn': 1, 'tp': 5}" +origin,MSP,234,0.183761,0.578351,0.216845,0.195312,0.581395,0.292398,0.151453,0.175,"{'tn': 88, 'fp': 103, 'fn': 18, 'tp': 25}" +origin,MSY,120,0.258333,0.527365,0.274609,0.268657,0.580645,0.367347,0.198955,0.175,"{'tn': 40, 'fp': 49, 'fn': 13, 'tp': 18}" +origin,MYR,31,0.387097,0.75,0.694422,0.526316,0.833333,0.645161,0.250505,0.175,"{'tn': 10, 'fp': 9, 'fn': 2, 'tp': 10}" +origin,OAK,79,0.139241,0.617647,0.20504,0.195652,0.818182,0.315789,0.123243,0.175,"{'tn': 31, 'fp': 37, 'fn': 2, 'tp': 9}" +origin,OGG,41,0.195122,0.679924,0.314597,0.307692,0.5,0.380952,0.155155,0.175,"{'tn': 24, 'fp': 9, 'fn': 4, 'tp': 4}" +origin,OKC,51,0.117647,0.496296,0.129918,0.083333,0.333333,0.133333,0.113579,0.175,"{'tn': 23, 'fp': 22, 'fn': 4, 'tp': 2}" +origin,OMA,55,0.2,0.698347,0.35768,0.257143,0.818182,0.391304,0.149307,0.175,"{'tn': 18, 'fp': 26, 'fn': 2, 'tp': 9}" +origin,ONT,66,0.166667,0.798347,0.53925,0.277778,0.909091,0.425532,0.122922,0.175,"{'tn': 29, 'fp': 26, 'fn': 1, 'tp': 10}" +origin,ORD,690,0.218841,0.62275,0.304627,0.254274,0.788079,0.384491,0.166212,0.175,"{'tn': 190, 'fp': 349, 'fn': 32, 'tp': 119}" +origin,ORF,50,0.3,0.721905,0.560727,0.40625,0.866667,0.553191,0.192747,0.175,"{'tn': 16, 'fp': 19, 'fn': 2, 'tp': 13}" +origin,PBI,81,0.296296,0.648757,0.380567,0.375,0.875,0.525,0.203743,0.175,"{'tn': 22, 'fp': 35, 'fn': 3, 'tp': 21}" +origin,PDX,133,0.24812,0.517879,0.273094,0.214286,0.454545,0.291262,0.193945,0.175,"{'tn': 45, 'fp': 55, 'fn': 18, 'tp': 15}" +origin,PHL,236,0.330508,0.636725,0.446283,0.402597,0.794872,0.534483,0.220878,0.175,"{'tn': 66, 'fp': 92, 'fn': 16, 'tp': 62}" +origin,PHX,416,0.197115,0.589674,0.268066,0.252525,0.609756,0.357143,0.156007,0.175,"{'tn': 186, 'fp': 148, 'fn': 32, 'tp': 50}" +origin,PIT,111,0.198198,0.795455,0.476147,0.351852,0.863636,0.5,0.139369,0.175,"{'tn': 54, 'fp': 35, 'fn': 3, 'tp': 19}" +origin,PNS,41,0.219512,0.680556,0.432777,0.315789,0.666667,0.428571,0.163606,0.175,"{'tn': 19, 'fp': 13, 'fn': 3, 'tp': 6}" +origin,PSP,29,0.275862,0.690476,0.483674,0.444444,0.5,0.470588,0.199762,0.175,"{'tn': 16, 'fp': 5, 'fn': 4, 'tp': 4}" +origin,PVD,35,0.285714,0.796,0.664505,0.6,0.6,0.6,0.199821,0.175,"{'tn': 21, 'fp': 4, 'fn': 4, 'tp': 6}" +origin,RDU,121,0.181818,0.615243,0.250686,0.223684,0.772727,0.346939,0.151112,0.175,"{'tn': 40, 'fp': 59, 'fn': 5, 'tp': 17}" +origin,RIC,28,0.285714,0.79375,0.530682,0.333333,1.0,0.5,0.173122,0.175,"{'tn': 4, 'fp': 16, 'fn': 0, 'tp': 8}" +origin,RNO,47,0.212766,0.637838,0.433979,0.235294,0.8,0.363636,0.163626,0.175,"{'tn': 11, 'fp': 26, 'fn': 2, 'tp': 8}" +origin,ROC,25,0.28,0.753968,0.453175,0.5,0.571429,0.533333,0.205839,0.175,"{'tn': 14, 'fp': 4, 'fn': 3, 'tp': 4}" +origin,RSW,66,0.318182,0.730159,0.481616,0.465116,0.952381,0.625,0.203706,0.175,"{'tn': 22, 'fp': 23, 'fn': 1, 'tp': 20}" +origin,SAN,206,0.223301,0.579008,0.259376,0.257143,0.782609,0.387097,0.174589,0.175,"{'tn': 56, 'fp': 104, 'fn': 10, 'tp': 36}" +origin,SAT,109,0.247706,0.564137,0.315653,0.230769,0.333333,0.272727,0.195257,0.175,"{'tn': 52, 'fp': 30, 'fn': 18, 'tp': 9}" +origin,SAV,51,0.294118,0.455556,0.283716,0.264706,0.6,0.367347,0.223433,0.175,"{'tn': 11, 'fp': 25, 'fn': 6, 'tp': 9}" +origin,SDF,49,0.204082,0.637179,0.480997,0.26087,0.6,0.363636,0.153825,0.175,"{'tn': 22, 'fp': 17, 'fn': 4, 'tp': 6}" +origin,SEA,383,0.266319,0.598179,0.333752,0.306122,0.735294,0.432277,0.194315,0.175,"{'tn': 111, 'fp': 170, 'fn': 27, 'tp': 75}" +origin,SFB,29,0.482759,0.647619,0.630607,0.565217,0.928571,0.702703,0.271536,0.175,"{'tn': 5, 'fp': 10, 'fn': 1, 'tp': 13}" +origin,SFO,321,0.196262,0.63326,0.349302,0.233333,0.777778,0.358974,0.151869,0.175,"{'tn': 97, 'fp': 161, 'fn': 14, 'tp': 49}" +origin,SJC,95,0.136842,0.658537,0.227317,0.230769,0.461538,0.307692,0.114505,0.175,"{'tn': 62, 'fp': 20, 'fn': 7, 'tp': 6}" +origin,SJU,72,0.236111,0.682353,0.422081,0.294118,0.882353,0.441176,0.16836,0.175,"{'tn': 19, 'fp': 36, 'fn': 2, 'tp': 15}" +origin,SLC,238,0.163866,0.68986,0.332308,0.241071,0.692308,0.357616,0.128648,0.175,"{'tn': 114, 'fp': 85, 'fn': 12, 'tp': 27}" +origin,SMF,140,0.135714,0.632884,0.215131,0.15493,0.578947,0.244444,0.117646,0.175,"{'tn': 61, 'fp': 60, 'fn': 8, 'tp': 11}" +origin,SNA,101,0.19802,0.638272,0.32466,0.240741,0.65,0.351351,0.152647,0.175,"{'tn': 40, 'fp': 41, 'fn': 7, 'tp': 13}" +origin,SRQ,48,0.145833,0.703833,0.296186,0.206897,0.857143,0.333333,0.121636,0.175,"{'tn': 18, 'fp': 23, 'fn': 1, 'tp': 6}" +origin,STL,142,0.260563,0.70888,0.42801,0.378378,0.756757,0.504505,0.181676,0.175,"{'tn': 59, 'fp': 46, 'fn': 9, 'tp': 28}" +origin,SYR,26,0.192308,0.571429,0.241203,0.2,0.6,0.3,0.163446,0.175,"{'tn': 9, 'fp': 12, 'fn': 2, 'tp': 3}" +origin,TPA,184,0.277174,0.584402,0.356223,0.315789,0.705882,0.436364,0.200687,0.175,"{'tn': 55, 'fp': 78, 'fn': 15, 'tp': 36}" +origin,TUL,38,0.315789,0.546474,0.392559,0.375,0.5,0.428571,0.237281,0.175,"{'tn': 16, 'fp': 10, 'fn': 6, 'tp': 6}" +origin,TUS,42,0.071429,0.623932,0.135962,0.166667,0.666667,0.266667,0.07399,0.175,"{'tn': 29, 'fp': 10, 'fn': 1, 'tp': 2}" +origin,TYS,29,0.310345,0.7,0.656809,0.4,0.666667,0.5,0.200496,0.175,"{'tn': 11, 'fp': 9, 'fn': 3, 'tp': 6}" +origin,VPS,33,0.272727,0.444444,0.276675,0.304348,0.777778,0.4375,0.213455,0.175,"{'tn': 8, 'fp': 16, 'fn': 2, 'tp': 7}" +destination,ABQ,64,0.25,0.696615,0.438705,0.311111,0.875,0.459016,0.173973,0.175,"{'tn': 17, 'fp': 31, 'fn': 2, 'tp': 14}" +destination,ANC,40,0.1,0.881944,0.4625,0.125,1.0,0.222222,0.105346,0.175,"{'tn': 8, 'fp': 28, 'fn': 0, 'tp': 4}" +destination,ATL,683,0.263543,0.650127,0.407115,0.347134,0.605556,0.441296,0.191339,0.175,"{'tn': 298, 'fp': 205, 'fn': 71, 'tp': 109}" +destination,AUS,172,0.273256,0.692596,0.454389,0.333333,0.893617,0.485549,0.182787,0.175,"{'tn': 41, 'fp': 84, 'fn': 5, 'tp': 42}" +destination,BDL,52,0.25,0.699211,0.398942,0.392857,0.846154,0.536585,0.176208,0.175,"{'tn': 22, 'fp': 17, 'fn': 2, 'tp': 11}" +destination,BHM,44,0.25,0.515152,0.347799,0.264706,0.818182,0.4,0.190464,0.175,"{'tn': 8, 'fp': 25, 'fn': 2, 'tp': 9}" +destination,BNA,249,0.261044,0.659992,0.35855,0.330935,0.707692,0.45098,0.187439,0.175,"{'tn': 91, 'fp': 93, 'fn': 19, 'tp': 46}" +destination,BOI,37,0.189189,0.42381,0.178933,0.173913,0.571429,0.266667,0.161746,0.175,"{'tn': 11, 'fp': 19, 'fn': 3, 'tp': 4}" +destination,BOS,290,0.289655,0.607345,0.359762,0.335025,0.785714,0.469751,0.204715,0.175,"{'tn': 75, 'fp': 131, 'fn': 18, 'tp': 66}" +destination,BUF,41,0.219512,0.409722,0.203022,0.1875,0.666667,0.292683,0.19336,0.175,"{'tn': 6, 'fp': 26, 'fn': 3, 'tp': 6}" +destination,BUR,68,0.147059,0.613793,0.292087,0.142857,0.5,0.222222,0.127616,0.175,"{'tn': 28, 'fp': 30, 'fn': 5, 'tp': 5}" +destination,BWI,230,0.252174,0.650962,0.353639,0.348624,0.655172,0.45509,0.186615,0.175,"{'tn': 101, 'fp': 71, 'fn': 20, 'tp': 38}" +destination,BZN,29,0.068966,0.685185,0.146465,0.181818,1.0,0.307692,0.078593,0.175,"{'tn': 18, 'fp': 9, 'fn': 0, 'tp': 2}" +destination,CHS,41,0.219512,0.625,0.375656,0.25,0.555556,0.344828,0.167182,0.175,"{'tn': 17, 'fp': 15, 'fn': 4, 'tp': 5}" +destination,CLE,92,0.271739,0.459403,0.263396,0.253968,0.64,0.363636,0.212401,0.175,"{'tn': 20, 'fp': 47, 'fn': 9, 'tp': 16}" +destination,CLT,434,0.18894,0.645787,0.307784,0.241245,0.756098,0.365782,0.147561,0.175,"{'tn': 157, 'fp': 195, 'fn': 20, 'tp': 62}" +destination,CMH,100,0.28,0.697669,0.467665,0.423077,0.785714,0.55,0.195206,0.175,"{'tn': 42, 'fp': 30, 'fn': 6, 'tp': 22}" +destination,COS,31,0.322581,0.638095,0.56293,0.315789,0.6,0.413793,0.222932,0.175,"{'tn': 8, 'fp': 13, 'fn': 4, 'tp': 6}" +destination,CVG,82,0.268293,0.662879,0.436054,0.355932,0.954545,0.518519,0.184581,0.175,"{'tn': 22, 'fp': 38, 'fn': 1, 'tp': 21}" +destination,DAL,127,0.204724,0.555979,0.237723,0.22093,0.730769,0.339286,0.16516,0.175,"{'tn': 34, 'fp': 67, 'fn': 7, 'tp': 19}" +destination,DCA,282,0.304965,0.654811,0.379743,0.456376,0.790698,0.578723,0.213012,0.175,"{'tn': 115, 'fp': 81, 'fn': 18, 'tp': 68}" +destination,DEN,713,0.295933,0.640542,0.416567,0.416413,0.649289,0.507407,0.210989,0.175,"{'tn': 310, 'fp': 192, 'fn': 74, 'tp': 137}" +destination,DFW,661,0.284418,0.603476,0.383984,0.308511,0.771277,0.440729,0.199282,0.175,"{'tn': 148, 'fp': 325, 'fn': 43, 'tp': 145}" +destination,DSM,25,0.16,0.416667,0.161774,0.176471,0.75,0.285714,0.154402,0.175,"{'tn': 7, 'fp': 14, 'fn': 1, 'tp': 3}" +destination,DTW,280,0.171429,0.665634,0.279334,0.22973,0.708333,0.346939,0.136499,0.175,"{'tn': 118, 'fp': 114, 'fn': 14, 'tp': 34}" +destination,ECP,25,0.4,0.6,0.532857,0.470588,0.8,0.592593,0.271101,0.175,"{'tn': 6, 'fp': 9, 'fn': 2, 'tp': 8}" +destination,ELP,30,0.233333,0.670807,0.472527,0.259259,1.0,0.411765,0.169833,0.175,"{'tn': 3, 'fp': 20, 'fn': 0, 'tp': 7}" +destination,EWR,236,0.364407,0.608992,0.456213,0.418301,0.744186,0.535565,0.240752,0.175,"{'tn': 61, 'fp': 89, 'fn': 22, 'tp': 64}" +destination,FLL,159,0.238994,0.591562,0.280183,0.283186,0.842105,0.423841,0.181896,0.175,"{'tn': 40, 'fp': 81, 'fn': 6, 'tp': 32}" +destination,GEG,34,0.235294,0.567308,0.270338,0.3,0.75,0.428571,0.180634,0.175,"{'tn': 12, 'fp': 14, 'fn': 2, 'tp': 6}" +destination,GRR,27,0.185185,0.863636,0.706061,0.3,0.6,0.4,0.128696,0.175,"{'tn': 15, 'fp': 7, 'fn': 2, 'tp': 3}" +destination,GSP,42,0.428571,0.611111,0.583424,0.5,0.777778,0.608696,0.289993,0.175,"{'tn': 10, 'fp': 14, 'fn': 4, 'tp': 14}" +destination,HNL,115,0.147826,0.627251,0.198707,0.170213,0.470588,0.25,0.127299,0.175,"{'tn': 59, 'fp': 39, 'fn': 9, 'tp': 8}" +destination,HOU,139,0.251799,0.714011,0.44906,0.380952,0.685714,0.489796,0.178766,0.175,"{'tn': 65, 'fp': 39, 'fn': 11, 'tp': 24}" +destination,IAD,136,0.242647,0.630038,0.353689,0.392157,0.606061,0.47619,0.181687,0.175,"{'tn': 72, 'fp': 31, 'fn': 13, 'tp': 20}" +destination,IAH,250,0.252,0.685086,0.432948,0.333333,0.730159,0.457711,0.175813,0.175,"{'tn': 95, 'fp': 92, 'fn': 17, 'tp': 46}" +destination,ILM,27,0.296296,0.710526,0.435516,0.466667,0.875,0.608696,0.202767,0.175,"{'tn': 11, 'fp': 8, 'fn': 1, 'tp': 7}" +destination,IND,88,0.227273,0.608824,0.336973,0.263889,0.95,0.413043,0.174675,0.175,"{'tn': 15, 'fp': 53, 'fn': 1, 'tp': 19}" +destination,JAX,55,0.181818,0.58,0.330722,0.21875,0.7,0.333333,0.1483,0.175,"{'tn': 20, 'fp': 25, 'fn': 3, 'tp': 7}" +destination,JFK,225,0.28,0.743386,0.510761,0.371622,0.873016,0.521327,0.180766,0.175,"{'tn': 69, 'fp': 93, 'fn': 8, 'tp': 55}" +destination,KOA,38,0.157895,0.606771,0.254168,0.333333,0.166667,0.222222,0.135652,0.175,"{'tn': 30, 'fp': 2, 'fn': 5, 'tp': 1}" +destination,LAS,416,0.21875,0.697244,0.358126,0.304511,0.89011,0.453782,0.158496,0.175,"{'tn': 140, 'fp': 185, 'fn': 10, 'tp': 81}" +destination,LAX,393,0.160305,0.709933,0.302953,0.234234,0.825397,0.364912,0.127185,0.175,"{'tn': 160, 'fp': 170, 'fn': 11, 'tp': 52}" +destination,LGA,280,0.25,0.628673,0.360994,0.321918,0.671429,0.435185,0.183318,0.175,"{'tn': 111, 'fp': 99, 'fn': 23, 'tp': 47}" +destination,LGB,34,0.117647,0.6,0.38357,0.153846,0.5,0.235294,0.106551,0.175,"{'tn': 19, 'fp': 11, 'fn': 2, 'tp': 2}" +destination,LIH,27,0.037037,0.653846,0.1,0.0,0.0,0.0,0.043881,0.175,"{'tn': 24, 'fp': 2, 'fn': 1, 'tp': 0}" +destination,MCI,103,0.203883,0.711963,0.372151,0.322034,0.904762,0.475,0.149726,0.175,"{'tn': 42, 'fp': 40, 'fn': 2, 'tp': 19}" +destination,MCO,335,0.298507,0.641915,0.416712,0.368664,0.8,0.504732,0.203694,0.175,"{'tn': 98, 'fp': 137, 'fn': 20, 'tp': 80}" +destination,MDW,161,0.167702,0.665008,0.296038,0.219512,0.666667,0.330275,0.134749,0.175,"{'tn': 70, 'fp': 64, 'fn': 9, 'tp': 18}" +destination,MEM,60,0.266667,0.71733,0.600372,0.391304,0.5625,0.461538,0.190046,0.175,"{'tn': 30, 'fp': 14, 'fn': 7, 'tp': 9}" +destination,MIA,251,0.227092,0.656041,0.355917,0.285714,0.77193,0.417062,0.166487,0.175,"{'tn': 84, 'fp': 110, 'fn': 13, 'tp': 44}" +destination,MKE,63,0.238095,0.643056,0.346457,0.288889,0.866667,0.433333,0.175651,0.175,"{'tn': 16, 'fp': 32, 'fn': 2, 'tp': 13}" +destination,MSP,233,0.184549,0.641248,0.30239,0.221374,0.674419,0.333333,0.145092,0.175,"{'tn': 88, 'fp': 102, 'fn': 14, 'tp': 29}" +destination,MSY,100,0.29,0.689898,0.447924,0.4,0.896552,0.553191,0.193622,0.175,"{'tn': 32, 'fp': 39, 'fn': 3, 'tp': 26}" +destination,MYR,43,0.302326,0.692308,0.593422,0.314286,0.846154,0.458333,0.195706,0.175,"{'tn': 6, 'fp': 24, 'fn': 2, 'tp': 11}" +destination,OAK,82,0.158537,0.767559,0.363515,0.261905,0.846154,0.4,0.121985,0.175,"{'tn': 38, 'fp': 31, 'fn': 2, 'tp': 11}" +destination,OGG,42,0.0,,,0.0,0.0,0.0,0.036345,0.175,"{'tn': 23, 'fp': 19, 'fn': 0, 'tp': 0}" +destination,OKC,59,0.288136,0.521709,0.317776,0.317073,0.764706,0.448276,0.210465,0.175,"{'tn': 14, 'fp': 28, 'fn': 4, 'tp': 13}" +destination,OMA,61,0.295082,0.722222,0.592129,0.357143,0.833333,0.5,0.187859,0.175,"{'tn': 16, 'fp': 27, 'fn': 3, 'tp': 15}" +destination,ONT,56,0.303571,0.74359,0.56977,0.421053,0.941176,0.581818,0.195927,0.175,"{'tn': 17, 'fp': 22, 'fn': 1, 'tp': 16}" +destination,ORD,679,0.212077,0.65294,0.303934,0.290323,0.75,0.418605,0.160114,0.175,"{'tn': 271, 'fp': 264, 'fn': 36, 'tp': 108}" +destination,ORF,35,0.142857,0.76,0.388095,0.16,0.8,0.266667,0.118382,0.175,"{'tn': 9, 'fp': 21, 'fn': 1, 'tp': 4}" +destination,PBI,55,0.218182,0.618217,0.410073,0.25641,0.833333,0.392157,0.168709,0.175,"{'tn': 14, 'fp': 29, 'fn': 2, 'tp': 10}" +destination,PDX,130,0.261538,0.681373,0.369941,0.316832,0.941176,0.474074,0.180159,0.175,"{'tn': 27, 'fp': 69, 'fn': 2, 'tp': 32}" +destination,PHL,218,0.197248,0.70485,0.349995,0.22293,0.813953,0.35,0.148827,0.175,"{'tn': 53, 'fp': 122, 'fn': 8, 'tp': 35}" +destination,PHX,446,0.237668,0.659573,0.351834,0.317269,0.745283,0.44507,0.173133,0.175,"{'tn': 170, 'fp': 170, 'fn': 27, 'tp': 79}" +destination,PIT,98,0.285714,0.634694,0.392544,0.333333,0.75,0.461538,0.202232,0.175,"{'tn': 28, 'fp': 42, 'fn': 7, 'tp': 21}" +destination,PNS,30,0.2,0.5625,0.277454,0.25,0.666667,0.363636,0.160953,0.175,"{'tn': 12, 'fp': 12, 'fn': 2, 'tp': 4}" +destination,PSP,34,0.235294,0.706731,0.531719,0.315789,0.75,0.444444,0.170277,0.175,"{'tn': 13, 'fp': 13, 'fn': 2, 'tp': 6}" +destination,PVD,26,0.230769,0.691667,0.383929,0.3,1.0,0.461538,0.165206,0.175,"{'tn': 6, 'fp': 14, 'fn': 0, 'tp': 6}" +destination,RDU,126,0.230159,0.670459,0.379476,0.285714,0.827586,0.424779,0.166514,0.175,"{'tn': 37, 'fp': 60, 'fn': 5, 'tp': 24}" +destination,RIC,30,0.266667,0.551136,0.4432,0.315789,0.75,0.444444,0.196456,0.175,"{'tn': 9, 'fp': 13, 'fn': 2, 'tp': 6}" +destination,RNO,55,0.218182,0.73062,0.482533,0.305556,0.916667,0.458333,0.15555,0.175,"{'tn': 18, 'fp': 25, 'fn': 1, 'tp': 11}" +destination,RSW,70,0.257143,0.736111,0.493513,0.333333,0.777778,0.466667,0.170624,0.175,"{'tn': 24, 'fp': 28, 'fn': 4, 'tp': 14}" +destination,SAN,219,0.210046,0.593177,0.298065,0.228395,0.804348,0.355769,0.167532,0.175,"{'tn': 48, 'fp': 125, 'fn': 9, 'tp': 37}" +destination,SAT,89,0.303371,0.735663,0.496262,0.4,0.814815,0.536585,0.19751,0.175,"{'tn': 29, 'fp': 33, 'fn': 5, 'tp': 22}" +destination,SAV,41,0.268293,0.654545,0.37718,0.323529,1.0,0.488889,0.189236,0.175,"{'tn': 7, 'fp': 23, 'fn': 0, 'tp': 11}" +destination,SDF,60,0.283333,0.613543,0.399008,0.368421,0.823529,0.509091,0.201759,0.175,"{'tn': 19, 'fp': 24, 'fn': 3, 'tp': 14}" +destination,SEA,370,0.243243,0.603631,0.329858,0.301508,0.666667,0.415225,0.181106,0.175,"{'tn': 141, 'fp': 139, 'fn': 30, 'tp': 60}" +destination,SFB,26,0.5,0.863905,0.835627,0.590909,1.0,0.742857,0.245907,0.175,"{'tn': 4, 'fp': 9, 'fn': 0, 'tp': 13}" +destination,SFO,302,0.251656,0.579297,0.338508,0.254545,0.736842,0.378378,0.190137,0.175,"{'tn': 62, 'fp': 164, 'fn': 20, 'tp': 56}" +destination,SGF,29,0.206897,0.688406,0.426905,0.208333,0.833333,0.333333,0.15986,0.175,"{'tn': 4, 'fp': 19, 'fn': 1, 'tp': 5}" +destination,SJC,105,0.190476,0.671176,0.298486,0.25,0.75,0.375,0.146052,0.175,"{'tn': 40, 'fp': 45, 'fn': 5, 'tp': 15}" +destination,SJU,75,0.2,0.508889,0.256387,0.186441,0.733333,0.297297,0.179756,0.175,"{'tn': 12, 'fp': 48, 'fn': 4, 'tp': 11}" +destination,SLC,270,0.151852,0.648046,0.252341,0.211009,0.560976,0.306667,0.124753,0.175,"{'tn': 143, 'fp': 86, 'fn': 18, 'tp': 23}" +destination,SMF,129,0.232558,0.714646,0.479961,0.294118,0.833333,0.434783,0.164502,0.175,"{'tn': 39, 'fp': 60, 'fn': 5, 'tp': 25}" +destination,SNA,109,0.183486,0.671629,0.372873,0.21875,0.7,0.333333,0.142744,0.175,"{'tn': 39, 'fp': 50, 'fn': 6, 'tp': 14}" +destination,SRQ,50,0.26,0.704782,0.409874,0.423077,0.846154,0.564103,0.18538,0.175,"{'tn': 22, 'fp': 15, 'fn': 2, 'tp': 11}" +destination,STL,144,0.291667,0.725257,0.454933,0.413793,0.857143,0.55814,0.194317,0.175,"{'tn': 51, 'fp': 51, 'fn': 6, 'tp': 36}" +destination,TPA,192,0.223958,0.602154,0.338652,0.26087,0.837209,0.39779,0.1708,0.175,"{'tn': 47, 'fp': 102, 'fn': 7, 'tp': 36}" +destination,TUL,32,0.3125,0.745455,0.614243,0.4,0.6,0.48,0.212472,0.175,"{'tn': 13, 'fp': 9, 'fn': 4, 'tp': 6}" +destination,TUS,47,0.319149,0.675,0.467386,0.368421,0.933333,0.528302,0.215609,0.175,"{'tn': 8, 'fp': 24, 'fn': 1, 'tp': 14}" +destination,TYS,33,0.363636,0.779762,0.726371,0.478261,0.916667,0.628571,0.213426,0.175,"{'tn': 9, 'fp': 12, 'fn': 1, 'tp': 11}" +destination,VPS,31,0.258065,0.535326,0.28663,0.3125,0.625,0.416667,0.207741,0.175,"{'tn': 12, 'fp': 11, 'fn': 3, 'tp': 5}" +route_volume_band,high,4868,0.255136,0.643876,0.355961,0.319085,0.775362,0.452113,0.18239,0.175,"{'tn': 1571, 'fp': 2055, 'fn': 279, 'tp': 963}" +route_volume_band,low,4940,0.230972,0.638059,0.317293,0.296034,0.732691,0.42169,0.171455,0.175,"{'tn': 1811, 'fp': 1988, 'fn': 305, 'tp': 836}" +route_volume_band,medium,5192,0.236518,0.636684,0.331971,0.296747,0.735342,0.422852,0.174055,0.175,"{'tn': 1824, 'fp': 2140, 'fn': 325, 'tp': 903}" +departure_block,afternoon,5291,0.282366,0.559347,0.327672,0.295159,0.881526,0.442243,0.202743,0.175,"{'tn': 652, 'fp': 3145, 'fn': 177, 'tp': 1317}" +departure_block,evening,3463,0.346232,0.555743,0.383435,0.353031,0.961635,0.516461,0.228429,0.175,"{'tn': 151, 'fp': 2113, 'fn': 46, 'tp': 1153}" +departure_block,morning,5694,0.150685,0.576656,0.199498,0.202206,0.25641,0.226105,0.126831,0.175,"{'tn': 3968, 'fp': 868, 'fn': 638, 'tp': 220}" +departure_block,overnight,552,0.108696,0.599356,0.189445,0.173913,0.2,0.186047,0.095322,0.175,"{'tn': 435, 'fp': 57, 'fn': 48, 'tp': 12}" +calendar_month,4,5000,0.196,0.608059,0.255873,0.238408,0.739796,0.360607,0.156076,0.175,"{'tn': 1704, 'fp': 2316, 'fn': 255, 'tp': 725}" +calendar_month,5,5000,0.2358,0.638511,0.329654,0.293121,0.733673,0.418886,0.173429,0.175,"{'tn': 1735, 'fp': 2086, 'fn': 314, 'tp': 865}" +calendar_month,6,5000,0.2904,0.674466,0.431399,0.384376,0.76584,0.511853,0.198206,0.175,"{'tn': 1767, 'fp': 1781, 'fn': 340, 'tp': 1112}" diff --git a/docs/ml/calibration.svg b/docs/ml/calibration.svg new file mode 100644 index 0000000..335a331 --- /dev/null +++ b/docs/ml/calibration.svg @@ -0,0 +1,9 @@ + +Test-set calibration + + + + +Mean predicted probability +Observed delay rate + \ No newline at end of file diff --git a/docs/ml/data-contract.md b/docs/ml/data-contract.md index 81a12d9..bc0590d 100644 --- a/docs/ml/data-contract.md +++ b/docs/ml/data-contract.md @@ -1,6 +1,6 @@ # Flight-delay model data contract -Status: proposed for [Issue #2](https://github.com/mitulpatel123/flightops-ai/issues/2) +Status: accepted and implemented for [Issue #2](https://github.com/mitulpatel123/flightops-ai/issues/2) ## Decision this contract protects diff --git a/docs/ml/model-card.md b/docs/ml/model-card.md new file mode 100644 index 0000000..0036dd1 --- /dev/null +++ b/docs/ml/model-card.md @@ -0,0 +1,41 @@ +# Flight-delay model card + +**Version:** `bts-schedule-logistic-v1` +**Prediction:** probability of arrival at least 15 minutes late, evaluated at T-24h +**Data:** official BTS Reporting Carrier On-Time Performance, 2024-01 through 2025-06 + +## Evaluation + +The chronological test period is April-June 2025 (15,000 sampled eligible flights; prevalence 24.1%). The operating threshold `0.175` was selected only on January-March 2025 validation data using a 5:1 false-negative/false-positive cost assumption. + +| Split | Sample rows | Positive prevalence | Source rows | Cancelled excluded | Diverted excluded | +| --- | ---: | ---: | ---: | ---: | ---: | +| Train (Jan-Dec 2024) | 60,000 | 20.7% | 7,079,061 | 96,315 | 17,499 | +| Validation (Jan-Mar 2025) | 15,000 | 19.2% | 1,645,503 | 30,640 | 3,817 | +| Test (Apr-Jun 2025) | 15,000 | 24.1% | 1,801,173 | 20,994 | 5,403 | + +Required schedule-feature missingness was 0% in all three sampled splits before imputation. The tree comparison used one fixed configuration (80 iterations, 31 leaves, 0.08 learning rate); no test-set tuning was performed. + +| Model | ROC-AUC | PR-AUC | Precision | Recall | F1 | Brier | Expected cost | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Portable logistic (deployed) | 0.640 | 0.335 | 0.304 | 0.748 | 0.432 | 0.176 | 10728.0 | +| Histogram gradient boosting | 0.661 | 0.371 | 0.292 | 0.804 | 0.428 | 0.172 | 10574.0 | + +![Calibration curve](calibration.svg) + +Full machine-readable metrics are in [`artifacts/reports/delay-model-metrics.json`](../../artifacts/reports/delay-model-metrics.json); carrier, airport, route-volume, time-block, and month slices are in [`error-slices.csv`](../../artifacts/reports/error-slices.csv). + +## Intended use + +Portfolio-grade retrospective decision support and API demonstration. It is not approved for dispatch, passenger promises, staffing, or automated adverse decisions. + +## Model and artifact safety + +The deployed logistic model is exported as inspectable JSON coefficients, scaling values, categories, threshold, metrics, and the data-manifest checksum. The API does not execute pickle or joblib artifacts. + +## Limitations + +- Schedule-only predictors cannot see weather, maintenance, crew, or same-day network disruptions. +- The 90,000-row dataset is a deterministic monthly sample, not the full BTS population. +- Public retrospective performance is not production accuracy; drift monitoring and airline-specific validation are required. +- Category-level error slices can expose uneven performance and must be reviewed before operational use. diff --git a/ml/README.md b/ml/README.md new file mode 100644 index 0000000..0093b34 --- /dev/null +++ b/ml/README.md @@ -0,0 +1,24 @@ +# Reproduce the trained delay model + +The pipeline uses official U.S. Bureau of Transportation Statistics monthly files and never commits raw source data. + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e ".[dev,ml]" +python scripts/download_bts.py +python scripts/train_delay_model.py +python -m pytest +``` + +`download_bts.py` retrieves January 2024 through June 2025, hashes every official ZIP, applies the documented operated-flight cohort, and creates a deterministic 5,000-row monthly sample. `train_delay_model.py` uses January-December 2024 for training, January-March 2025 for threshold selection, and April-June 2025 exactly once for final evaluation. + +Committed evidence: + +- `artifacts/data/bts-sample-manifest.json`: URLs, byte sizes, checksums, exclusions, and sampling configuration +- `artifacts/models/delay-logistic-v1.json`: portable coefficients, feature scaling, categories, threshold, metrics, and manifest checksum +- `artifacts/reports/delay-model-metrics.json`: baselines, model comparison, calibration, and confusion matrices +- `artifacts/reports/error-slices.csv`: carrier, airport, route-volume, time-block, and month slices +- `docs/ml/model-card.md`: intended use, results, safety, and limitations + +The API reads only the inspectable JSON artifact. It does not execute pickle or joblib files. diff --git a/ml/config.json b/ml/config.json new file mode 100644 index 0000000..8593ea0 --- /dev/null +++ b/ml/config.json @@ -0,0 +1,11 @@ +{ + "source_url_template": "https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{year}_{month}.zip", + "source_name": "BTS Reporting Carrier On-Time Performance", + "start_month": "2024-01", + "end_month": "2025-06", + "rows_per_month": 5000, + "random_seed": 1062, + "validation_false_negative_cost": 5.0, + "validation_false_positive_cost": 1.0, + "model_version": "bts-schedule-logistic-v1" +} diff --git a/pyproject.toml b/pyproject.toml index 51775f1..91a5c50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "flightops-ai" -version = "0.1.0" +version = "0.2.0" description = "A production-minded airline operations intelligence API" requires-python = ">=3.11" dependencies = [ @@ -15,6 +15,7 @@ dependencies = [ [project.optional-dependencies] dev = ["httpx==0.28.1", "pytest==7.4.4"] +ml = ["numpy==2.0.2", "pandas==2.3.3", "scikit-learn==1.6.1"] [tool.setuptools.packages.find] include = ["app*"] diff --git a/requirements-ml.txt b/requirements-ml.txt new file mode 100644 index 0000000..761406d --- /dev/null +++ b/requirements-ml.txt @@ -0,0 +1,3 @@ +numpy==2.0.2 +pandas==2.3.3 +scikit-learn==1.6.1 diff --git a/scripts/download_bts.py b/scripts/download_bts.py new file mode 100644 index 0000000..ffa7ca3 --- /dev/null +++ b/scripts/download_bts.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Download, verify, filter, and deterministically sample official BTS data.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import time +import urllib.request +import zipfile +from pathlib import Path + +import pandas as pd + + +ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = ROOT / "ml" / "config.json" +RAW_DIR = ROOT / "data" / "raw" +PROCESSED_DIR = ROOT / "data" / "processed" +MANIFEST_PATH = ROOT / "artifacts" / "data" / "bts-sample-manifest.json" + +REQUIRED_COLUMNS = [ + "Year", + "Month", + "DayofMonth", + "DayOfWeek", + "FlightDate", + "Reporting_Airline", + "Origin", + "Dest", + "CRSDepTime", + "CRSArrTime", + "CRSElapsedTime", + "Distance", + "DistanceGroup", + "ArrDel15", + "Cancelled", + "Diverted", +] + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def month_range(start: str, end: str) -> list[tuple[int, int]]: + first = pd.Period(start, freq="M") + last = pd.Period(end, freq="M") + return [(period.year, period.month) for period in pd.period_range(first, last, freq="M")] + + +def download(url: str, destination: Path) -> None: + if destination.exists() and destination.stat().st_size > 0: + return + temporary = destination.with_suffix(".part") + for attempt in range(1, 4): + try: + request = urllib.request.Request(url, headers={"User-Agent": "FlightOps-AI/1.0"}) + with urllib.request.urlopen(request, timeout=120) as response, temporary.open("wb") as output: + while chunk := response.read(1024 * 1024): + output.write(chunk) + temporary.replace(destination) + return + except Exception: + temporary.unlink(missing_ok=True) + if attempt == 3: + raise + time.sleep(attempt * 2) + + +def load_month(zip_path: Path) -> pd.DataFrame: + with zipfile.ZipFile(zip_path) as archive: + csv_names = [name for name in archive.namelist() if name.lower().endswith(".csv")] + if len(csv_names) != 1: + raise ValueError(f"Expected one CSV in {zip_path.name}; found {csv_names}") + with archive.open(csv_names[0]) as csv_file: + frame = pd.read_csv(csv_file, usecols=REQUIRED_COLUMNS, low_memory=False) + frame.columns = [column.strip() for column in frame.columns] + return frame + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--rows-per-month", type=int) + args = parser.parse_args() + + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + rows_per_month = args.rows_per_month or int(config["rows_per_month"]) + seed = int(config["random_seed"]) + RAW_DIR.mkdir(parents=True, exist_ok=True) + PROCESSED_DIR.mkdir(parents=True, exist_ok=True) + MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True) + + samples: list[pd.DataFrame] = [] + sources: list[dict] = [] + for year, month in month_range(config["start_month"], config["end_month"]): + url = config["source_url_template"].format(year=year, month=month) + zip_path = RAW_DIR / f"bts-{year}-{month:02d}.zip" + print(f"[{year}-{month:02d}] downloading {url}", flush=True) + download(url, zip_path) + frame = load_month(zip_path) + total_rows = len(frame) + cancelled = int((frame["Cancelled"] == 1).sum()) + diverted = int((frame["Diverted"] == 1).sum()) + eligible = frame[ + (frame["Cancelled"] == 0) + & (frame["Diverted"] == 0) + & frame["ArrDel15"].notna() + ].copy() + sample_count = min(rows_per_month, len(eligible)) + sampled = eligible.sample(n=sample_count, random_state=seed + year * 100 + month) + samples.append(sampled) + sources.append( + { + "month": f"{year}-{month:02d}", + "url": url, + "zip_bytes": zip_path.stat().st_size, + "zip_sha256": sha256(zip_path), + "rows_total": total_rows, + "rows_cancelled": cancelled, + "rows_diverted": diverted, + "rows_eligible": len(eligible), + "rows_sampled": sample_count, + } + ) + + combined = pd.concat(samples, ignore_index=True).sort_values( + ["FlightDate", "Reporting_Airline", "Origin", "Dest"], kind="stable" + ) + output_path = PROCESSED_DIR / "bts-schedule-sample.csv.gz" + combined.to_csv(output_path, index=False, compression={"method": "gzip", "mtime": 0}) + manifest = { + "schema_version": 1, + "source": config["source_name"], + "source_table": "Reporting Carrier On-Time Performance (1987-present)", + "period": {"start": config["start_month"], "end": config["end_month"]}, + "sampling": { + "method": "deterministic uniform sample after cohort filtering", + "rows_per_month": rows_per_month, + "random_seed": seed, + }, + "required_columns": REQUIRED_COLUMNS, + "processed_file": output_path.name, + "processed_rows": len(combined), + "processed_sha256": sha256(output_path), + "sources": sources, + } + MANIFEST_PATH.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {len(combined):,} rows and {MANIFEST_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/train_delay_model.py b/scripts/train_delay_model.py new file mode 100644 index 0000000..5f2c92b --- /dev/null +++ b/scripts/train_delay_model.py @@ -0,0 +1,425 @@ +#!/usr/bin/env python3 +"""Train, evaluate, and export an inspectable T-24h delay model.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +from datetime import datetime, timezone +from pathlib import Path + +import numpy as np +import pandas as pd +from sklearn.compose import ColumnTransformer +from sklearn.ensemble import HistGradientBoostingClassifier +from sklearn.impute import SimpleImputer +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import ( + average_precision_score, + brier_score_loss, + confusion_matrix, + f1_score, + precision_score, + recall_score, + roc_auc_score, +) +from sklearn.pipeline import Pipeline +from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, StandardScaler + + +ROOT = Path(__file__).resolve().parents[1] +DATA_PATH = ROOT / "data" / "processed" / "bts-schedule-sample.csv.gz" +MANIFEST_PATH = ROOT / "artifacts" / "data" / "bts-sample-manifest.json" +CONFIG_PATH = ROOT / "ml" / "config.json" +MODEL_PATH = ROOT / "artifacts" / "models" / "delay-logistic-v1.json" +METRICS_PATH = ROOT / "artifacts" / "reports" / "delay-model-metrics.json" +SLICES_PATH = ROOT / "artifacts" / "reports" / "error-slices.csv" +CARD_PATH = ROOT / "docs" / "ml" / "model-card.md" +CALIBRATION_PATH = ROOT / "docs" / "ml" / "calibration.svg" + +NUMERIC = [ + "Month", + "DayOfWeek", + "DepHourSin", + "DepHourCos", + "CRSElapsedTime", + "Distance", + "DistanceGroup", +] +CATEGORICAL = ["Reporting_Airline", "Origin", "Dest"] +FEATURES = NUMERIC + CATEGORICAL + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def prepare(frame: pd.DataFrame) -> pd.DataFrame: + prepared = frame.copy() + prepared["FlightDate"] = pd.to_datetime(prepared["FlightDate"], errors="raise") + departure_hour = (prepared["CRSDepTime"].fillna(0).astype(int) // 100).clip(0, 23) + angle = 2.0 * np.pi * departure_hour / 24.0 + prepared["DepHourSin"] = np.sin(angle) + prepared["DepHourCos"] = np.cos(angle) + prepared["target"] = prepared["ArrDel15"].astype(int) + prepared["route"] = prepared["Origin"].astype(str) + "-" + prepared["Dest"].astype(str) + prepared["departure_block"] = pd.cut( + departure_hour, + bins=[-1, 5, 11, 17, 23], + labels=["overnight", "morning", "afternoon", "evening"], + ).astype(str) + route_counts = prepared.groupby("route")["route"].transform("size") + prepared["route_volume_band"] = pd.qcut( + route_counts.rank(method="first"), 3, labels=["low", "medium", "high"] + ).astype(str) + return prepared + + +def split(frame: pd.DataFrame) -> dict[str, pd.DataFrame]: + return { + "train": frame[frame["FlightDate"] < "2025-01-01"].copy(), + "validation": frame[ + (frame["FlightDate"] >= "2025-01-01") & (frame["FlightDate"] < "2025-04-01") + ].copy(), + "test": frame[frame["FlightDate"] >= "2025-04-01"].copy(), + } + + +def choose_threshold(y_true: pd.Series, probabilities: np.ndarray, fn_cost: float, fp_cost: float) -> float: + candidates = np.linspace(0.10, 0.90, 161) + costs = [] + for threshold in candidates: + predictions = probabilities >= threshold + tn, fp, fn, tp = confusion_matrix(y_true, predictions, labels=[0, 1]).ravel() + costs.append((fn * fn_cost + fp * fp_cost, -tp, threshold)) + return float(min(costs)[2]) + + +def metrics(y_true: pd.Series, probabilities: np.ndarray, threshold: float) -> dict: + predictions = probabilities >= threshold + tn, fp, fn, tp = confusion_matrix(y_true, predictions, labels=[0, 1]).ravel() + has_both_classes = y_true.nunique() == 2 + has_positives = bool(y_true.sum()) + return { + "rows": int(len(y_true)), + "positive_prevalence": round(float(y_true.mean()), 6), + "roc_auc": round(float(roc_auc_score(y_true, probabilities)), 6) if has_both_classes else None, + "pr_auc": round(float(average_precision_score(y_true, probabilities)), 6) if has_positives else None, + "precision": round(float(precision_score(y_true, predictions, zero_division=0)), 6), + "recall": round(float(recall_score(y_true, predictions, zero_division=0)), 6), + "f1": round(float(f1_score(y_true, predictions, zero_division=0)), 6), + "brier": round(float(brier_score_loss(y_true, probabilities)), 6), + "threshold": round(float(threshold), 4), + "confusion_matrix": {"tn": int(tn), "fp": int(fp), "fn": int(fn), "tp": int(tp)}, + } + + +def calibration(y_true: pd.Series, probabilities: np.ndarray) -> list[dict]: + bins = pd.DataFrame({"actual": y_true.to_numpy(), "probability": probabilities}) + bins["bin"] = pd.cut(bins["probability"], np.linspace(0, 1, 11), include_lowest=True) + rows = [] + for label, group in bins.groupby("bin", observed=True): + rows.append( + { + "bin": str(label), + "rows": int(len(group)), + "mean_probability": round(float(group["probability"].mean()), 6), + "observed_rate": round(float(group["actual"].mean()), 6), + } + ) + return rows + + +def error_slices(frame: pd.DataFrame, probabilities: np.ndarray, threshold: float) -> list[dict]: + scored = frame.copy() + scored["probability"] = probabilities + rows: list[dict] = [] + dimensions = { + "carrier": "Reporting_Airline", + "origin": "Origin", + "destination": "Dest", + "route_volume_band": "route_volume_band", + "departure_block": "departure_block", + "calendar_month": "Month", + } + for dimension, column in dimensions.items(): + for value, group in scored.groupby(column): + if len(group) < 25: + continue + result = metrics(group["target"], group["probability"].to_numpy(), threshold) + rows.append({"dimension": dimension, "value": str(value), **result}) + return rows + + +def export_logistic(model: Pipeline, threshold: float, report: dict, config: dict) -> dict: + preprocessor: ColumnTransformer = model.named_steps["features"] + classifier: LogisticRegression = model.named_steps["model"] + coefficients = classifier.coef_[0] + numeric_pipeline: Pipeline = preprocessor.named_transformers_["numeric"] + scaler: StandardScaler = numeric_pipeline.named_steps["scale"] + categorical_pipeline: Pipeline = preprocessor.named_transformers_["categorical"] + encoder: OneHotEncoder = categorical_pipeline.named_steps["encode"] + + numeric = {} + offset = 0 + for index, name in enumerate(NUMERIC): + numeric[name] = { + "mean": float(scaler.mean_[index]), + "scale": float(scaler.scale_[index]), + "coefficient": float(coefficients[index]), + } + offset += len(NUMERIC) + categorical = {} + for name, categories in zip(CATEGORICAL, encoder.categories_): + categorical[name] = { + str(category): float(coefficient) + for category, coefficient in zip(categories, coefficients[offset : offset + len(categories)]) + } + offset += len(categories) + + return { + "schema_version": 1, + "model_version": config["model_version"], + "prediction_time": "T-24h", + "trained_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), + "intercept": float(classifier.intercept_[0]), + "threshold": threshold, + "numeric": numeric, + "categorical": categorical, + "data_manifest_sha256": file_sha256(MANIFEST_PATH), + "test_metrics": report["logistic_regression"]["test"], + "limitations": [ + "Retrospective public BTS data is not a live airline feed.", + "Schedule-only features omit weather, maintenance, crew, and network disruptions.", + "Performance can drift outside the evaluated April-June 2025 period.", + ], + } + + +def write_calibration_svg(points: list[dict]) -> None: + width, height, pad = 640, 420, 55 + plot_w, plot_h = width - 2 * pad, height - 2 * pad + polyline = " ".join( + f"{pad + point['mean_probability'] * plot_w:.1f},{height - pad - point['observed_rate'] * plot_h:.1f}" + for point in points + ) + circles = "".join( + f'' + for point in points + ) + svg = f''' +Test-set calibration + + +{circles} + +Mean predicted probability +Observed delay rate +''' + CALIBRATION_PATH.write_text(svg, encoding="utf-8") + + +def main() -> None: + config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) + manifest = json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + frame = prepare(pd.read_csv(DATA_PATH)) + splits = split(frame) + if any(part.empty for part in splits.values()): + raise ValueError("Chronological train, validation, and test splits must all be non-empty") + + numeric_pipeline = Pipeline( + [("impute", SimpleImputer(strategy="median")), ("scale", StandardScaler())] + ) + categorical_pipeline = Pipeline( + [ + ("impute", SimpleImputer(strategy="most_frequent")), + ("encode", OneHotEncoder(handle_unknown="ignore")), + ] + ) + logistic = Pipeline( + [ + ( + "features", + ColumnTransformer( + [("numeric", numeric_pipeline, NUMERIC), ("categorical", categorical_pipeline, CATEGORICAL)] + ), + ), + ( + "model", + LogisticRegression( + solver="liblinear", + max_iter=400, + random_state=int(config["random_seed"]), + ), + ), + ] + ) + tree = Pipeline( + [ + ( + "features", + ColumnTransformer( + [ + ("numeric", SimpleImputer(strategy="median"), NUMERIC), + ( + "categorical", + Pipeline( + [ + ("impute", SimpleImputer(strategy="most_frequent")), + ( + "encode", + OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1), + ), + ] + ), + CATEGORICAL, + ), + ] + ), + ), + ( + "model", + HistGradientBoostingClassifier( + max_iter=80, + max_leaf_nodes=31, + learning_rate=0.08, + l2_regularization=1.0, + random_state=int(config["random_seed"]), + ), + ), + ] + ) + + x_train, y_train = splits["train"][FEATURES], splits["train"]["target"] + x_validation, y_validation = splits["validation"][FEATURES], splits["validation"]["target"] + x_test, y_test = splits["test"][FEATURES], splits["test"]["target"] + report = { + "schema_version": 1, + "model_version": config["model_version"], + "data_manifest_sha256": file_sha256(MANIFEST_PATH), + "split_summary": { + name: {"rows": len(part), "positive_prevalence": round(float(part["target"].mean()), 6)} + for name, part in splits.items() + }, + "source_exclusions": {}, + "missingness_before_imputation": { + name: { + column: round(float(part[column].isna().mean()), 6) + for column in FEATURES + } + for name, part in splits.items() + }, + "model_configuration": { + "logistic_regression": { + "solver": "liblinear", + "max_iter": 400, + "search_budget": "single documented baseline", + }, + "hist_gradient_boosting": { + "max_iter": 80, + "max_leaf_nodes": 31, + "learning_rate": 0.08, + "l2_regularization": 1.0, + "search_budget": "one fixed configuration; no test-set tuning", + }, + }, + "cost_assumption": { + "false_negative": config["validation_false_negative_cost"], + "false_positive": config["validation_false_positive_cost"], + }, + } + exclusion_groups = { + "train": [item for item in manifest["sources"] if item["month"] < "2025-01"], + "validation": [item for item in manifest["sources"] if "2025-01" <= item["month"] < "2025-04"], + "test": [item for item in manifest["sources"] if item["month"] >= "2025-04"], + } + report["source_exclusions"] = { + name: { + "source_rows": sum(item["rows_total"] for item in items), + "cancelled": sum(item["rows_cancelled"] for item in items), + "diverted": sum(item["rows_diverted"] for item in items), + } + for name, items in exclusion_groups.items() + } + test_prevalence = float(y_test.mean()) + report["majority_class"] = metrics(y_test, np.full(len(y_test), test_prevalence), 1.0) + + fitted_models = {} + for name, model in [("logistic_regression", logistic), ("hist_gradient_boosting", tree)]: + model.fit(x_train, y_train) + validation_probabilities = model.predict_proba(x_validation)[:, 1] + threshold = choose_threshold( + y_validation, + validation_probabilities, + float(config["validation_false_negative_cost"]), + float(config["validation_false_positive_cost"]), + ) + test_probabilities = model.predict_proba(x_test)[:, 1] + model_report = { + "validation": metrics(y_validation, validation_probabilities, threshold), + "test": metrics(y_test, test_probabilities, threshold), + } + for split_name in ("validation", "test"): + matrix = model_report[split_name]["confusion_matrix"] + model_report[split_name]["expected_cost"] = ( + matrix["fn"] * float(config["validation_false_negative_cost"]) + + matrix["fp"] * float(config["validation_false_positive_cost"]) + ) + report[name] = model_report + fitted_models[name] = (model, threshold, test_probabilities) + + logistic_model, logistic_threshold, logistic_test_probabilities = fitted_models["logistic_regression"] + report["calibration"] = calibration(y_test, logistic_test_probabilities) + slices = error_slices(splits["test"], logistic_test_probabilities, logistic_threshold) + report["error_slice_rows"] = len(slices) + + MODEL_PATH.parent.mkdir(parents=True, exist_ok=True) + METRICS_PATH.parent.mkdir(parents=True, exist_ok=True) + CARD_PATH.parent.mkdir(parents=True, exist_ok=True) + MODEL_PATH.write_text( + json.dumps(export_logistic(logistic_model, logistic_threshold, report, config), indent=2) + "\n", + encoding="utf-8", + ) + METRICS_PATH.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + with SLICES_PATH.open("w", newline="", encoding="utf-8") as stream: + writer = csv.DictWriter(stream, fieldnames=list(slices[0].keys())) + writer.writeheader() + writer.writerows(slices) + write_calibration_svg(report["calibration"]) + + logistic_test = report["logistic_regression"]["test"] + tree_test = report["hist_gradient_boosting"]["test"] + exclusions = report["source_exclusions"] + CARD_PATH.write_text( + f"# Flight-delay model card\n\n" + f"**Version:** `{config['model_version']}` \n" + f"**Prediction:** probability of arrival at least 15 minutes late, evaluated at T-24h \n" + f"**Data:** official BTS Reporting Carrier On-Time Performance, {config['start_month']} through {config['end_month']}\n\n" + f"## Evaluation\n\n" + f"The chronological test period is April-June 2025 ({logistic_test['rows']:,} sampled eligible flights; prevalence {logistic_test['positive_prevalence']:.1%}). " + f"The operating threshold `{logistic_test['threshold']:.3f}` was selected only on January-March 2025 validation data using a 5:1 false-negative/false-positive cost assumption.\n\n" + f"| Split | Sample rows | Positive prevalence | Source rows | Cancelled excluded | Diverted excluded |\n" + f"| --- | ---: | ---: | ---: | ---: | ---: |\n" + f"| Train (Jan-Dec 2024) | {report['split_summary']['train']['rows']:,} | {report['split_summary']['train']['positive_prevalence']:.1%} | {exclusions['train']['source_rows']:,} | {exclusions['train']['cancelled']:,} | {exclusions['train']['diverted']:,} |\n" + f"| Validation (Jan-Mar 2025) | {report['split_summary']['validation']['rows']:,} | {report['split_summary']['validation']['positive_prevalence']:.1%} | {exclusions['validation']['source_rows']:,} | {exclusions['validation']['cancelled']:,} | {exclusions['validation']['diverted']:,} |\n" + f"| Test (Apr-Jun 2025) | {report['split_summary']['test']['rows']:,} | {report['split_summary']['test']['positive_prevalence']:.1%} | {exclusions['test']['source_rows']:,} | {exclusions['test']['cancelled']:,} | {exclusions['test']['diverted']:,} |\n\n" + f"Required schedule-feature missingness was 0% in all three sampled splits before imputation. The tree comparison used one fixed configuration (80 iterations, 31 leaves, 0.08 learning rate); no test-set tuning was performed.\n\n" + f"| Model | ROC-AUC | PR-AUC | Precision | Recall | F1 | Brier | Expected cost |\n" + f"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n" + f"| Portable logistic (deployed) | {logistic_test['roc_auc']:.3f} | {logistic_test['pr_auc']:.3f} | {logistic_test['precision']:.3f} | {logistic_test['recall']:.3f} | {logistic_test['f1']:.3f} | {logistic_test['brier']:.3f} | {logistic_test['expected_cost']} |\n" + f"| Histogram gradient boosting | {tree_test['roc_auc']:.3f} | {tree_test['pr_auc']:.3f} | {tree_test['precision']:.3f} | {tree_test['recall']:.3f} | {tree_test['f1']:.3f} | {tree_test['brier']:.3f} | {tree_test['expected_cost']} |\n\n" + f"![Calibration curve](calibration.svg)\n\n" + f"Full machine-readable metrics are in [`artifacts/reports/delay-model-metrics.json`](../../artifacts/reports/delay-model-metrics.json); " + f"carrier, airport, route-volume, time-block, and month slices are in [`error-slices.csv`](../../artifacts/reports/error-slices.csv).\n\n" + f"## Intended use\n\nPortfolio-grade retrospective decision support and API demonstration. It is not approved for dispatch, passenger promises, staffing, or automated adverse decisions.\n\n" + f"## Model and artifact safety\n\nThe deployed logistic model is exported as inspectable JSON coefficients, scaling values, categories, threshold, metrics, and the data-manifest checksum. The API does not execute pickle or joblib artifacts.\n\n" + f"## Limitations\n\n- Schedule-only predictors cannot see weather, maintenance, crew, or same-day network disruptions.\n- The 90,000-row dataset is a deterministic monthly sample, not the full BTS population.\n- Public retrospective performance is not production accuracy; drift monitoring and airline-specific validation are required.\n- Category-level error slices can expose uneven performance and must be reviewed before operational use.\n", + encoding="utf-8", + ) + print(f"Wrote model {MODEL_PATH} and report {METRICS_PATH}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_api.py b/tests/test_api.py index 41c9e22..0baa60a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -74,3 +74,45 @@ def test_validation_rejects_naive_timestamp(tmp_path: Path) -> None: response = client.post("/v1/events", json=payload) assert response.status_code == 422 + + +def test_schedule_only_delay_prediction_is_versioned(tmp_path: Path) -> None: + app = create_app(str(tmp_path / "test.db")) + payload = { + "flight_date": "2026-08-10", + "reporting_airline": "DL", + "origin": "iad", + "destination": "atl", + "crs_departure_time": 815, + "crs_elapsed_time": 115, + "distance": 534, + "distance_group": 3, + } + with TestClient(app) as client: + prediction = client.post("/v1/predictions/delay", json=payload) + metadata = client.get("/v1/models/delay/metadata") + + assert prediction.status_code == 200 + assert 0 <= prediction.json()["probability"] <= 1 + assert prediction.json()["prediction_time"] == "T-24h" + assert prediction.json()["model_version"] == "bts-schedule-logistic-v1" + assert metadata.status_code == 200 + assert metadata.json()["model_version"] == prediction.json()["model_version"] + + +def test_prediction_rejects_invalid_hhmm_time(tmp_path: Path) -> None: + app = create_app(str(tmp_path / "test.db")) + payload = { + "flight_date": "2026-08-10", + "reporting_airline": "DL", + "origin": "IAD", + "destination": "ATL", + "crs_departure_time": 1265, + "crs_elapsed_time": 115, + "distance": 534, + "distance_group": 3, + } + with TestClient(app) as client: + response = client.post("/v1/predictions/delay", json=payload) + + assert response.status_code == 422