feat(hle): add hle_250 — fixed 250-question subset, 10 epochs, OpenRouter judge default

docs/snippets/benchmarks.data.mdx

@@ -5158,6 +5158,20 @@ export const benchmarksData = [
     "function_name": "hle",
     "is_alpha": false
   },
+  {
+    "name": "Humanity's Last Exam (250)",
+    "description": "Fixed 250-question text-only HLE subset, stratified over categories, 10 epochs averaged at temperature=1.0/top_p=0.95 - mirrors the shape of Artificial Analysis's Endpoint Accuracy Index HLE-250",
+    "category": "core",
+    "tags": [
+      "knowledge",
+      "reasoning",
+      "text-only",
+      "graded",
+      "frontier"
+    ],
+    "function_name": "hle_250",
+    "is_alpha": false
+  },
   {
     "name": "Humanity's Last Exam (Text-Only)",
     "description": "Text-only variant of HLE with multi-modal questions filtered out - evaluates models without vision capabilities on text-based questions from the frontier of human knowledge",

src/openbench/config.py

@@ -670,6 +670,14 @@ class EvalGroup:
         function_name="hle_text",
         subtask=True,
     ),
+    "hle_250": BenchmarkMetadata(
+        name="Humanity's Last Exam (250)",
+        description="Fixed 250-question text-only HLE subset, stratified over categories, 10 epochs averaged at temperature=1.0/top_p=0.95 - mirrors the shape of Artificial Analysis's Endpoint Accuracy Index HLE-250",
+        category="core",
+        tags=["knowledge", "reasoning", "text-only", "graded", "frontier"],
+        module_path="openbench.evals.hle",
+        function_name="hle_250",
+    ),
     "mmstar": BenchmarkMetadata(
         name="MMStar",
         description="MMStar benchmark for measuring multi-modal gain and leakage via coordinated vision and text ablations",

src/openbench/datasets/hle.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Union, cast
+from typing import Any, Collection, List, Optional, Union, cast
 
 from inspect_ai.dataset import Dataset, Sample, MemoryDataset, hf_dataset
 from inspect_ai.model import ChatMessageUser, ContentText, ContentImage
@@ -47,11 +47,17 @@ def record_to_sample(record: dict) -> Sample:
     )
 
 
-def get_dataset(text_only: bool = False) -> Dataset:
+def get_dataset(
+    text_only: bool = False,
+    ids: Optional[Collection[str]] = None,
+    name: Optional[str] = None,
+) -> Dataset:
     """Load the HLE (Humanity's Last Exam) dataset.
 
     Args:
         text_only: If True, filter out multi-modal questions with images
+        ids: If given, keep only the questions with these ids
+        name: Dataset name override (defaults to hle/hle_text)
 
     Returns:
         Dataset with HLE questions and answers
@@ -75,4 +81,14 @@ def get_dataset(text_only: bool = False) -> Dataset:
     else:
         dataset_name = "hle"
 
-    return MemoryDataset(samples=samples, name=dataset_name)
+    if ids is not None:
+        wanted = set(ids)
+        samples = [s for s in samples if s.id in wanted]
+        if len(samples) != len(wanted):
+            missing = wanted - {s.id for s in samples}
+            raise ValueError(
+                f"HLE subset ids not found in the dataset: {sorted(missing)[:5]}"
+                f" ({len(missing)} missing)"
+            )
+
+    return MemoryDataset(samples=samples, name=name or dataset_name)

src/openbench/datasets/hle_250_ids.py

@@ -0,0 +1,262 @@
+"""The fixed hle_250 question set: 250 text-only HLE questions.
+
+Selected once and committed as data so the subset can never drift with
+dataset reordering: proportional stratification over the 8 HLE categories
+(2,158 text-only questions, May 2025 revision), sampled per-category with
+random.Random(20260807) over category-sorted id lists, largest-remainder
+rounding to reach exactly 250. Regenerating requires deliberately editing
+this file; scores before and after are not comparable.
+"""
+
+HLE_250_IDS = frozenset({
+    "66b727d367968fa27f2dddda",
+    "66e4cdec11c64a7e4051b2d9",
+    "66e883265ab37f0a7da089be",
+    "66e8d3ed713a83e8aeddc2f5",
+    "66e94bf0d61a79cca258c675",
+    "66e962f814e5657e81349bfa",
+    "66ea12d21684c1846037a28d",
+    "66ea36e7aa2256fb4c995faf",
+    "66ea542e4509a12ed7db1d7b",
+    "66eae565b2e7406d498f8cc9",
+    "66eaeaa8afe364e4f477b777",
+    "66eb105d5f6e64fbae56b0fe",
+    "66eb206f99363a31f68414bc",
+    "66eb2d7378d12ffd1e4d123d",
+    "66eb35e9e3d7202c68806272",
+    "66eb36fbb38dbcd7910ceafc",
+    "66ed28dfde312a049a78b3ba",
+    "66ed5611d94c2fb87ded8826",
+    "66ed5e6a1d24f687ee9b06d1",
+    "66ed93471cbe5da13351cd67",
+    "66ed985a7b0ffebd9fae6993",
+    "66ede973564cb32b38cc8a4a",
+    "66edf49c46d28f04618d6498",
+    "66ee93ba02314f06feb186cf",
+    "66eea759f76fda99cec44de6",
+    "66eeea0187ec4a6a04092e78",
+    "66f05a0644e8181fd0e3f69b",
+    "66f05d93454a69621ec1badc",
+    "66f0bc36c6a29a8fa8010813",
+    "66f0dab31f3d7fb98443ad69",
+    "66f1b5cec75c3ece07124859",
+    "66f1e0138e382acde4df9a95",
+    "66f28a89f1256af80477e15d",
+    "66f28cc8b866ea3f1f4e95f5",
+    "66f2cda3b508188b6e7328a8",
+    "66f3117acff7b10052fe1e3d",
+    "66f3c52f6d142335350bb6e5",
+    "66f402add1c77d20ca3338ef",
+    "66f47471bba0ce38e1ca559f",
+    "66f4979d41a8777b61cf391a",
+    "66f5c96b4ae309d465eeca47",
+    "66f5fd3235147083021a66c6",
+    "66f6b73a1b586571e550784f",
+    "66f79e7259706e88d28a23b5",
+    "66f864c8f7c10f2d2ba396df",
+    "66fa67dc29908114d8954b55",
+    "66fb8135483861eb2d0252a3",
+    "66fc539cfb0c1cf50794a0e2",
+    "66fc5e8f98a7264ef58309b9",
+    "66fc5ed440e3b3e56869687f",
+    "66fc82b2bb24336da864daef",
+    "66fc8c821d39fbf6d8bcdd11",
+    "66fcbe5b68476a89da553c97",
+    "66fcde117e54294eb4a8fbba",
+    "66fe796bc12cbe6aeaa26e69",
+    "66feef700e977ac289068c27",
+    "66ffcfa0864258b2f971a80c",
+    "67015a7f6a2b21f149f3aaba",
+    "6701c3cb38b706939b30cb0b",
+    "6702db18a423c5b9f9c1c49c",
+    "670417b84f1cdb9711ec68d6",
+    "670426b4d831bdf30d3c03cb",
+    "670477044580e464f8bff2a3",
+    "6704d297f486e216becd613f",
+    "670663d687c53b9e6fa1dc8f",
+    "67073b43de74df1472536be3",
+    "67085c82e22de5e37e883351",
+    "67099b940109535a956a14ab",
+    "670b02d5560fcdf78354fad0",
+    "670c8b10148f2a113537c8f6",
+    "670ca1456731aa001b9ba021",
+    "670daa682c5d078fa33fba12",
+    "670f511063b765ce8edde772",
+    "670fb58f2ca6bea76e9354a7",
+    "670fe01af99389b3c7942185",
+    "670fe03ef99389b3c7942186",
+    "670ffefcdf4931d858723e36",
+    "671021ffd741deec4ddbfc48",
+    "6712c2616313b64cf8499927",
+    "67149cf883cc7720158c1110",
+    "67153bd7f588f3f15b038f5b",
+    "6715b2dab3287d3dd4db03d6",
+    "6715beb7b3287d3dd4db03e6",
+    "671683479e5d26a2905120dc",
+    "671689ae68b8747c96268936",
+    "6717cfeee8995e9e60504f33",
+    "6718c479034f8ddb2f67a346",
+    "6719e946a81614142be02bec",
+    "671ae4d6fa568baec7785817",
+    "671bb0b0863984716ffe8850",
+    "671bb1348b80a27571baf0d3",
+    "671bea6afd01315eb4f1c376",
+    "671bef23b55b44cbffbf0972",
+    "671c0d6f66154e5cf55994de",
+    "671d999f18a4da3122fd2118",
+    "671d9c62355c956ce7de5a89",
+    "671db266fe1146e348ef1267",
+    "671e3d672637abea9c147ba1",
+    "671ebaf92a7c16b748fd2709",
+    "671f612d12bc18b3bf57dd89",
+    "671f941e919b83ce036e80d9",
+    "671fb0b7298c0d11670fc561",
+    "671fc9508c7c5056012df16a",
+    "67202419dbc5086c0bdd4e0b",
+    "67202822dbc5086c0bdd4e1c",
+    "6720449622c03e062e242dd2",
+    "67205ffef58a25138266a364",
+    "6720feb00d9c62225dd12bd5",
+    "6720ff8a198910fd26b01224",
+    "672102b6febecf1a8b9b0845",
+    "67216f0abddbccdce9955e93",
+    "672170613659e79885449d5a",
+    "672179e4c8cc0ac89395e8d0",
+    "67217fba262eafa82562cc2c",
+    "6721a513823e82a699228953",
+    "6721ac0547bc48850f17f779",
+    "6721c8e11b5a8e4cb0e9079b",
+    "6722039010af9c566d42b825",
+    "6722198c0628f6647e3d591e",
+    "672235a01e36ca7369b8b157",
+    "672262d988e407d7eb07744d",
+    "67229ab808748295331b3e1b",
+    "6722a52340918481038cda8e",
+    "6722c0ddd60ad77b5374fdc9",
+    "6722d9de6bc44598e1fd09d6",
+    "67232241f19eb2a69ee319fc",
+    "672378554b5264ad52901028",
+    "6723977a2460a4b2c86caee8",
+    "6723a613f747d32c6b0b65dc",
+    "6723bf0d71d8a82752075279",
+    "6723c2226e47cec0509b5cbb",
+    "6723d5524a5a9552dc3d8836",
+    "6723f4e0cd09615a3afc3d0d",
+    "6723fd7d1e085509e462323e",
+    "67241990a1bd6b06dafb7052",
+    "67243ab4c287b71df6f486c0",
+    "672458e95f170d6ed834f340",
+    "672473c50c7275f41d6d1e63",
+    "6724970787a3fb903b6a4663",
+    "6724d70af70a476bbcaa32e1",
+    "6724d81d42ec04c22a24aa83",
+    "6724ea8fef7ca78ae041e22d",
+    "6724fe91ea5926938a631b9c",
+    "6724ff0dea5926938a631b9e",
+    "6725107c97743d26179494c6",
+    "6725255ee6807ea2c8372c3c",
+    "672536736bc3970bc3e12ca5",
+    "67254b9b3f37dfb9cf080a95",
+    "67256b14ac4f9591b137e180",
+    "67258391e0340e3face2bc7b",
+    "6725adefc6537022833a1e04",
+    "6725b1b58255543498bdcbd1",
+    "6725bb348255543498bdcbed",
+    "6725cc85569cf0c62da64d29",
+    "6725e382086428ce4e2fa8d6",
+    "6725ed80de551b21db6a0f29",
+    "6725f6cebbcb646b2076e64b",
+    "672600b226992c47ce3a7efe",
+    "67260a2a8f9ce8a272ed4e9a",
+    "6726b934d4553efaee99c556",
+    "6726ee5f8825660211cd8a70",
+    "6727716f45a3c3a9020e2732",
+    "6727dd5c5e987b1e8c17f8d9",
+    "67283cfe8a55429d9a6f5a56",
+    "6728696b7c5ceb52207dc9d8",
+    "6728c038c556bb2fdda61dd7",
+    "6728d1e3a8053eddd7a7f24a",
+    "6729d4704648ce900654fb6d",
+    "672a30472091cee6de17ebd1",
+    "672e6368958c5d0efe9d037d",
+    "672f4434e9c13daba078d693",
+    "672ff71e724ca22f42c0ff85",
+    "6730a9be58ef965949f1faa4",
+    "67325d2d189b1ffa84c76d55",
+    "6732ca5730bf2e525136b688",
+    "67332b7198af9f49ad5d743a",
+    "67335820c7d8c66591e6dfc7",
+    "673364562c46590b377046dd",
+    "67339e9c25c864e84bf1a351",
+    "6733fca10a354898fb579c2e",
+    "67345cb7fe6cc0ffd493d6fe",
+    "673484483ad1f0cf44dd1ca6",
+    "6734f24a08351f5bdfd38627",
+    "673586ad6ea2c89a57e3bfb7",
+    "6735a2d69caf928ddf5957ed",
+    "6735bfec24a805ed5fc0d055",
+    "6735cd220c703cedb6f1646a",
+    "673634d36299ad951a699b97",
+    "67367af29ba049ad64182820",
+    "67367cbd06a61a42cae30293",
+    "6736ab1af0b1f35585e14f7d",
+    "6736d80ac3870d903d810cb3",
+    "6736ddc7ab70ca0b4ee6a2e6",
+    "6736e17253926b38af8c205e",
+    "6736efd7c62053058bf5eef6",
+    "67370a26144baed9dcabbe11",
+    "6737382a90a20eb348edbe23",
+    "673797594656f5343e5d35db",
+    "6737cd443b2291333fbb48b6",
+    "6737d3148ebd5a3e08cacd4d",
+    "67380ecdb808e1bf292d214e",
+    "6738243f9b3842b348242430",
+    "673829d59b3842b348242450",
+    "67387af4aa63aa9bc8096f48",
+    "6738d6a4475748524584b23c",
+    "67393de74954110962f03245",
+    "6739479743b2a5ca5cf1a9dd",
+    "6739674739118cf30f5f1075",
+    "67396779f2c1f4ba83e0a0a2",
+    "673971a55c3de09264d6d373",
+    "6739e82ba0d19bb8d127ad6c",
+    "6739ec066a53385958bca5e8",
+    "673a5d4f6bd877b81c665adc",
+    "673a73d1addb9293360a5d07",
+    "673a7987334115f5123f9c64",
+    "673ae99a3022b9e17f89d1b6",
+    "673b631505be2302d4445ece",
+    "673b6ca1a646afb63047219f",
+    "673b7ee0b22ebd24e26d4ece",
+    "673cd3821cf7bd0b7cc91fe5",
+    "673e64979c1787f33fd0b983",
+    "673e66f59f77eeca0292a3b8",
+    "673eb1cfadce15d9254eb2ac",
+    "673f8934ccb23d2ccc360e4e",
+    "67435ecfa59a37c4a0f5d579",
+    "674362ab6b7aa212dfb90595",
+    "67458d2109d194da0dfd7cef",
+    "6749b01bf1f47be487dd2508",
+    "6749b2994fd3108e46cd57d7",
+    "674f07ffffb28de6c182c458",
+    "67526499b42d785cf1cb1025",
+    "67581f18abd39842c40bd2fd",
+    "675b894146d55c2ffdebd8f5",
+    "67631f198b502023c4567380",
+    "67643038c1cda8ef39debd4b",
+    "67666a593e38774fc651f9f3",
+    "6769bce8a4435c3c69035510",
+    "676b5d9ef4feb105f601d808",
+    "676d4db9cd83de685f85ed01",
+    "676d66a775870eef1bf0f9ce",
+    "6771857cb5c33d92de2a4bb4",
+    "6775acff518cb15c3772295e",
+    "6775b1ab76b69969709e80a7",
+    "677b991a7304702305835cd6",
+    "677bd8f5247a21d7a948db6c",
+    "677c9bc140c35493a85c3c33",
+    "677e6a6841ef4b6937c21b7c",
+    "67838fb04c753fa6da434e1c",
+    "678ae26a0b0bc44cadbb173f",
+    "67ae17cf1509e1d55b4a0eec",
+})

src/openbench/evals/hle.py

@@ -1,7 +1,8 @@
-from inspect_ai import task, Task
+from inspect_ai import task, Task, Epochs
 from inspect_ai.solver import generate, system_message
 from inspect_ai.model import GenerateConfig
 from openbench.datasets.hle import get_dataset
+from openbench.datasets.hle_250_ids import HLE_250_IDS
 from openbench.scorers.hle import hle_scorer
 
 
@@ -70,3 +71,49 @@ def hle_text(
             max_tokens=max_tokens,  # HLE recommends at least 8192 for reasoning models
         ),
     )
+
+
+@task
+def hle_250(
+    grader_model: str = "openrouter/openai/gpt-5.6-luna",
+    grader_reasoning_effort: str = "medium",
+    max_tokens: int = 8192,
+    temperature: float = 1.0,
+    top_p: float = 0.95,
+) -> Task:
+    """Humanity's Last Exam (250): a fixed 250-question text-only HLE subset.
+
+    Mirrors the shape of Artificial Analysis's Endpoint Accuracy Index HLE-250
+    (their exact subset is private): 250 text-only questions stratified over the
+    HLE categories, 10 epochs averaged, sampled at temperature=1.0/top_p=0.95,
+    graded by an equality-checker judge. The question ids live in
+    hle_250_ids.py; the judge defaults to GPT-5.6 Luna (medium) via OpenRouter,
+    which needs OPENROUTER_API_KEY.
+
+    Args:
+        grader_model: Model to use for grading responses
+        grader_reasoning_effort: Reasoning effort for the grader model
+        max_tokens: Maximum tokens for model response
+        temperature: Sampling temperature (non-zero, averaged over epochs)
+        top_p: Nucleus sampling bound
+
+    Returns:
+        Task configured for HLE-250 evaluation
+    """
+    return Task(
+        dataset=get_dataset(text_only=True, ids=HLE_250_IDS, name="hle_250"),
+        solver=[
+            system_message(HLE_SYSTEM_PROMPT),
+            generate(),
+        ],
+        epochs=Epochs(10, "mean"),
+        scorer=hle_scorer(
+            model=grader_model, reasoning_effort=grader_reasoning_effort
+        ),
+        name="hle_250",
+        config=GenerateConfig(
+            temperature=temperature,
+            top_p=top_p,
+            max_tokens=max_tokens,
+        ),
+    )