diff --git a/docs/snippets/benchmarks.data.mdx b/docs/snippets/benchmarks.data.mdx index d734fc77..8b0d5a40 100644 --- a/docs/snippets/benchmarks.data.mdx +++ b/docs/snippets/benchmarks.data.mdx @@ -5158,6 +5158,20 @@ export const benchmarksData = [ "function_name": "hle", "is_alpha": false }, + { + "name": "Humanity's Last Exam (250)", + "description": "Fixed 250-question text-only HLE subset, stratified over categories, 10 epochs averaged at temperature=1.0/top_p=0.95 - mirrors the shape of Artificial Analysis's Endpoint Accuracy Index HLE-250", + "category": "core", + "tags": [ + "knowledge", + "reasoning", + "text-only", + "graded", + "frontier" + ], + "function_name": "hle_250", + "is_alpha": false + }, { "name": "Humanity's Last Exam (Text-Only)", "description": "Text-only variant of HLE with multi-modal questions filtered out - evaluates models without vision capabilities on text-based questions from the frontier of human knowledge", diff --git a/src/openbench/config.py b/src/openbench/config.py index 9070ee7d..0d9679b4 100644 --- a/src/openbench/config.py +++ b/src/openbench/config.py @@ -670,6 +670,14 @@ class EvalGroup: function_name="hle_text", subtask=True, ), + "hle_250": BenchmarkMetadata( + name="Humanity's Last Exam (250)", + description="Fixed 250-question text-only HLE subset, stratified over categories, 10 epochs averaged at temperature=1.0/top_p=0.95 - mirrors the shape of Artificial Analysis's Endpoint Accuracy Index HLE-250", + category="core", + tags=["knowledge", "reasoning", "text-only", "graded", "frontier"], + module_path="openbench.evals.hle", + function_name="hle_250", + ), "mmstar": BenchmarkMetadata( name="MMStar", description="MMStar benchmark for measuring multi-modal gain and leakage via coordinated vision and text ablations", diff --git a/src/openbench/datasets/hle.py b/src/openbench/datasets/hle.py index 463099e2..8f5d5e64 100644 --- a/src/openbench/datasets/hle.py +++ b/src/openbench/datasets/hle.py @@ -1,4 +1,4 @@ -from typing import Any, List, Union, cast +from typing import Any, Collection, List, Optional, Union, cast from inspect_ai.dataset import Dataset, Sample, MemoryDataset, hf_dataset from inspect_ai.model import ChatMessageUser, ContentText, ContentImage @@ -47,11 +47,17 @@ def record_to_sample(record: dict) -> Sample: ) -def get_dataset(text_only: bool = False) -> Dataset: +def get_dataset( + text_only: bool = False, + ids: Optional[Collection[str]] = None, + name: Optional[str] = None, +) -> Dataset: """Load the HLE (Humanity's Last Exam) dataset. Args: text_only: If True, filter out multi-modal questions with images + ids: If given, keep only the questions with these ids + name: Dataset name override (defaults to hle/hle_text) Returns: Dataset with HLE questions and answers @@ -75,4 +81,14 @@ def get_dataset(text_only: bool = False) -> Dataset: else: dataset_name = "hle" - return MemoryDataset(samples=samples, name=dataset_name) + if ids is not None: + wanted = set(ids) + samples = [s for s in samples if s.id in wanted] + if len(samples) != len(wanted): + missing = wanted - {s.id for s in samples} + raise ValueError( + f"HLE subset ids not found in the dataset: {sorted(missing)[:5]}" + f" ({len(missing)} missing)" + ) + + return MemoryDataset(samples=samples, name=name or dataset_name) diff --git a/src/openbench/datasets/hle_250_ids.py b/src/openbench/datasets/hle_250_ids.py new file mode 100644 index 00000000..f65b098b --- /dev/null +++ b/src/openbench/datasets/hle_250_ids.py @@ -0,0 +1,262 @@ +"""The fixed hle_250 question set: 250 text-only HLE questions. + +Selected once and committed as data so the subset can never drift with +dataset reordering: proportional stratification over the 8 HLE categories +(2,158 text-only questions, May 2025 revision), sampled per-category with +random.Random(20260807) over category-sorted id lists, largest-remainder +rounding to reach exactly 250. Regenerating requires deliberately editing +this file; scores before and after are not comparable. +""" + +HLE_250_IDS = frozenset({ + "66b727d367968fa27f2dddda", + "66e4cdec11c64a7e4051b2d9", + "66e883265ab37f0a7da089be", + "66e8d3ed713a83e8aeddc2f5", + "66e94bf0d61a79cca258c675", + "66e962f814e5657e81349bfa", + "66ea12d21684c1846037a28d", + "66ea36e7aa2256fb4c995faf", + "66ea542e4509a12ed7db1d7b", + "66eae565b2e7406d498f8cc9", + "66eaeaa8afe364e4f477b777", + "66eb105d5f6e64fbae56b0fe", + "66eb206f99363a31f68414bc", + "66eb2d7378d12ffd1e4d123d", + "66eb35e9e3d7202c68806272", + "66eb36fbb38dbcd7910ceafc", + "66ed28dfde312a049a78b3ba", + "66ed5611d94c2fb87ded8826", + "66ed5e6a1d24f687ee9b06d1", + "66ed93471cbe5da13351cd67", + "66ed985a7b0ffebd9fae6993", + "66ede973564cb32b38cc8a4a", + "66edf49c46d28f04618d6498", + "66ee93ba02314f06feb186cf", + "66eea759f76fda99cec44de6", + "66eeea0187ec4a6a04092e78", + "66f05a0644e8181fd0e3f69b", + "66f05d93454a69621ec1badc", + "66f0bc36c6a29a8fa8010813", + "66f0dab31f3d7fb98443ad69", + "66f1b5cec75c3ece07124859", + "66f1e0138e382acde4df9a95", + "66f28a89f1256af80477e15d", + "66f28cc8b866ea3f1f4e95f5", + "66f2cda3b508188b6e7328a8", + "66f3117acff7b10052fe1e3d", + "66f3c52f6d142335350bb6e5", + "66f402add1c77d20ca3338ef", + "66f47471bba0ce38e1ca559f", + "66f4979d41a8777b61cf391a", + "66f5c96b4ae309d465eeca47", + "66f5fd3235147083021a66c6", + "66f6b73a1b586571e550784f", + "66f79e7259706e88d28a23b5", + "66f864c8f7c10f2d2ba396df", + "66fa67dc29908114d8954b55", + "66fb8135483861eb2d0252a3", + "66fc539cfb0c1cf50794a0e2", + "66fc5e8f98a7264ef58309b9", + "66fc5ed440e3b3e56869687f", + "66fc82b2bb24336da864daef", + "66fc8c821d39fbf6d8bcdd11", + "66fcbe5b68476a89da553c97", + "66fcde117e54294eb4a8fbba", + "66fe796bc12cbe6aeaa26e69", + "66feef700e977ac289068c27", + "66ffcfa0864258b2f971a80c", + "67015a7f6a2b21f149f3aaba", + "6701c3cb38b706939b30cb0b", + "6702db18a423c5b9f9c1c49c", + "670417b84f1cdb9711ec68d6", + "670426b4d831bdf30d3c03cb", + "670477044580e464f8bff2a3", + "6704d297f486e216becd613f", + "670663d687c53b9e6fa1dc8f", + "67073b43de74df1472536be3", + "67085c82e22de5e37e883351", + "67099b940109535a956a14ab", + "670b02d5560fcdf78354fad0", + "670c8b10148f2a113537c8f6", + "670ca1456731aa001b9ba021", + "670daa682c5d078fa33fba12", + "670f511063b765ce8edde772", + "670fb58f2ca6bea76e9354a7", + "670fe01af99389b3c7942185", + "670fe03ef99389b3c7942186", + "670ffefcdf4931d858723e36", + "671021ffd741deec4ddbfc48", + "6712c2616313b64cf8499927", + "67149cf883cc7720158c1110", + "67153bd7f588f3f15b038f5b", + "6715b2dab3287d3dd4db03d6", + "6715beb7b3287d3dd4db03e6", + "671683479e5d26a2905120dc", + "671689ae68b8747c96268936", + "6717cfeee8995e9e60504f33", + "6718c479034f8ddb2f67a346", + "6719e946a81614142be02bec", + "671ae4d6fa568baec7785817", + "671bb0b0863984716ffe8850", + "671bb1348b80a27571baf0d3", + "671bea6afd01315eb4f1c376", + "671bef23b55b44cbffbf0972", + "671c0d6f66154e5cf55994de", + "671d999f18a4da3122fd2118", + "671d9c62355c956ce7de5a89", + "671db266fe1146e348ef1267", + "671e3d672637abea9c147ba1", + "671ebaf92a7c16b748fd2709", + "671f612d12bc18b3bf57dd89", + "671f941e919b83ce036e80d9", + "671fb0b7298c0d11670fc561", + "671fc9508c7c5056012df16a", + "67202419dbc5086c0bdd4e0b", + "67202822dbc5086c0bdd4e1c", + "6720449622c03e062e242dd2", + "67205ffef58a25138266a364", + "6720feb00d9c62225dd12bd5", + "6720ff8a198910fd26b01224", + "672102b6febecf1a8b9b0845", + "67216f0abddbccdce9955e93", + "672170613659e79885449d5a", + "672179e4c8cc0ac89395e8d0", + "67217fba262eafa82562cc2c", + "6721a513823e82a699228953", + "6721ac0547bc48850f17f779", + "6721c8e11b5a8e4cb0e9079b", + "6722039010af9c566d42b825", + "6722198c0628f6647e3d591e", + "672235a01e36ca7369b8b157", + "672262d988e407d7eb07744d", + "67229ab808748295331b3e1b", + "6722a52340918481038cda8e", + "6722c0ddd60ad77b5374fdc9", + "6722d9de6bc44598e1fd09d6", + "67232241f19eb2a69ee319fc", + "672378554b5264ad52901028", + "6723977a2460a4b2c86caee8", + "6723a613f747d32c6b0b65dc", + "6723bf0d71d8a82752075279", + "6723c2226e47cec0509b5cbb", + "6723d5524a5a9552dc3d8836", + "6723f4e0cd09615a3afc3d0d", + "6723fd7d1e085509e462323e", + "67241990a1bd6b06dafb7052", + "67243ab4c287b71df6f486c0", + "672458e95f170d6ed834f340", + "672473c50c7275f41d6d1e63", + "6724970787a3fb903b6a4663", + "6724d70af70a476bbcaa32e1", + "6724d81d42ec04c22a24aa83", + "6724ea8fef7ca78ae041e22d", + "6724fe91ea5926938a631b9c", + "6724ff0dea5926938a631b9e", + "6725107c97743d26179494c6", + "6725255ee6807ea2c8372c3c", + "672536736bc3970bc3e12ca5", + "67254b9b3f37dfb9cf080a95", + "67256b14ac4f9591b137e180", + "67258391e0340e3face2bc7b", + "6725adefc6537022833a1e04", + "6725b1b58255543498bdcbd1", + "6725bb348255543498bdcbed", + "6725cc85569cf0c62da64d29", + "6725e382086428ce4e2fa8d6", + "6725ed80de551b21db6a0f29", + "6725f6cebbcb646b2076e64b", + "672600b226992c47ce3a7efe", + "67260a2a8f9ce8a272ed4e9a", + "6726b934d4553efaee99c556", + "6726ee5f8825660211cd8a70", + "6727716f45a3c3a9020e2732", + "6727dd5c5e987b1e8c17f8d9", + "67283cfe8a55429d9a6f5a56", + "6728696b7c5ceb52207dc9d8", + "6728c038c556bb2fdda61dd7", + "6728d1e3a8053eddd7a7f24a", + "6729d4704648ce900654fb6d", + "672a30472091cee6de17ebd1", + "672e6368958c5d0efe9d037d", + "672f4434e9c13daba078d693", + "672ff71e724ca22f42c0ff85", + "6730a9be58ef965949f1faa4", + "67325d2d189b1ffa84c76d55", + "6732ca5730bf2e525136b688", + "67332b7198af9f49ad5d743a", + "67335820c7d8c66591e6dfc7", + "673364562c46590b377046dd", + "67339e9c25c864e84bf1a351", + "6733fca10a354898fb579c2e", + "67345cb7fe6cc0ffd493d6fe", + "673484483ad1f0cf44dd1ca6", + "6734f24a08351f5bdfd38627", + "673586ad6ea2c89a57e3bfb7", + "6735a2d69caf928ddf5957ed", + "6735bfec24a805ed5fc0d055", + "6735cd220c703cedb6f1646a", + "673634d36299ad951a699b97", + "67367af29ba049ad64182820", + "67367cbd06a61a42cae30293", + "6736ab1af0b1f35585e14f7d", + "6736d80ac3870d903d810cb3", + "6736ddc7ab70ca0b4ee6a2e6", + "6736e17253926b38af8c205e", + "6736efd7c62053058bf5eef6", + "67370a26144baed9dcabbe11", + "6737382a90a20eb348edbe23", + "673797594656f5343e5d35db", + "6737cd443b2291333fbb48b6", + "6737d3148ebd5a3e08cacd4d", + "67380ecdb808e1bf292d214e", + "6738243f9b3842b348242430", + "673829d59b3842b348242450", + "67387af4aa63aa9bc8096f48", + "6738d6a4475748524584b23c", + "67393de74954110962f03245", + "6739479743b2a5ca5cf1a9dd", + "6739674739118cf30f5f1075", + "67396779f2c1f4ba83e0a0a2", + "673971a55c3de09264d6d373", + "6739e82ba0d19bb8d127ad6c", + "6739ec066a53385958bca5e8", + "673a5d4f6bd877b81c665adc", + "673a73d1addb9293360a5d07", + "673a7987334115f5123f9c64", + "673ae99a3022b9e17f89d1b6", + "673b631505be2302d4445ece", + "673b6ca1a646afb63047219f", + "673b7ee0b22ebd24e26d4ece", + "673cd3821cf7bd0b7cc91fe5", + "673e64979c1787f33fd0b983", + "673e66f59f77eeca0292a3b8", + "673eb1cfadce15d9254eb2ac", + "673f8934ccb23d2ccc360e4e", + "67435ecfa59a37c4a0f5d579", + "674362ab6b7aa212dfb90595", + "67458d2109d194da0dfd7cef", + "6749b01bf1f47be487dd2508", + "6749b2994fd3108e46cd57d7", + "674f07ffffb28de6c182c458", + "67526499b42d785cf1cb1025", + "67581f18abd39842c40bd2fd", + "675b894146d55c2ffdebd8f5", + "67631f198b502023c4567380", + "67643038c1cda8ef39debd4b", + "67666a593e38774fc651f9f3", + "6769bce8a4435c3c69035510", + "676b5d9ef4feb105f601d808", + "676d4db9cd83de685f85ed01", + "676d66a775870eef1bf0f9ce", + "6771857cb5c33d92de2a4bb4", + "6775acff518cb15c3772295e", + "6775b1ab76b69969709e80a7", + "677b991a7304702305835cd6", + "677bd8f5247a21d7a948db6c", + "677c9bc140c35493a85c3c33", + "677e6a6841ef4b6937c21b7c", + "67838fb04c753fa6da434e1c", + "678ae26a0b0bc44cadbb173f", + "67ae17cf1509e1d55b4a0eec", +}) diff --git a/src/openbench/evals/hle.py b/src/openbench/evals/hle.py index 92b877c6..6baab481 100644 --- a/src/openbench/evals/hle.py +++ b/src/openbench/evals/hle.py @@ -1,7 +1,8 @@ -from inspect_ai import task, Task +from inspect_ai import task, Task, Epochs from inspect_ai.solver import generate, system_message from inspect_ai.model import GenerateConfig from openbench.datasets.hle import get_dataset +from openbench.datasets.hle_250_ids import HLE_250_IDS from openbench.scorers.hle import hle_scorer @@ -70,3 +71,49 @@ def hle_text( max_tokens=max_tokens, # HLE recommends at least 8192 for reasoning models ), ) + + +@task +def hle_250( + grader_model: str = "openrouter/openai/gpt-5.6-luna", + grader_reasoning_effort: str = "medium", + max_tokens: int = 8192, + temperature: float = 1.0, + top_p: float = 0.95, +) -> Task: + """Humanity's Last Exam (250): a fixed 250-question text-only HLE subset. + + Mirrors the shape of Artificial Analysis's Endpoint Accuracy Index HLE-250 + (their exact subset is private): 250 text-only questions stratified over the + HLE categories, 10 epochs averaged, sampled at temperature=1.0/top_p=0.95, + graded by an equality-checker judge. The question ids live in + hle_250_ids.py; the judge defaults to GPT-5.6 Luna (medium) via OpenRouter, + which needs OPENROUTER_API_KEY. + + Args: + grader_model: Model to use for grading responses + grader_reasoning_effort: Reasoning effort for the grader model + max_tokens: Maximum tokens for model response + temperature: Sampling temperature (non-zero, averaged over epochs) + top_p: Nucleus sampling bound + + Returns: + Task configured for HLE-250 evaluation + """ + return Task( + dataset=get_dataset(text_only=True, ids=HLE_250_IDS, name="hle_250"), + solver=[ + system_message(HLE_SYSTEM_PROMPT), + generate(), + ], + epochs=Epochs(10, "mean"), + scorer=hle_scorer( + model=grader_model, reasoning_effort=grader_reasoning_effort + ), + name="hle_250", + config=GenerateConfig( + temperature=temperature, + top_p=top_p, + max_tokens=max_tokens, + ), + ) diff --git a/src/openbench/scorers/hle.py b/src/openbench/scorers/hle.py index 30418752..ac3d35a6 100644 --- a/src/openbench/scorers/hle.py +++ b/src/openbench/scorers/hle.py @@ -1,5 +1,5 @@ import re -from typing import Callable +from typing import Callable, Optional from inspect_ai.scorer import ( accuracy, scorer, @@ -8,7 +8,7 @@ Target, ) from inspect_ai.solver import TaskState -from inspect_ai.model import get_model, ChatMessageUser, Model +from inspect_ai.model import GenerateConfig, get_model, ChatMessageUser, Model from openbench.utils.text import extract_confidence_score from openbench.metrics.hle import hle_metrics @@ -56,17 +56,26 @@ def parse_judge_response(judge_response: str) -> tuple[str, str, int]: @scorer(metrics=[accuracy(), stderr(), hle_metrics()]) -def hle_scorer(model: str = "openai/o3-mini-2025-01-31") -> Callable: +def hle_scorer( + model: str = "openai/o3-mini-2025-01-31", + reasoning_effort: Optional[str] = None, +) -> Callable: """HLE scorer using model grading. Args: model: Model to use for grading (defaults to o3-mini-2025-01-31 as per HLE repo) + reasoning_effort: Reasoning effort for the grader model, when it supports one """ + grader_config = ( + GenerateConfig(reasoning_effort=reasoning_effort) + if reasoning_effort + else GenerateConfig() + ) async def score(state: TaskState, target: Target) -> Score: # Get the grader model - try default first, fallback if not available try: - grader_model: Model = get_model(model) + grader_model: Model = get_model(model, config=grader_config) except Exception: # Fallback to previous default judge model used in HLE try: